diff --git a/src/agent_env/env/envs/service_db.py b/src/agent_env/env/envs/service_db.py index c976e8bf3..d1bb6fb95 100644 --- a/src/agent_env/env/envs/service_db.py +++ b/src/agent_env/env/envs/service_db.py @@ -7,6 +7,7 @@ from agent_env.artifact import Artifact, DockerImageArtifact from agent_env.env.env import Env +from agent_env.env.gateway.constants import PGWEB_CONSOLE SERVICE_DB_PORT = 5432 DB_USER = "agentenv" @@ -15,7 +16,7 @@ # The service db's containers: compose service names and ports. DATABASE_SERVICE_NAME = "servicedb" -PGWEB_SERVICE_NAME = "pgweb" +PGWEB_SERVICE_NAME = PGWEB_CONSOLE DB_WEB_PORT = 8081 # pgweb: same port host-side and in-container (8081:8081) DB_MCP_SERVICE_NAME = "db-mcp" DB_MCP_PORT = 18767 # db-mcp host/tunnel port diff --git a/src/agent_env/env/gateway/console_session.py b/src/agent_env/env/gateway/console_session.py new file mode 100644 index 000000000..734b38f32 --- /dev/null +++ b/src/agent_env/env/gateway/console_session.py @@ -0,0 +1,123 @@ +"""A browser's session on the env's pgweb console, in place of the control token, which never reaches a browser. + +The deployment's owner mints a short link from the control token (``ConsoleSession.link``); the gateway trades it for a +cookie that opens ``/svc/pgweb/`` only. Both are ``.``: unix seconds, and the HMAC-SHA256 of +``link:`` or ``cookie:``, base64url without padding, under ``HMAC-SHA256(control token, +"pgweb-console-v1")``. A link or cookie minted for another deployment fails its signature. + +Every Modal tunnel is same-site with every other (``modal.host`` is not a public suffix), so SameSite alone would +not keep the cookie to its own deployment's pages. The cookie therefore counts only on a request from a page of the +gateway's own origin, or on a navigation to the console's index. +""" +from __future__ import annotations + +import base64 +import hashlib +import hmac +import logging +import re +import time +from collections.abc import Callable, Iterable +from urllib.parse import parse_qsl + +from .constants import PGWEB_CONSOLE + +LINK_PARAM = "pgweb_link" +COOKIE_NAME = "__Host-agentenv-pgweb" +CONSOLE_PREFIX = f"/svc/{PGWEB_CONSOLE}/" +LINK_MAX_SECONDS = 300 +SESSION_SECONDS = 3600 +_KEY_LABEL = b"pgweb-console-v1" +_SIGNED = re.compile(r"([0-9]{1,12})\.([A-Za-z0-9_-]{43})") +_LINK_IN_LOG = re.compile(rf"({LINK_PARAM}=)[^&\s\"']+") + + +class ConsoleSession: + """Mints and checks a deployment's pgweb links and cookies. ``now`` is the clock, in unix seconds.""" + + def __init__(self, control_token: str, now: Callable[[], float] = time.time): + self._key = hmac.new(control_token.encode(), _KEY_LABEL, hashlib.sha256).digest() + self._now = now + + def _sign(self, purpose: str, expires_at: int) -> str: + digest = hmac.new(self._key, f"{purpose}:{expires_at}".encode(), hashlib.sha256).digest() + return f"{expires_at}.{base64.urlsafe_b64encode(digest).rstrip(b'=').decode()}" + + def _valid(self, purpose: str, value: str, longest: int) -> bool: + match = _SIGNED.fullmatch(value) + if not match: + return False + expires_at = int(match[1]) + now = int(self._now()) + return now < expires_at <= now + longest and hmac.compare_digest(self._sign(purpose, expires_at), value) + + def link(self, seconds: int = 120) -> str: + return self._sign("link", int(self._now()) + seconds) + + def opens(self, link: str) -> bool: + return self._valid("link", link, LINK_MAX_SECONDS) + + def set_cookie(self) -> str: + value = self._sign("cookie", int(self._now()) + SESSION_SECONDS) + return (f"{COOKIE_NAME}={value}; Max-Age={SESSION_SECONDS}; Path=/; Secure; HttpOnly; SameSite=None; " + "Partitioned") + + def admits(self, scope: dict) -> bool: + """Whether ``scope``, a request to the pgweb console, carries a live cookie and comes from where one counts.""" + headers = scope.get("headers", ()) + if not any(self._valid("cookie", value, SESSION_SECONDS) for value in cookie_values(headers)): + return False + fetch = {name: value for name, value in headers if name in (b"sec-fetch-site", b"sec-fetch-mode")} + if fetch.get(b"sec-fetch-site") == b"same-origin": + return True + return (fetch.get(b"sec-fetch-mode") == b"navigate" and scope["method"] == "GET" + and scope["path"] == CONSOLE_PREFIX) + + def opening_link(self, scope: dict) -> str | None: + """The link a GET of the console's index carries, or None.""" + if scope["method"] != "GET" or scope["path"] != CONSOLE_PREFIX: + return None + return dict(parse_qsl(scope.get("query_string", b"").decode("latin-1"))).get(LINK_PARAM) + + +def cookie_values(headers: Iterable[tuple[bytes, bytes]]) -> list[str]: + """Every value the request's Cookie headers give the session cookie.""" + found = [] + for name, value in headers: + if name != b"cookie": + continue + for pair in value.decode("latin-1").split(";"): + key, _, cookie = pair.strip().partition("=") + if key == COOKIE_NAME: + found.append(cookie) + return found + + +def without_session_cookie(headers: Iterable[tuple[bytes, bytes]]) -> list[tuple[bytes, bytes]]: + """``headers`` with the session cookie taken out of each Cookie header, and a header left empty dropped.""" + kept = [] + for name, value in headers: + if name == b"cookie": + pairs = [pair for pair in value.decode("latin-1").split(";") + if pair.strip().partition("=")[0] != COOKIE_NAME] + if not any(pair.strip() for pair in pairs): + continue + value = ";".join(pairs).strip().encode("latin-1") + kept.append((name, value)) + return kept + + +class LinkRedaction(logging.Filter): + """Takes the link out of each log line that names one. It rewrites the argument that holds it, since uvicorn's + access log formatter reads its request line from the record's arguments.""" + + def filter(self, record: logging.LogRecord) -> bool: + if isinstance(record.msg, str) and LINK_PARAM in record.msg: + record.msg = _redacted(record.msg) + if isinstance(record.args, tuple): + record.args = tuple(_redacted(str(arg)) if LINK_PARAM in str(arg) else arg for arg in record.args) + return True + + +def _redacted(text: str) -> str: + return _LINK_IN_LOG.sub(r"\1", text) diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index 01ea981c2..149fbf64e 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -80,6 +80,8 @@ def data_plane_load_timeout_s(size_bytes: int | None) -> int: # The env's SQL consoles (pgweb, db-mcp), each as `key=url` on the env's network, comma-separated. The gateway serves # each at /svc//, every path of it behind its control token, and none of them when it has no token. CONSOLE_URLS_ENV = "CONSOLE_URLS" +# The console a browser may open with a link instead of the control token (`console_session`): pgweb's key. +PGWEB_CONSOLE = "pgweb" # The route a clock follower long-polls, with its read token. CLOCK_WATCH_PATH = "/clock/watch" CLOCK_FOLLOWERS_PATH = "/clock/followers" diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index 31e3460da..fbf79fbaa 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -72,6 +72,7 @@ WEBSITE_BROWSE_URL_PARAM, WELL_KNOWN_PATH, WILDCARD, + PGWEB_CONSOLE, GatewayMode, tool_name_clashes_text, ) @@ -81,6 +82,7 @@ from .child_sessions import ChildSessions, is_dead_session from .clock import Clock, ClockError, _iso from .clock_followers import ClockFollowers, parse_wait +from .console_session import ConsoleSession, LinkRedaction, without_session_cookie from .get_time import GET_TIME_TOOL_NAME, build_get_time_tool from .rest_proxy import ( ROLE_HEADER_BYTES, @@ -212,11 +214,13 @@ def __init__( log = logging.getLogger(name) if not any(isinstance(f, TokenRedaction) for f in log.filters): log.addFilter(TokenRedaction()) + log.addFilter(LinkRedaction()) self.gateway_mode = gateway_mode self._service_db_url = service_db_url self._control_bearer = f"Bearer {control_token}".encode() if control_token else None self._control_token_children = frozenset(control_token_children) self._consoles = dict(console_urls or {}) if control_token else {} + self._console_session = ConsoleSession(control_token) if PGWEB_CONSOLE in self._consoles else None if clashes := sorted(self._consoles.keys() & self.rest_proxy_urls.keys()): raise ValueError(f"console keys {clashes} are also backing servers' /svc keys") self._computers = frozenset(computers) @@ -620,8 +624,13 @@ def _asgi_app(self): async def app(scope, receive, send): if self._control_bearer and scope["type"] in ("http", "websocket"): headers = scope.get("headers", ()) + if scope["type"] == "http" and self._console_session and ( + link := self._console_session.opening_link(scope)) is not None: + await self._open_console(link)(scope, receive, send) + return if scope["type"] == "http" and self._takes_control_token(scope) and not hmac.compare_digest( - dict(headers).get(b"authorization", b""), self._control_bearer): + dict(headers).get(b"authorization", b""), self._control_bearer) and not ( + self._console_session and self._is_pgweb(scope) and self._console_session.admits(scope)): logger.warning("refused %s %s: it takes the env's control token", scope["method"], scope["path"]) response = self._json_response({"ok": False, "error": "this route takes the env's control token"}, 401) await response(scope, receive, send) @@ -629,8 +638,9 @@ async def app(scope, receive, send): # Only the clock watch reads the token, which the computers the gateway fronts send it, and only they # receive it, through their /svc routes; every other request goes on without it. if not self._keeps_control_token(scope["path"]): - scope = {**scope, "headers": [(name, value) for name, value in headers if not ( - name == b"authorization" and hmac.compare_digest(value, self._control_bearer))]} + headers = [(name, value) for name, value in headers if not ( + name == b"authorization" and hmac.compare_digest(value, self._control_bearer))] + scope = {**scope, "headers": without_session_cookie(headers)} if scope["type"] != "http": await inner(scope, receive, send) return @@ -647,6 +657,21 @@ async def app(scope, receive, send): return app + def _open_console(self, link: str) -> Response: + """The answer to a link to the pgweb console: its session cookie and the console, or 401 for a link this + deployment did not mint or that has expired.""" + if not self._console_session.opens(link): + logger.warning("refused a pgweb link: not this deployment's, or expired") + return self._json_response({"ok": False, "error": "this link is not valid for this env"}, 401) + return Response(status_code=303, headers={"location": "./", "set-cookie": self._console_session.set_cookie(), + "cache-control": "no-store", "referrer-policy": "no-referrer"}) + + @staticmethod + def _is_pgweb(scope: dict) -> bool: + """Whether ``scope`` is routed to the pgweb console: the service its path names, as the /svc route reads it.""" + prefix, _, rest = scope["path"].lstrip("/").partition("/") + return prefix == "svc" and rest.partition("/")[0] == PGWEB_CONSOLE + def _takes_control_token(self, scope: dict) -> bool: """Whether a request takes the deploy's control token: a control route, the env's data plane, any path of a SQL console behind /svc, or one of ``token_paths`` behind /svc, by the path as routed or by the one the backing diff --git a/tst/unit/env/gateway/console_routes_test.py b/tst/unit/env/gateway/console_routes_test.py index 2cdc74547..aa175cbcd 100644 --- a/tst/unit/env/gateway/console_routes_test.py +++ b/tst/unit/env/gateway/console_routes_test.py @@ -3,12 +3,17 @@ from __future__ import annotations import asyncio -from urllib.parse import unquote, urlsplit +import logging +import time +from urllib.parse import quote, unquote, urlsplit import httpx import pytest +from uvicorn.logging import AccessFormatter, DefaultFormatter from agent_env.env.gateway import rest_proxy as rest_proxy_module +from agent_env.env.gateway.console_session import COOKIE_NAME, ConsoleSession, LinkRedaction +from agent_env.env.gateway.constants import CONTROL_ROUTES from agent_env.env.gateway.gateway import Gateway TOKEN = "s3cret" @@ -31,11 +36,13 @@ class _Upstream: def __init__(self, monkeypatch): self.requests: list[tuple[str, str, str | None]] = [] + self.cookies: list[str | None] = [] transport = httpx.MockTransport(self._answer) monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", lambda **kw: _RealAsyncClient(transport=transport, **kw)) def _answer(self, request: httpx.Request) -> httpx.Response: self.requests.append((request.url.host, request.url.path, request.headers.get("authorization"))) + self.cookies.append(request.headers.get("cookie")) async def body(): yield b'event: message\ndata: {"jsonrpc": "2.0", "id": 1, "result": {}}\n\n' @@ -74,6 +81,18 @@ async def send(message): return started[0] +def _mint(token: str = TOKEN, shift: float = 0) -> ConsoleSession: + """A deployment's session minter on a clock ``shift`` seconds off the gateway's.""" + return ConsoleSession(token, now=lambda: time.time() + shift) + + +def _cookie(session: ConsoleSession | None = None) -> str: + return (session or _mint()).set_cookie().split(";")[0] + + +SAME_ORIGIN = {"Sec-Fetch-Site": "same-origin", "Sec-Fetch-Mode": "cors"} + + def _gateway(token: str | None, consoles: dict[str, str] | None = None) -> Gateway: return Gateway(host="127.0.0.1", port=0, server_name="unit", internal_mcp_servers=[], control_token=token, rest_proxy_urls=dict(BACKING), website_urls={"shop": BACKING["shop"]}, @@ -147,3 +166,143 @@ async def test_consoles_alone_get_the_svc_route(monkeypatch): def test_a_console_cannot_share_a_backing_servers_key(): with pytest.raises(ValueError, match="shop"): _gateway(TOKEN, {"shop": "http://pgweb:8081"}) + + +@pytest.mark.asyncio +async def test_a_link_sets_a_pgweb_only_cookie_and_sends_the_browser_to_the_console(monkeypatch): + upstream = _Upstream(monkeypatch) + status, headers = await _send(_gateway(TOKEN), "GET", f"/svc/pgweb/?pgweb_link={_mint().link()}", {}) + assert status == 303 + assert headers[b"location"] == b"./" + cookie = headers[b"set-cookie"].decode() + assert cookie.startswith(f"{COOKIE_NAME}=") + assert {"Path=/", "Secure", "HttpOnly", "SameSite=None", "Partitioned", "Max-Age=3600"} <= { + part.strip() for part in cookie.split(";")} + assert upstream.requests == [] + + +def _in_arabic_indic_digits(link: str) -> str: + """``link`` with its expiry's digits written as Arabic-Indic digits, which Python's int() reads as the same number.""" + expiry, _, signature = link.partition(".") + return "".join(chr(0x0660 + int(digit)) for digit in expiry) + "." + signature + + +@pytest.mark.asyncio +@pytest.mark.parametrize("link", [ + pytest.param(lambda: _mint("another deployment's token").link(), id="other-deployment"), + pytest.param(lambda: _mint(shift=-200).link(120), id="expired"), + pytest.param(lambda: _mint(shift=1000).link(120), id="too-far-ahead"), + pytest.param(lambda: _mint().link()[:-1] + "A", id="forged"), + pytest.param(lambda: _cookie().partition("=")[2], id="a-cookie-is-not-a-link"), + pytest.param(lambda: "", id="empty"), + pytest.param(lambda: _in_arabic_indic_digits(_mint().link()), id="non-ascii-digits")]) +async def test_a_link_this_deployment_did_not_mint_or_that_expired_is_refused(monkeypatch, link): + upstream = _Upstream(monkeypatch) + status, headers = await _send(_gateway(TOKEN), "GET", f"/svc/pgweb/?pgweb_link={quote(link())}", {}) + assert status == 401 + assert b"set-cookie" not in headers + assert upstream.requests == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method, path, sent", [ + ("GET", "/svc/pgweb/", "/"), ("GET", "/svc/pgweb/api/query?query=SELECT%201", "/api/query"), + ("POST", "/svc/pgweb/api/query", "/api/query"), ("GET", "/svc/pgweb/static/js/app.js", "/static/js/app.js")]) +async def test_the_cookie_opens_pgweb_from_its_own_page_and_never_reaches_it(monkeypatch, method, path, sent): + upstream = _Upstream(monkeypatch) + status, _ = await _send(_gateway(TOKEN), method, path, {**SAME_ORIGIN, "Cookie": f"theme=dark; {_cookie()}"}) + assert status == 200 + assert upstream.requests == [("pgweb", sent, None)] + assert upstream.cookies == ["theme=dark"] + + +@pytest.mark.asyncio +async def test_a_navigation_to_the_console_index_takes_the_cookie_from_any_site(monkeypatch): + upstream = _Upstream(monkeypatch) + status, _ = await _send(_gateway(TOKEN), "GET", "/svc/pgweb/", + {"Sec-Fetch-Site": "cross-site", "Sec-Fetch-Mode": "navigate", "Cookie": _cookie()}) + assert status == 200 + assert upstream.requests == [("pgweb", "/", None)] + assert upstream.cookies == [None] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method, path, fetch", [ + ("POST", "/svc/pgweb/api/query", {"Sec-Fetch-Site": "same-site", "Sec-Fetch-Mode": "cors"}), + ("POST", "/svc/pgweb/api/query", {"Sec-Fetch-Site": "cross-site", "Sec-Fetch-Mode": "no-cors"}), + ("POST", "/svc/pgweb/api/query", {}), + ("GET", "/svc/pgweb/api/query?query=DROP%20TABLE%20t", {"Sec-Fetch-Site": "same-site", "Sec-Fetch-Mode": "navigate"}), + ("POST", "/svc/pgweb/", {"Sec-Fetch-Site": "same-site", "Sec-Fetch-Mode": "navigate"})]) +async def test_the_cookie_does_not_count_on_a_request_another_site_sends(monkeypatch, method, path, fetch): + """Every Modal tunnel is same-site with every other, so another deployment's page could otherwise write here.""" + upstream = _Upstream(monkeypatch) + status, _ = await _send(_gateway(TOKEN), method, path, {**fetch, "Cookie": _cookie()}) + assert status == 401 + assert upstream.requests == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cookie", [ + pytest.param(lambda: _cookie(_mint("another deployment's token")), id="other-deployment"), + pytest.param(lambda: _cookie(_mint(shift=-4000)), id="expired"), + pytest.param(lambda: f"{COOKIE_NAME}={_mint().link()}", id="a-link-is-not-a-cookie"), + pytest.param(lambda: f"{COOKIE_NAME}=1.{'A' * 43}", id="forged")]) +async def test_a_cookie_this_deployment_did_not_set_or_that_expired_opens_nothing(monkeypatch, cookie): + upstream = _Upstream(monkeypatch) + status, _ = await _send(_gateway(TOKEN), "POST", "/svc/pgweb/api/query", {**SAME_ORIGIN, "Cookie": cookie()}) + assert status == 401 + assert upstream.requests == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method, path", [ + *[(method, path) for method, path in CONSOLE_PATHS if "db-" in path], + *[("POST", route) for route in sorted(CONTROL_ROUTES)], + ("POST", "/svc/db-mcp/../pgweb/api/query")]) +async def test_the_cookie_opens_no_route_but_pgweb(monkeypatch, method, path): + upstream = _Upstream(monkeypatch) + status, _ = await _send(_gateway(TOKEN), method, path, {**SAME_ORIGIN, "Cookie": _cookie()}) + assert status in (401, 404) + assert upstream.console_hosts == [] + + +@pytest.mark.asyncio +async def test_the_bearer_token_still_opens_pgweb_and_the_cookie_still_does_not_reach_it(monkeypatch): + upstream = _Upstream(monkeypatch) + status, _ = await _send(_gateway(TOKEN), "POST", "/svc/pgweb/api/query", + {"Authorization": f"Bearer {TOKEN}", "Cookie": _cookie()}) + assert status == 200 + assert upstream.requests == [("pgweb", "/api/query", None)] + assert upstream.cookies == [None] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("token, consoles", [(TOKEN, {"db-mcp": CONSOLES["db-mcp"]}), (None, CONSOLES)], + ids=["no-pgweb", "no-token"]) +async def test_a_gateway_with_no_pgweb_behind_its_token_takes_no_link(monkeypatch, token, consoles): + upstream = _Upstream(monkeypatch) + status, headers = await _send(_gateway(token, consoles), "GET", f"/svc/pgweb/?pgweb_link={_mint().link()}", {}) + assert status in (401, 404) + assert b"set-cookie" not in headers + assert upstream.console_hosts == [] + + +@pytest.mark.parametrize("logger, fmt, args, line", [ + pytest.param("uvicorn.access", '%s - "%s %s HTTP/%s" %d', + lambda link: ("agent:1", "GET", f"/svc/pgweb/?pgweb_link={link}&x=1", "1.1", 303), + 'INFO: agent:1 - "GET /svc/pgweb/?pgweb_link=&x=1 HTTP/1.1" 303 See Other', + id="uvicorn-access"), + pytest.param("httpx", 'HTTP Request: %s %s "%s %d %s"', + lambda link: ("GET", httpx.URL(f"http://pgweb:8081/?pgweb_link={link}"), "HTTP/1.1", 200, "OK"), + 'INFO: HTTP Request: GET http://pgweb:8081/?pgweb_link= "HTTP/1.1 200 OK"', id="httpx")]) +def test_the_log_names_no_link_and_still_formats_its_line(logger, fmt, args, line): + """Through the formatters uvicorn's log config gives these loggers: its access formatter reads the request line + from the record's arguments.""" + link = _mint().link() + record = logging.LogRecord(logger, logging.INFO, __file__, 0, fmt, args(link), None) + assert LinkRedaction().filter(record) + formatter = (AccessFormatter('%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s', use_colors=False) + if logger == "uvicorn.access" else DefaultFormatter("%(levelprefix)s %(message)s", use_colors=False)) + formatted = formatter.format(record) + assert link not in formatted + assert formatted == line \ No newline at end of file