diff --git a/pyproject.toml b/pyproject.toml index e6af0e443..5c5439fd0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -162,7 +162,7 @@ dev = [ "moto>=5.0.0", "psycopg2-binary>=2.9.0", # the gateway server module's driver; outside its container only the tests import it "regex>=2023.12.25", # the gateway's trigger predicates, matched under a timeout; outside its container only the tests import it - "websockets>=13", # the gateway's WebSocket relay to a website; outside its container only the tests import it + "websockets>=15", # the gateway's WebSocket relay to a website; outside its container only the tests import it "pytest>=9.0", "pytest-asyncio>=0.23.0", "pytest-dependency>=0.6.0", diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index 0a8e48ab6..88165db20 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -158,6 +158,10 @@ def sources(names: list[str]) -> str: # Sent by a computer's site proxy, which serves a website's route at the root of the site's own name. AGENT_ENV_SITE_ROOT_HEADER = "AgentEnv-Site-Root" GATEWAY_PROVIDER_HEADERS_ENV = "AGENTENV_PROVIDER_REQUEST_HEADERS" +# Each synthetic site's names as its env stores them, `shop=shop.test|www.shop.test,...`, one of which a site takes as +# its Host when a client reaches it there; and the sites list_website_urls gives at their first name, `shop,docs`. +WEBSITE_NAMES_ENV = "WEBSITE_NAMES" +WEBSITE_LISTED_BY_NAME_ENV = "WEBSITE_LISTED_BY_NAME" # MCP request `_meta` keys the gateway stamps on every child tools/call: the caller's role, and on the MCP path a # key naming the inbound session, so a child can keep per-caller state behind the gateway's one shared session. AGENT_ENV_ROLE_META_KEY = "agentenv.io/role" diff --git a/src/agent_env/env/gateway/entrypoint.py b/src/agent_env/env/gateway/entrypoint.py index 0843b75b1..81d3cad3c 100644 --- a/src/agent_env/env/gateway/entrypoint.py +++ b/src/agent_env/env/gateway/entrypoint.py @@ -8,8 +8,9 @@ import httpx from . import AGENT_ENV_GATEWAY_MCP_PORT, GatewayMode, InternalMCPServer -from .constants import (CLOCK_FOLLOWERS_ENV, CONTROL_TOKEN_ENV, DEFAULT_MCP_SERVER_NAME, GATEWAY_PROVIDER_HEADERS_ENV, MCP_TRANSPORT, - WELL_KNOWN_PATH) +from .constants import (CLOCK_FOLLOWERS_ENV, CONTROL_TOKEN_ENV, DEFAULT_MCP_SERVER_NAME, + GATEWAY_PROVIDER_HEADERS_ENV, MCP_TRANSPORT, WEBSITE_LISTED_BY_NAME_ENV, + WEBSITE_NAMES_ENV, WELL_KNOWN_PATH) from .gateway import Gateway, card_url logger = logging.getLogger(__name__) @@ -201,6 +202,17 @@ def parse_clock_followers(spec: str) -> dict[str, str]: return followers +def parse_site_names(env_str: str) -> dict[str, tuple[str, ...]]: + """Each site's names from ``WEBSITE_NAMES``: ``shop=shop.test|www.shop.test,docs=docs.test``.""" + names: dict[str, tuple[str, ...]] = {} + for entry in filter(None, (part.strip() for part in env_str.split(","))): + site, sep, listed = entry.partition("=") + if not sep or not site.strip(): + raise ValueError(f"Invalid {WEBSITE_NAMES_ENV} entry: '{entry}' (expected 'site=name|name')") + names[site.strip()] = tuple(name.strip() for name in listed.split("|") if name.strip()) + return names + + if __name__ == "__main__": logging.basicConfig( level=logging.INFO, @@ -232,5 +244,8 @@ def parse_clock_followers(spec: str) -> dict[str, str]: control_token=os.environ.get(CONTROL_TOKEN_ENV), provider_headers=os.environ.get(GATEWAY_PROVIDER_HEADERS_ENV, "").split(","), clock_followers=parse_clock_followers(os.environ.get(CLOCK_FOLLOWERS_ENV, "")), + site_names=parse_site_names(os.environ.get(WEBSITE_NAMES_ENV, "")), + listed_by_name=[site.strip() for site in os.environ.get(WEBSITE_LISTED_BY_NAME_ENV, "").split(",") + if site.strip()], ) gateway.run() diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index bbf0296a9..1c3058b41 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -82,8 +82,11 @@ ROLE_HEADER_BYTES, UpstreamResponse, WebsiteRoute, + declared_site_names, fetch_child_card, forward, + listed_url, + lowered, operations, refuse_websocket, relay_websocket, @@ -107,6 +110,14 @@ _STEP_API = {"source": "step_api"} +class SiteCard(NamedTuple): + """What the gateway keeps from a site's card: the paths it advertises for agentenv, which its route refuses, and + the names it declares.""" + + reserved: frozenset[str] + names: tuple[str, ...] + + class ToolNameClash(ValueError): """More than one source serves a tool under one name, so an agent couldn't tell which it calls: the gateway serves none of its servers' tools rather than a set missing some.""" @@ -145,18 +156,25 @@ def __init__( control_token: str | None = None, provider_headers: Iterable[str] = (), clock_followers: Mapping[str, str] | None = None, + site_names: dict[str, Iterable[str]] | None = None, + listed_by_name: Iterable[str] = (), ): """`provider_headers` names the headers the sandbox provider in front of the gateway is sent, its credentials, which the gateway takes off every request it forwards to a service behind it. `clock_followers` names the - followers of its clock, each with the read token its watch carries.""" + followers of its clock, each with the read token its watch carries. `site_names` are each site's names as its + env stores them, the names its card declares standing in for a site's that stores none: a site reached at one + takes it as its ``Host``. ``list_website_urls`` gives each site `listed_by_name` at its first name, as a browser + computer opens it, and the others at their ``website_urls``.""" self.host = host self.port = port self.server_name = server_name self.internal_mcp_servers = internal_mcp_servers self.website_urls = website_urls or {} self.rest_proxy_urls = rest_proxy_urls or {} + self._site_names = {site: lowered(names) for site, names in (site_names or {}).items()} + self._listed_by_name = frozenset(listed_by_name) self._site_mcp_endpoints = site_mcp_endpoints(internal_mcp_servers, self.website_urls, self.rest_proxy_urls) - self._site_reserved: dict[str, frozenset[str]] = {} + self._site_cards: dict[str, SiteCard] = {} self._withheld = frozenset(name.strip().lower().encode() for name in provider_headers if name.strip()) self._browsers = BrowserRoutes(self.rest_proxy_urls, self.REST_PROXY_TIMEOUT, self._log_event, self._attribution, self._request_role, withheld=self._withheld) @@ -199,7 +217,7 @@ def __init__( if self.website_urls: @self._mcp.tool(name="list_website_urls", description="List available website URLs that can be browsed.") async def list_website_urls() -> list[str]: - return list(self.website_urls.values()) + return [await self._listed_url(site, url) for site, url in self.website_urls.items()] self._server_tools.setdefault("gateway", []).append( MCPTool( name="list_website_urls", @@ -686,8 +704,10 @@ def _query_changelog_id(self) -> int | None: self._db_conn = None return None - async def _proxy_rest_request(self, request: Request, url_map: dict[str, str] | None = None) -> Response: - """Reverse-proxy an HTTP request to an internal service, streaming the response through.""" + async def _proxy_rest_request(self, request: Request, url_map: dict[str, str] | None = None, + site_names: tuple[str, ...] = ()) -> Response: + """Reverse-proxy an HTTP request to an internal service, streaming the response through; a site reached at one + of its `site_names` takes it as its ``Host``.""" url_map = self.rest_proxy_urls if url_map is None else url_map service_name = request.path_params["service_name"] base_url = url_map.get(service_name) @@ -701,7 +721,7 @@ async def _proxy_rest_request(self, request: Request, url_map: dict[str, str] | return cdp if request.method in _SAFE_METHODS: return await forward(request, base_url, role=role, timeout=self.REST_PROXY_TIMEOUT, mcp_server=mcp_server, - withheld=self._withheld, mcp_endpoint=mcp_endpoint) + withheld=self._withheld, mcp_endpoint=mcp_endpoint, site_names=site_names) changelog_before = self._query_changelog_id() attribution = self._attribution(role) # at the start: a fire may be gone by the time the response ends @@ -726,20 +746,21 @@ async def log_write(status: int) -> None: }) return await forward(request, base_url, role=role, timeout=self.REST_PROXY_TIMEOUT, mcp_server=mcp_server, - on_done=log_write, withheld=self._withheld, mcp_endpoint=mcp_endpoint) + on_done=log_write, withheld=self._withheld, mcp_endpoint=mcp_endpoint, + site_names=site_names) async def _proxy_website_request(self, request: Request) -> Response: """Proxy a website request; with a website or state trigger registered, feed it to the engine and hold barriers. A site's data plane and card, and every path its card advertises for agentenv (``site_reserved_paths``), are never reached through its route, so an agent browsing the site can't read its data or what grades it; agentenv reaches them through /svc//.""" - reserved = await self._site_reserved_paths(request.path_params["service_name"]) - if reserved is None: + app = request.path_params["service_name"] + card = await self._site_card(app) + if card is None: return self._json_response({"error": "website card unavailable"}, 503) - if reserved_site_request(request, reserved): + if reserved_site_request(request, card.reserved): return self._json_response({"error": "not found"}, 404) - response = await self._proxy_rest_request(request, url_map=self.website_urls) - app = request.path_params["service_name"] + response = await self._proxy_rest_request(request, url_map=self.website_urls, site_names=self._names(app, card)) engine = self._trigger_engine if app in self.website_urls and (engine.has_website_triggers or engine.has_state_triggers): path = site_request_path(request) or "/" + request.path_params.get("path", "") @@ -757,20 +778,21 @@ async def _proxy_website_socket(self, websocket: WebSocket) -> None: """Relay a WebSocket on a site's route to its server, as its HTTP requests are: its data plane and card, and every path its card advertises for agentenv (``site_reserved_paths``), are refused, and it goes upstream with the role, role key and forwarded address an HTTP request would carry.""" - reserved = await self._site_reserved_paths(websocket.path_params["service_name"]) - if reserved is None: + service_name = websocket.path_params["service_name"] + card = await self._site_card(service_name) + if card is None: await refuse_websocket(websocket, self._json_response({"error": "website card unavailable"}, 503)) return - if reserved_site_request(websocket, reserved): + if reserved_site_request(websocket, card.reserved): await refuse_websocket(websocket, self._json_response({"error": "not found"}, 404)) return - service_name = websocket.path_params["service_name"] base_url = self.website_urls.get(service_name) if base_url is None: await refuse_websocket(websocket, self._json_response({"error": f"Unknown service: {service_name}"}, 404)) return await relay_websocket(websocket, base_url, role=self._request_role(websocket), - open_timeout=self.REST_PROXY_CONNECT_TIMEOUT_S, withheld=self._withheld) + open_timeout=self.REST_PROXY_CONNECT_TIMEOUT_S, withheld=self._withheld, + site_names=self._names(service_name, card)) async def _fire_website_triggers(self, request: Request, app: str, status: int) -> tuple[list[asyncio.Task], float | None]: @@ -819,16 +841,17 @@ def _website_op(self, app: str, method: str, path: str) -> str | None: read of its own, so answering a request waits on no card.""" return route_name(self._website_routes.get(app) or [], method, path) - async def _site_reserved_paths(self, app: str) -> frozenset[str] | None: - """The paths ``app``'s card advertises for agentenv (``site_reserved_paths``), read once from its server, the one - its /svc/ route reaches (else the one its website route does), and kept; none for a site serving no card, one - whose card path answers a 4xx or what isn't a JSON object, and None, read again next time, while its card can't - be read, unreachable or answering a 5xx, so its route serves nothing until it knows what to refuse.""" - if app in self._site_reserved: - return self._site_reserved[app] + async def _site_card(self, app: str) -> SiteCard | None: + """What ``app``'s card says (``SiteCard``): the paths it advertises for agentenv (``site_reserved_paths``) and + the names it declares, read once from its server, the one its /svc/ route reaches (else the one its website + route does), and kept; neither for a site serving no card, one whose card path answers a 4xx or what isn't a + JSON object, and None, read again next time, while its card can't be read, unreachable or answering a 5xx, so + its route serves nothing until it knows what to refuse.""" + if app in self._site_cards: + return self._site_cards[app] base_url = self.rest_proxy_urls.get(app) or self.website_urls.get(app) if base_url is None: - return frozenset() + return SiteCard(frozenset(), ()) try: async with httpx.AsyncClient() as client: response = await client.get(f"{base_url.rstrip('/')}{WELL_KNOWN_PATH}", timeout=CARD_FETCH_TIMEOUT_S) @@ -843,8 +866,21 @@ async def _site_reserved_paths(self, app: str) -> frozenset[str] | None: except ValueError: card = None address = self.website_urls.get(app) or base_url - self._site_reserved[app] = site_reserved_paths(card, address) if isinstance(card, dict) else frozenset() - return self._site_reserved[app] + self._site_cards[app] = (SiteCard(site_reserved_paths(card, address), declared_site_names(card)) + if isinstance(card, dict) else SiteCard(frozenset(), ())) + return self._site_cards[app] + + def _names(self, app: str, card: SiteCard | None) -> tuple[str, ...]: + """The names website ``app`` is reached at: those its env stores, else those its ``card`` declares.""" + return self._site_names.get(app) or (card.names if card else ()) + + async def _listed_url(self, site: str, url: str) -> str: + """The address ``list_website_urls`` gives ``site``, at ``url`` behind the gateway: for a site listed by name, + its first name (``listed_url``), its card read for a site whose env stores none; else, or with none, ``url``.""" + if site not in self._listed_by_name: + return url + card = None if site in self._site_names else await self._site_card(site) + return listed_url(self._names(site, card)) or url def _backing_keys(self) -> list[str]: # rest_proxy_urls entries are the env's backing servers: mcp- (MCP) + (website diff --git a/src/agent_env/env/gateway/requirements.txt b/src/agent_env/env/gateway/requirements.txt index 2d5541b98..e4d7ac9c8 100644 --- a/src/agent_env/env/gateway/requirements.txt +++ b/src/agent_env/env/gateway/requirements.txt @@ -4,4 +4,4 @@ mcp>=1.25,<2 pydantic>=2.0 regex>=2023.12.25 tzdata -websockets>=13 +websockets>=15 diff --git a/src/agent_env/env/gateway/rest_proxy.py b/src/agent_env/env/gateway/rest_proxy.py index 54235b290..73d783374 100644 --- a/src/agent_env/env/gateway/rest_proxy.py +++ b/src/agent_env/env/gateway/rest_proxy.py @@ -15,7 +15,8 @@ own, when one before the gateway sent them, else the request's ``Host`` and scheme) and ``X-Forwarded-Prefix``, the route prefix (``/svc/{service}`` or ``/website/{service}``) the gateway strips, which a client never sets: a client that serves the route at a root of its own (a computer's site proxy, at ``https://notes.test/``) says so with - ``AgentEnv-Site-Root``, and the upstream is then told no prefix; + ``AgentEnv-Site-Root``, and the upstream is then told no prefix; a site reached at one of its own names gets that + name as its ``Host`` too, and any other host leaves ``Host`` its internal address, so a forged one never reaches it; - the upstream response and the client it came through are closed when the response ends, iterated or not. A WebSocket on ``/website/{service}/{path}`` is relayed under the same rules (:func:`relay_websocket`): the gateway @@ -36,9 +37,9 @@ import logging import posixpath import re -from collections.abc import AsyncIterator, Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable, Iterable from typing import Any, NamedTuple -from urllib.parse import unquote, urlsplit +from urllib.parse import unquote, urlsplit, urlunsplit import httpx from starlette.requests import HTTPConnection, Request @@ -71,6 +72,10 @@ ROLE_HEADER_BYTES = AGENT_ENV_ROLE_HEADER.lower().encode() FORWARDED_HOST, FORWARDED_PROTO, FORWARDED_PREFIX = "x-forwarded-host", "x-forwarded-proto", "x-forwarded-prefix" +# A host as a site's name: a DNS name, an optional port. +_LABEL = r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?" +_SITE_HOST = re.compile(rf"({_LABEL}(?:\.{_LABEL})*)(?::([0-9]{{1,5}}))?") +_MAX_PORT = 65535 _FORWARDED = frozenset(name.encode() for name in (FORWARDED_HOST, FORWARDED_PROTO, FORWARDED_PREFIX, AGENT_ENV_SITE_ROOT_HEADER.lower())) # A WebSocket's handshake fields, which the gateway's own connection upstream writes for itself. @@ -86,9 +91,10 @@ async def forward(request: Request, base_url: str, *, role: str, timeout: httpx.Timeout, mcp_server: bool, on_done: Callable[[int], Awaitable[None]] | None = None, - withheld: frozenset[bytes] = frozenset(), mcp_endpoint: str = "mcp") -> Response: + withheld: frozenset[bytes] = frozenset(), mcp_endpoint: str = "mcp", + site_names: tuple[str, ...] = ()) -> Response: """Forward `request` to the service at `base_url` as `role`, without the headers named in `withheld`, and stream - the response through. `mcp_server` says + the response through; a site's `site_names` say which hosts it takes as its ``Host``. `mcp_server` says the service is a backing MCP server, whose own endpoint, `mcp_endpoint` under it, takes the role in its JSON-RPC `_meta` as well as in the header. A 400 names a request-target the proxy cannot route; a 502 names an upstream that never answered. `on_done` is awaited with the status once the response has ended, after its last byte when streamed.""" @@ -97,7 +103,7 @@ async def forward(request: Request, base_url: str, *, role: str, timeout: httpx. except ValueError as e: # an escaped slash inside the route prefix; refuse rather than forward a wrong path return await _done(_json_response({"error": str(e)}, 400), on_done) content = await request.body() # /svc and /website request bodies are small (JSON-RPC, forms) - headers = upstream_headers(request, role, withheld=withheld) + headers = upstream_headers(request, role, withheld=withheld, site_names=site_names) if mcp_server and request.method == "POST" and is_child_mcp_path(request.path_params.get("path", ""), mcp_endpoint): # The gateway is the trust boundary for identity: a child's proxied MCP endpoint stays reachable, but the # role in its JSON-RPC `_meta` is the gateway's to write, like the header above. @@ -142,15 +148,19 @@ def client_headers(connection: HTTPConnection, withheld: frozenset[bytes] = froz return [(k, v) for k, v in connection.headers.raw if k.lower() not in HOP_BY_HOP and k.lower() not in withheld] -def upstream_headers(request: HTTPConnection, role: str, *, withheld: frozenset[bytes] = frozenset()) -> httpx.Headers: +def upstream_headers(request: HTTPConnection, role: str, *, withheld: frozenset[bytes] = frozenset(), + site_names: tuple[str, ...] = ()) -> httpx.Headers: """The client's headers (``client_headers``) as a proxied request sends them. The role header is the gateway's to set: the client's copies are dropped and the role the gateway attributed the request to goes in their place, so the upstream never reads a value the gateway did not write. So are the forwarded-address headers - (`forwarded_address`).""" + (`forwarded_address`), and ``Host`` is the site's own name when the client reached it at one of `site_names` + (``site_host``), else the upstream's address.""" headers = httpx.Headers([(k, v) for k, v in client_headers(request, withheld) if k.lower() != ROLE_HEADER_BYTES and k.lower() not in _FORWARDED]) headers[AGENT_ENV_ROLE_HEADER] = role headers.update(forwarded_address(request)) + if host := site_host(request, site_names): + headers["host"] = host # Bytes are forwarded undecoded, so only ask upstream for encodings the caller can take. headers.setdefault("accept-encoding", "identity") return headers @@ -165,10 +175,11 @@ def forwarded_address(request: HTTPConnection) -> dict[str, str]: (``https://notes.test``) is that root's address, ahead of any ``X-Forwarded-Host``/``-Proto``, which a front between the client and the gateway may have rewritten to its own. Like the host, that only changes the links the client's own answer names; what the route reaches is the path's alone. A header with nothing to say is left - out.""" + out, as is a host that isn't printable ASCII, which no header upstream could carry.""" root = request.headers.get(AGENT_ENV_SITE_ROOT_HEADER, "").strip() origin = _origin(root) - host = origin[1] if origin else _first(request.headers.get(FORWARDED_HOST, "")) or request.headers.get("host", "") + host = origin[1] if origin else (_plain(_first(request.headers.get(FORWARDED_HOST, ""))) + or _plain(request.headers.get("host", ""))) proto = origin[0] if origin else _first(request.headers.get(FORWARDED_PROTO, "")).lower() at_root = bool(root) scheme = _HTTP_SCHEME.get(request.url.scheme, request.url.scheme) @@ -179,17 +190,64 @@ def forwarded_address(request: HTTPConnection) -> dict[str, str]: def _origin(value: str) -> tuple[str, str] | None: """The scheme and host of ``value`` when it is an origin (``https://notes.test``, a port allowed), else None.""" - parts = urlsplit(value) + try: + parts = urlsplit(value) + except ValueError: # an unbalanced bracket, as in ``https://]x[`` + return None if (parts.scheme in ("http", "https") and parts.netloc and parts.path in ("", "/") and not parts.query - and not parts.fragment and "@" not in parts.netloc and value.isprintable()): + and not parts.fragment and "@" not in parts.netloc and _plain(value)): return parts.scheme, parts.netloc return None +def _plain(value: str) -> str: + return value if value.isascii() and value.isprintable() else "" + + def _first(value: str) -> str: return value.split(",")[0].strip() +def site_host(request: HTTPConnection, names: tuple[str, ...]) -> str | None: + """The host the client reached a site at (``forwarded_address``), when its name is one the site's ``names`` name, + compared without case or port: what the site's ``Host`` says, so a site that builds its links from ``Host`` names + itself as the client reached it. None for any other host, which a site never sees as its ``Host``.""" + host = forwarded_address(request).get(FORWARDED_HOST, "") + found = _SITE_HOST.fullmatch(host) + if not found or int(found.group(2) or 0) > _MAX_PORT: + return None + return host if named(found.group(1).lower(), names) else None + + +def named(host: str, names: tuple[str, ...]) -> bool: + """Whether lowercase ``host`` is one ``names`` name: one of them, or one label more than a ``*.`` one, as a + computer's site proxy matches a site's names.""" + head, _, rest = host.partition(".") + return host in names or bool(head and rest) and f"*.{rest}" in names + + +def lowered(names: Iterable[str]) -> tuple[str, ...]: + return tuple(dict.fromkeys(name.lower() for name in names if isinstance(name, str) and name)) + + +def declared_site_names(card: dict) -> tuple[str, ...]: + """The names a site's card declares, its ``website/v1`` ``domains``, lowercase; none when it declares none.""" + for ext in (card.get("capabilities") or {}).get("extensions") or []: + params = ext.get("params") if isinstance(ext, dict) and ext.get("uri") == EXT_WEBSITE_URI else None + if isinstance(params, dict) and isinstance(params.get("domains"), list): + return lowered(params["domains"]) + return () + + +def listed_url(names: tuple[str, ...]) -> str | None: + """The address a browser computer opens a site with ``names`` at: its first name that isn't a ``*.`` one, over + HTTPS but for a ``.localhost`` one, as its site proxy serves them. None with no such name.""" + name = next((name for name in names if not name.startswith("*.")), None) + if name is None: + return None + return f"{'http' if name == 'localhost' or name.endswith('.localhost') else 'https'}://{name}/" + + def route_prefix(request: HTTPConnection) -> str: """The path the client reached the proxied service under, as it sent it: the request-target up to the `{path}` remainder, without its trailing slash, e.g. ``/website/notes``, or under a root path ``/env/website/notes``.""" @@ -254,7 +312,8 @@ def raw_remainder(raw_path: bytes, decoded_path: str, decoded_rest: str) -> byte async def relay_websocket(websocket: WebSocket, base_url: str, *, role: str, - open_timeout: float = 10.0, withheld: frozenset[bytes] = frozenset()) -> None: + open_timeout: float = 10.0, withheld: frozenset[bytes] = frozenset(), + site_names: tuple[str, ...] = ()) -> None: """Relay `websocket` to the service at `base_url` as `role`: a WebSocket of the gateway's own is opened to the same path, with the headers ``upstream_headers`` sends, asking for the subprotocols the client offered; `websocket` is accepted with the one the service chose and the headers its answer carried (a ``Set-Cookie`` among them, each @@ -262,20 +321,30 @@ async def relay_websocket(websocket: WebSocket, base_url: str, *, role: str, close, with its code, closes the other; a message from the service larger than ``WEBSOCKET_MESSAGE_MAX_BYTES``, the size uvicorn takes from a client, closes both with 1009 rather than be held whole. A service that refuses the handshake has its answer relayed as the refusal, its headers as an accepted one's are; one that never answers is a - 502. The headers named in `withheld` don't go upstream, as with ``upstream_headers``.""" + 502. The headers named in `withheld` don't go upstream, as with ``upstream_headers``, and a site reached at one of + its `site_names` takes it as its ``Host``.""" try: target_url = upstream_url(base_url, websocket) except ValueError as e: await refuse_websocket(websocket, _json_response({"error": str(e)}, 400)) return - headers = [(key, value) for key, value in upstream_headers(websocket, role, withheld=withheld).multi_items() - if key.lower() not in _WEBSOCKET_HANDSHAKE] + headers = [(key, value) for key, value in + upstream_headers(websocket, role, withheld=withheld, site_names=site_names).multi_items() + if key.lower() not in _WEBSOCKET_HANDSHAKE and key.lower() != "host"] + uri, destination = "ws" + target_url.removeprefix("http"), {} + if host := site_host(websocket, site_names): # websockets writes Host from the URI; TCP and TLS go to the upstream + parts = urlsplit(uri) + secure = parts.scheme == "wss" + destination = {"host": parts.hostname, "port": parts.port or (443 if secure else 80), + **({"server_hostname": parts.hostname} if secure else {})} + uri = urlunsplit(parts._replace(netloc=host)) offered = [protocol.strip() for value in websocket.headers.getlist("sec-websocket-protocol") for protocol in value.split(",") if protocol.strip()] try: - upstream = await connect_websocket("ws" + target_url.removeprefix("http"), additional_headers=headers, + upstream = await connect_websocket(uri, additional_headers=headers, # never through a proxy: it's internal subprotocols=offered or None, user_agent_header=None, - open_timeout=open_timeout, max_size=WEBSOCKET_MESSAGE_MAX_BYTES) + open_timeout=open_timeout, max_size=WEBSOCKET_MESSAGE_MAX_BYTES, + proxy=None, **destination) except InvalidStatus as e: refusal = Response(content=bytes(e.response.body or b""), status_code=e.response.status_code) refusal.raw_headers = [*(item for item in refusal.raw_headers if item[0] == b"content-length"), diff --git a/src/agent_env/providers/env_providers/env_gateway_provider.py b/src/agent_env/providers/env_providers/env_gateway_provider.py index 4c7a01b7c..16437dc0d 100644 --- a/src/agent_env/providers/env_providers/env_gateway_provider.py +++ b/src/agent_env/providers/env_providers/env_gateway_provider.py @@ -38,6 +38,8 @@ DEFAULT_MCP_SERVER_NAME, GATEWAY_PROVIDER_HEADERS_ENV, STATE_TOOL_NAME_CLASHES_KEY, + WEBSITE_LISTED_BY_NAME_ENV, + WEBSITE_NAMES_ENV, WELL_KNOWN_PATH, random_mcp_server_name, tool_name_clashes_text, @@ -169,6 +171,10 @@ class WebsiteConfig: # a failure and the deploy waits, and its server's /dev/shm in MiB; None keeps the gateway's own and Docker's. ready_timeout_s: float | None = None shm_mb: int | None = None + # The names its env stores, one of which it takes as its Host when a client reaches it there (its card's stand in + # for none), and whether list_website_urls gives it at its first name, as a browser computer opens it. + domains: tuple[str, ...] = () + listed_by_name: bool = False @dataclass @@ -525,6 +531,11 @@ def create_docker_compose( ) if website_urls: gateway_env.append(f" - WEBSITE_URLS={website_urls}") + if names := ",".join(f"{wc.environment_name}={'|'.join(wc.domains)}" + for wc in website_configs if wc.domains and not wc.live): + gateway_env.append(f" - {WEBSITE_NAMES_ENV}={names}") + if listed := ",".join(wc.environment_name for wc in website_configs if wc.listed_by_name and not wc.live): + gateway_env.append(f" - {WEBSITE_LISTED_BY_NAME_ENV}={listed}") for wc in website_configs: rest_proxy_urls.append( f"{wc.environment_name}=http://{wc.environment_name}-{AGENT_ENV_WEBSITE_BACKEND_SUFFIX}:{AGENT_ENV_WEBSITE_BACKEND_PORT}" @@ -1584,14 +1595,15 @@ class _GatewayTopology: website_browser: "MCPServerEnv | None | object" # the browser it runs among its MCP servers, or _THE_RUNS -def website_config(env: WebsiteEnv) -> WebsiteConfig: +def website_config(env: WebsiteEnv, *, listed_by_name: bool = False) -> WebsiteConfig: """A website env as the gateway runs it: its server image, any deprecated frontend image, its health path, how long - it may take to be ready and its shared memory.""" + it may take to be ready, its shared memory and the names it stores, listed by its first when ``listed_by_name``.""" frontend = env.frontend_docker_image_artifact return WebsiteConfig(backend_image=env.server_docker_image_artifact.image_name, frontend_image=frontend.image_name if frontend else None, environment_name=env.environment_name, health_path=env.health_path, live=env.live, - settings=dict(env.settings), ready_timeout_s=env.ready_timeout_s, shm_mb=env.shm_mb) + settings=dict(env.settings), ready_timeout_s=env.ready_timeout_s, shm_mb=env.shm_mb, + domains=tuple(env.domains), listed_by_name=listed_by_name) def run_website_browser() -> "MCPServerEnv": @@ -1622,16 +1634,18 @@ def _with_browser(mcp_servers: list[MCPServerConfig], images: list[DockerImageAr def _gateway_topology(env: Env) -> _GatewayTopology: """What a gateway fronts for env: a MultiEnv's MCP servers and websites (whatever their own env_provider_type) under its - name and the website browser it names or the run's, a WebsiteEnv's website and the run's browser, or else the env's - one MCP server.""" + name and the website browser it names or the run's, its websites listed by name when that's a browser computer, a + WebsiteEnv's website and the run's browser, or else the env's one MCP server.""" from agent_env.env.envs.multi_env import MultiEnv from agent_env.env.envs.website import WebsiteEnv is_multi = isinstance(env, MultiEnv) + by_name = False if is_multi: servers, websites, name = env.mcp_server_envs, env.website_envs, env.name placed = env.website_browser() browser = _THE_RUNS if placed.runs else placed.server + by_name = placed.computer is not None # it opens the sites at their names, on a sandbox of its own elif isinstance(env, WebsiteEnv): servers, websites, name, browser = [], [env], None, _THE_RUNS else: @@ -1639,7 +1653,7 @@ def _gateway_topology(env: Env) -> _GatewayTopology: return _GatewayTopology( mcp_servers=[MCPServerConfig(image=s.docker_image_artifact.image_name, environment_name=s.environment_name) for s in servers], mcp_server_images=[s.docker_image_artifact for s in servers], - website_configs=[website_config(w) for w in websites] or None, + website_configs=[website_config(w, listed_by_name=by_name) for w in websites] or None, website_images=[image for w in websites for image in (w.server_docker_image_artifact, w.frontend_docker_image_artifact) if image] or None, mcp_server_name=name, diff --git a/tst/data/host_website/Dockerfile b/tst/data/host_website/Dockerfile new file mode 100644 index 000000000..7080524ee --- /dev/null +++ b/tst/data/host_website/Dockerfile @@ -0,0 +1,12 @@ +FROM mirror.gcr.io/library/python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* +WORKDIR /app +RUN pip install --no-cache-dir starlette uvicorn pydantic httpx +COPY agentenv_protocol /app/agentenv_protocol +COPY server.py /app/server.py +ENV PYTHONPATH=/app +ENV PORT=8000 +EXPOSE 8000 +RUN useradd --create-home --uid 10001 appuser +USER appuser +CMD ["python", "server.py"] diff --git a/tst/data/host_website/server.py b/tst/data/host_website/server.py new file mode 100644 index 000000000..e2b3781f2 --- /dev/null +++ b/tst/data/host_website/server.py @@ -0,0 +1,49 @@ +"""A plain website, not on WebsiteEnvironment, that builds its addresses from its Host header, as many real apps do, for +integration-testing the gateway's site routes: /login redirects to /home at its Host, and /api/host says the Host and +X-Forwarded-Host it was reached with. Its setting ``card_domains``, names split by commas, makes it serve a card +declaring them; without it, it serves none.""" +import html +import json +import os + +import uvicorn +from starlette.applications import Starlette +from starlette.responses import HTMLResponse, JSONResponse, RedirectResponse, Response +from starlette.routing import Route + +from agentenv_protocol.website import WEBSITE_SETTINGS_ENV +from agentenv_protocol.website.readiness import WEBSITE_EXTENSION_URI + +DOMAINS = [name for name in str(json.loads(os.environ.get(WEBSITE_SETTINGS_ENV) or "{}").get("card_domains", "") + ).split(",") if name] + + +async def card(_): + if not DOMAINS: + return Response(status_code=404) + return JSONResponse({"name": os.environ.get("ENVIRONMENT_NAME", "shop"), "capabilities": {"extensions": [ + {"uri": WEBSITE_EXTENSION_URI, "params": {"domains": DOMAINS}}]}}) + + +async def health(_): + return JSONResponse({"status": "ok"}) + + +async def login(request): + return RedirectResponse(f"https://{request.headers.get('host', '')}/home", status_code=302) + + +async def home(request): + return HTMLResponse(f"home

{html.escape(request.headers.get('host', ''))}

") + + +async def host(request): + return JSONResponse({"host": request.headers.get("host"), + "forwarded_host": request.headers.get("x-forwarded-host")}) + + +app = Starlette(routes=[Route("/.well-known/agent-env.json", card), Route("/api/health", health), + Route("/login", login), Route("/home", home), Route("/api/host", host)]) + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", "8000")), proxy_headers=False, log_level="warning") diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index 805922da3..09da773e2 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -81,6 +81,7 @@ from agent_env.env.envs.website import WebsiteEnv from agent_env.env.envs.computer.utils import inside from agent_env.env.store import get_env_store +from agent_env.providers.env_providers.constants import AGENT_ENV_WEBSITE_BACKEND_PORT, AGENT_ENV_WEBSITE_BACKEND_SUFFIX from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider from agent_env.store.object_store.local.tls import LocalCA, local_ca from agent_env.store.routing import namespace_routing @@ -1612,3 +1613,97 @@ async def test_a_multi_env_naming_no_browser_deploys_none(browser_computer, shop assert not report.failed and not deployed.child_envs assert "shop" in stated and PLAYWRIGHT_MCP_ENVIRONMENT_NAME not in stated and "chromium" not in stated, stated + + +# --- a site reached by its names through a multi env's gateway --- + +HOST_START = "https://shop.test/api/host" +HOST_CHROMIUM = {**MULTI_CHROMIUM, + "env_params": {"form": "container", "app_settings": {"chromium": {"start_urls": [HOST_START]}}}} +#: Where the gateway reaches the site, which is the site's Host when a client reaches it at any other name. +SITE_INTERNAL = f"shop-{AGENT_ENV_WEBSITE_BACKEND_SUFFIX}:{AGENT_ENV_WEBSITE_BACKEND_PORT}" + + +@pytest.fixture(scope="module") +def host_shops(browser_computer): + """Two sites named shop.test that build their addresses from their Host, as many real apps do, under local ids: + one whose env stores the name and that serves no card, and one that stores none and whose card declares it.""" + with namespace_routing(): + _seed_bootstrap_envs() + uid = uuid.uuid4().hex[:8] + image = _image("host_website", f"host-shop-{uid}") + return {"stored": WebsiteEnv.put(id=f"@local/~/host-{uid}/stored/shop", environment_name="shop", + domains=["shop.test"], server_docker_image_artifact=image), + "card": WebsiteEnv.put(id=f"@local/~/host-{uid}/card/shop", environment_name="shop", + settings={"card_domains": "shop.test"}, server_docker_image_artifact=image)} + + +async def _landed(deployed, browser: DeployedEnv, url: str) -> str: + """Where chromium lands, driven through the gateway's /mcp to ``url``: the first tab at the site's /home, else + every tab's URL.""" + async with streamable_http_client(deployed.mcp_url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + assert not (await session.call_tool("chromium_computer_navigate", {"url": url})).isError + deadline = asyncio.get_running_loop().time() + WAIT_S + while True: + urls = [tab.get("url", "") for tab in await tabs_of(browser)] + if (home := next((u for u in urls if u.endswith("/home")), None)) or \ + asyncio.get_running_loop().time() > deadline: + return home or " ".join(urls) + await asyncio.sleep(POLL_S) + + +async def _listed(deployed) -> list[str]: + async with streamable_http_client(deployed.mcp_url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + result = await session.call_tool("list_website_urls", {}) + assert not result.isError and result.structuredContent, result + return result.structuredContent["result"] + + +async def _seen(deployed, root: str | None) -> dict: + """The Host and X-Forwarded-Host the site sees for a request straight to the gateway's site route naming ``root`` + as the origin it was reached at, or naming none.""" + async with httpx.AsyncClient(timeout=30) as http: + answer = await http.get(f"{deployed.gateway_url}/website/shop/api/host", + headers={"AgentEnv-Site-Root": root} if root else {}) + assert answer.status_code == 200, answer.text + return answer.json() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("names", ["stored", "card"]) +async def test_a_site_chromium_reaches_by_name_through_the_gateway_takes_that_name_as_its_host_and_no_forged_one( + browser_computer, host_shops, made, names): + with namespace_routing(): + office = _office(host_shops[names], [Env.get(browser_computer.id)]) + step = DeployEnvTaskStep(id="office", version=None, env_id=office.id, sandbox_type="local", ttl_seconds=900, + child_envs={"chromium": HOST_CHROMIUM}) + context = TaskStepContext() + try: + [deployed] = (await step.execute(context)).deployed_envs + browser = DeployedEnv.from_dict(deployed.child_envs["chromium"]) + landed, listed = await _landed(deployed, browser, "https://shop.test/login"), await _listed(deployed) + seen = [await _seen(deployed, root) for root in ("https://Shop.Test", "https://evil.example", None)] + subprocess.run(["docker", "restart", _gateway(deployed, made)], capture_output=True, check=True) + again = await _answers_again(deployed) + after_restart = (await _landed(deployed, browser, "https://shop.test/login"), await _listed(deployed), + await _seen(deployed, "https://evil.example")) + finally: + report = await teardown_run(context) + + assert not report.failed + # chromium follows the site's redirect, built from its Host, to the name it reached it at. + assert landed == "https://shop.test/home", landed + # An agent beside chromium is given the site at the name chromium opens it at. + assert listed == ["https://shop.test/"], listed + # A client naming any other origin, or none, leaves the site its internal Host; X-Forwarded-Host is as it was. + gateway_host = urllib.parse.urlsplit(deployed.gateway_url).netloc + assert seen == [{"host": "Shop.Test", "forwarded_host": "Shop.Test"}, + {"host": SITE_INTERNAL, "forwarded_host": "evil.example"}, + {"host": SITE_INTERNAL, "forwarded_host": gateway_host}], seen + # A restarted gateway, which reads a card's names again, keeps all of it. + assert again and after_restart == ("https://shop.test/home", ["https://shop.test/"], + {"host": SITE_INTERNAL, "forwarded_host": "evil.example"}), after_restart diff --git a/tst/unit/env/envs/test_multi_env_computers.py b/tst/unit/env/envs/test_multi_env_computers.py index 98fcc4f59..903c91a88 100644 --- a/tst/unit/env/envs/test_multi_env_computers.py +++ b/tst/unit/env/envs/test_multi_env_computers.py @@ -854,6 +854,23 @@ def test_the_gateway_runs_an_mcp_server_browser_and_leaves_a_browser_computer_to assert getattr(browser, "id", None) == expected +@pytest.mark.parametrize("browser_envs, runs, by_name", [ + (None, _playwright, False), + (None, FakeBrowser, True), + ([], _playwright, False), + ("chromium", _playwright, True), + ("playwright", FakeBrowser, False), +], ids=["runs-mcp-server", "runs-computer", "none", "named-computer", "named-mcp-server"]) +def test_the_gateway_lists_the_sites_by_name_only_for_a_browser_computer(browser_envs, runs, by_name): + named = {"chromium": [FakeBrowser()], "playwright": [_playwright()]}[browser_envs] \ + if isinstance(browser_envs, str) else browser_envs + office = _browsing(browser_envs=named) + with patch("agent_env.env.env.Env.get", return_value=runs()), \ + patch.object(env_gateway_provider, "website_config", return_value=MagicMock()) as config: + env_gateway_provider._gateway_topology(office) + assert config.call_args_list and {call.kwargs["listed_by_name"] for call in config.call_args_list} == {by_name} + + def test_a_lone_website_with_a_browser_computer_as_the_runs_browser_is_refused_naming_a_multi_env(): with patch("agent_env.env.env.Env.get", return_value=FakeBrowser()), \ pytest.raises(ValueError, match="'chromium-computer' is a computer env, which runs on a sandbox of its own " diff --git a/tst/unit/env/gateway/rest_proxy_stream_test.py b/tst/unit/env/gateway/rest_proxy_stream_test.py index 27a82c258..921187199 100644 --- a/tst/unit/env/gateway/rest_proxy_stream_test.py +++ b/tst/unit/env/gateway/rest_proxy_stream_test.py @@ -527,7 +527,9 @@ async def test_a_client_serving_the_route_at_its_own_root_is_told_no_prefix_and_ ("javascript://notes.test", (["front.example"], ["https"], [])), ("https://notes.test/elsewhere", (["front.example"], ["https"], [])), ("https://user@notes.test", (["front.example"], ["https"], [])), -], ids=["origin", "port", "not-http", "a-path", "userinfo"]) + ("https://]notes.test[", (["front.example"], ["https"], [])), + ("https://nötes.test", (["front.example"], ["https"], [])), +], ids=["origin", "port", "not-http", "a-path", "userinfo", "unbalanced-bracket", "not-ascii"]) async def test_a_site_root_naming_an_origin_is_the_address_though_a_front_rewrote_the_forwarded_host( monkeypatch, root, expected): """A front between a computer's site proxy and the gateway, as a provider's ingress is, may set its own @@ -535,11 +537,26 @@ async def test_a_site_root_naming_an_origin_is_the_address_though_a_front_rewrot up = _Upstream([b"ok"]) up.install(monkeypatch) gw = _gateway(website_urls={"web": "http://web"}) - await _call(gw, "/website/web/api/x", headers={"host": "gw.example", "x-forwarded-host": "front.example", - "x-forwarded-proto": "https", "agentenv-site-root": root}) + await _call(gw, "/website/web/api/x", raw_headers=[ + (b"host", b"gw.example"), (b"x-forwarded-host", b"front.example"), (b"x-forwarded-proto", b"https"), + (b"agentenv-site-root", root.encode())]) assert [_forwarded(r) for r in up.forwarded] == [expected] +@pytest.mark.asyncio +async def test_a_host_that_isnt_printable_ascii_names_nothing_and_the_request_still_goes_through(monkeypatch): + up = _Upstream([b"ok"]) + up.install(monkeypatch) + gw = _gateway(website_urls={"web": "http://web"}, site_names={"web": ["shop.test"]}) + answered = [(await _call(gw, "/website/web/x", raw_headers=raw))[0] for raw in ( + [(b"host", "shöp.test".encode())], + [(b"host", b"shop.test"), (b"x-forwarded-host", "shöp.test".encode())], + [(b"host", b"gw.example"), (b"x-forwarded-host", b"shop.test\x7f")])] + assert answered == [200, 200, 200] + assert [(r.headers["host"], _forwarded(r)[0]) for r in up.forwarded] == [ + ("web", []), ("shop.test", ["shop.test"]), ("web", ["gw.example"])] + + @pytest.mark.asyncio async def test_a_front_s_own_forwarded_host_and_scheme_win_and_reach_the_upstream_once(monkeypatch): up = _Upstream([b"ok"]) @@ -556,6 +573,119 @@ async def test_a_front_s_own_forwarded_host_and_scheme_win_and_reach_the_upstrea assert up.forwarded[0].headers.get_list("x-forwarded-for") == ["10.0.0.1"] +@pytest.mark.asyncio +@pytest.mark.parametrize("method, headers, host", [ + ("GET", {"agentenv-site-root": "https://shop.test"}, "shop.test"), + ("HEAD", {"agentenv-site-root": "https://shop.test"}, "shop.test"), + ("POST", {"agentenv-site-root": "https://Shop.TEST:8443"}, "Shop.TEST:8443"), + ("GET", {"x-forwarded-host": "www.shop.test, gw.example", "x-forwarded-proto": "https"}, "www.shop.test"), + ("GET", {"host": "shop.test"}, "shop.test"), + ("GET", {"agentenv-site-root": "https://api.shop.test"}, "api.shop.test"), + ("GET", {"agentenv-site-root": "https://a.b.shop.test"}, "web"), + ("GET", {"agentenv-site-root": "https://evil.example"}, "web"), + ("GET", {"x-forwarded-host": "shop.test.evil.example"}, "web"), + ("GET", {"host": "evil.example"}, "web"), + ("GET", {"x-forwarded-host": "shop.test:65536"}, "web"), + ("GET", {"x-forwarded-host": "shop.test@evil.example"}, "web"), + ("GET", {"x-forwarded-host": "shop..test"}, "web"), + ("GET", {"x-forwarded-host": "[::1]:80"}, "web"), +], ids=["site-root", "head", "case-and-port", "front", "host", "wildcard", "two-labels-under-a-wildcard", "forged-root", + "suffix", "forged-host", "bad-port", "userinfo", "empty-label", "address"]) +async def test_a_site_reached_at_one_of_its_names_takes_it_as_its_host_and_at_any_other_keeps_its_own( + monkeypatch, method, headers, host): + up = _Upstream([b"ok"]) + up.install(monkeypatch) + gw = _gateway(website_urls={"web": "http://web"}, site_names={"web": ["shop.test", "www.shop.test", "*.shop.test"]}) + await _call(gw, "/website/web/api/x", method=method, headers={"host": "gw.example", **headers}) + assert [r.headers["host"] for r in up.forwarded] == [host] + + +@pytest.mark.asyncio +async def test_a_site_with_no_names_and_a_svc_route_keep_their_own_host_wherever_the_client_was(monkeypatch): + up = _Upstream([b"ok"]) + up.install(monkeypatch) + gw = Gateway(host="127.0.0.1", port=0, server_name="t", internal_mcp_servers=[], + website_urls={"web": "http://web", "docs": "http://docs"}, rest_proxy_urls={"web": "http://web"}, + site_names={"web": ["shop.test"]}) + for path in ("/website/docs/x", "/svc/web/x"): + await _call(gw, path, headers={"host": "shop.test", "agentenv-site-root": "https://shop.test"}) + assert [r.headers["host"] for r in up.forwarded] == ["docs", "web"] + + +def _card_site(seen: list[str], declared: dict[str, list[str]]): + """A site at each host of ``declared`` whose card declares those names; ``seen`` takes each other request's Host, + and a host ``declared`` doesn't name serves no card.""" + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/.well-known/agent-env.json": + if request.url.host not in declared: + return httpx.Response(404) + return httpx.Response(200, json={"name": request.url.host, "capabilities": {"extensions": [ + {"uri": "urn:agentenv:website/v1", "params": {"domains": declared[request.url.host]}}]}}) + seen.append(request.headers["host"]) + + async def body(): + yield b"ok" + return httpx.Response(200, content=body()) + return httpx.MockTransport(handler) + + +@pytest.mark.asyncio +async def test_a_site_whose_env_stores_no_names_takes_its_host_from_the_names_its_card_declares(monkeypatch): + seen: list[str] = [] + transport = _card_site(seen, {"notes": ["Notes.Test"], "kept": ["notes.test"]}) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", + lambda **kw: _RealAsyncClient(transport=transport, **kw)) + gw = _gateway(website_urls={"notes": "http://notes", "kept": "http://kept"}, site_names={"kept": ["kept.test"]}) + for path, root in (("notes", "https://notes.test"), ("kept", "https://notes.test"), ("kept", "https://kept.test")): + await _call(gw, f"/website/{path}/x", headers={"host": "gw.example", "agentenv-site-root": root}) + assert seen == ["notes.test", "kept", "kept.test"] + + +@pytest.mark.parametrize("upstream, destination", [ + ("https://notes.internal", {"host": "notes.internal", "port": 443, "server_hostname": "notes.internal"}), + ("http://notes.internal", {"host": "notes.internal", "port": 80, "server_hostname": "unset"}), + ("https://notes.internal:8443", {"host": "notes.internal", "port": 8443, "server_hostname": "notes.internal"}), +], ids=["https", "http", "https-port"]) +def test_a_websocket_to_a_site_reached_by_name_connects_to_its_upstream_and_checks_the_upstreams_certificate( + monkeypatch, upstream, destination): + """Straight to the upstream, never through a proxy the environment names, which would tunnel to the site's + name: a real domain for a site named after one.""" + monkeypatch.setenv("HTTPS_PROXY", "http://proxy.example:3128") + monkeypatch.setenv("HTTP_PROXY", "http://proxy.example:3128") + transport = _card_site([], {}) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", + lambda **kw: _RealAsyncClient(transport=transport, **kw)) + asked = [] + + async def connect(uri, **kwargs): + asked.append((uri, {key: kwargs.get(key, "unset") for key in ("host", "port", "server_hostname", "proxy")})) + raise OSError("refused") + monkeypatch.setattr(rest_proxy_module, "connect_websocket", connect) + gw = _gateway(website_urls={"notes": upstream}, site_names={"notes": ["notes.test"]}) + with pytest.raises(Exception): + with TestClient(gw._asgi_app()).websocket_connect("/website/notes/live", + headers={"AgentEnv-Site-Root": "https://notes.test"}): + pass + scheme = "wss" if upstream.startswith("https") else "ws" + assert asked == [(f"{scheme}://notes.test/live", {**destination, "proxy": None})] + + +@pytest.mark.asyncio +async def test_list_website_urls_gives_a_site_listed_by_name_at_its_first_name_and_any_other_behind_the_gateway( + monkeypatch): + transport = _card_site([], {"carded": ["carded.test"]}) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", + lambda **kw: _RealAsyncClient(transport=transport, **kw)) + sites = ("shop", "notes", "wild", "bare", "carded", "other") + gw = _gateway(website_urls={site: f"http://{site}" for site in sites}, + site_names={"shop": ["*.shop.test", "www.shop.test", "shop.test"], "notes": ["notes.localhost"], + "wild": ["*.wild.test"], "other": ["other.test"]}, + listed_by_name=["shop", "notes", "wild", "bare", "carded"]) + listed = await gw._mcp._tool_manager.get_tool("list_website_urls").fn() + assert listed == ["https://www.shop.test/", "http://notes.localhost/", "http://wild", "http://bare", + "https://carded.test/", "http://other"] + + @pytest.mark.asyncio async def test_an_sdk_site_behind_the_gateway_names_the_gateway_s_address(monkeypatch): @website(domains=["notes.example"]) @@ -565,7 +695,8 @@ def whoami(self, ctx: ApiContext) -> dict: return {"base_url": ctx.base_url} transport = httpx.ASGITransport(app=Notes().create_app()) - monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", lambda **kw: _RealAsyncClient(transport=transport, **kw)) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", + lambda **kw: _RealAsyncClient(transport=transport, **kw)) gw = _gateway(website_urls={"notes": "http://notes"}) async def whoami(path: str) -> tuple[int, dict, bytes]: diff --git a/tst/unit/env/gateway/website_mcp_test.py b/tst/unit/env/gateway/website_mcp_test.py index 8f24864ed..c62be9ea1 100644 --- a/tst/unit/env/gateway/website_mcp_test.py +++ b/tst/unit/env/gateway/website_mcp_test.py @@ -198,3 +198,11 @@ async def test_the_gateway_serves_a_sites_tools_and_its_route_refuses_the_sites_ assert site.orders == [{"item": "milk", "by": "alice@example.com"}, {"item": "eggs", "by": "bob@example.com"}] assert browsing.status_code == 404 assert data.json()["result"]["parts"][0]["data"]["orders"][0]["item"] == "milk" + + +def test_each_sites_names_are_read_from_their_variable_and_a_malformed_entry_is_refused(): + assert entrypoint_module.parse_site_names(" shop=shop.test|*.shop.test , docs=docs.test,") == { + "shop": ("shop.test", "*.shop.test"), "docs": ("docs.test",)} + assert entrypoint_module.parse_site_names("") == {} + with pytest.raises(ValueError, match="WEBSITE_NAMES entry: 'shop.test'"): + entrypoint_module.parse_site_names("shop.test") diff --git a/tst/unit/env/gateway/website_websocket_test.py b/tst/unit/env/gateway/website_websocket_test.py index 002e7791b..3eff57ce6 100644 --- a/tst/unit/env/gateway/website_websocket_test.py +++ b/tst/unit/env/gateway/website_websocket_test.py @@ -114,6 +114,27 @@ async def test_messages_and_the_sites_close_are_relayed_with_the_gateways_header assert (headers["x-forwarded-proto"], headers["x-forwarded-prefix"]) == ("http", "/website/notes") +@pytest.mark.asyncio +async def test_a_websocket_reached_at_the_sites_name_takes_it_as_its_host_and_at_any_other_keeps_its_own(): + site = EchoSite() + site_server, site_port = await serve(site.app) + gateway = Gateway(host="127.0.0.1", port=0, server_name="t", internal_mcp_servers=[], + rest_proxy_urls={"notes": f"http://127.0.0.1:{site_port}"}, + website_urls={"notes": f"http://127.0.0.1:{site_port}"}, site_names={"notes": ["notes.test"]}) + gateway_server, gateway_port = await serve(gateway._asgi_app()) + try: + for root in ("https://notes.test", "https://evil.example"): + async with connect(f"ws://127.0.0.1:{gateway_port}/website/notes/live", + additional_headers=[("AgentEnv-Site-Root", root)]) as websocket: + await websocket.send("hi") + assert await asyncio.wait_for(websocket.recv(), 5) == "HI" + finally: + await gateway_server.shutdown() + await site_server.shutdown() + assert [(headers["host"], headers.get("x-forwarded-host")) for _, headers in site.handshakes] == [ + ("notes.test", "notes.test"), (f"127.0.0.1:{site_port}", "evil.example")] + + @pytest.mark.asyncio async def test_the_sites_handshake_answer_reaches_the_client_with_each_cookie_apart(stack): _, route = stack diff --git a/tst/unit/providers/env_providers/env_gateway_provider_test.py b/tst/unit/providers/env_providers/env_gateway_provider_test.py index a7014ee02..7ddc4a0d4 100644 --- a/tst/unit/providers/env_providers/env_gateway_provider_test.py +++ b/tst/unit/providers/env_providers/env_gateway_provider_test.py @@ -31,6 +31,7 @@ from agent_env.env.envs.multi_env import MultiEnv from agent_env.env.envs.service_db import SERVICE_DB_PORT from agent_env.env.envs.website import WebsiteEnv +from agent_env.env.gateway.constants import WEBSITE_LISTED_BY_NAME_ENV, WEBSITE_NAMES_ENV from agent_env.providers.env_providers import env_gateway_provider, env_provider from agent_env.providers.env_providers.constants import GATEWAY_APP_DIR, GATEWAY_SERVICE_NAME from agent_env.providers.env_providers.env_gateway_provider import ( @@ -1577,6 +1578,23 @@ def test_each_website_backend_gets_a_session_secret_of_its_own_which_the_logged_ assert shop not in _redact_compose_secrets(compose) and notes not in _redact_compose_secrets(compose) +def test_the_gateway_is_handed_each_synthetic_sites_stored_names_and_the_sites_it_lists_by_name(): + compose = _compose_for([], website_configs=[ + WebsiteConfig(backend_image="b", frontend_image=None, environment_name="shop", + domains=("shop.test", "*.shop.test"), listed_by_name=True), + WebsiteConfig(backend_image="b", frontend_image=None, environment_name="notes", listed_by_name=True), + WebsiteConfig(backend_image="b", frontend_image=None, environment_name="docs", domains=("docs.test",)), + WebsiteConfig(backend_image="b", frontend_image=None, environment_name="github", live=True, + domains=("github.com",), listed_by_name=True), + ]) + gateway = _service_block(compose, "gateway") + assert _env_values(gateway, WEBSITE_NAMES_ENV) == ["shop=shop.test|*.shop.test,docs=docs.test"] + assert _env_values(gateway, WEBSITE_LISTED_BY_NAME_ENV) == ["shop,notes"] + plain = _service_block(_compose_for([], website_configs=[ + WebsiteConfig(backend_image="b", frontend_image=None, environment_name="notes")]), "gateway") + assert _env_values(plain, WEBSITE_NAMES_ENV) == _env_values(plain, WEBSITE_LISTED_BY_NAME_ENV) == [] + + def test_injected_names_are_the_environment_name_not_the_image_or_container_name(): """Both vars carry environment_name verbatim. The image tags here differ from the names, and the website backend's container name is `-website-backend`, so substituting the wrong diff --git a/uv.lock b/uv.lock index 10b74e13c..6f5528239 100644 --- a/uv.lock +++ b/uv.lock @@ -275,7 +275,7 @@ requires-dist = [ { name = "urllib3", marker = "extra == 'boat'", specifier = ">=2.1.0,<3" }, { name = "uvicorn", specifier = ">=0.47" }, { name = "vercel-sandbox", marker = "extra == 'vercel'", specifier = ">=0.7.0,<0.8" }, - { name = "websockets", marker = "extra == 'dev'", specifier = ">=13" }, + { name = "websockets", marker = "extra == 'dev'", specifier = ">=15" }, ] provides-extras = ["openenv", "harness", "trl", "nemo-gym", "verl", "skyrl", "miles", "slime", "aws", "gcp", "sail", "vercel", "tensorlake", "tenki", "boxd", "daytona", "boat", "coreweave", "dev", "smol"]