From 3ed3350ea379ef87fdd986fb0e461e1fcb0c177f Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 11 Nov 2026 14:46:41 -0800 Subject: [PATCH 1/7] Gateway site routes by real names (M8.12) A site reached at one of its names through the gateway's /website/ route gets that name as its Host, so a site that builds its links and redirects from Host names itself as the browser reached it. Any other host leaves Host the site's internal address, so a forged one never reaches it; X-Forwarded-Host is unchanged. The names are the ones the site's env stores (WEBSITE_NAMES), else the ones its card declares, matched as a computer's site proxy matches them. WebSockets on the route follow the same rule. list_website_urls gives each site at its first name when the multi env's browser is a browser computer, which opens the sites there (WEBSITE_LISTED_BY_NAME), and at its internal address when it's playwright-mcp. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/constants.py | 4 + src/agent_env/env/gateway/entrypoint.py | 16 +++- src/agent_env/env/gateway/gateway.py | 38 +++++++- src/agent_env/env/gateway/rest_proxy.py | 85 ++++++++++++++--- .../env_providers/env_gateway_provider.py | 26 +++-- tst/data/host_website/Dockerfile | 12 +++ tst/data/host_website/server.py | 48 ++++++++++ .../computer/browser_computer_test.py | 95 +++++++++++++++++++ tst/unit/env/envs/test_multi_env_computers.py | 17 ++++ .../env/gateway/rest_proxy_stream_test.py | 82 ++++++++++++++++ tst/unit/env/gateway/website_mcp_test.py | 8 ++ .../env/gateway/website_websocket_test.py | 21 ++++ .../env_gateway_provider_test.py | 18 ++++ 13 files changed, 444 insertions(+), 26 deletions(-) create mode 100644 tst/data/host_website/Dockerfile create mode 100644 tst/data/host_website/server.py diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index 37af9456c..58b8698df 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -150,6 +150,10 @@ def sources(names: list[str]) -> str: # Sent by a computer's site proxy, which serves a website's route at the root of the site's own name. AGENT_ENV_SITE_ROOT_HEADER = "AgentEnv-Site-Root" GATEWAY_PROVIDER_HEADERS_ENV = "AGENTENV_PROVIDER_REQUEST_HEADERS" +# Each synthetic site's names as its env stores them, `shop=shop.test|www.shop.test,...`, one of which a site takes as +# its Host when a client reaches it there; and the sites list_website_urls gives at their first name, `shop,docs`. +WEBSITE_NAMES_ENV = "WEBSITE_NAMES" +WEBSITE_LISTED_BY_NAME_ENV = "WEBSITE_LISTED_BY_NAME" # MCP request `_meta` keys the gateway stamps on every child tools/call: the caller's role, and on the MCP path a # key naming the inbound session, so a child can keep per-caller state behind the gateway's one shared session. AGENT_ENV_ROLE_META_KEY = "agentenv.io/role" diff --git a/src/agent_env/env/gateway/entrypoint.py b/src/agent_env/env/gateway/entrypoint.py index 1437372ff..311facfe2 100644 --- a/src/agent_env/env/gateway/entrypoint.py +++ b/src/agent_env/env/gateway/entrypoint.py @@ -9,7 +9,7 @@ from . import AGENT_ENV_GATEWAY_MCP_PORT, GatewayMode, InternalMCPServer from .constants import (CONTROL_TOKEN_ENV, DEFAULT_MCP_SERVER_NAME, GATEWAY_PROVIDER_HEADERS_ENV, MCP_TRANSPORT, - WELL_KNOWN_PATH) + WEBSITE_LISTED_BY_NAME_ENV, WEBSITE_NAMES_ENV, WELL_KNOWN_PATH) from .gateway import Gateway, card_url logger = logging.getLogger(__name__) @@ -188,6 +188,17 @@ def parse_name_url_map(env_str: str) -> dict[str, str]: return urls +def parse_site_names(env_str: str) -> dict[str, tuple[str, ...]]: + """Each site's names from ``WEBSITE_NAMES``: ``shop=shop.test|www.shop.test,docs=docs.test``.""" + names: dict[str, tuple[str, ...]] = {} + for entry in filter(None, (part.strip() for part in env_str.split(","))): + site, sep, listed = entry.partition("=") + if not sep or not site.strip(): + raise ValueError(f"Invalid {WEBSITE_NAMES_ENV} entry: '{entry}' (expected 'site=name|name')") + names[site.strip()] = tuple(name.strip() for name in listed.split("|") if name.strip()) + return names + + if __name__ == "__main__": logging.basicConfig( level=logging.INFO, @@ -218,5 +229,8 @@ def parse_name_url_map(env_str: str) -> dict[str, str]: service_db_url=service_db_url, control_token=os.environ.get(CONTROL_TOKEN_ENV), provider_headers=os.environ.get(GATEWAY_PROVIDER_HEADERS_ENV, "").split(","), + site_names=parse_site_names(os.environ.get(WEBSITE_NAMES_ENV, "")), + listed_by_name=[site.strip() for site in os.environ.get(WEBSITE_LISTED_BY_NAME_ENV, "").split(",") + if site.strip()], ) gateway.run() diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index 3b257b056..c2837e3ec 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -80,8 +80,11 @@ ROLE_HEADER_BYTES, UpstreamResponse, WebsiteRoute, + declared_site_names, fetch_child_card, forward, + listed_url, + lowered, operations, refuse_websocket, relay_websocket, @@ -142,15 +145,23 @@ def __init__( service_db_url: str | None = None, control_token: str | None = None, provider_headers: Iterable[str] = (), + site_names: dict[str, Iterable[str]] | None = None, + listed_by_name: Iterable[str] = (), ): """`provider_headers` names the headers the sandbox provider in front of the gateway is sent, its credentials, - which the gateway takes off every request it forwards to a service behind it.""" + which the gateway takes off every request it forwards to a service behind it. `site_names` are each site's names + as its env stores them, the names its card declares standing in for a site's that stores none: a site reached at + one takes it as its ``Host``. ``list_website_urls`` gives each site `listed_by_name` at its first name, as a + browser computer opens it, and the others at their ``website_urls``.""" self.host = host self.port = port self.server_name = server_name self.internal_mcp_servers = internal_mcp_servers self.website_urls = website_urls or {} self.rest_proxy_urls = rest_proxy_urls or {} + self._site_names = {site: lowered(names) for site, names in (site_names or {}).items()} + self._card_site_names: dict[str, tuple[str, ...]] = {} + self._listed_by_name = frozenset(listed_by_name) self._site_mcp_endpoints = site_mcp_endpoints(internal_mcp_servers, self.website_urls, self.rest_proxy_urls) self._site_reserved: dict[str, frozenset[str]] = {} self._withheld = frozenset(name.strip().lower().encode() for name in provider_headers if name.strip()) @@ -195,7 +206,7 @@ def __init__( if self.website_urls: @self._mcp.tool(name="list_website_urls", description="List available website URLs that can be browsed.") async def list_website_urls() -> list[str]: - return list(self.website_urls.values()) + return [await self._listed_url(site, url) for site, url in self.website_urls.items()] self._server_tools.setdefault("gateway", []).append( MCPTool( name="list_website_urls", @@ -659,9 +670,10 @@ async def _proxy_rest_request(self, request: Request, url_map: dict[str, str] | mcp_endpoint = site_mcp or "mcp" if url_map is self.rest_proxy_urls and (cdp := await self._browsers.cdp_answer(request, base_url, role)) is not None: return cdp + site_names = self._names(service_name) if url_map is self.website_urls else () if request.method in _SAFE_METHODS: return await forward(request, base_url, role=role, timeout=self.REST_PROXY_TIMEOUT, mcp_server=mcp_server, - withheld=self._withheld, mcp_endpoint=mcp_endpoint) + withheld=self._withheld, mcp_endpoint=mcp_endpoint, site_names=site_names) changelog_before = self._query_changelog_id() attribution = self._attribution(role) # at the start: a fire may be gone by the time the response ends @@ -686,7 +698,8 @@ async def log_write(status: int) -> None: }) return await forward(request, base_url, role=role, timeout=self.REST_PROXY_TIMEOUT, mcp_server=mcp_server, - on_done=log_write, withheld=self._withheld, mcp_endpoint=mcp_endpoint) + on_done=log_write, withheld=self._withheld, mcp_endpoint=mcp_endpoint, + site_names=site_names) async def _proxy_website_request(self, request: Request) -> Response: """Proxy a website request; with a website or state trigger registered, feed it to the engine and hold barriers. @@ -730,7 +743,8 @@ async def _proxy_website_socket(self, websocket: WebSocket) -> None: await refuse_websocket(websocket, self._json_response({"error": f"Unknown service: {service_name}"}, 404)) return await relay_websocket(websocket, base_url, role=self._request_role(websocket), - open_timeout=self.REST_PROXY_CONNECT_TIMEOUT_S, withheld=self._withheld) + open_timeout=self.REST_PROXY_CONNECT_TIMEOUT_S, withheld=self._withheld, + site_names=self._names(service_name)) async def _fire_website_triggers(self, request: Request, app: str, status: int) -> tuple[list[asyncio.Task], float | None]: @@ -803,9 +817,23 @@ async def _site_reserved_paths(self, app: str) -> frozenset[str] | None: except ValueError: card = None address = self.website_urls.get(app) or base_url + self._card_site_names[app] = declared_site_names(card) if isinstance(card, dict) else () self._site_reserved[app] = site_reserved_paths(card, address) if isinstance(card, dict) else frozenset() return self._site_reserved[app] + def _names(self, app: str) -> tuple[str, ...]: + """The names website ``app`` is reached at: those its env stores, else those its card declares, as last read.""" + return self._site_names.get(app) or self._card_site_names.get(app, ()) + + async def _listed_url(self, site: str, url: str) -> str: + """The address ``list_website_urls`` gives ``site``, at ``url`` behind the gateway: for a site listed by name, + its first name (``listed_url``), its card read for a site whose env stores none; else, or with none, ``url``.""" + if site not in self._listed_by_name: + return url + if site not in self._site_names: + await self._site_reserved_paths(site) + return listed_url(self._names(site)) or url + def _backing_keys(self) -> list[str]: # rest_proxy_urls entries are the env's backing servers: mcp- (MCP) + (website # backends). Non-v1 backers (e.g. the injected website_browser) self-omit from the card via 404. diff --git a/src/agent_env/env/gateway/rest_proxy.py b/src/agent_env/env/gateway/rest_proxy.py index 54235b290..2543d4ac6 100644 --- a/src/agent_env/env/gateway/rest_proxy.py +++ b/src/agent_env/env/gateway/rest_proxy.py @@ -15,7 +15,8 @@ own, when one before the gateway sent them, else the request's ``Host`` and scheme) and ``X-Forwarded-Prefix``, the route prefix (``/svc/{service}`` or ``/website/{service}``) the gateway strips, which a client never sets: a client that serves the route at a root of its own (a computer's site proxy, at ``https://notes.test/``) says so with - ``AgentEnv-Site-Root``, and the upstream is then told no prefix; + ``AgentEnv-Site-Root``, and the upstream is then told no prefix; a site reached at one of its own names gets that + name as its ``Host`` too, and any other host leaves ``Host`` its internal address, so a forged one never reaches it; - the upstream response and the client it came through are closed when the response ends, iterated or not. A WebSocket on ``/website/{service}/{path}`` is relayed under the same rules (:func:`relay_websocket`): the gateway @@ -36,9 +37,9 @@ import logging import posixpath import re -from collections.abc import AsyncIterator, Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable, Iterable from typing import Any, NamedTuple -from urllib.parse import unquote, urlsplit +from urllib.parse import unquote, urlsplit, urlunsplit import httpx from starlette.requests import HTTPConnection, Request @@ -71,6 +72,9 @@ ROLE_HEADER_BYTES = AGENT_ENV_ROLE_HEADER.lower().encode() FORWARDED_HOST, FORWARDED_PROTO, FORWARDED_PREFIX = "x-forwarded-host", "x-forwarded-proto", "x-forwarded-prefix" +# A host as a site's name: a DNS name, an optional port. +_LABEL = r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?" +_SITE_HOST = re.compile(rf"({_LABEL}(?:\.{_LABEL})*)(?::([0-9]{{1,5}}))?") _FORWARDED = frozenset(name.encode() for name in (FORWARDED_HOST, FORWARDED_PROTO, FORWARDED_PREFIX, AGENT_ENV_SITE_ROOT_HEADER.lower())) # A WebSocket's handshake fields, which the gateway's own connection upstream writes for itself. @@ -86,9 +90,10 @@ async def forward(request: Request, base_url: str, *, role: str, timeout: httpx.Timeout, mcp_server: bool, on_done: Callable[[int], Awaitable[None]] | None = None, - withheld: frozenset[bytes] = frozenset(), mcp_endpoint: str = "mcp") -> Response: + withheld: frozenset[bytes] = frozenset(), mcp_endpoint: str = "mcp", + site_names: tuple[str, ...] = ()) -> Response: """Forward `request` to the service at `base_url` as `role`, without the headers named in `withheld`, and stream - the response through. `mcp_server` says + the response through; a site's `site_names` say which hosts it takes as its ``Host``. `mcp_server` says the service is a backing MCP server, whose own endpoint, `mcp_endpoint` under it, takes the role in its JSON-RPC `_meta` as well as in the header. A 400 names a request-target the proxy cannot route; a 502 names an upstream that never answered. `on_done` is awaited with the status once the response has ended, after its last byte when streamed.""" @@ -97,7 +102,7 @@ async def forward(request: Request, base_url: str, *, role: str, timeout: httpx. except ValueError as e: # an escaped slash inside the route prefix; refuse rather than forward a wrong path return await _done(_json_response({"error": str(e)}, 400), on_done) content = await request.body() # /svc and /website request bodies are small (JSON-RPC, forms) - headers = upstream_headers(request, role, withheld=withheld) + headers = upstream_headers(request, role, withheld=withheld, site_names=site_names) if mcp_server and request.method == "POST" and is_child_mcp_path(request.path_params.get("path", ""), mcp_endpoint): # The gateway is the trust boundary for identity: a child's proxied MCP endpoint stays reachable, but the # role in its JSON-RPC `_meta` is the gateway's to write, like the header above. @@ -142,15 +147,19 @@ def client_headers(connection: HTTPConnection, withheld: frozenset[bytes] = froz return [(k, v) for k, v in connection.headers.raw if k.lower() not in HOP_BY_HOP and k.lower() not in withheld] -def upstream_headers(request: HTTPConnection, role: str, *, withheld: frozenset[bytes] = frozenset()) -> httpx.Headers: +def upstream_headers(request: HTTPConnection, role: str, *, withheld: frozenset[bytes] = frozenset(), + site_names: tuple[str, ...] = ()) -> httpx.Headers: """The client's headers (``client_headers``) as a proxied request sends them. The role header is the gateway's to set: the client's copies are dropped and the role the gateway attributed the request to goes in their place, so the upstream never reads a value the gateway did not write. So are the forwarded-address headers - (`forwarded_address`).""" + (`forwarded_address`), and ``Host`` is the site's own name when the client reached it at one of `site_names` + (``site_host``), else the upstream's address.""" headers = httpx.Headers([(k, v) for k, v in client_headers(request, withheld) if k.lower() != ROLE_HEADER_BYTES and k.lower() not in _FORWARDED]) headers[AGENT_ENV_ROLE_HEADER] = role headers.update(forwarded_address(request)) + if host := site_host(request, site_names): + headers["host"] = host # Bytes are forwarded undecoded, so only ask upstream for encodings the caller can take. headers.setdefault("accept-encoding", "identity") return headers @@ -190,6 +199,46 @@ def _first(value: str) -> str: return value.split(",")[0].strip() +def site_host(request: HTTPConnection, names: tuple[str, ...]) -> str | None: + """The host the client reached a site at (``forwarded_address``), when its name is one the site's ``names`` name, + compared without case or port: what the site's ``Host`` says, so a site that builds its links from ``Host`` names + itself as the client reached it. None for any other host, which a site never sees as its ``Host``.""" + host = forwarded_address(request).get(FORWARDED_HOST, "") + found = _SITE_HOST.fullmatch(host) + if not found or int(found.group(2) or 0) > 65535: + return None + return host if named(found.group(1).lower(), names) else None + + +def named(host: str, names: tuple[str, ...]) -> bool: + """Whether lowercase ``host`` is one ``names`` name: one of them, or one label more than a ``*.`` one, as a + computer's site proxy matches a site's names.""" + head, _, rest = host.partition(".") + return host in names or bool(head and rest) and f"*.{rest}" in names + + +def lowered(names: Iterable[str]) -> tuple[str, ...]: + return tuple(dict.fromkeys(name.lower() for name in names if isinstance(name, str) and name)) + + +def declared_site_names(card: dict) -> tuple[str, ...]: + """The names a site's card declares, its ``website/v1`` ``domains``, lowercase; none when it declares none.""" + for ext in (card.get("capabilities") or {}).get("extensions") or []: + params = ext.get("params") if isinstance(ext, dict) and ext.get("uri") == EXT_WEBSITE_URI else None + if isinstance(params, dict) and isinstance(params.get("domains"), list): + return lowered(params["domains"]) + return () + + +def listed_url(names: tuple[str, ...]) -> str | None: + """The address a browser computer opens a site with ``names`` at: its first name that isn't a ``*.`` one, over + HTTPS but for a ``.localhost`` one, as its site proxy serves them. None with no such name.""" + name = next((name for name in names if not name.startswith("*.")), None) + if name is None: + return None + return f"{'http' if name == 'localhost' or name.endswith('.localhost') else 'https'}://{name}/" + + def route_prefix(request: HTTPConnection) -> str: """The path the client reached the proxied service under, as it sent it: the request-target up to the `{path}` remainder, without its trailing slash, e.g. ``/website/notes``, or under a root path ``/env/website/notes``.""" @@ -254,7 +303,8 @@ def raw_remainder(raw_path: bytes, decoded_path: str, decoded_rest: str) -> byte async def relay_websocket(websocket: WebSocket, base_url: str, *, role: str, - open_timeout: float = 10.0, withheld: frozenset[bytes] = frozenset()) -> None: + open_timeout: float = 10.0, withheld: frozenset[bytes] = frozenset(), + site_names: tuple[str, ...] = ()) -> None: """Relay `websocket` to the service at `base_url` as `role`: a WebSocket of the gateway's own is opened to the same path, with the headers ``upstream_headers`` sends, asking for the subprotocols the client offered; `websocket` is accepted with the one the service chose and the headers its answer carried (a ``Set-Cookie`` among them, each @@ -262,20 +312,27 @@ async def relay_websocket(websocket: WebSocket, base_url: str, *, role: str, close, with its code, closes the other; a message from the service larger than ``WEBSOCKET_MESSAGE_MAX_BYTES``, the size uvicorn takes from a client, closes both with 1009 rather than be held whole. A service that refuses the handshake has its answer relayed as the refusal, its headers as an accepted one's are; one that never answers is a - 502. The headers named in `withheld` don't go upstream, as with ``upstream_headers``.""" + 502. The headers named in `withheld` don't go upstream, as with ``upstream_headers``, and a site reached at one of + its `site_names` takes it as its ``Host``.""" try: target_url = upstream_url(base_url, websocket) except ValueError as e: await refuse_websocket(websocket, _json_response({"error": str(e)}, 400)) return - headers = [(key, value) for key, value in upstream_headers(websocket, role, withheld=withheld).multi_items() - if key.lower() not in _WEBSOCKET_HANDSHAKE] + headers = [(key, value) for key, value in + upstream_headers(websocket, role, withheld=withheld, site_names=site_names).multi_items() + if key.lower() not in _WEBSOCKET_HANDSHAKE and key.lower() != "host"] + uri, destination = "ws" + target_url.removeprefix("http"), {} + if host := site_host(websocket, site_names): # websockets writes Host from the URI; the TCP goes to the upstream + parts = urlsplit(uri) + uri, destination = urlunsplit(parts._replace(netloc=host)), {"host": parts.hostname, "port": parts.port or 80} offered = [protocol.strip() for value in websocket.headers.getlist("sec-websocket-protocol") for protocol in value.split(",") if protocol.strip()] try: - upstream = await connect_websocket("ws" + target_url.removeprefix("http"), additional_headers=headers, + upstream = await connect_websocket(uri, additional_headers=headers, subprotocols=offered or None, user_agent_header=None, - open_timeout=open_timeout, max_size=WEBSOCKET_MESSAGE_MAX_BYTES) + open_timeout=open_timeout, max_size=WEBSOCKET_MESSAGE_MAX_BYTES, + **destination) except InvalidStatus as e: refusal = Response(content=bytes(e.response.body or b""), status_code=e.response.status_code) refusal.raw_headers = [*(item for item in refusal.raw_headers if item[0] == b"content-length"), diff --git a/src/agent_env/providers/env_providers/env_gateway_provider.py b/src/agent_env/providers/env_providers/env_gateway_provider.py index 6537a4a99..4d1043d65 100644 --- a/src/agent_env/providers/env_providers/env_gateway_provider.py +++ b/src/agent_env/providers/env_providers/env_gateway_provider.py @@ -36,6 +36,8 @@ DEFAULT_MCP_SERVER_NAME, GATEWAY_PROVIDER_HEADERS_ENV, STATE_TOOL_NAME_CLASHES_KEY, + WEBSITE_LISTED_BY_NAME_ENV, + WEBSITE_NAMES_ENV, WELL_KNOWN_PATH, random_mcp_server_name, tool_name_clashes_text, @@ -167,6 +169,10 @@ class WebsiteConfig: # a failure and the deploy waits, and its server's /dev/shm in MiB; None keeps the gateway's own and Docker's. ready_timeout_s: float | None = None shm_mb: int | None = None + # The names its env stores, one of which it takes as its Host when a client reaches it there (its card's stand in + # for none), and whether list_website_urls gives it at its first name, as a browser computer opens it. + domains: tuple[str, ...] = () + listed_by_name: bool = False @dataclass @@ -523,6 +529,11 @@ def create_docker_compose( ) if website_urls: gateway_env.append(f" - WEBSITE_URLS={website_urls}") + if names := ",".join(f"{wc.environment_name}={'|'.join(wc.domains)}" + for wc in website_configs if wc.domains and not wc.live): + gateway_env.append(f" - {WEBSITE_NAMES_ENV}={names}") + if listed := ",".join(wc.environment_name for wc in website_configs if wc.listed_by_name and not wc.live): + gateway_env.append(f" - {WEBSITE_LISTED_BY_NAME_ENV}={listed}") for wc in website_configs: rest_proxy_urls.append( f"{wc.environment_name}=http://{wc.environment_name}-{AGENT_ENV_WEBSITE_BACKEND_SUFFIX}:{AGENT_ENV_WEBSITE_BACKEND_PORT}" @@ -1580,14 +1591,15 @@ class _GatewayTopology: website_browser: "MCPServerEnv | None | object" # the browser it runs among its MCP servers, or _THE_RUNS -def website_config(env: WebsiteEnv) -> WebsiteConfig: +def website_config(env: WebsiteEnv, *, listed_by_name: bool = False) -> WebsiteConfig: """A website env as the gateway runs it: its server image, any deprecated frontend image, its health path, how long - it may take to be ready and its shared memory.""" + it may take to be ready, its shared memory and the names it stores, listed by its first when ``listed_by_name``.""" frontend = env.frontend_docker_image_artifact return WebsiteConfig(backend_image=env.server_docker_image_artifact.image_name, frontend_image=frontend.image_name if frontend else None, environment_name=env.environment_name, health_path=env.health_path, live=env.live, - settings=dict(env.settings), ready_timeout_s=env.ready_timeout_s, shm_mb=env.shm_mb) + settings=dict(env.settings), ready_timeout_s=env.ready_timeout_s, shm_mb=env.shm_mb, + domains=tuple(env.domains), listed_by_name=listed_by_name) def run_website_browser() -> "MCPServerEnv": @@ -1618,16 +1630,18 @@ def _with_browser(mcp_servers: list[MCPServerConfig], images: list[DockerImageAr def _gateway_topology(env: Env) -> _GatewayTopology: """What a gateway fronts for env: a MultiEnv's MCP servers and websites (whatever their own env_provider_type) under its - name and the website browser it names or the run's, a WebsiteEnv's website and the run's browser, or else the env's - one MCP server.""" + name and the website browser it names or the run's, its websites listed by name when that's a browser computer, a + WebsiteEnv's website and the run's browser, or else the env's one MCP server.""" from agent_env.env.envs.multi_env import MultiEnv from agent_env.env.envs.website import WebsiteEnv is_multi = isinstance(env, MultiEnv) + by_name = False if is_multi: servers, websites, name = env.mcp_server_envs, env.website_envs, env.name placed = env.website_browser() browser = _THE_RUNS if placed.runs else placed.server + by_name = placed.computer is not None # it opens the sites at their names, on a sandbox of its own elif isinstance(env, WebsiteEnv): servers, websites, name, browser = [], [env], None, _THE_RUNS else: @@ -1635,7 +1649,7 @@ def _gateway_topology(env: Env) -> _GatewayTopology: return _GatewayTopology( mcp_servers=[MCPServerConfig(image=s.docker_image_artifact.image_name, environment_name=s.environment_name) for s in servers], mcp_server_images=[s.docker_image_artifact for s in servers], - website_configs=[website_config(w) for w in websites] or None, + website_configs=[website_config(w, listed_by_name=by_name) for w in websites] or None, website_images=[image for w in websites for image in (w.server_docker_image_artifact, w.frontend_docker_image_artifact) if image] or None, mcp_server_name=name, diff --git a/tst/data/host_website/Dockerfile b/tst/data/host_website/Dockerfile new file mode 100644 index 000000000..7080524ee --- /dev/null +++ b/tst/data/host_website/Dockerfile @@ -0,0 +1,12 @@ +FROM mirror.gcr.io/library/python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* +WORKDIR /app +RUN pip install --no-cache-dir starlette uvicorn pydantic httpx +COPY agentenv_protocol /app/agentenv_protocol +COPY server.py /app/server.py +ENV PYTHONPATH=/app +ENV PORT=8000 +EXPOSE 8000 +RUN useradd --create-home --uid 10001 appuser +USER appuser +CMD ["python", "server.py"] diff --git a/tst/data/host_website/server.py b/tst/data/host_website/server.py new file mode 100644 index 000000000..aa71c7794 --- /dev/null +++ b/tst/data/host_website/server.py @@ -0,0 +1,48 @@ +"""A plain website, not on WebsiteEnvironment, that builds its addresses from its Host header, as many real apps do, for +integration-testing the gateway's site routes: /login redirects to /home at its Host, and /api/host says the Host and +X-Forwarded-Host it was reached with. Its setting ``card_domains``, names split by commas, makes it serve a card +declaring them; without it, it serves none.""" +import html +import json +import os + +import uvicorn +from starlette.applications import Starlette +from starlette.responses import HTMLResponse, JSONResponse, RedirectResponse, Response +from starlette.routing import Route + +from agentenv_protocol.website import WEBSITE_SETTINGS_ENV +from agentenv_protocol.website.readiness import WEBSITE_EXTENSION_URI + +DOMAINS = [name for name in str(json.loads(os.environ.get(WEBSITE_SETTINGS_ENV) or "{}").get("card_domains", "") + ).split(",") if name] + + +async def card(_): + if not DOMAINS: + return Response(status_code=404) + return JSONResponse({"name": os.environ.get("ENVIRONMENT_NAME", "shop"), "capabilities": {"extensions": [ + {"uri": WEBSITE_EXTENSION_URI, "params": {"domains": DOMAINS}}]}}) + + +async def health(_): + return JSONResponse({"status": "ok"}) + + +async def login(request): + return RedirectResponse(f"https://{request.headers.get('host', '')}/home", status_code=302) + + +async def home(request): + return HTMLResponse(f"home

{html.escape(request.headers.get('host', ''))}

") + + +async def host(request): + return JSONResponse({"host": request.headers.get("host"), "forwarded_host": request.headers.get("x-forwarded-host")}) + + +app = Starlette(routes=[Route("/.well-known/agent-env.json", card), Route("/api/health", health), + Route("/login", login), Route("/home", home), Route("/api/host", host)]) + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", "8000")), proxy_headers=False, log_level="warning") diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index fa54c53ff..d6984b5f8 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -77,6 +77,7 @@ from agent_env.env.envs.website import WebsiteEnv from agent_env.env.envs.computer.utils import inside from agent_env.env.store import get_env_store +from agent_env.providers.env_providers.constants import AGENT_ENV_WEBSITE_BACKEND_PORT, AGENT_ENV_WEBSITE_BACKEND_SUFFIX from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider from agent_env.store.object_store.local.tls import LocalCA, local_ca from agent_env.store.routing import namespace_routing @@ -1515,3 +1516,97 @@ async def test_a_multi_env_naming_no_browser_deploys_none(browser_computer, shop assert not report.failed and not deployed.child_envs assert "shop" in stated and PLAYWRIGHT_MCP_ENVIRONMENT_NAME not in stated and "chromium" not in stated, stated + + +# --- a site reached by its names through a multi env's gateway --- + +HOST_START = "https://shop.test/api/host" +HOST_CHROMIUM = {**MULTI_CHROMIUM, + "env_params": {"form": "container", "app_settings": {"chromium": {"start_urls": [HOST_START]}}}} +#: Where the gateway reaches the site, which is the site's Host when a client reaches it at any other name. +SITE_INTERNAL = f"shop-{AGENT_ENV_WEBSITE_BACKEND_SUFFIX}:{AGENT_ENV_WEBSITE_BACKEND_PORT}" + + +@pytest.fixture(scope="module") +def host_shops(browser_computer): + """Two sites named shop.test that build their addresses from their Host, as many real apps do, under local ids: + one whose env stores the name and that serves no card, and one that stores none and whose card declares it.""" + with namespace_routing(): + _seed_bootstrap_envs() + uid = uuid.uuid4().hex[:8] + image = _image("host_website", f"host-shop-{uid}") + return {"stored": WebsiteEnv.put(id=f"@local/~/host-{uid}/stored/shop", environment_name="shop", + domains=["shop.test"], server_docker_image_artifact=image), + "card": WebsiteEnv.put(id=f"@local/~/host-{uid}/card/shop", environment_name="shop", + settings={"card_domains": "shop.test"}, server_docker_image_artifact=image)} + + +async def _landed(deployed, browser: DeployedEnv, url: str) -> str: + """Where chromium lands, driven through the gateway's /mcp to ``url``: the first tab at the site's /home, else + every tab's URL.""" + async with streamable_http_client(deployed.mcp_url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + assert not (await session.call_tool("chromium_computer_navigate", {"url": url})).isError + deadline = asyncio.get_running_loop().time() + WAIT_S + while True: + urls = [tab.get("url", "") for tab in await tabs_of(browser)] + if (home := next((u for u in urls if u.endswith("/home")), None)) or \ + asyncio.get_running_loop().time() > deadline: + return home or " ".join(urls) + await asyncio.sleep(POLL_S) + + +async def _listed(deployed) -> list[str]: + async with streamable_http_client(deployed.mcp_url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + result = await session.call_tool("list_website_urls", {}) + assert not result.isError and result.structuredContent, result + return result.structuredContent["result"] + + +async def _seen(deployed, root: str | None) -> dict: + """The Host and X-Forwarded-Host the site sees for a request straight to the gateway's site route naming ``root`` + as the origin it was reached at, or naming none.""" + async with httpx.AsyncClient(timeout=30) as http: + answer = await http.get(f"{deployed.gateway_url}/website/shop/api/host", + headers={"AgentEnv-Site-Root": root} if root else {}) + assert answer.status_code == 200, answer.text + return answer.json() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("names", ["stored", "card"]) +async def test_a_site_chromium_reaches_by_name_through_the_gateway_takes_that_name_as_its_host_and_no_forged_one( + browser_computer, host_shops, made, names): + with namespace_routing(): + office = _office(host_shops[names], [Env.get(browser_computer.id)]) + step = DeployEnvTaskStep(id="office", version=None, env_id=office.id, sandbox_type="local", ttl_seconds=900, + child_envs={"chromium": HOST_CHROMIUM}) + context = TaskStepContext() + try: + [deployed] = (await step.execute(context)).deployed_envs + browser = DeployedEnv.from_dict(deployed.child_envs["chromium"]) + landed, listed = await _landed(deployed, browser, "https://shop.test/login"), await _listed(deployed) + seen = [await _seen(deployed, root) for root in ("https://Shop.Test", "https://evil.example", None)] + subprocess.run(["docker", "restart", _gateway(deployed, made)], capture_output=True, check=True) + again = await _answers_again(deployed) + after_restart = (await _landed(deployed, browser, "https://shop.test/login"), await _listed(deployed), + await _seen(deployed, "https://evil.example")) + finally: + report = await teardown_run(context) + + assert not report.failed + # chromium follows the site's redirect, built from its Host, to the name it reached it at. + assert landed == "https://shop.test/home", landed + # An agent beside chromium is given the site at the name chromium opens it at. + assert listed == ["https://shop.test/"], listed + # A client naming any other origin, or none, leaves the site its internal Host; X-Forwarded-Host is as it was. + gateway_host = urllib.parse.urlsplit(deployed.gateway_url).netloc + assert seen == [{"host": "Shop.Test", "forwarded_host": "Shop.Test"}, + {"host": SITE_INTERNAL, "forwarded_host": "evil.example"}, + {"host": SITE_INTERNAL, "forwarded_host": gateway_host}], seen + # A restarted gateway, which reads a card's names again, keeps all of it. + assert again and after_restart == ("https://shop.test/home", ["https://shop.test/"], + {"host": SITE_INTERNAL, "forwarded_host": "evil.example"}), after_restart diff --git a/tst/unit/env/envs/test_multi_env_computers.py b/tst/unit/env/envs/test_multi_env_computers.py index ffaeb9563..686cc2425 100644 --- a/tst/unit/env/envs/test_multi_env_computers.py +++ b/tst/unit/env/envs/test_multi_env_computers.py @@ -846,6 +846,23 @@ def test_the_gateway_runs_an_mcp_server_browser_and_leaves_a_browser_computer_to assert getattr(browser, "id", None) == expected +@pytest.mark.parametrize("browser_envs, runs, by_name", [ + (None, _playwright, False), + (None, FakeBrowser, True), + ([], _playwright, False), + ("chromium", _playwright, True), + ("playwright", FakeBrowser, False), +], ids=["runs-mcp-server", "runs-computer", "none", "named-computer", "named-mcp-server"]) +def test_the_gateway_lists_the_sites_by_name_only_for_a_browser_computer(browser_envs, runs, by_name): + named = {"chromium": [FakeBrowser()], "playwright": [_playwright()]}[browser_envs] \ + if isinstance(browser_envs, str) else browser_envs + office = _browsing(browser_envs=named) + with patch("agent_env.env.env.Env.get", return_value=runs()), \ + patch.object(env_gateway_provider, "website_config", return_value=MagicMock()) as config: + env_gateway_provider._gateway_topology(office) + assert config.call_args_list and {call.kwargs["listed_by_name"] for call in config.call_args_list} == {by_name} + + def test_a_lone_website_with_a_browser_computer_as_the_runs_browser_is_refused_naming_a_multi_env(): with patch("agent_env.env.env.Env.get", return_value=FakeBrowser()), \ pytest.raises(ValueError, match="'chromium-computer' is a computer env, which runs on a sandbox of its own " diff --git a/tst/unit/env/gateway/rest_proxy_stream_test.py b/tst/unit/env/gateway/rest_proxy_stream_test.py index 27a82c258..059763cc6 100644 --- a/tst/unit/env/gateway/rest_proxy_stream_test.py +++ b/tst/unit/env/gateway/rest_proxy_stream_test.py @@ -556,6 +556,88 @@ async def test_a_front_s_own_forwarded_host_and_scheme_win_and_reach_the_upstrea assert up.forwarded[0].headers.get_list("x-forwarded-for") == ["10.0.0.1"] +@pytest.mark.asyncio +@pytest.mark.parametrize("method, headers, host", [ + ("GET", {"agentenv-site-root": "https://shop.test"}, "shop.test"), + ("HEAD", {"agentenv-site-root": "https://shop.test"}, "shop.test"), + ("POST", {"agentenv-site-root": "https://Shop.TEST:8443"}, "Shop.TEST:8443"), + ("GET", {"x-forwarded-host": "www.shop.test, gw.example", "x-forwarded-proto": "https"}, "www.shop.test"), + ("GET", {"host": "shop.test"}, "shop.test"), + ("GET", {"agentenv-site-root": "https://api.shop.test"}, "api.shop.test"), + ("GET", {"agentenv-site-root": "https://a.b.shop.test"}, "web"), + ("GET", {"agentenv-site-root": "https://evil.example"}, "web"), + ("GET", {"x-forwarded-host": "shop.test.evil.example"}, "web"), + ("GET", {"host": "evil.example"}, "web"), + ("GET", {"x-forwarded-host": "shop.test:65536"}, "web"), + ("GET", {"x-forwarded-host": "shop.test@evil.example"}, "web"), + ("GET", {"x-forwarded-host": "shop..test"}, "web"), + ("GET", {"x-forwarded-host": "[::1]:80"}, "web"), +], ids=["site-root", "head", "case-and-port", "front", "host", "wildcard", "two-labels-under-a-wildcard", "forged-root", + "suffix", "forged-host", "bad-port", "userinfo", "empty-label", "address"]) +async def test_a_site_reached_at_one_of_its_names_takes_it_as_its_host_and_at_any_other_keeps_its_own( + monkeypatch, method, headers, host): + up = _Upstream([b"ok"]) + up.install(monkeypatch) + gw = _gateway(website_urls={"web": "http://web"}, site_names={"web": ["shop.test", "www.shop.test", "*.shop.test"]}) + await _call(gw, "/website/web/api/x", method=method, headers={"host": "gw.example", **headers}) + assert [r.headers["host"] for r in up.forwarded] == [host] + + +@pytest.mark.asyncio +async def test_a_site_with_no_names_and_a_svc_route_keep_their_own_host_wherever_the_client_was(monkeypatch): + up = _Upstream([b"ok"]) + up.install(monkeypatch) + gw = Gateway(host="127.0.0.1", port=0, server_name="t", internal_mcp_servers=[], + website_urls={"web": "http://web", "docs": "http://docs"}, rest_proxy_urls={"web": "http://web"}, + site_names={"web": ["shop.test"]}) + for path in ("/website/docs/x", "/svc/web/x"): + await _call(gw, path, headers={"host": "shop.test", "agentenv-site-root": "https://shop.test"}) + assert [r.headers["host"] for r in up.forwarded] == ["docs", "web"] + + +def _card_site(seen: list[str], declared: dict[str, list[str]]): + """A site at each host of ``declared`` whose card declares those names; ``seen`` takes each other request's Host, + and a host ``declared`` doesn't name serves no card.""" + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/.well-known/agent-env.json": + if request.url.host not in declared: + return httpx.Response(404) + return httpx.Response(200, json={"name": request.url.host, "capabilities": {"extensions": [ + {"uri": "urn:agentenv:website/v1", "params": {"domains": declared[request.url.host]}}]}}) + seen.append(request.headers["host"]) + + async def body(): + yield b"ok" + return httpx.Response(200, content=body()) + return httpx.MockTransport(handler) + + +@pytest.mark.asyncio +async def test_a_site_whose_env_stores_no_names_takes_its_host_from_the_names_its_card_declares(monkeypatch): + seen: list[str] = [] + transport = _card_site(seen, {"notes": ["Notes.Test"], "kept": ["notes.test"]}) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", lambda **kw: _RealAsyncClient(transport=transport, **kw)) + gw = _gateway(website_urls={"notes": "http://notes", "kept": "http://kept"}, site_names={"kept": ["kept.test"]}) + for path, root in (("notes", "https://notes.test"), ("kept", "https://notes.test"), ("kept", "https://kept.test")): + await _call(gw, f"/website/{path}/x", headers={"host": "gw.example", "agentenv-site-root": root}) + assert seen == ["notes.test", "kept", "kept.test"] + + +@pytest.mark.asyncio +async def test_list_website_urls_gives_a_site_listed_by_name_at_its_first_name_and_any_other_behind_the_gateway( + monkeypatch): + transport = _card_site([], {"carded": ["carded.test"]}) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", lambda **kw: _RealAsyncClient(transport=transport, **kw)) + sites = ("shop", "notes", "wild", "bare", "carded", "other") + gw = _gateway(website_urls={site: f"http://{site}" for site in sites}, + site_names={"shop": ["*.shop.test", "www.shop.test", "shop.test"], "notes": ["notes.localhost"], + "wild": ["*.wild.test"], "other": ["other.test"]}, + listed_by_name=["shop", "notes", "wild", "bare", "carded"]) + listed = await gw._mcp._tool_manager.get_tool("list_website_urls").fn() + assert listed == ["https://www.shop.test/", "http://notes.localhost/", "http://wild", "http://bare", + "https://carded.test/", "http://other"] + + @pytest.mark.asyncio async def test_an_sdk_site_behind_the_gateway_names_the_gateway_s_address(monkeypatch): @website(domains=["notes.example"]) diff --git a/tst/unit/env/gateway/website_mcp_test.py b/tst/unit/env/gateway/website_mcp_test.py index 8f24864ed..c62be9ea1 100644 --- a/tst/unit/env/gateway/website_mcp_test.py +++ b/tst/unit/env/gateway/website_mcp_test.py @@ -198,3 +198,11 @@ async def test_the_gateway_serves_a_sites_tools_and_its_route_refuses_the_sites_ assert site.orders == [{"item": "milk", "by": "alice@example.com"}, {"item": "eggs", "by": "bob@example.com"}] assert browsing.status_code == 404 assert data.json()["result"]["parts"][0]["data"]["orders"][0]["item"] == "milk" + + +def test_each_sites_names_are_read_from_their_variable_and_a_malformed_entry_is_refused(): + assert entrypoint_module.parse_site_names(" shop=shop.test|*.shop.test , docs=docs.test,") == { + "shop": ("shop.test", "*.shop.test"), "docs": ("docs.test",)} + assert entrypoint_module.parse_site_names("") == {} + with pytest.raises(ValueError, match="WEBSITE_NAMES entry: 'shop.test'"): + entrypoint_module.parse_site_names("shop.test") diff --git a/tst/unit/env/gateway/website_websocket_test.py b/tst/unit/env/gateway/website_websocket_test.py index 002e7791b..3eff57ce6 100644 --- a/tst/unit/env/gateway/website_websocket_test.py +++ b/tst/unit/env/gateway/website_websocket_test.py @@ -114,6 +114,27 @@ async def test_messages_and_the_sites_close_are_relayed_with_the_gateways_header assert (headers["x-forwarded-proto"], headers["x-forwarded-prefix"]) == ("http", "/website/notes") +@pytest.mark.asyncio +async def test_a_websocket_reached_at_the_sites_name_takes_it_as_its_host_and_at_any_other_keeps_its_own(): + site = EchoSite() + site_server, site_port = await serve(site.app) + gateway = Gateway(host="127.0.0.1", port=0, server_name="t", internal_mcp_servers=[], + rest_proxy_urls={"notes": f"http://127.0.0.1:{site_port}"}, + website_urls={"notes": f"http://127.0.0.1:{site_port}"}, site_names={"notes": ["notes.test"]}) + gateway_server, gateway_port = await serve(gateway._asgi_app()) + try: + for root in ("https://notes.test", "https://evil.example"): + async with connect(f"ws://127.0.0.1:{gateway_port}/website/notes/live", + additional_headers=[("AgentEnv-Site-Root", root)]) as websocket: + await websocket.send("hi") + assert await asyncio.wait_for(websocket.recv(), 5) == "HI" + finally: + await gateway_server.shutdown() + await site_server.shutdown() + assert [(headers["host"], headers.get("x-forwarded-host")) for _, headers in site.handshakes] == [ + ("notes.test", "notes.test"), (f"127.0.0.1:{site_port}", "evil.example")] + + @pytest.mark.asyncio async def test_the_sites_handshake_answer_reaches_the_client_with_each_cookie_apart(stack): _, route = stack diff --git a/tst/unit/providers/env_providers/env_gateway_provider_test.py b/tst/unit/providers/env_providers/env_gateway_provider_test.py index 0de98b99d..d71d2a201 100644 --- a/tst/unit/providers/env_providers/env_gateway_provider_test.py +++ b/tst/unit/providers/env_providers/env_gateway_provider_test.py @@ -31,6 +31,7 @@ from agent_env.env.envs.multi_env import MultiEnv from agent_env.env.envs.service_db import SERVICE_DB_PORT from agent_env.env.envs.website import WebsiteEnv +from agent_env.env.gateway.constants import WEBSITE_LISTED_BY_NAME_ENV, WEBSITE_NAMES_ENV from agent_env.providers.env_providers import env_gateway_provider, env_provider from agent_env.providers.env_providers.constants import GATEWAY_APP_DIR, GATEWAY_SERVICE_NAME from agent_env.providers.env_providers.env_gateway_provider import ( @@ -1577,6 +1578,23 @@ def test_each_website_backend_gets_a_session_secret_of_its_own_which_the_logged_ assert shop not in _redact_compose_secrets(compose) and notes not in _redact_compose_secrets(compose) +def test_the_gateway_is_handed_each_synthetic_sites_stored_names_and_the_sites_it_lists_by_name(): + compose = _compose_for([], website_configs=[ + WebsiteConfig(backend_image="b", frontend_image=None, environment_name="shop", + domains=("shop.test", "*.shop.test"), listed_by_name=True), + WebsiteConfig(backend_image="b", frontend_image=None, environment_name="notes", listed_by_name=True), + WebsiteConfig(backend_image="b", frontend_image=None, environment_name="docs", domains=("docs.test",)), + WebsiteConfig(backend_image="b", frontend_image=None, environment_name="github", live=True, + domains=("github.com",), listed_by_name=True), + ]) + gateway = _service_block(compose, "gateway") + assert _env_values(gateway, WEBSITE_NAMES_ENV) == ["shop=shop.test|*.shop.test,docs=docs.test"] + assert _env_values(gateway, WEBSITE_LISTED_BY_NAME_ENV) == ["shop,notes"] + plain = _service_block(_compose_for([], website_configs=[ + WebsiteConfig(backend_image="b", frontend_image=None, environment_name="notes")]), "gateway") + assert _env_values(plain, WEBSITE_NAMES_ENV) == _env_values(plain, WEBSITE_LISTED_BY_NAME_ENV) == [] + + def test_injected_names_are_the_environment_name_not_the_image_or_container_name(): """Both vars carry environment_name verbatim. The image tags here differ from the names, and the website backend's container name is `-website-backend`, so substituting the wrong From cf03c336121cad17a66fa5ea0b00f74add5e6858 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 11 Nov 2026 15:11:31 -0800 Subject: [PATCH 2/7] Gateway: a forwarded host it can't read names nothing, rather than a 500 An AgentEnv-Site-Root that urlsplit can't parse (an unbalanced bracket) and a Host or X-Forwarded-Host that isn't printable ASCII, which httpx can't send upstream, each failed the request with a 500. Each now names nothing, as an empty one does: the request falls back to the next header and goes through. Found by M8.12's chaos fuzz; main answers the same 500s. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/rest_proxy.py | 16 +++++++++---- .../env/gateway/rest_proxy_stream_test.py | 23 ++++++++++++++++--- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/agent_env/env/gateway/rest_proxy.py b/src/agent_env/env/gateway/rest_proxy.py index 2543d4ac6..b2ddce968 100644 --- a/src/agent_env/env/gateway/rest_proxy.py +++ b/src/agent_env/env/gateway/rest_proxy.py @@ -174,10 +174,11 @@ def forwarded_address(request: HTTPConnection) -> dict[str, str]: (``https://notes.test``) is that root's address, ahead of any ``X-Forwarded-Host``/``-Proto``, which a front between the client and the gateway may have rewritten to its own. Like the host, that only changes the links the client's own answer names; what the route reaches is the path's alone. A header with nothing to say is left - out.""" + out, as is a host that isn't printable ASCII, which no header upstream could carry.""" root = request.headers.get(AGENT_ENV_SITE_ROOT_HEADER, "").strip() origin = _origin(root) - host = origin[1] if origin else _first(request.headers.get(FORWARDED_HOST, "")) or request.headers.get("host", "") + host = origin[1] if origin else (_plain(_first(request.headers.get(FORWARDED_HOST, ""))) + or _plain(request.headers.get("host", ""))) proto = origin[0] if origin else _first(request.headers.get(FORWARDED_PROTO, "")).lower() at_root = bool(root) scheme = _HTTP_SCHEME.get(request.url.scheme, request.url.scheme) @@ -188,13 +189,20 @@ def forwarded_address(request: HTTPConnection) -> dict[str, str]: def _origin(value: str) -> tuple[str, str] | None: """The scheme and host of ``value`` when it is an origin (``https://notes.test``, a port allowed), else None.""" - parts = urlsplit(value) + try: + parts = urlsplit(value) + except ValueError: # an unbalanced bracket, as in ``https://]x[`` + return None if (parts.scheme in ("http", "https") and parts.netloc and parts.path in ("", "/") and not parts.query - and not parts.fragment and "@" not in parts.netloc and value.isprintable()): + and not parts.fragment and "@" not in parts.netloc and _plain(value)): return parts.scheme, parts.netloc return None +def _plain(value: str) -> str: + return value if value.isascii() and value.isprintable() else "" + + def _first(value: str) -> str: return value.split(",")[0].strip() diff --git a/tst/unit/env/gateway/rest_proxy_stream_test.py b/tst/unit/env/gateway/rest_proxy_stream_test.py index 059763cc6..e8169917a 100644 --- a/tst/unit/env/gateway/rest_proxy_stream_test.py +++ b/tst/unit/env/gateway/rest_proxy_stream_test.py @@ -527,7 +527,9 @@ async def test_a_client_serving_the_route_at_its_own_root_is_told_no_prefix_and_ ("javascript://notes.test", (["front.example"], ["https"], [])), ("https://notes.test/elsewhere", (["front.example"], ["https"], [])), ("https://user@notes.test", (["front.example"], ["https"], [])), -], ids=["origin", "port", "not-http", "a-path", "userinfo"]) + ("https://]notes.test[", (["front.example"], ["https"], [])), + ("https://nötes.test", (["front.example"], ["https"], [])), +], ids=["origin", "port", "not-http", "a-path", "userinfo", "unbalanced-bracket", "not-ascii"]) async def test_a_site_root_naming_an_origin_is_the_address_though_a_front_rewrote_the_forwarded_host( monkeypatch, root, expected): """A front between a computer's site proxy and the gateway, as a provider's ingress is, may set its own @@ -535,11 +537,26 @@ async def test_a_site_root_naming_an_origin_is_the_address_though_a_front_rewrot up = _Upstream([b"ok"]) up.install(monkeypatch) gw = _gateway(website_urls={"web": "http://web"}) - await _call(gw, "/website/web/api/x", headers={"host": "gw.example", "x-forwarded-host": "front.example", - "x-forwarded-proto": "https", "agentenv-site-root": root}) + await _call(gw, "/website/web/api/x", raw_headers=[ + (b"host", b"gw.example"), (b"x-forwarded-host", b"front.example"), (b"x-forwarded-proto", b"https"), + (b"agentenv-site-root", root.encode())]) assert [_forwarded(r) for r in up.forwarded] == [expected] +@pytest.mark.asyncio +async def test_a_host_that_isnt_printable_ascii_names_nothing_and_the_request_still_goes_through(monkeypatch): + up = _Upstream([b"ok"]) + up.install(monkeypatch) + gw = _gateway(website_urls={"web": "http://web"}, site_names={"web": ["shop.test"]}) + answered = [(await _call(gw, "/website/web/x", raw_headers=raw))[0] for raw in ( + [(b"host", "shöp.test".encode())], + [(b"host", b"shop.test"), (b"x-forwarded-host", "shöp.test".encode())], + [(b"host", b"gw.example"), (b"x-forwarded-host", b"shop.test\x7f")])] + assert answered == [200, 200, 200] + assert [(r.headers["host"], _forwarded(r)[0]) for r in up.forwarded] == [ + ("web", []), ("shop.test", ["shop.test"]), ("web", ["gw.example"])] + + @pytest.mark.asyncio async def test_a_front_s_own_forwarded_host_and_scheme_win_and_reach_the_upstream_once(monkeypatch): up = _Upstream([b"ok"]) From 93c9d03dad03ef0bef50fe7c7265295a2055e278 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 11 Nov 2026 15:22:08 -0800 Subject: [PATCH 3/7] Gateway: name the highest port a site host may name Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/rest_proxy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/agent_env/env/gateway/rest_proxy.py b/src/agent_env/env/gateway/rest_proxy.py index b2ddce968..80b70288d 100644 --- a/src/agent_env/env/gateway/rest_proxy.py +++ b/src/agent_env/env/gateway/rest_proxy.py @@ -75,6 +75,7 @@ # A host as a site's name: a DNS name, an optional port. _LABEL = r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?" _SITE_HOST = re.compile(rf"({_LABEL}(?:\.{_LABEL})*)(?::([0-9]{{1,5}}))?") +_MAX_PORT = 65535 _FORWARDED = frozenset(name.encode() for name in (FORWARDED_HOST, FORWARDED_PROTO, FORWARDED_PREFIX, AGENT_ENV_SITE_ROOT_HEADER.lower())) # A WebSocket's handshake fields, which the gateway's own connection upstream writes for itself. @@ -213,7 +214,7 @@ def site_host(request: HTTPConnection, names: tuple[str, ...]) -> str | None: itself as the client reached it. None for any other host, which a site never sees as its ``Host``.""" host = forwarded_address(request).get(FORWARDED_HOST, "") found = _SITE_HOST.fullmatch(host) - if not found or int(found.group(2) or 0) > 65535: + if not found or int(found.group(2) or 0) > _MAX_PORT: return None return host if named(found.group(1).lower(), names) else None From ffb4f88b0632ecb7f2c8797311a30924fbe0b1cd Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 11 Nov 2026 16:10:36 -0800 Subject: [PATCH 4/7] Gateway: read a site's card into one SiteCard, its reserved paths and names together The names a card declares were a side effect of the reserved-path read, and list_website_urls called that read only to fill them in. Now one cached read returns both, and each caller passes the names on explicitly. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/gateway.py | 72 +++++++++++++++------------- 1 file changed, 40 insertions(+), 32 deletions(-) diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index c2837e3ec..b072bcbce 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -108,6 +108,14 @@ _STEP_API = {"source": "step_api"} +class SiteCard(NamedTuple): + """What the gateway keeps from a site's card: the paths it advertises for agentenv, which its route refuses, and + the names it declares.""" + + reserved: frozenset[str] + names: tuple[str, ...] + + class ToolNameClash(ValueError): """More than one source serves a tool under one name, so an agent couldn't tell which it calls: the gateway serves none of its servers' tools rather than a set missing some.""" @@ -160,10 +168,9 @@ def __init__( self.website_urls = website_urls or {} self.rest_proxy_urls = rest_proxy_urls or {} self._site_names = {site: lowered(names) for site, names in (site_names or {}).items()} - self._card_site_names: dict[str, tuple[str, ...]] = {} self._listed_by_name = frozenset(listed_by_name) self._site_mcp_endpoints = site_mcp_endpoints(internal_mcp_servers, self.website_urls, self.rest_proxy_urls) - self._site_reserved: dict[str, frozenset[str]] = {} + self._site_cards: dict[str, SiteCard] = {} self._withheld = frozenset(name.strip().lower().encode() for name in provider_headers if name.strip()) self._browsers = BrowserRoutes(self.rest_proxy_urls, self.REST_PROXY_TIMEOUT, self._log_event, self._attribution, self._request_role, withheld=self._withheld) @@ -657,8 +664,10 @@ def _query_changelog_id(self) -> int | None: self._db_conn = None return None - async def _proxy_rest_request(self, request: Request, url_map: dict[str, str] | None = None) -> Response: - """Reverse-proxy an HTTP request to an internal service, streaming the response through.""" + async def _proxy_rest_request(self, request: Request, url_map: dict[str, str] | None = None, + site_names: tuple[str, ...] = ()) -> Response: + """Reverse-proxy an HTTP request to an internal service, streaming the response through; a site reached at one + of its `site_names` takes it as its ``Host``.""" url_map = self.rest_proxy_urls if url_map is None else url_map service_name = request.path_params["service_name"] base_url = url_map.get(service_name) @@ -670,7 +679,6 @@ async def _proxy_rest_request(self, request: Request, url_map: dict[str, str] | mcp_endpoint = site_mcp or "mcp" if url_map is self.rest_proxy_urls and (cdp := await self._browsers.cdp_answer(request, base_url, role)) is not None: return cdp - site_names = self._names(service_name) if url_map is self.website_urls else () if request.method in _SAFE_METHODS: return await forward(request, base_url, role=role, timeout=self.REST_PROXY_TIMEOUT, mcp_server=mcp_server, withheld=self._withheld, mcp_endpoint=mcp_endpoint, site_names=site_names) @@ -706,13 +714,13 @@ async def _proxy_website_request(self, request: Request) -> Response: A site's data plane and card, and every path its card advertises for agentenv (``site_reserved_paths``), are never reached through its route, so an agent browsing the site can't read its data or what grades it; agentenv reaches them through /svc//.""" - reserved = await self._site_reserved_paths(request.path_params["service_name"]) - if reserved is None: + app = request.path_params["service_name"] + card = await self._site_card(app) + if card is None: return self._json_response({"error": "website card unavailable"}, 503) - if reserved_site_request(request, reserved): + if reserved_site_request(request, card.reserved): return self._json_response({"error": "not found"}, 404) - response = await self._proxy_rest_request(request, url_map=self.website_urls) - app = request.path_params["service_name"] + response = await self._proxy_rest_request(request, url_map=self.website_urls, site_names=self._names(app, card)) engine = self._trigger_engine if app in self.website_urls and (engine.has_website_triggers or engine.has_state_triggers): path = site_request_path(request) or "/" + request.path_params.get("path", "") @@ -730,21 +738,21 @@ async def _proxy_website_socket(self, websocket: WebSocket) -> None: """Relay a WebSocket on a site's route to its server, as its HTTP requests are: its data plane and card, and every path its card advertises for agentenv (``site_reserved_paths``), are refused, and it goes upstream with the role, role key and forwarded address an HTTP request would carry.""" - reserved = await self._site_reserved_paths(websocket.path_params["service_name"]) - if reserved is None: + service_name = websocket.path_params["service_name"] + card = await self._site_card(service_name) + if card is None: await refuse_websocket(websocket, self._json_response({"error": "website card unavailable"}, 503)) return - if reserved_site_request(websocket, reserved): + if reserved_site_request(websocket, card.reserved): await refuse_websocket(websocket, self._json_response({"error": "not found"}, 404)) return - service_name = websocket.path_params["service_name"] base_url = self.website_urls.get(service_name) if base_url is None: await refuse_websocket(websocket, self._json_response({"error": f"Unknown service: {service_name}"}, 404)) return await relay_websocket(websocket, base_url, role=self._request_role(websocket), open_timeout=self.REST_PROXY_CONNECT_TIMEOUT_S, withheld=self._withheld, - site_names=self._names(service_name)) + site_names=self._names(service_name, card)) async def _fire_website_triggers(self, request: Request, app: str, status: int) -> tuple[list[asyncio.Task], float | None]: @@ -793,16 +801,17 @@ def _website_op(self, app: str, method: str, path: str) -> str | None: read of its own, so answering a request waits on no card.""" return route_name(self._website_routes.get(app) or [], method, path) - async def _site_reserved_paths(self, app: str) -> frozenset[str] | None: - """The paths ``app``'s card advertises for agentenv (``site_reserved_paths``), read once from its server, the one - its /svc/ route reaches (else the one its website route does), and kept; none for a site serving no card, one - whose card path answers a 4xx or what isn't a JSON object, and None, read again next time, while its card can't - be read, unreachable or answering a 5xx, so its route serves nothing until it knows what to refuse.""" - if app in self._site_reserved: - return self._site_reserved[app] + async def _site_card(self, app: str) -> SiteCard | None: + """What ``app``'s card says (``SiteCard``): the paths it advertises for agentenv (``site_reserved_paths``) and + the names it declares, read once from its server, the one its /svc/ route reaches (else the one its website + route does), and kept; neither for a site serving no card, one whose card path answers a 4xx or what isn't a + JSON object, and None, read again next time, while its card can't be read, unreachable or answering a 5xx, so + its route serves nothing until it knows what to refuse.""" + if app in self._site_cards: + return self._site_cards[app] base_url = self.rest_proxy_urls.get(app) or self.website_urls.get(app) if base_url is None: - return frozenset() + return SiteCard(frozenset(), ()) try: async with httpx.AsyncClient() as client: response = await client.get(f"{base_url.rstrip('/')}{WELL_KNOWN_PATH}", timeout=CARD_FETCH_TIMEOUT_S) @@ -817,22 +826,21 @@ async def _site_reserved_paths(self, app: str) -> frozenset[str] | None: except ValueError: card = None address = self.website_urls.get(app) or base_url - self._card_site_names[app] = declared_site_names(card) if isinstance(card, dict) else () - self._site_reserved[app] = site_reserved_paths(card, address) if isinstance(card, dict) else frozenset() - return self._site_reserved[app] + self._site_cards[app] = (SiteCard(site_reserved_paths(card, address), declared_site_names(card)) + if isinstance(card, dict) else SiteCard(frozenset(), ())) + return self._site_cards[app] - def _names(self, app: str) -> tuple[str, ...]: - """The names website ``app`` is reached at: those its env stores, else those its card declares, as last read.""" - return self._site_names.get(app) or self._card_site_names.get(app, ()) + def _names(self, app: str, card: SiteCard | None) -> tuple[str, ...]: + """The names website ``app`` is reached at: those its env stores, else those its ``card`` declares.""" + return self._site_names.get(app) or (card.names if card else ()) async def _listed_url(self, site: str, url: str) -> str: """The address ``list_website_urls`` gives ``site``, at ``url`` behind the gateway: for a site listed by name, its first name (``listed_url``), its card read for a site whose env stores none; else, or with none, ``url``.""" if site not in self._listed_by_name: return url - if site not in self._site_names: - await self._site_reserved_paths(site) - return listed_url(self._names(site)) or url + card = None if site in self._site_names else await self._site_card(site) + return listed_url(self._names(site, card)) or url def _backing_keys(self) -> list[str]: # rest_proxy_urls entries are the env's backing servers: mcp- (MCP) + (website From 5f6371850b6d95f8c5b54b5008cf944ba62c3648 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 11 Nov 2026 16:27:46 -0800 Subject: [PATCH 5/7] Gateway: a WebSocket to an https site reached by name goes to its 443 and checks its own certificate The URI names the site, so websockets would default the port to 80 and verify TLS against the site's name; the upstream's port now defaults by its scheme, and a wss upstream is verified as the upstream. Also wraps two lines over 120 characters. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/rest_proxy.py | 7 ++-- tst/data/host_website/server.py | 3 +- .../env/gateway/rest_proxy_stream_test.py | 34 +++++++++++++++++-- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/agent_env/env/gateway/rest_proxy.py b/src/agent_env/env/gateway/rest_proxy.py index 80b70288d..ab67eba61 100644 --- a/src/agent_env/env/gateway/rest_proxy.py +++ b/src/agent_env/env/gateway/rest_proxy.py @@ -332,9 +332,12 @@ async def relay_websocket(websocket: WebSocket, base_url: str, *, role: str, upstream_headers(websocket, role, withheld=withheld, site_names=site_names).multi_items() if key.lower() not in _WEBSOCKET_HANDSHAKE and key.lower() != "host"] uri, destination = "ws" + target_url.removeprefix("http"), {} - if host := site_host(websocket, site_names): # websockets writes Host from the URI; the TCP goes to the upstream + if host := site_host(websocket, site_names): # websockets writes Host from the URI; TCP and TLS go to the upstream parts = urlsplit(uri) - uri, destination = urlunsplit(parts._replace(netloc=host)), {"host": parts.hostname, "port": parts.port or 80} + secure = parts.scheme == "wss" + destination = {"host": parts.hostname, "port": parts.port or (443 if secure else 80), + **({"server_hostname": parts.hostname} if secure else {})} + uri = urlunsplit(parts._replace(netloc=host)) offered = [protocol.strip() for value in websocket.headers.getlist("sec-websocket-protocol") for protocol in value.split(",") if protocol.strip()] try: diff --git a/tst/data/host_website/server.py b/tst/data/host_website/server.py index aa71c7794..e2b3781f2 100644 --- a/tst/data/host_website/server.py +++ b/tst/data/host_website/server.py @@ -38,7 +38,8 @@ async def home(request): async def host(request): - return JSONResponse({"host": request.headers.get("host"), "forwarded_host": request.headers.get("x-forwarded-host")}) + return JSONResponse({"host": request.headers.get("host"), + "forwarded_host": request.headers.get("x-forwarded-host")}) app = Starlette(routes=[Route("/.well-known/agent-env.json", card), Route("/api/health", health), diff --git a/tst/unit/env/gateway/rest_proxy_stream_test.py b/tst/unit/env/gateway/rest_proxy_stream_test.py index e8169917a..18697a1a6 100644 --- a/tst/unit/env/gateway/rest_proxy_stream_test.py +++ b/tst/unit/env/gateway/rest_proxy_stream_test.py @@ -633,18 +633,45 @@ async def body(): async def test_a_site_whose_env_stores_no_names_takes_its_host_from_the_names_its_card_declares(monkeypatch): seen: list[str] = [] transport = _card_site(seen, {"notes": ["Notes.Test"], "kept": ["notes.test"]}) - monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", lambda **kw: _RealAsyncClient(transport=transport, **kw)) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", + lambda **kw: _RealAsyncClient(transport=transport, **kw)) gw = _gateway(website_urls={"notes": "http://notes", "kept": "http://kept"}, site_names={"kept": ["kept.test"]}) for path, root in (("notes", "https://notes.test"), ("kept", "https://notes.test"), ("kept", "https://kept.test")): await _call(gw, f"/website/{path}/x", headers={"host": "gw.example", "agentenv-site-root": root}) assert seen == ["notes.test", "kept", "kept.test"] +@pytest.mark.parametrize("upstream, destination", [ + ("https://notes.internal", {"host": "notes.internal", "port": 443, "server_hostname": "notes.internal"}), + ("http://notes.internal", {"host": "notes.internal", "port": 80, "server_hostname": None}), + ("https://notes.internal:8443", {"host": "notes.internal", "port": 8443, "server_hostname": "notes.internal"}), +], ids=["https", "http", "https-port"]) +def test_a_websocket_to_a_site_reached_by_name_connects_to_its_upstream_and_checks_the_upstreams_certificate( + monkeypatch, upstream, destination): + transport = _card_site([], {}) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", + lambda **kw: _RealAsyncClient(transport=transport, **kw)) + asked = [] + + async def connect(uri, **kwargs): + asked.append((uri, {key: kwargs.get(key) for key in ("host", "port", "server_hostname")})) + raise OSError("refused") + monkeypatch.setattr(rest_proxy_module, "connect_websocket", connect) + gw = _gateway(website_urls={"notes": upstream}, site_names={"notes": ["notes.test"]}) + with pytest.raises(Exception): + with TestClient(gw._asgi_app()).websocket_connect("/website/notes/live", + headers={"AgentEnv-Site-Root": "https://notes.test"}): + pass + scheme = "wss" if upstream.startswith("https") else "ws" + assert asked == [(f"{scheme}://notes.test/live", destination)] + + @pytest.mark.asyncio async def test_list_website_urls_gives_a_site_listed_by_name_at_its_first_name_and_any_other_behind_the_gateway( monkeypatch): transport = _card_site([], {"carded": ["carded.test"]}) - monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", lambda **kw: _RealAsyncClient(transport=transport, **kw)) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", + lambda **kw: _RealAsyncClient(transport=transport, **kw)) sites = ("shop", "notes", "wild", "bare", "carded", "other") gw = _gateway(website_urls={site: f"http://{site}" for site in sites}, site_names={"shop": ["*.shop.test", "www.shop.test", "shop.test"], "notes": ["notes.localhost"], @@ -664,7 +691,8 @@ def whoami(self, ctx: ApiContext) -> dict: return {"base_url": ctx.base_url} transport = httpx.ASGITransport(app=Notes().create_app()) - monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", lambda **kw: _RealAsyncClient(transport=transport, **kw)) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", + lambda **kw: _RealAsyncClient(transport=transport, **kw)) gw = _gateway(website_urls={"notes": "http://notes"}) async def whoami(path: str) -> tuple[int, dict, bytes]: From db664a07e3cb0d94316270bbcd9f6fa44259f304 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 11 Nov 2026 18:49:41 -0800 Subject: [PATCH 6/7] Gateway: a site's WebSocket goes straight to its upstream, never through an environment proxy The relay's URI now names the site, so websockets' default proxy=True would decide no_proxy by the site's name and, with a proxy in the gateway's environment, tunnel to that name: the real domain, for a site named after one. Its upstream is always internal, so the relay connects directly. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/rest_proxy.py | 4 ++-- tst/unit/env/gateway/rest_proxy_stream_test.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/agent_env/env/gateway/rest_proxy.py b/src/agent_env/env/gateway/rest_proxy.py index ab67eba61..73d783374 100644 --- a/src/agent_env/env/gateway/rest_proxy.py +++ b/src/agent_env/env/gateway/rest_proxy.py @@ -341,10 +341,10 @@ async def relay_websocket(websocket: WebSocket, base_url: str, *, role: str, offered = [protocol.strip() for value in websocket.headers.getlist("sec-websocket-protocol") for protocol in value.split(",") if protocol.strip()] try: - upstream = await connect_websocket(uri, additional_headers=headers, + upstream = await connect_websocket(uri, additional_headers=headers, # never through a proxy: it's internal subprotocols=offered or None, user_agent_header=None, open_timeout=open_timeout, max_size=WEBSOCKET_MESSAGE_MAX_BYTES, - **destination) + proxy=None, **destination) except InvalidStatus as e: refusal = Response(content=bytes(e.response.body or b""), status_code=e.response.status_code) refusal.raw_headers = [*(item for item in refusal.raw_headers if item[0] == b"content-length"), diff --git a/tst/unit/env/gateway/rest_proxy_stream_test.py b/tst/unit/env/gateway/rest_proxy_stream_test.py index 18697a1a6..921187199 100644 --- a/tst/unit/env/gateway/rest_proxy_stream_test.py +++ b/tst/unit/env/gateway/rest_proxy_stream_test.py @@ -643,18 +643,22 @@ async def test_a_site_whose_env_stores_no_names_takes_its_host_from_the_names_it @pytest.mark.parametrize("upstream, destination", [ ("https://notes.internal", {"host": "notes.internal", "port": 443, "server_hostname": "notes.internal"}), - ("http://notes.internal", {"host": "notes.internal", "port": 80, "server_hostname": None}), + ("http://notes.internal", {"host": "notes.internal", "port": 80, "server_hostname": "unset"}), ("https://notes.internal:8443", {"host": "notes.internal", "port": 8443, "server_hostname": "notes.internal"}), ], ids=["https", "http", "https-port"]) def test_a_websocket_to_a_site_reached_by_name_connects_to_its_upstream_and_checks_the_upstreams_certificate( monkeypatch, upstream, destination): + """Straight to the upstream, never through a proxy the environment names, which would tunnel to the site's + name: a real domain for a site named after one.""" + monkeypatch.setenv("HTTPS_PROXY", "http://proxy.example:3128") + monkeypatch.setenv("HTTP_PROXY", "http://proxy.example:3128") transport = _card_site([], {}) monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", lambda **kw: _RealAsyncClient(transport=transport, **kw)) asked = [] async def connect(uri, **kwargs): - asked.append((uri, {key: kwargs.get(key) for key in ("host", "port", "server_hostname")})) + asked.append((uri, {key: kwargs.get(key, "unset") for key in ("host", "port", "server_hostname", "proxy")})) raise OSError("refused") monkeypatch.setattr(rest_proxy_module, "connect_websocket", connect) gw = _gateway(website_urls={"notes": upstream}, site_names={"notes": ["notes.test"]}) @@ -663,7 +667,7 @@ async def connect(uri, **kwargs): headers={"AgentEnv-Site-Root": "https://notes.test"}): pass scheme = "wss" if upstream.startswith("https") else "ws" - assert asked == [(f"{scheme}://notes.test/live", destination)] + assert asked == [(f"{scheme}://notes.test/live", {**destination, "proxy": None})] @pytest.mark.asyncio From f3dfd5940633bfe1dba6f16b7085f1093ed285f3 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 11 Nov 2026 18:55:40 -0800 Subject: [PATCH 7/7] Gateway: require websockets 15, the first that takes the relay's proxy=None websockets 13 and 14 hand an unknown proxy argument to asyncio's create_connection, which raises TypeError, so every site WebSocket would fail there. The gateway image and the dev extra now ask for 15; the lock already resolves 17.2, so only its recorded floor changes. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 2 +- src/agent_env/env/gateway/requirements.txt | 2 +- uv.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e6af0e443..5c5439fd0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -162,7 +162,7 @@ dev = [ "moto>=5.0.0", "psycopg2-binary>=2.9.0", # the gateway server module's driver; outside its container only the tests import it "regex>=2023.12.25", # the gateway's trigger predicates, matched under a timeout; outside its container only the tests import it - "websockets>=13", # the gateway's WebSocket relay to a website; outside its container only the tests import it + "websockets>=15", # the gateway's WebSocket relay to a website; outside its container only the tests import it "pytest>=9.0", "pytest-asyncio>=0.23.0", "pytest-dependency>=0.6.0", diff --git a/src/agent_env/env/gateway/requirements.txt b/src/agent_env/env/gateway/requirements.txt index 2d5541b98..e4d7ac9c8 100644 --- a/src/agent_env/env/gateway/requirements.txt +++ b/src/agent_env/env/gateway/requirements.txt @@ -4,4 +4,4 @@ mcp>=1.25,<2 pydantic>=2.0 regex>=2023.12.25 tzdata -websockets>=13 +websockets>=15 diff --git a/uv.lock b/uv.lock index 10b74e13c..6f5528239 100644 --- a/uv.lock +++ b/uv.lock @@ -275,7 +275,7 @@ requires-dist = [ { name = "urllib3", marker = "extra == 'boat'", specifier = ">=2.1.0,<3" }, { name = "uvicorn", specifier = ">=0.47" }, { name = "vercel-sandbox", marker = "extra == 'vercel'", specifier = ">=0.7.0,<0.8" }, - { name = "websockets", marker = "extra == 'dev'", specifier = ">=13" }, + { name = "websockets", marker = "extra == 'dev'", specifier = ">=15" }, ] provides-extras = ["openenv", "harness", "trl", "nemo-gym", "verl", "skyrl", "miles", "slime", "aws", "gcp", "sail", "vercel", "tensorlake", "tenki", "boxd", "daytona", "boat", "coreweave", "dev", "smol"]