diff --git a/pyproject.toml b/pyproject.toml index e7e6f6245..511d66725 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,12 @@ novita = [ inspect = [ "inspect-ai>=0.3.0", ] +security = [ + # Optional request-security middleware (IP lists, rate limiting, + # penetration detection) for env servers. No effect unless + # OPENENV_GUARD_ENABLED=1. + "fastapi-guard>=8.0.3,<9.0.0", +] harbor = [ # Every sandbox backend. Bare `harbor` gives one that lists all 23 backends and can # instantiate none of them: each raises MissingExtraError from its constructor, which diff --git a/src/openenv/core/env_server/http_server.py b/src/openenv/core/env_server/http_server.py index 8410a6d45..4e83901e1 100644 --- a/src/openenv/core/env_server/http_server.py +++ b/src/openenv/core/env_server/http_server.py @@ -2350,4 +2350,8 @@ def create_fastapi_app( if mode is None: mode = os.environ.get("OPENENV_MODE", ServerMode.SIMULATION.value) server.register_routes(app, mode=mode) + + from .security import attach_guard + + attach_guard(app) return app diff --git a/src/openenv/core/env_server/security.py b/src/openenv/core/env_server/security.py new file mode 100644 index 000000000..e54cbb767 --- /dev/null +++ b/src/openenv/core/env_server/security.py @@ -0,0 +1,247 @@ +# SPDX-License-Identifier: BSD-3-Clause + +"""Optional fastapi-guard security middleware for environment servers. + +Opt-in via the `OPENENV_GUARD_*` environment variables. When disabled (the +default), `attach_guard` is a no-op and environment servers behave exactly as +before. + +Install the extra with `pip install "openenv[security]"` and enable with +`OPENENV_GUARD_ENABLED=1` to get IP block/allow lists, rate limiting with +auto-ban, user-agent blocking, penetration-attempt detection, and optional +Redis-backed distributed state (fastapi-guard). + +Health and schema endpoints are excluded from detection checks by default so +orchestrators can probe the server; global IP allow/block lists are still +enforced on those paths. + +WebSocket connections (`/ws`, `/mcp`, `/harness`, `/ws/ui`) get the same +checks on the handshake: ban, IP lists, a separate rate-limit bucket and +penetration detection on the path, query and headers. A refused handshake is +closed before `accept()`, so the client gets HTTP 403 on the upgrade and no +socket. Messages on an accepted connection are not screened, for the same +reason request bodies are not by default. With Redis configured, the guard +initializes at startup; while Redis is unreachable a handshake is closed with +1013 and HTTP requests are refused. + +In passive mode a request or handshake that would be refused is logged and let +through, and so is one that arrives while Redis is unreachable (fastapi-guard's +`redis_fail_open`), at startup or later. +""" + +from __future__ import annotations + +import logging +import os +from contextlib import asynccontextmanager +from typing import Any + +from starlette import status +from starlette.exceptions import WebSocketException +from starlette.websockets import WebSocket + +DEFAULT_EXCLUDED_PATHS = "/health,/schema,/metadata,/docs,/redoc,/openapi.json" +# Empty by default on purpose: trusting private-range peers would let +# any of them spoof X-Forwarded-For past the blocklist and rate limits. +# Behind a reverse proxy, list the proxy's addresses in +# OPENENV_GUARD_TRUSTED_PROXIES. +DEFAULT_TRUSTED_PROXIES = "" + + +def _env_list(name: str, default: str = "") -> list[str]: + raw = os.environ.get(name, default) + return [item.strip() for item in raw.split(",") if item.strip()] + + +def _env_str(name: str, default: str | None = None) -> str | None: + raw = os.environ.get(name) + return raw if raw not in (None, "") else default + + +def _env_bool(name: str, *, default: bool = False) -> bool: + raw = os.environ.get(name) + if raw is None or not raw.strip(): + return default + normalized = raw.strip().lower() + if normalized in ("1", "true", "yes", "on"): + return True + if normalized in ("0", "false", "no", "off"): + return False + msg = f"{name} must be a boolean (1/true/yes/on or 0/false/no/off), got {raw!r}" + raise ValueError(msg) + + +def _build_security_config() -> Any: + from guard import SecurityConfig + + passive_mode = _env_bool("OPENENV_GUARD_PASSIVE_MODE") + kwargs: dict[str, Any] = { + "enable_rate_limiting": True, + "rate_limit": int(os.environ.get("OPENENV_GUARD_RATE_LIMIT", "100")), + "rate_limit_window": int( + os.environ.get("OPENENV_GUARD_RATE_LIMIT_WINDOW", "60") + ), + "enable_ip_banning": True, + "auto_ban_threshold": int( + os.environ.get("OPENENV_GUARD_AUTO_BAN_THRESHOLD", "10") + ), + "auto_ban_duration": int( + os.environ.get("OPENENV_GUARD_AUTO_BAN_DURATION", "300") + ), + # On by default: repeated rate-limit violations feed the ban engine (OPENENV_GUARD_RATE_LIMIT_AUTO_BAN=0 keeps 429-only enforcement). + "enable_rate_limit_auto_ban": _env_bool( + "OPENENV_GUARD_RATE_LIMIT_AUTO_BAN", default=True + ), + # Environment/request bodies are model input (observations, prompts, + # completions), not commands this server executes: signature-scanning + # them false-positives on ordinary content. URL, query, and header + # screening stay on; body scanning is opt-in. + "detection_scan_body": _env_bool("OPENENV_GUARD_SCAN_BODY"), + "enable_penetration_detection": True, + # In-memory state unless a Redis URL is configured: never implicitly + # depend on a Redis server being reachable at localhost. + "enable_redis": False, + "blacklist": tuple(_env_list("OPENENV_GUARD_BLOCKED_IPS")), + "blocked_user_agents": _env_list("OPENENV_GUARD_BLOCKED_USER_AGENTS"), + "trusted_proxies": tuple( + _env_list("OPENENV_GUARD_TRUSTED_PROXIES", DEFAULT_TRUSTED_PROXIES) + ), + "trusted_proxy_depth": int( + os.environ.get("OPENENV_GUARD_TRUSTED_PROXY_DEPTH", "1") + ), + "exclude_paths": _env_list( + "OPENENV_GUARD_EXCLUDED_PATHS", DEFAULT_EXCLUDED_PATHS + ), + "passive_mode": passive_mode, + # Passive mode lets requests through during a Redis outage too. + "redis_fail_open": passive_mode, + "custom_log_file": os.environ.get("OPENENV_GUARD_LOG_FILE") or None, + "log_format": os.environ.get("OPENENV_GUARD_LOG_FORMAT", "text"), + "security_headers": ( + { + "enabled": True, + "hsts": {"max_age": 31536000, "include_subdomains": True}, + "frame_options": "SAMEORIGIN", + "content_type_options": "nosniff", + "referrer_policy": "strict-origin-when-cross-origin", + } + if _env_bool("OPENENV_GUARD_SECURITY_HEADERS") + else None + ), + "enforce_https": _env_bool("OPENENV_GUARD_ENFORCE_HTTPS"), + } + + # Behind a TLS-terminating proxy, enforce_https must read the forwarded + # scheme or every request looks like plain HTTP. Follow enforce_https + # unless explicitly overridden. + explicit_xfp = _env_str("OPENENV_GUARD_TRUST_X_FORWARDED_PROTO") + if explicit_xfp is not None: + kwargs["trust_x_forwarded_proto"] = _env_bool( + "OPENENV_GUARD_TRUST_X_FORWARDED_PROTO" + ) + else: + kwargs["trust_x_forwarded_proto"] = _env_bool("OPENENV_GUARD_ENFORCE_HTTPS") + + if blocked_countries := _env_list("OPENENV_GUARD_BLOCKED_COUNTRIES"): + kwargs["blocked_countries"] = frozenset(blocked_countries) + if allowed_countries := _env_list("OPENENV_GUARD_ALLOWED_COUNTRIES"): + kwargs["whitelist_countries"] = frozenset(allowed_countries) + if cloud_providers := _env_list("OPENENV_GUARD_BLOCK_CLOUD_PROVIDERS"): + kwargs["block_cloud_providers"] = frozenset(cloud_providers) + + if ipinfo_token := os.environ.get("IPINFO_TOKEN"): + kwargs["ipinfo_token"] = ipinfo_token + + allowed_ips = _env_list("OPENENV_GUARD_ALLOWED_IPS") + if allowed_ips: + kwargs["whitelist"] = tuple(allowed_ips) + + redis_url = os.environ.get("OPENENV_GUARD_REDIS_URL") + if redis_url: + kwargs["enable_redis"] = True + kwargs["redis_url"] = redis_url + kwargs["redis_prefix"] = os.environ.get( + "OPENENV_GUARD_REDIS_PREFIX", "openenv_guard:" + ) + + return SecurityConfig(**kwargs) + + +class _WebSocketGuardMiddleware: + # SecurityMiddleware is a BaseHTTPMiddleware, which never sees websocket + # scopes, so every handshake goes through fastapi-guard's guard_websocket. + def __init__(self, app: Any, config: Any) -> None: + from guard import guard_websocket + from guard.lifespan import _find_security_redis_handler, guard_startup + + self.app = app + self.config = config + self.guard_websocket = guard_websocket + # guard_websocket reads Redis through this; None means not connected. + self.find_redis = _find_security_redis_handler + self.guard_startup = guard_startup + + async def __call__(self, scope: Any, receive: Any, send: Any) -> None: + if scope["type"] == "websocket": + websocket = WebSocket(scope, receive, send) + try: + if not await self._redis_ready(scope["app"]): + # The HTTP side refuses too until Redis is reachable. + raise WebSocketException( + code=status.WS_1013_TRY_AGAIN_LATER, + reason="Service temporarily unavailable", + ) + await self.guard_websocket(websocket) + except WebSocketException as exc: + if not self.config.passive_mode: + await websocket.close(code=exc.code, reason=exc.reason) + return + logging.getLogger(__name__).warning( + "Passive mode: WebSocket %s from %s would be refused: %s", + websocket.url.path, + getattr(websocket.state, "client_ip", None), + exc.reason, + ) + await self.app(scope, receive, send) + + async def _redis_ready(self, app: Any) -> bool: + if not self.config.enable_redis or self.find_redis(app) is not None: + return True + # Retry on each handshake, like the HTTP side's first request. + await self.guard_startup(app) + return self.find_redis(app) is not None + + +def attach_guard(app: Any) -> None: + """Add the fastapi-guard middleware to `app` when enabled via env. + + No-op unless OPENENV_GUARD_ENABLED is set to 1/true/yes/on. + """ + if not _env_bool("OPENENV_GUARD_ENABLED"): + return + if getattr(app.state, "guard_attached", False): + return + try: + from guard import SecurityMiddleware + from guard.lifespan import guard_lifespan + except ImportError as exc: + raise ImportError( + "OPENENV_GUARD_ENABLED requires fastapi-guard. " + 'Install it with: pip install "openenv[security]"' + ) from exc + + config = _build_security_config() + app.add_middleware(SecurityMiddleware, config=config) + app.add_middleware(_WebSocketGuardMiddleware, config=config) + # Initialize the guard (Redis included) at startup, so a WebSocket-only + # worker never runs on in-memory state. + existing_lifespan = app.router.lifespan_context + + @asynccontextmanager + async def lifespan(app: Any): + async with guard_lifespan(app): + async with existing_lifespan(app) as state: + yield state + + app.router.lifespan_context = lifespan + app.state.guard_attached = True diff --git a/src/openenv/core/env_server/web_interface.py b/src/openenv/core/env_server/web_interface.py index 5ecc27f4c..7dcd2f283 100644 --- a/src/openenv/core/env_server/web_interface.py +++ b/src/openenv/core/env_server/web_interface.py @@ -643,6 +643,8 @@ async def web_state(): css=OPENENV_GRADIO_CSS, ) + # Security middleware is attached inside create_fastapi_app; the gradio + # mount at /web happens before that call, so it is covered too. return app diff --git a/tests/core/test_guard_security.py b/tests/core/test_guard_security.py new file mode 100644 index 000000000..03a57f5ad --- /dev/null +++ b/tests/core/test_guard_security.py @@ -0,0 +1,381 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Tests for the optional fastapi-guard security middleware wiring.""" + +import asyncio +import itertools +import json +import os +from contextlib import asynccontextmanager + +import anyio +import httpx +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from openenv.core.env_server.http_server import create_fastapi_app +from openenv.core.env_server.interfaces import Environment +from openenv.core.env_server.security import _build_security_config, attach_guard +from openenv.core.env_server.types import Action, Observation, State +from starlette.websockets import WebSocketDisconnect + +# Rate limiting state in guard-core is process-wide, so every scenario gets +# its own TEST-NET client IP to stay hermetic. +_IP = itertools.count(1) + + +@pytest.fixture(autouse=True) +def _optional_security_dependency(request): + if request.node.originalname in ( + "test_disabled_by_default", + "test_websocket_unguarded_when_disabled", + "test_invalid_enabled_flag_raises", + ): + return + pytest.importorskip("guard", reason="requires the optional security extra") + + +def test_invalid_enabled_flag_raises(monkeypatch): + with pytest.raises(ValueError, match="OPENENV_GUARD_ENABLED"): + _make_app(monkeypatch, OPENENV_GUARD_ENABLED="sometimes") + + +def _unique_ip() -> str: + n = next(_IP) + return f"198.51.{n // 250}.{(n % 250) + 1}" + + +class GuardAction(Action): + value: str = "" + + +class GuardObservation(Observation): + message: str = "" + + +class GuardEnvironment(Environment): + def reset(self, **kwargs) -> GuardObservation: + return GuardObservation(message="ready") + + def step(self, action: GuardAction, **kwargs) -> GuardObservation: + return GuardObservation(message=action.value, reward=1.0) + + @property + def state(self) -> State: + return State() + + +def _make_app(monkeypatch, **env): + for name in list(os.environ): + if name.startswith("OPENENV_GUARD_"): + monkeypatch.delenv(name, raising=False) + for name, value in env.items(): + monkeypatch.setenv(name, value) + return create_fastapi_app(GuardEnvironment, GuardAction, GuardObservation) + + +async def _request(app, path, client_ip=None): + transport = httpx.ASGITransport(app=app, client=(client_ip or _unique_ip(), 50000)) + async with httpx.AsyncClient( + transport=transport, base_url="http://testserver" + ) as client: + return await client.get(path) + + +async def test_disabled_by_default(monkeypatch): + app = _make_app(monkeypatch) + assert len(app.user_middleware) == 0 + response = await _request(app, "/health") + assert response.status_code == 200 + + +@pytest.mark.parametrize("enabled", ["1", "true", "yes", "on"]) +async def test_enabled_adds_middleware(monkeypatch, enabled): + app = _make_app(monkeypatch, OPENENV_GUARD_ENABLED=enabled) + # SecurityMiddleware for HTTP plus the websocket handshake guard. + assert len(app.user_middleware) == 2 + response = await _request(app, "/health") + assert response.status_code == 200 + + +async def test_blocked_ip_is_rejected(monkeypatch): + blocked = _unique_ip() + app = _make_app( + monkeypatch, + OPENENV_GUARD_ENABLED="1", + OPENENV_GUARD_BLOCKED_IPS=blocked, + ) + response = await _request(app, "/health", client_ip=blocked) + assert response.status_code == 403 + + +async def test_ip_lists_enforced_on_excluded_paths(monkeypatch): + blocked = _unique_ip() + app = _make_app( + monkeypatch, + OPENENV_GUARD_ENABLED="1", + OPENENV_GUARD_BLOCKED_IPS=blocked, + ) + # /health is in the default exclusion list, but global IP lists are + # enforced there too. + response = await _request(app, "/health", client_ip=blocked) + assert response.status_code == 403 + + +def _ws_reset(app, client_ip): + with TestClient(app, client=(client_ip, 50000)) as client: + with client.websocket_connect("/ws") as websocket: + websocket.send_text(json.dumps({"type": "reset", "data": {}})) + return websocket.receive_json()["type"] + + +def _ws_refusal_code(app, client_ip, path="/ws"): + with TestClient(app, client=(client_ip, 50000)) as client: + with pytest.raises(WebSocketDisconnect) as closed: + with client.websocket_connect(path): + pass + return closed.value.code + + +def test_websocket_handshake_refused_for_blocked_ip(monkeypatch): + blocked = _unique_ip() + app = _make_app( + monkeypatch, + OPENENV_GUARD_ENABLED="1", + OPENENV_GUARD_BLOCKED_IPS=blocked, + ) + for path in ("/ws", "/mcp"): + assert _ws_refusal_code(app, blocked, path) == 1008 + assert _ws_reset(app, _unique_ip()) == "observation" + + +def test_websocket_handshake_is_rate_limited(monkeypatch): + ip = _unique_ip() + app = _make_app( + monkeypatch, + OPENENV_GUARD_ENABLED="1", + OPENENV_GUARD_RATE_LIMIT="1", + OPENENV_GUARD_RATE_LIMIT_AUTO_BAN="0", + ) + assert _ws_reset(app, ip) == "observation" + assert _ws_refusal_code(app, ip) == 1008 + + +def test_websocket_unguarded_when_disabled(monkeypatch): + app = _make_app(monkeypatch, OPENENV_GUARD_BLOCKED_IPS="198.51.100.1") + assert _ws_reset(app, "198.51.100.1") == "observation" + + +def test_websocket_passive_mode_logs_and_accepts(monkeypatch, caplog): + blocked = _unique_ip() + app = _make_app( + monkeypatch, + OPENENV_GUARD_ENABLED="1", + OPENENV_GUARD_PASSIVE_MODE="1", + OPENENV_GUARD_BLOCKED_IPS=blocked, + ) + assert _ws_reset(app, blocked) == "observation" + assert "would be refused" in caplog.text + + +def test_guard_initializes_at_startup(monkeypatch, caplog): + app = _make_app( + monkeypatch, + OPENENV_GUARD_ENABLED="1", + OPENENV_GUARD_REDIS_URL="redis://127.0.0.1:1", + ) + with TestClient(app): + assert "Redis unavailable during startup initialization" in caplog.text + + +def test_guard_preserves_existing_lifespan_state_and_cleanup(monkeypatch): + for name in list(os.environ): + if name.startswith("OPENENV_GUARD_"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("OPENENV_GUARD_ENABLED", "1") + events = [] + resource = object() + + @asynccontextmanager + async def lifespan(app): + events.append("startup") + try: + yield {"resource": resource} + finally: + events.append("shutdown") + + app = FastAPI(lifespan=lifespan) + attach_guard(app) + attach_guard(app) + with TestClient(app) as client: + assert events == ["startup"] + assert client.app_state["resource"] is resource + assert events == ["startup", "shutdown"] + + +def test_websocket_refused_while_redis_unreachable(monkeypatch): + ip = _unique_ip() + app = _make_app( + monkeypatch, + OPENENV_GUARD_ENABLED="1", + OPENENV_GUARD_REDIS_URL="redis://127.0.0.1:1", + ) + # A WebSocket as the first request must not fall back to in-memory state. + assert _ws_refusal_code(app, ip) == 1013 + with TestClient(app, client=(ip, 50000)) as client: + assert client.get("/health").status_code == 503 + + +def test_passive_mode_admits_while_redis_unreachable(monkeypatch, caplog): + ip = _unique_ip() + app = _make_app( + monkeypatch, + OPENENV_GUARD_ENABLED="1", + OPENENV_GUARD_PASSIVE_MODE="1", + OPENENV_GUARD_REDIS_URL="redis://127.0.0.1:1", + ) + assert _ws_reset(app, ip) == "observation" + assert "would be refused" in caplog.text + with TestClient(app, client=(ip, 50000)) as client: + assert client.get("/health").status_code == 200 + + +async def test_full_bundle_mapping(monkeypatch): + for name in list(os.environ): + if name.startswith("OPENENV_GUARD_"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("OPENENV_GUARD_ENABLED", "1") + monkeypatch.setenv("OPENENV_GUARD_PASSIVE_MODE", "1") + monkeypatch.setenv("OPENENV_GUARD_SECURITY_HEADERS", "1") + monkeypatch.setenv("OPENENV_GUARD_ENFORCE_HTTPS", "1") + monkeypatch.setenv("OPENENV_GUARD_BLOCKED_COUNTRIES", "RU") + monkeypatch.setenv("IPINFO_TOKEN", "test-token") + monkeypatch.setenv("OPENENV_GUARD_BLOCK_CLOUD_PROVIDERS", "AWS") + monkeypatch.setenv("OPENENV_GUARD_LOG_FILE", "guard.log") + config = _build_security_config() + assert config.passive_mode is True + assert config.redis_fail_open is True + assert config.enforce_https is True + assert config.security_headers["enabled"] is True + assert config.blocked_countries == frozenset({"RU"}) + assert config.block_cloud_providers == frozenset({"AWS"}) + assert config.custom_log_file == "guard.log" + assert config.enable_redis is False + + +async def test_rate_limit_returns_429(monkeypatch): + ip = _unique_ip() + app = _make_app( + monkeypatch, + OPENENV_GUARD_ENABLED="1", + OPENENV_GUARD_RATE_LIMIT="2", + OPENENV_GUARD_RATE_LIMIT_WINDOW="60", + ) + assert (await _request(app, "/metadata", client_ip=ip)).status_code == 200 + assert (await _request(app, "/metadata", client_ip=ip)).status_code == 200 + assert (await _request(app, "/metadata", client_ip=ip)).status_code == 429 + + +async def test_default_config_zero_trust(monkeypatch): + for name in list(os.environ): + if name.startswith("OPENENV_GUARD_"): + monkeypatch.delenv(name, raising=False) + config = _build_security_config() + # No implicit trust in RFC1918 peers: an unlisted proxy means + # X-Forwarded-For is not honored, so LAN clients cannot spoof their + # way past the blocklist or rate limits. + assert config.trusted_proxies == () + # Request bodies are model input, not commands: scanning them is opt-in. + assert config.detection_scan_body is False + # Repeated rate-limit violations feed the ban engine by default. + assert config.enable_rate_limit_auto_ban is True + + +async def test_trusted_proxies_from_env(monkeypatch): + for name in list(os.environ): + if name.startswith("OPENENV_GUARD_"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("OPENENV_GUARD_TRUSTED_PROXIES", "10.0.0.1,10.0.0.2") + config = _build_security_config() + assert config.trusted_proxies == ("10.0.0.1", "10.0.0.2") + + +async def test_invalid_boolean_env_raises(monkeypatch): + for name in list(os.environ): + if name.startswith("OPENENV_GUARD_"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("OPENENV_GUARD_SCAN_BODY", "sometimes") + with pytest.raises(ValueError, match="OPENENV_GUARD_SCAN_BODY"): + _build_security_config() + + +@pytest.mark.parametrize("startup_fails", [False, True]) +def test_guard_lifespan_exit_preserves_task_and_unwinds( + monkeypatch, tmp_path, startup_fails +): + import guard.lifespan + + monkeypatch.setenv("OPENENV_GUARD_ENABLED", "1") + for name in list(os.environ): + if name.startswith("OPENENV_GUARD_") and name != "OPENENV_GUARD_ENABLED": + monkeypatch.delenv(name) + original_guard_lifespan = guard.lifespan.guard_lifespan + events = [] + resource = None + owner = None + + @asynccontextmanager + async def guard_context(app): + nonlocal resource, owner + owner = asyncio.current_task() + # A real task group detects exit in a different task/cancel scope. + async with anyio.create_task_group(): + with (tmp_path / "resource").open("w") as resource: + async with original_guard_lifespan(app): + events.append("guard startup") + try: + yield + finally: + assert asyncio.current_task() is owner + events.append("guard shutdown") + + @asynccontextmanager + async def app_context(app): + assert asyncio.current_task() is owner + assert not resource.closed + events.append("app startup") + if startup_fails: + raise RuntimeError("app startup failed") + try: + yield {"resource": resource} + finally: + assert asyncio.current_task() is owner + assert not resource.closed + events.append("app shutdown") + + monkeypatch.setattr(guard.lifespan, "guard_lifespan", guard_context) + app = FastAPI(lifespan=app_context) + attach_guard(app) + if startup_fails: + with pytest.raises(Exception, match="unhandled errors") as caught: + with TestClient(app): + pytest.fail("startup should fail") + assert len(caught.value.exceptions) == 1 + assert isinstance(caught.value.exceptions[0], RuntimeError) + assert str(caught.value.exceptions[0]) == "app startup failed" + assert events == ["guard startup", "app startup", "guard shutdown"] + else: + with TestClient(app) as client: + assert client.app_state["resource"] is resource + assert not resource.closed + assert events == [ + "guard startup", + "app startup", + "app shutdown", + "guard shutdown", + ] + assert resource.closed