diff --git a/src/late/mcp/auth.py b/src/late/mcp/auth.py index 2a865cee..dd855d1f 100644 --- a/src/late/mcp/auth.py +++ b/src/late/mcp/auth.py @@ -1,8 +1,10 @@ """Authentication module for Zernio MCP HTTP server.""" +import asyncio import hashlib import logging import os +import threading import time from collections import OrderedDict from enum import Enum @@ -88,11 +90,11 @@ def is_allowed_origin(request: Request) -> bool: # --workers), and a shared store would put a new dependency in front of auth, # which is how you rebuild this same outage. _VERIFIED_AT: OrderedDict[str, float] = OrderedDict() +_VERIFIED_CACHE_LOCK = threading.Lock() _VERIFIED_CACHE_MAX = 10_000 -# Mirrors the 60s Redis TTL the API itself keeps on apikey:{keyHash} -# (Schedule-Posts-API libs/api-auth.ts:471), so this adds no revocation window -# the platform does not already have. -_FRESH_TTL_SECONDS = 60.0 +_FRESH_TTL_SECONDS = 600.0 +_VERIFY_KEEPALIVE_SECONDS = 60.0 +_IN_FLIGHT: dict[str, "asyncio.Task[Verification]"] = {} # Only used when upstream could not answer. Blast radius is bounded to # nothing: the MCP stores no data and every tool call re-presents the same # key to the same API, so a revoked key honoured here still cannot read or @@ -140,10 +142,32 @@ def token_cache_key(token: str) -> str: def _remember(token_key: str, now: float) -> None: - _VERIFIED_AT[token_key] = now - _VERIFIED_AT.move_to_end(token_key) - while len(_VERIFIED_AT) > _VERIFIED_CACHE_MAX: - _VERIFIED_AT.popitem(last=False) + with _VERIFIED_CACHE_LOCK: + _VERIFIED_AT[token_key] = now + _VERIFIED_AT.move_to_end(token_key) + while len(_VERIFIED_AT) > _VERIFIED_CACHE_MAX: + _VERIFIED_AT.popitem(last=False) + + +def _touch(token_key: str) -> None: + with _VERIFIED_CACHE_LOCK: + if token_key in _VERIFIED_AT: + _VERIFIED_AT.move_to_end(token_key) + + +def _forget(token_key: str) -> None: + with _VERIFIED_CACHE_LOCK: + _VERIFIED_AT.pop(token_key, None) + + +def forget_rejected_token(token: str) -> None: + _forget(token_cache_key(token)) + + +def _new_verify_client() -> httpx.AsyncClient: + return httpx.AsyncClient( + limits=httpx.Limits(keepalive_expiry=_VERIFY_KEEPALIVE_SECONDS) + ) async def verify_late_api_key( @@ -155,8 +179,8 @@ async def verify_late_api_key( Returns a three-valued Verification, NOT a bool: never write ``if await verify_late_api_key(...)``, every Enum member is truthy. - client is injected by tests (httpx.MockTransport); production opens a - per-call client and must not close an injected one. + Without a client this opens and closes one per call; ZernioTokenVerifier + passes its long-lived client so verifications reuse warm connections. """ owns_client = client is None client = client or httpx.AsyncClient() @@ -198,7 +222,7 @@ class ZernioTokenVerifier(TokenVerifier): def __init__(self, *, client: httpx.AsyncClient | None = None) -> None: super().__init__() - self._client = client + self._client = client or _new_verify_client() async def verify_token(self, token: str) -> AccessToken | None: token_key = token_cache_key(token) @@ -206,17 +230,20 @@ async def verify_token(self, token: str) -> AccessToken | None: verified_at = _VERIFIED_AT.get(token_key) if verified_at is not None and now - verified_at < _FRESH_TTL_SECONDS: - _VERIFIED_AT.move_to_end(token_key) - return self._grant(token) + _touch(token_key) + return self._grant(token, resolved_by="cache") - verdict = await verify_late_api_key(token, client=self._client) + joined = token_key in _IN_FLIGHT + verdict = await self._verify_once(token, token_key) + waited_ms = round((time.monotonic() - now) * 1000) + resolved_by = "joined" if joined else "upstream" if verdict is Verification.VALID: _remember(token_key, now) - return self._grant(token) + return self._grant(token, resolved_by=resolved_by, waited_ms=waited_ms) if verdict is Verification.INVALID: - _VERIFIED_AT.pop(token_key, None) + _forget(token_key) return None # UNKNOWN. Honour a token upstream confirmed recently; refuse to @@ -228,24 +255,41 @@ async def verify_token(self, token: str) -> AccessToken | None: # type OTHER than AuthenticationError escapes as a 500 with the # message swallowed. if verified_at is not None and now - verified_at < _GRACE_TTL_SECONDS: - # Guarded because the await above can suspend for the full verify - # timeout, and a concurrent request may LRU-evict this key while - # we are parked. A bare move_to_end would raise KeyError, which is - # not AuthenticationError and so escapes as a 500. - if token_key in _VERIFIED_AT: - _VERIFIED_AT.move_to_end(token_key) + _touch(token_key) logger.warning( "Granting %s... from cache: upstream unreachable", token_key[:12], ) - return self._grant(token) + return self._grant(token, resolved_by="grace", waited_ms=waited_ms) raise AuthenticationError( "Zernio API is temporarily unreachable, so this token could not " "be verified. Your credentials are fine; retry in a moment." ) - def _grant(self, token: str) -> AccessToken: - return AccessToken(token=token, client_id="zernio", scopes=list(OAUTH_SCOPES)) + async def _verify_once(self, token: str, token_key: str) -> Verification: + task = _IN_FLIGHT.get(token_key) + if task is None: + task = asyncio.ensure_future( + verify_late_api_key(token, client=self._client) + ) + _IN_FLIGHT[token_key] = task + task.add_done_callback(lambda done: _release_in_flight(token_key, done)) + return await asyncio.shield(task) + + def _grant( + self, token: str, *, resolved_by: str, waited_ms: int = 0 + ) -> AccessToken: + return AccessToken( + token=token, + client_id="zernio", + scopes=list(OAUTH_SCOPES), + claims={"resolved_by": resolved_by, "waited_ms": waited_ms}, + ) + + +def _release_in_flight(token_key: str, task: "asyncio.Task[Verification]") -> None: + if _IN_FLIGHT.get(token_key) is task: + del _IN_FLIGHT[token_key] class VerbatimIssuerRemoteAuthProvider(RemoteAuthProvider): diff --git a/src/late/mcp/chatgpt_server.py b/src/late/mcp/chatgpt_server.py index 42264773..c32f743f 100644 --- a/src/late/mcp/chatgpt_server.py +++ b/src/late/mcp/chatgpt_server.py @@ -23,7 +23,7 @@ from mcp.types import TextContent, ToolAnnotations from pydantic import BaseModel, Field -from late.mcp.auth import build_auth_provider +from late.mcp.auth import build_auth_provider, forget_rejected_token API_BASE = ( os.getenv("ZERNIO_BASE_URL") @@ -231,11 +231,14 @@ def _error_message(response: httpx.Response) -> str: async def _api( method: str, path: str, *, params: dict[str, Any] | None = None, json: Any = None ) -> Any: - headers = {"Authorization": f"Bearer {_bearer()}", "Accept": "application/json"} + bearer = _bearer() + headers = {"Authorization": f"Bearer {bearer}", "Accept": "application/json"} async with httpx.AsyncClient(base_url=API_BASE, timeout=_TIMEOUT) as client: response = await client.request( method, path, params=params, json=json, headers=headers ) + if response.status_code == 401: + forget_rejected_token(bearer) if response.status_code >= 400: raise ToolError(_error_message(response)) return response.json() if response.content else {} diff --git a/src/late/mcp/server.py b/src/late/mcp/server.py index 1f7b3a05..7f0927a0 100644 --- a/src/late/mcp/server.py +++ b/src/late/mcp/server.py @@ -47,7 +47,7 @@ from late import Late, MediaType, PostStatus from late.client.exceptions import LateAPIError, LateError, LateRateLimitError -from .auth import build_auth_provider +from .auth import build_auth_provider, forget_rejected_token from .feedback_source import FEEDBACK_TOOL, FeedbackSourceMiddleware from .idempotency import same_arguments_key from .resources import register_resources @@ -86,10 +86,25 @@ _thread_state = threading.local() -def _api_transport() -> httpx.HTTPTransport: +class _ApiTransport(httpx.BaseTransport): + def __init__(self, inner: httpx.BaseTransport | None = None) -> None: + self._inner = inner or httpx.HTTPTransport() + + def handle_request(self, request: httpx.Request) -> httpx.Response: + response = self._inner.handle_request(request) + if response.status_code == 401: + bearer = request.headers.get("Authorization", "") + forget_rejected_token(bearer.removeprefix("Bearer ")) + return response + + def close(self) -> None: + self._inner.close() + + +def _api_transport() -> httpx.BaseTransport: transport = getattr(_thread_state, "transport", None) if transport is None: - transport = _thread_state.transport = httpx.HTTPTransport() + transport = _thread_state.transport = _ApiTransport() return transport # Always-visible core: the 20 hand-written ergonomic tools (guardrails + diff --git a/src/late/mcp/telemetry.py b/src/late/mcp/telemetry.py index 8ebd19ae..c074b116 100644 --- a/src/late/mcp/telemetry.py +++ b/src/late/mcp/telemetry.py @@ -11,6 +11,7 @@ import sentry_sdk from fastmcp.exceptions import FastMCPError, NotFoundError, ToolError +from fastmcp.server.auth import AccessToken from fastmcp.server.dependencies import get_access_token, get_http_headers from fastmcp.server.middleware import CallNext, Middleware, MiddlewareContext from mcp.types import CallToolRequestParams, TextContent @@ -115,14 +116,26 @@ def quiet_routine_logs() -> None: logging.getLogger("fastmcp.server.server").addFilter(ToolFailureLogFilter()) -def _caller_key() -> str | None: +def _current_token() -> AccessToken | None: try: - token = get_access_token() + return get_access_token() except Exception: return None + + +def _caller_key() -> str | None: + token = _current_token() return token_cache_key(token.token)[:12] if token and token.token else None +def _auth_fields() -> dict[str, Any]: + token = _current_token() + claims = token.claims if token else {} + if "resolved_by" not in claims: + return {} + return {"auth": claims["resolved_by"], "auth_wait_ms": claims.get("waited_ms", 0)} + + def _request_fields(arguments: dict[str, Any] | None) -> dict[str, Any]: fields: dict[str, Any] = {} limit = (arguments or {}).get("limit") @@ -198,6 +211,7 @@ async def _record( tool = context.message.name caller = _caller_key() request = _request_fields(context.message.arguments) + auth = _auth_fields() fields: dict[str, Any] = {"outcome": "ok"} stats: dict[str, Any] = {} stats_token = format_stats.set(stats) @@ -212,6 +226,9 @@ async def _record( transaction.set_tag("mcp.caller", caller) for name, value in request.items(): transaction.set_tag(f"mcp.{name}", value) + if auth: + transaction.set_tag("mcp.auth", auth["auth"]) + transaction.set_data("mcp.auth.wait_ms", auth["auth_wait_ms"]) try: result = await call_next(context) if _is_rejected(result): @@ -258,6 +275,7 @@ async def _record( ), "caller": caller, **request, + **auth, **fields, } ) diff --git a/tests/test_chatgpt_server.py b/tests/test_chatgpt_server.py index 6973cd2c..600eb83f 100644 --- a/tests/test_chatgpt_server.py +++ b/tests/test_chatgpt_server.py @@ -17,7 +17,7 @@ import respx from fastmcp import Client -from late.mcp import chatgpt_server +from late.mcp import auth, chatgpt_server from late.mcp.chatgpt_server import chatgpt_mcp API = chatgpt_server.API_BASE @@ -328,3 +328,16 @@ async def test_best_time_returns_named_days_best_first(): "post_count": 1, } assert result.structured_content["based_on_posts"] == 3 + + +@respx.mock +async def test_an_api_401_evicts_the_cached_token_so_chatgpt_gets_a_401_and_refreshes(): + auth._VERIFIED_AT[auth.token_cache_key("sk_test")] = 0.0 + respx.get(f"{API}/v1/accounts").mock( + return_value=httpx.Response(401, json={"error": "Invalid API key"}) + ) + + result = await _call("list_social_accounts") + + assert result.is_error + assert auth.token_cache_key("sk_test") not in auth._VERIFIED_AT diff --git a/tests/test_mcp_auth_cache.py b/tests/test_mcp_auth_cache.py new file mode 100644 index 00000000..fa9195b7 --- /dev/null +++ b/tests/test_mcp_auth_cache.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import asyncio +import time + +import httpx +import pytest + +from late.mcp import auth, server +from late.mcp.auth import ZernioTokenVerifier + + +@pytest.fixture(autouse=True) +def _clear_verification_cache(): + auth._VERIFIED_AT.clear() + auth._IN_FLIGHT.clear() + yield + auth._VERIFIED_AT.clear() + auth._IN_FLIGHT.clear() + + +class SlowVerifyUpstream: + def __init__(self, status_code: int = 200) -> None: + self.calls = 0 + self.release = asyncio.Event() + self.status_code = status_code + + async def __call__(self, _request: httpx.Request) -> httpx.Response: + self.calls += 1 + await self.release.wait() + return httpx.Response(self.status_code) + + +async def test_concurrent_requests_with_one_token_share_a_single_upstream_verification(): + upstream = SlowVerifyUpstream() + client = httpx.AsyncClient(transport=httpx.MockTransport(upstream)) + verifier = ZernioTokenVerifier(client=client) + + pending = [asyncio.create_task(verifier.verify_token("sk_live")) for _ in range(5)] + await asyncio.sleep(0) + upstream.release.set() + granted = await asyncio.gather(*pending) + + assert upstream.calls == 1 + assert sorted(t.claims["resolved_by"] for t in granted) == ["joined"] * 4 + [ + "upstream" + ] + assert auth._IN_FLIGHT == {} + await client.aclose() + + +async def test_a_cancelled_request_does_not_cancel_the_verification_others_wait_on(): + upstream = SlowVerifyUpstream() + client = httpx.AsyncClient(transport=httpx.MockTransport(upstream)) + verifier = ZernioTokenVerifier(client=client) + + first = asyncio.create_task(verifier.verify_token("sk_live")) + second = asyncio.create_task(verifier.verify_token("sk_live")) + await asyncio.sleep(0) + first.cancel() + upstream.release.set() + + assert (await second) is not None + assert upstream.calls == 1 + await client.aclose() + + +async def test_a_token_verified_nine_minutes_ago_is_served_from_cache(): + upstream = SlowVerifyUpstream() + client = httpx.AsyncClient(transport=httpx.MockTransport(upstream)) + verifier = ZernioTokenVerifier(client=client) + auth._VERIFIED_AT[auth.token_cache_key("sk_live")] = time.monotonic() - 540 + + granted = await verifier.verify_token("sk_live") + + assert granted is not None + assert granted.claims == {"resolved_by": "cache", "waited_ms": 0} + assert upstream.calls == 0 + await client.aclose() + + +async def test_an_upstream_verification_reports_how_long_the_request_waited(): + upstream = SlowVerifyUpstream() + upstream.release.set() + client = httpx.AsyncClient(transport=httpx.MockTransport(upstream)) + + granted = await ZernioTokenVerifier(client=client).verify_token("sk_live") + + assert granted is not None + assert granted.claims["resolved_by"] == "upstream" + assert isinstance(granted.claims["waited_ms"], int) + await client.aclose() + + +def test_the_verifier_keeps_one_client_with_warm_connections_for_every_verification(): + verifier = ZernioTokenVerifier() + + assert isinstance(verifier._client, httpx.AsyncClient) + assert verifier._client._transport._pool._keepalive_expiry == 60.0 + + +def test_an_api_401_during_a_tool_call_evicts_the_cached_token_so_the_client_can_refresh(): + auth._VERIFIED_AT[auth.token_cache_key("sk_revoked")] = time.monotonic() + auth._VERIFIED_AT[auth.token_cache_key("sk_other")] = time.monotonic() + transport = server._ApiTransport( + httpx.MockTransport(lambda _request: httpx.Response(401)) + ) + + httpx.Client(transport=transport).get( + "https://zernio.test/api/v1/posts", + headers={"Authorization": "Bearer sk_revoked"}, + ) + + assert auth.token_cache_key("sk_revoked") not in auth._VERIFIED_AT + assert auth.token_cache_key("sk_other") in auth._VERIFIED_AT + + +def test_a_successful_tool_call_keeps_the_cached_token(): + auth._VERIFIED_AT[auth.token_cache_key("sk_live")] = time.monotonic() + transport = server._ApiTransport( + httpx.MockTransport(lambda _request: httpx.Response(200)) + ) + + httpx.Client(transport=transport).get( + "https://zernio.test/api/v1/posts", + headers={"Authorization": "Bearer sk_live"}, + ) + + assert auth.token_cache_key("sk_live") in auth._VERIFIED_AT diff --git a/tests/test_mcp_telemetry.py b/tests/test_mcp_telemetry.py index 2c903d1c..ace36ea3 100644 --- a/tests/test_mcp_telemetry.py +++ b/tests/test_mcp_telemetry.py @@ -13,6 +13,7 @@ import sentry_sdk from fastmcp import Client, FastMCP from fastmcp.exceptions import ToolError +from fastmcp.server.auth import AccessToken from fastmcp.server.transforms.search import BM25SearchTransform from sentry_sdk.transport import Transport @@ -282,6 +283,28 @@ async def test_a_direct_call_is_tagged_direct_with_its_caller( assert transaction["tags"]["mcp.caller"] == "a1b2c3d4e5f6" +async def test_a_call_records_whether_its_request_waited_on_token_verification( + sentry_events, caplog, monkeypatch +) -> None: + caplog.set_level(logging.INFO, logger="late.mcp.tools") + token = AccessToken( + token="sk_live", + client_id="zernio", + scopes=[], + claims={"resolved_by": "upstream", "waited_ms": 312}, + ) + monkeypatch.setattr(telemetry, "get_access_token", lambda: token) + + async with Client(_server()) as client: + await client.call_tool("fine", {}) + + [transaction] = sentry_events.payloads("transaction") + assert transaction["tags"]["mcp.auth"] == "upstream" + assert transaction["contexts"]["trace"]["data"]["mcp.auth.wait_ms"] == 312 + [line] = _logged(caplog) + assert (line["auth"], line["auth_wait_ms"]) == ("upstream", 312) + + async def test_call_tool_failing_before_it_dispatches_is_still_recorded(caplog) -> None: caplog.set_level(logging.INFO, logger="late.mcp.tools") diff --git a/uv.lock b/uv.lock index e1a7a6cb..c8af8ed2 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -2411,7 +2411,7 @@ wheels = [ [[package]] name = "zernio-sdk" -version = "1.4.1746" +version = "1.4.1747" source = { editable = "." } dependencies = [ { name = "httpx" },