From 262b4d45281d92626d592684ef05483a7d0276eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gemma=20Torell=C3=B3?= Date: Thu, 12 Nov 2026 17:10:17 +0100 Subject: [PATCH 1/2] feat(mcp): send each tool call line to Axiom when AXIOM_TOKEN and AXIOM_DATASET are set Sentry stopped accepting spans when the organization's quota ran out, so tool-call latency, size and auth outcomes could not be measured. The same mcp_tool_call line printed to stdout now also goes to an Axiom dataset, batched every 5 s from a background thread, dropped (and counted) instead of ever slowing a tool call. Co-Authored-By: Claude Opus 5.5 --- docs/HTTP_DEPLOYMENT.md | 1 + src/late/mcp/axiom_sink.py | 124 ++++++++++++++++++++ src/late/mcp/http_server.py | 2 + src/late/mcp/telemetry.py | 9 ++ tests/test_mcp_axiom_sink.py | 220 +++++++++++++++++++++++++++++++++++ 5 files changed, 356 insertions(+) create mode 100644 src/late/mcp/axiom_sink.py create mode 100644 tests/test_mcp_axiom_sink.py diff --git a/docs/HTTP_DEPLOYMENT.md b/docs/HTTP_DEPLOYMENT.md index 36678c59..2adb5880 100644 --- a/docs/HTTP_DEPLOYMENT.md +++ b/docs/HTTP_DEPLOYMENT.md @@ -69,6 +69,7 @@ Optional variables: - `MCP_ALLOWED_ORIGINS` — comma-separated hostnames appended to the browser Origin allowlist (DNS-rebinding protection). - `SENTRY_DSN` — sends one transaction per tool call (`tools/call `) and unexpected errors to Sentry. Unset, nothing is sent. - `SENTRY_TRACES_SAMPLE_RATE` (default: 1.0) — share of tool calls traced in Sentry. +- `AXIOM_TOKEN` and `AXIOM_DATASET` — with both set, every `mcp_tool_call` line below is also sent to that Axiom dataset, in batches every 5 s from a background thread, with `release` (commit), `environment` and `worker` (process id) added. Unset, nothing is sent. The token needs ingest permission on that dataset only. A failed batch is dropped and logged, never retried, and never slows a tool call; when more than 20,000 lines wait, new ones are dropped and counted in an `mcp_axiom_dropped` event. - `WEB_CONCURRENCY` (default: 1) — number of server processes. One process uses one CPU core at most. Above 1, `/health` returns 503 until every process is up, and legacy `/sse` breaks: its `/messages` POSTs can reach a process that does not hold the stream. ## Logging diff --git a/src/late/mcp/axiom_sink.py b/src/late/mcp/axiom_sink.py new file mode 100644 index 00000000..66cbd53b --- /dev/null +++ b/src/late/mcp/axiom_sink.py @@ -0,0 +1,124 @@ +import json +import logging +import os +import threading +from typing import Any + +import httpx + +_INGEST_URL = "https://api.axiom.co/v1/datasets/{dataset}/ingest" +_FLUSH_INTERVAL_SECONDS = 5.0 +_MAX_BATCH = 1_000 +_MAX_BUFFERED = 20_000 +_INGEST_TIMEOUT_SECONDS = 10.0 + +logger = logging.getLogger(__name__) + + +class AxiomIngestHandler(logging.Handler): + def __init__( + self, + *, + dataset: str, + token: str, + static_fields: dict[str, Any] | None = None, + client: httpx.Client | None = None, + flush_interval: float = _FLUSH_INTERVAL_SECONDS, + ) -> None: + super().__init__(logging.INFO) + self._url = _INGEST_URL.format(dataset=dataset) + self._headers = {"Authorization": f"Bearer {token}"} + self._static_fields = static_fields or {} + self._client = client or httpx.Client(timeout=_INGEST_TIMEOUT_SECONDS) + self._flush_interval = flush_interval + self._buffer: list[dict[str, Any]] = [] + self._buffer_lock = threading.Lock() + self._send_lock = threading.Lock() + self._dropped = 0 + self._wake = threading.Event() + threading.Thread( + target=self._flush_forever, name="axiom-ingest", daemon=True + ).start() + + def emit(self, record: logging.LogRecord) -> None: + try: + event = json.loads(record.getMessage()) + except ValueError: + return + if not isinstance(event, dict): + return + with self._buffer_lock: + if len(self._buffer) >= _MAX_BUFFERED: + self._dropped += 1 + return + self._buffer.append({**self._static_fields, **event}) + if len(self._buffer) >= _MAX_BATCH: + self._wake.set() + + def flush(self) -> None: + with self._send_lock: + batch, dropped = self._take_buffer() + if dropped: + batch.append( + { + **self._static_fields, + "event": "mcp_axiom_dropped", + "dropped": dropped, + } + ) + for start in range(0, len(batch), _MAX_BATCH): + self._send(batch[start : start + _MAX_BATCH]) + + def close(self) -> None: + self.flush() + super().close() + + def _take_buffer(self) -> tuple[list[dict[str, Any]], int]: + with self._buffer_lock: + batch, self._buffer = self._buffer, [] + dropped, self._dropped = self._dropped, 0 + return batch, dropped + + def _send(self, events: list[dict[str, Any]]) -> None: + try: + response = self._client.post( + self._url, + params={"timestamp-field": "ts"}, + headers=self._headers, + json=events, + ) + except httpx.HTTPError as exc: + logger.warning( + "Axiom ingest failed, %d events lost: %s", + len(events), + type(exc).__name__, + ) + return + if response.status_code >= 400: + logger.warning( + "Axiom ingest rejected %d events: HTTP %s", + len(events), + response.status_code, + ) + + def _flush_forever(self) -> None: + while True: + self._wake.wait(self._flush_interval) + self._wake.clear() + self.flush() + + +def axiom_handler_from_env() -> AxiomIngestHandler | None: + token = os.getenv("AXIOM_TOKEN") + dataset = os.getenv("AXIOM_DATASET") + if not token or not dataset: + return None + return AxiomIngestHandler( + dataset=dataset, + token=token, + static_fields={ + "release": (os.getenv("RAILWAY_GIT_COMMIT_SHA") or "")[:12] or None, + "environment": os.getenv("RAILWAY_ENVIRONMENT_NAME") or "production", + "worker": os.getpid(), + }, + ) diff --git a/src/late/mcp/http_server.py b/src/late/mcp/http_server.py index b0395edf..5d3b1126 100644 --- a/src/late/mcp/http_server.py +++ b/src/late/mcp/http_server.py @@ -60,6 +60,7 @@ init_sentry, log_tool_calls_to_stdout, quiet_routine_logs, + ship_tool_calls_to_axiom, ) from late.mcp.workers import all_workers_ready, mark_worker_ready, worker_count @@ -392,6 +393,7 @@ def parse_args() -> argparse.Namespace: def _configure_process(debug: bool) -> None: init_sentry() log_tool_calls_to_stdout() + ship_tool_calls_to_axiom() if not debug: quiet_routine_logs() diff --git a/src/late/mcp/telemetry.py b/src/late/mcp/telemetry.py index c074b116..25847ae5 100644 --- a/src/late/mcp/telemetry.py +++ b/src/late/mcp/telemetry.py @@ -20,6 +20,7 @@ from late.client.exceptions import LateAPIError, LateError from late.mcp.auth import token_cache_key +from late.mcp.axiom_sink import axiom_handler_from_env from late.mcp.constants import SERVICE_VERSION from late.mcp.response_format import format_stats @@ -74,6 +75,14 @@ def init_sentry() -> bool: return True +def ship_tool_calls_to_axiom() -> bool: + handler = axiom_handler_from_env() + if handler is None: + return False + _tool_logger.addHandler(handler) + return True + + def log_tool_calls_to_stdout() -> None: handler = logging.StreamHandler(sys.stdout) handler.setFormatter(logging.Formatter("%(message)s")) diff --git a/tests/test_mcp_axiom_sink.py b/tests/test_mcp_axiom_sink.py new file mode 100644 index 00000000..9fc9b357 --- /dev/null +++ b/tests/test_mcp_axiom_sink.py @@ -0,0 +1,220 @@ +from __future__ import annotations + +import json +import logging + +import httpx +import pytest +from fastmcp import Client, FastMCP + +from late.mcp import axiom_sink, telemetry +from late.mcp.axiom_sink import AxiomIngestHandler, axiom_handler_from_env +from late.mcp.telemetry import ToolTelemetryMiddleware + + +class RecordingIngest: + def __init__(self, status_code: int = 200) -> None: + self.requests: list[httpx.Request] = [] + self.status_code = status_code + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + return httpx.Response(self.status_code, json={"ingested": 1}) + + def events(self) -> list[dict]: + return [ + event for request in self.requests for event in json.loads(request.content) + ] + + +def _handler(ingest: RecordingIngest, **kwargs) -> AxiomIngestHandler: + return AxiomIngestHandler( + dataset="zernio-mcp", + token="xaat-test", + client=httpx.Client(transport=httpx.MockTransport(ingest)), + flush_interval=3600, + **kwargs, + ) + + +def _record(message: str) -> logging.LogRecord: + return logging.makeLogRecord({"msg": message, "levelno": logging.INFO}) + + +def test_tool_call_lines_reach_the_dataset_in_one_batch_keyed_on_their_own_timestamp() -> ( + None +): + ingest = RecordingIngest() + handler = _handler(ingest, static_fields={"release": "abc123", "worker": 7}) + + handler.emit( + _record( + json.dumps( + { + "event": "mcp_tool_call", + "tool": "posts_list", + "ts": "2026-11-12T16:00:00.000+00:00", + } + ) + ) + ) + handler.emit( + _record( + json.dumps( + { + "event": "mcp_tool_call", + "tool": "accounts_list", + "ts": "2026-11-12T16:00:01.000+00:00", + } + ) + ) + ) + handler.flush() + + [request] = ingest.requests + assert ( + str(request.url) + == "https://api.axiom.co/v1/datasets/zernio-mcp/ingest?timestamp-field=ts" + ) + assert request.headers["Authorization"] == "Bearer xaat-test" + assert [e["tool"] for e in ingest.events()] == ["posts_list", "accounts_list"] + assert ingest.events()[0]["release"] == "abc123" + assert ingest.events()[0]["worker"] == 7 + + +def test_lines_that_are_not_json_objects_are_skipped() -> None: + ingest = RecordingIngest() + handler = _handler(ingest) + + handler.emit(_record("plain text")) + handler.emit(_record("[1, 2]")) + handler.flush() + + assert ingest.requests == [] + + +def test_a_full_buffer_drops_new_lines_and_reports_how_many( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(axiom_sink, "_MAX_BUFFERED", 2) + ingest = RecordingIngest() + handler = _handler(ingest) + + for tool in ("a", "b", "c", "d"): + handler.emit(_record(json.dumps({"event": "mcp_tool_call", "tool": tool}))) + handler.flush() + + events = ingest.events() + assert [e.get("tool") for e in events if e["event"] == "mcp_tool_call"] == [ + "a", + "b", + ] + assert {"event": "mcp_axiom_dropped", "dropped": 2} in events + + +def test_a_failed_ingest_loses_the_batch_without_raising( + caplog: pytest.LogCaptureFixture, +) -> None: + def unreachable(_request: httpx.Request) -> httpx.Response: + raise httpx.ConnectTimeout("boom") + + handler = AxiomIngestHandler( + dataset="zernio-mcp", + token="xaat-test", + client=httpx.Client(transport=httpx.MockTransport(unreachable)), + flush_interval=3600, + ) + handler.emit(_record(json.dumps({"event": "mcp_tool_call"}))) + + handler.flush() + handler.flush() + + assert "Axiom ingest failed, 1 events lost: ConnectTimeout" in caplog.text + + +def test_a_rejected_ingest_is_logged(caplog: pytest.LogCaptureFixture) -> None: + handler = _handler(RecordingIngest(status_code=403)) + handler.emit(_record(json.dumps({"event": "mcp_tool_call"}))) + + handler.flush() + + assert "Axiom ingest rejected 1 events: HTTP 403" in caplog.text + + +def test_batches_are_split_at_the_ingest_size_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(axiom_sink, "_MAX_BATCH", 2) + ingest = RecordingIngest() + handler = _handler(ingest) + + for tool in ("a", "b", "c"): + handler.emit(_record(json.dumps({"event": "mcp_tool_call", "tool": tool}))) + handler.flush() + + assert [len(json.loads(r.content)) for r in ingest.requests] == [2, 1] + + +def test_closing_the_handler_sends_what_is_buffered() -> None: + ingest = RecordingIngest() + handler = _handler(ingest) + handler.emit(_record(json.dumps({"event": "mcp_tool_call", "tool": "last"}))) + + handler.close() + + assert [e["tool"] for e in ingest.events()] == ["last"] + + +@pytest.mark.parametrize( + "env", [{}, {"AXIOM_TOKEN": "xaat"}, {"AXIOM_DATASET": "zernio-mcp"}] +) +def test_nothing_is_shipped_without_both_token_and_dataset( + monkeypatch: pytest.MonkeyPatch, env: dict +) -> None: + monkeypatch.delenv("AXIOM_TOKEN", raising=False) + monkeypatch.delenv("AXIOM_DATASET", raising=False) + for name, value in env.items(): + monkeypatch.setenv(name, value) + + assert axiom_handler_from_env() is None + + +@pytest.fixture +def tool_logger_at_info(monkeypatch: pytest.MonkeyPatch): + tool_logger = logging.getLogger("late.mcp.tools") + original_level = tool_logger.level + monkeypatch.setattr(tool_logger, "handlers", []) + monkeypatch.setattr(tool_logger, "propagate", False) + monkeypatch.setattr(tool_logger, "disabled", False) + tool_logger.setLevel(logging.INFO) + yield tool_logger + tool_logger.setLevel(original_level) + + +@pytest.mark.usefixtures("tool_logger_at_info") +async def test_a_tool_call_ends_up_as_one_event_in_the_dataset( + monkeypatch: pytest.MonkeyPatch, +) -> None: + ingest = RecordingIngest() + handler = _handler(ingest) + monkeypatch.setattr(telemetry, "axiom_handler_from_env", lambda: handler) + + assert telemetry.ship_tool_calls_to_axiom() + server = FastMCP("t") + server.add_middleware(ToolTelemetryMiddleware("mcp")) + + @server.tool + def fine() -> str: + return "ok" + + async with Client(server) as client: + await client.call_tool("fine", {}) + handler.flush() + + [event] = ingest.events() + assert (event["event"], event["tool"], event["outcome"]) == ( + "mcp_tool_call", + "fine", + "ok", + ) + assert isinstance(event["duration_ms"], int) From 47e911ee9b9c4a8f7fe2f9f937fce60553f3c69a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gemma=20Torell=C3=B3?= Date: Thu, 12 Nov 2026 17:12:44 +0100 Subject: [PATCH 2/2] fix(mcp): the Axiom ingest thread survives any send error and skips the log pump Co-Authored-By: Claude Opus 5.5 --- src/late/mcp/axiom_sink.py | 4 ++- src/late/mcp/non_blocking_logging.py | 1 + tests/test_mcp_axiom_sink.py | 40 ++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/late/mcp/axiom_sink.py b/src/late/mcp/axiom_sink.py index 66cbd53b..bec4e4b4 100644 --- a/src/late/mcp/axiom_sink.py +++ b/src/late/mcp/axiom_sink.py @@ -16,6 +16,8 @@ class AxiomIngestHandler(logging.Handler): + emits_without_blocking = True + def __init__( self, *, @@ -87,7 +89,7 @@ def _send(self, events: list[dict[str, Any]]) -> None: headers=self._headers, json=events, ) - except httpx.HTTPError as exc: + except Exception as exc: logger.warning( "Axiom ingest failed, %d events lost: %s", len(events), diff --git a/src/late/mcp/non_blocking_logging.py b/src/late/mcp/non_blocking_logging.py index 8a31765e..e25e0e5b 100644 --- a/src/late/mcp/non_blocking_logging.py +++ b/src/late/mcp/non_blocking_logging.py @@ -87,6 +87,7 @@ def _wrap_handlers(logger: logging.Logger, pump: LogPump) -> None: logger.handlers = [ handler if isinstance(handler, NonBlockingHandler) + or getattr(handler, "emits_without_blocking", False) else NonBlockingHandler(handler, pump) for handler in logger.handlers ] diff --git a/tests/test_mcp_axiom_sink.py b/tests/test_mcp_axiom_sink.py index 9fc9b357..3477b8c2 100644 --- a/tests/test_mcp_axiom_sink.py +++ b/tests/test_mcp_axiom_sink.py @@ -179,6 +179,46 @@ def test_nothing_is_shipped_without_both_token_and_dataset( assert axiom_handler_from_env() is None +def test_an_unexpected_send_error_is_logged_and_the_next_flush_still_sends( + caplog: pytest.LogCaptureFixture, +) -> None: + attempts: list[int] = [] + + def first_call_explodes(_request: httpx.Request) -> httpx.Response: + attempts.append(1) + if len(attempts) == 1: + raise RuntimeError("unexpected") + return httpx.Response(200) + + handler = AxiomIngestHandler( + dataset="zernio-mcp", + token="xaat-test", + client=httpx.Client(transport=httpx.MockTransport(first_call_explodes)), + flush_interval=3600, + ) + handler.emit(_record(json.dumps({"event": "mcp_tool_call"}))) + handler.flush() + handler.emit(_record(json.dumps({"event": "mcp_tool_call"}))) + handler.flush() + + assert "Axiom ingest failed, 1 events lost: RuntimeError" in caplog.text + assert len(attempts) == 2 + + +def test_the_log_pump_does_not_wrap_the_axiom_handler( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from late.mcp.non_blocking_logging import LogPump, _wrap_handlers + + handler = _handler(RecordingIngest()) + probe = logging.getLogger("late.mcp.tests.axiom-pump-probe") + monkeypatch.setattr(probe, "handlers", [handler]) + + _wrap_handlers(probe, LogPump()) + + assert probe.handlers == [handler] + + @pytest.fixture def tool_logger_at_info(monkeypatch: pytest.MonkeyPatch): tool_logger = logging.getLogger("late.mcp.tools")