Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/HTTP_DEPLOYMENT.md
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ Optional variables:
- `MCP_ALLOWED_ORIGINS` — comma-separated hostnames appended to the browser Origin allowlist (DNS-rebinding protection).
- `SENTRY_DSN` — sends one transaction per tool call (`tools/call <tool>`) and unexpected errors to Sentry. Unset, nothing is sent.
- `SENTRY_TRACES_SAMPLE_RATE` (default: 1.0) — share of tool calls traced in Sentry.
- `AXIOM_TOKEN` and `AXIOM_DATASET` — with both set, every `mcp_tool_call` line below is also sent to that Axiom dataset, in batches every 5 s from a background thread, with `release` (commit), `environment` and `worker` (process id) added. Unset, nothing is sent. The token needs ingest permission on that dataset only. A failed batch is dropped and logged, never retried, and never slows a tool call; when more than 20,000 lines wait, new ones are dropped and counted in an `mcp_axiom_dropped` event.
- `WEB_CONCURRENCY` (default: 1) — number of server processes. One process uses one CPU core at most. Above 1, `/health` returns 503 until every process is up, and legacy `/sse` breaks: its `/messages` POSTs can reach a process that does not hold the stream.

## Logging
Expand Down
126 changes: 126 additions & 0 deletions src/late/mcp/axiom_sink.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
import json
import logging
import os
import threading
from typing import Any

import httpx

_INGEST_URL = "https://api.axiom.co/v1/datasets/{dataset}/ingest"
_FLUSH_INTERVAL_SECONDS = 5.0
_MAX_BATCH = 1_000
_MAX_BUFFERED = 20_000
_INGEST_TIMEOUT_SECONDS = 10.0

logger = logging.getLogger(__name__)


class AxiomIngestHandler(logging.Handler):
emits_without_blocking = True

def __init__(
self,
*,
dataset: str,
token: str,
static_fields: dict[str, Any] | None = None,
client: httpx.Client | None = None,
flush_interval: float = _FLUSH_INTERVAL_SECONDS,
) -> None:
super().__init__(logging.INFO)
self._url = _INGEST_URL.format(dataset=dataset)
self._headers = {"Authorization": f"Bearer {token}"}
self._static_fields = static_fields or {}
self._client = client or httpx.Client(timeout=_INGEST_TIMEOUT_SECONDS)
self._flush_interval = flush_interval
self._buffer: list[dict[str, Any]] = []
self._buffer_lock = threading.Lock()
self._send_lock = threading.Lock()
self._dropped = 0
self._wake = threading.Event()
threading.Thread(
target=self._flush_forever, name="axiom-ingest", daemon=True
).start()

def emit(self, record: logging.LogRecord) -> None:
try:
event = json.loads(record.getMessage())
except ValueError:
return
if not isinstance(event, dict):
return
with self._buffer_lock:
if len(self._buffer) >= _MAX_BUFFERED:
self._dropped += 1
return
self._buffer.append({**self._static_fields, **event})
if len(self._buffer) >= _MAX_BATCH:
self._wake.set()

def flush(self) -> None:
with self._send_lock:
batch, dropped = self._take_buffer()
if dropped:
batch.append(
{
**self._static_fields,
"event": "mcp_axiom_dropped",
"dropped": dropped,
}
)
for start in range(0, len(batch), _MAX_BATCH):
self._send(batch[start : start + _MAX_BATCH])

def close(self) -> None:
self.flush()
super().close()

def _take_buffer(self) -> tuple[list[dict[str, Any]], int]:
with self._buffer_lock:
batch, self._buffer = self._buffer, []
dropped, self._dropped = self._dropped, 0
return batch, dropped

def _send(self, events: list[dict[str, Any]]) -> None:
try:
response = self._client.post(
self._url,
params={"timestamp-field": "ts"},
headers=self._headers,
json=events,
)
except Exception as exc:
logger.warning(
"Axiom ingest failed, %d events lost: %s",
len(events),
type(exc).__name__,
)
return
if response.status_code >= 400:
logger.warning(
"Axiom ingest rejected %d events: HTTP %s",
len(events),
response.status_code,
)

def _flush_forever(self) -> None:
while True:
self._wake.wait(self._flush_interval)
self._wake.clear()
self.flush()


def axiom_handler_from_env() -> AxiomIngestHandler | None:
token = os.getenv("AXIOM_TOKEN")
dataset = os.getenv("AXIOM_DATASET")
if not token or not dataset:
return None
return AxiomIngestHandler(
dataset=dataset,
token=token,
static_fields={
"release": (os.getenv("RAILWAY_GIT_COMMIT_SHA") or "")[:12] or None,
"environment": os.getenv("RAILWAY_ENVIRONMENT_NAME") or "production",
"worker": os.getpid(),
},
)
2 changes: 2 additions & 0 deletions src/late/mcp/http_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
init_sentry,
log_tool_calls_to_stdout,
quiet_routine_logs,
ship_tool_calls_to_axiom,
)
from late.mcp.workers import all_workers_ready, mark_worker_ready, worker_count

Expand Down Expand Up @@ -392,6 +393,7 @@ def parse_args() -> argparse.Namespace:
def _configure_process(debug: bool) -> None:
init_sentry()
log_tool_calls_to_stdout()
ship_tool_calls_to_axiom()
if not debug:
quiet_routine_logs()

Expand Down
1 change: 1 addition & 0 deletions src/late/mcp/non_blocking_logging.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ def _wrap_handlers(logger: logging.Logger, pump: LogPump) -> None:
logger.handlers = [
handler
if isinstance(handler, NonBlockingHandler)
or getattr(handler, "emits_without_blocking", False)
else NonBlockingHandler(handler, pump)
for handler in logger.handlers
]
Expand Down
9 changes: 9 additions & 0 deletions src/late/mcp/telemetry.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

from late.client.exceptions import LateAPIError, LateError
from late.mcp.auth import token_cache_key
from late.mcp.axiom_sink import axiom_handler_from_env
from late.mcp.constants import SERVICE_VERSION
from late.mcp.response_format import format_stats

Expand Down Expand Up @@ -74,6 +75,14 @@ def init_sentry() -> bool:
return True


def ship_tool_calls_to_axiom() -> bool:
handler = axiom_handler_from_env()
if handler is None:
return False
_tool_logger.addHandler(handler)
return True


def log_tool_calls_to_stdout() -> None:
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(logging.Formatter("%(message)s"))
Expand Down
Loading
Loading