feat(observability): add Prometheus /metrics endpoint (Phase 10)

app/observability/__init__.py

@@ -0,0 +1,20 @@
+"""Prometheus metrics: registry, request counters and the /metrics route."""
+
+from __future__ import annotations
+
+from flask import Flask, request
+
+from .metrics import record_request, record_usage
+from .routes import bp
+
+__all__ = ["init_observability", "record_usage"]
+
+
+def init_observability(app: Flask) -> None:
+    app.register_blueprint(bp)
+
+    @app.after_request
+    def _count_request(response):
+        rule = request.url_rule.rule if request.url_rule else "unmatched"
+        record_request(request.method, rule, response.status_code)
+        return response

app/observability/metrics.py

@@ -0,0 +1,142 @@
+"""In-process Prometheus metrics with bounded label cardinality."""
+
+from __future__ import annotations
+
+import threading
+
+CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8"
+MAX_SERIES_PER_METRIC = 500
+OVERFLOW_LABEL = "other"
+MAX_LABEL_LENGTH = 64
+
+
+def _escape(value: str) -> str:
+    return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
+
+
+def _clean(value: object) -> str:
+    text = str(value) if value not in (None, "") else "unknown"
+    return text[:MAX_LABEL_LENGTH]
+
+
+def _format_value(value: float) -> str:
+    if value == int(value):
+        return str(int(value))
+    return repr(float(value))
+
+
+class Counter:
+    """A monotonic counter. The number of label combinations is capped."""
+
+    def __init__(self, name: str, help_text: str, label_names: tuple[str, ...] = ()):
+        self.name = name
+        self.help_text = help_text
+        self.label_names = label_names
+        self._values: dict[tuple[str, ...], float] = {}
+        self._lock = threading.Lock()
+
+    def inc(self, amount: float = 1.0, **labels: object) -> None:
+        if amount < 0:
+            raise ValueError("counters can only increase")
+        if set(labels) != set(self.label_names):
+            raise ValueError(f"{self.name} expects labels {self.label_names}")
+        key = tuple(_clean(labels[name]) for name in self.label_names)
+        with self._lock:
+            if key not in self._values and len(self._values) >= MAX_SERIES_PER_METRIC:
+                key = tuple(OVERFLOW_LABEL for _ in self.label_names)
+            self._values[key] = self._values.get(key, 0.0) + amount
+
+    def reset(self) -> None:
+        with self._lock:
+            self._values.clear()
+
+    def render(self) -> list[str]:
+        lines = [
+            f"# HELP {self.name} {self.help_text}",
+            f"# TYPE {self.name} counter",
+        ]
+        with self._lock:
+            items = sorted(self._values.items())
+        for key, value in items:
+            if self.label_names:
+                pairs = ",".join(
+                    f'{name}="{_escape(val)}"'
+                    for name, val in zip(self.label_names, key)
+                )
+                lines.append(f"{self.name}{{{pairs}}} {_format_value(value)}")
+            else:
+                lines.append(f"{self.name} {_format_value(value)}")
+        return lines
+
+
+class Registry:
+    def __init__(self) -> None:
+        self._counters: list[Counter] = []
+
+    def counter(
+        self, name: str, help_text: str, label_names: tuple[str, ...] = ()
+    ) -> Counter:
+        counter = Counter(name, help_text, label_names)
+        self._counters.append(counter)
+        return counter
+
+    def reset(self) -> None:
+        for counter in self._counters:
+            counter.reset()
+
+    def render(self) -> str:
+        lines: list[str] = []
+        for counter in self._counters:
+            lines.extend(counter.render())
+        return "\n".join(lines) + "\n"
+
+
+REGISTRY = Registry()
+
+HTTP_REQUESTS = REGISTRY.counter(
+    "http_requests_total",
+    "Total HTTP requests processed.",
+    ("method", "endpoint", "status"),
+)
+LLM_TOKENS = REGISTRY.counter(
+    "llm_tokens_total",
+    "Total LLM tokens consumed.",
+    ("provider", "model", "direction"),
+)
+LLM_COST = REGISTRY.counter(
+    "llm_cost_usd_total",
+    "Estimated LLM cost in USD.",
+    ("provider", "model"),
+)
+
+
+def record_request(method: str, endpoint: str, status_code: int) -> None:
+    HTTP_REQUESTS.inc(
+        method=method,
+        endpoint=endpoint,
+        status=f"{status_code // 100}xx",
+    )
+
+
+def record_usage(
+    provider: str,
+    model: str,
+    prompt_tokens: int | None = None,
+    completion_tokens: int | None = None,
+    cost_usd: float | None = None,
+) -> None:
+    """Record LLM usage. Never raises on missing values."""
+    if prompt_tokens and prompt_tokens > 0:
+        LLM_TOKENS.inc(
+            prompt_tokens, provider=provider, model=model, direction="in"
+        )
+    if completion_tokens and completion_tokens > 0:
+        LLM_TOKENS.inc(
+            completion_tokens, provider=provider, model=model, direction="out"
+        )
+    if cost_usd and cost_usd > 0:
+        LLM_COST.inc(cost_usd, provider=provider, model=model)
+
+
+def render_metrics() -> str:
+    return REGISTRY.render()

app/observability/routes.py

@@ -0,0 +1,18 @@
+"""The /metrics endpoint."""
+
+from __future__ import annotations
+
+from flask import Blueprint, Response, current_app
+
+from .metrics import CONTENT_TYPE, render_metrics
+
+bp = Blueprint("observability", __name__)
+
+
+@bp.get("/metrics")
+def metrics() -> Response:
+    if not current_app.config.get("METRICS_ENABLED", True):
+        return Response("not found\n", status=404, mimetype="text/plain")
+    response = Response(render_metrics(), content_type=CONTENT_TYPE)
+    response.headers["Cache-Control"] = "no-store"
+    return response

docs/observability.md

@@ -1,7 +1,7 @@
 # Observability design
 
 > **Implementation status:** The ledger, metrics registry, endpoints, settings
-> and module/test paths below are proposed architecture, not implemented
+> and module/test paths below are proposed architecture, have been  implemented  in app/observability/
 > interfaces in this revision. In particular, `LEDGER_MAX_RECORDS` is not an
 > active retention control. For what is stored and accessible today, see
 > [Usage and billing data privacy](usage-privacy.md).

tests/test_metrics.py

@@ -0,0 +1,126 @@
+import re
+
+import pytest
+from flask import Flask
+
+from app.observability import init_observability, record_usage
+from app.observability.metrics import (
+    MAX_SERIES_PER_METRIC,
+    REGISTRY,
+    Counter,
+)
+
+SAMPLE_LINE = re.compile(
+    r"^[a-zA-Z_:][a-zA-Z0-9_:]*(\{[^}]*\})? -?\d+(\.\d+)?([eE][+-]?\d+)?$"
+)
+
+
+def make_app(**config):
+    app = Flask(__name__)
+    app.config["TESTING"] = True
+    app.config.update(config)
+    init_observability(app)
+
+    @app.get("/items/<int:item_id>")
+    def item(item_id):
+        return {"id": item_id}
+
+    return app
+
+
+@pytest.fixture()
+def client():
+    REGISTRY.reset()
+    yield make_app().test_client()
+    REGISTRY.reset()
+
+
+def test_counter_renders_help_type_and_sample():
+    counter = Counter("demo_total", "A demo.", ("kind",))
+    counter.inc(2, kind="a")
+    lines = counter.render()
+    assert lines[0] == "# HELP demo_total A demo."
+    assert lines[1] == "# TYPE demo_total counter"
+    assert 'demo_total{kind="a"} 2' in lines
+
+
+def test_counter_escapes_label_values():
+    counter = Counter("esc_total", "Escaping.", ("kind",))
+    counter.inc(kind='a"b\\c\nd')
+    assert 'esc_total{kind="a\\"b\\\\c\\nd"} 1' in counter.render()
+
+
+def test_counter_rejects_negative_and_wrong_labels():
+    counter = Counter("neg_total", "Negative.", ("kind",))
+    with pytest.raises(ValueError):
+        counter.inc(-1, kind="a")
+    with pytest.raises(ValueError):
+        counter.inc(1, other="a")
+
+
+def test_counter_cardinality_is_bounded():
+    counter = Counter("cap_total", "Cap.", ("kind",))
+    for i in range(MAX_SERIES_PER_METRIC + 50):
+        counter.inc(kind=f"k{i}")
+    samples = [line for line in counter.render() if not line.startswith("#")]
+    assert len(samples) == MAX_SERIES_PER_METRIC + 1
+    assert 'cap_total{kind="other"} 50' in samples
+
+
+def test_metrics_endpoint_returns_valid_exposition(client):
+    client.get("/items/1")
+    response = client.get("/metrics")
+    assert response.status_code == 200
+    assert response.content_type.startswith("text/plain")
+    assert "version=0.0.4" in response.content_type
+    body = response.get_data(as_text=True)
+    assert body.endswith("\n")
+    for line in body.strip().splitlines():
+        assert line.startswith("#") or SAMPLE_LINE.match(line), line
+
+
+def test_metrics_declares_all_families(client):
+    body = client.get("/metrics").get_data(as_text=True)
+    for name in ("http_requests_total", "llm_tokens_total", "llm_cost_usd_total"):
+        assert f"# TYPE {name} counter" in body
+
+
+def test_request_counter_uses_route_pattern_not_raw_path(client):
+    client.get("/items/1")
+    client.get("/items/2")
+    body = client.get("/metrics").get_data(as_text=True)
+    assert (
+        'http_requests_total{method="GET",endpoint="/items/<int:item_id>",'
+        'status="2xx"} 2'
+    ) in body
+    assert "/items/1" not in body
+
+
+def test_unknown_paths_share_one_series(client):
+    client.get("/nope-1")
+    client.get("/nope-2")
+    body = client.get("/metrics").get_data(as_text=True)
+    assert (
+        'http_requests_total{method="GET",endpoint="unmatched",status="4xx"} 2'
+    ) in body
+
+
+def test_token_and_cost_counters_are_published(client):
+    record_usage("openai", "gpt-x", 10, 5, 0.25)
+    record_usage("openai", "gpt-x", 2, 1, 0.5)
+    body = client.get("/metrics").get_data(as_text=True)
+    assert 'llm_tokens_total{provider="openai",model="gpt-x",direction="in"} 12' in body
+    assert 'llm_tokens_total{provider="openai",model="gpt-x",direction="out"} 6' in body
+    assert 'llm_cost_usd_total{provider="openai",model="gpt-x"} 0.75' in body
+
+
+def test_record_usage_ignores_missing_values(client):
+    record_usage("openai", "gpt-x", None, None, None)
+    body = client.get("/metrics").get_data(as_text=True)
+    assert 'provider="openai"' not in body
+
+
+def test_metrics_endpoint_can_be_disabled():
+    REGISTRY.reset()
+    response = make_app(METRICS_ENABLED=False).test_client().get("/metrics")
+    assert response.status_code == 404