diff --git a/app/observability/__init__.py b/app/observability/__init__.py new file mode 100644 index 0000000..e2a179d --- /dev/null +++ b/app/observability/__init__.py @@ -0,0 +1,20 @@ +"""Prometheus metrics: registry, request counters and the /metrics route.""" + +from __future__ import annotations + +from flask import Flask, request + +from .metrics import record_request, record_usage +from .routes import bp + +__all__ = ["init_observability", "record_usage"] + + +def init_observability(app: Flask) -> None: + app.register_blueprint(bp) + + @app.after_request + def _count_request(response): + rule = request.url_rule.rule if request.url_rule else "unmatched" + record_request(request.method, rule, response.status_code) + return response \ No newline at end of file diff --git a/app/observability/metrics.py b/app/observability/metrics.py new file mode 100644 index 0000000..b4821a9 --- /dev/null +++ b/app/observability/metrics.py @@ -0,0 +1,142 @@ +"""In-process Prometheus metrics with bounded label cardinality.""" + +from __future__ import annotations + +import threading + +CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8" +MAX_SERIES_PER_METRIC = 500 +OVERFLOW_LABEL = "other" +MAX_LABEL_LENGTH = 64 + + +def _escape(value: str) -> str: + return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"') + + +def _clean(value: object) -> str: + text = str(value) if value not in (None, "") else "unknown" + return text[:MAX_LABEL_LENGTH] + + +def _format_value(value: float) -> str: + if value == int(value): + return str(int(value)) + return repr(float(value)) + + +class Counter: + """A monotonic counter. The number of label combinations is capped.""" + + def __init__(self, name: str, help_text: str, label_names: tuple[str, ...] = ()): + self.name = name + self.help_text = help_text + self.label_names = label_names + self._values: dict[tuple[str, ...], float] = {} + self._lock = threading.Lock() + + def inc(self, amount: float = 1.0, **labels: object) -> None: + if amount < 0: + raise ValueError("counters can only increase") + if set(labels) != set(self.label_names): + raise ValueError(f"{self.name} expects labels {self.label_names}") + key = tuple(_clean(labels[name]) for name in self.label_names) + with self._lock: + if key not in self._values and len(self._values) >= MAX_SERIES_PER_METRIC: + key = tuple(OVERFLOW_LABEL for _ in self.label_names) + self._values[key] = self._values.get(key, 0.0) + amount + + def reset(self) -> None: + with self._lock: + self._values.clear() + + def render(self) -> list[str]: + lines = [ + f"# HELP {self.name} {self.help_text}", + f"# TYPE {self.name} counter", + ] + with self._lock: + items = sorted(self._values.items()) + for key, value in items: + if self.label_names: + pairs = ",".join( + f'{name}="{_escape(val)}"' + for name, val in zip(self.label_names, key) + ) + lines.append(f"{self.name}{{{pairs}}} {_format_value(value)}") + else: + lines.append(f"{self.name} {_format_value(value)}") + return lines + + +class Registry: + def __init__(self) -> None: + self._counters: list[Counter] = [] + + def counter( + self, name: str, help_text: str, label_names: tuple[str, ...] = () + ) -> Counter: + counter = Counter(name, help_text, label_names) + self._counters.append(counter) + return counter + + def reset(self) -> None: + for counter in self._counters: + counter.reset() + + def render(self) -> str: + lines: list[str] = [] + for counter in self._counters: + lines.extend(counter.render()) + return "\n".join(lines) + "\n" + + +REGISTRY = Registry() + +HTTP_REQUESTS = REGISTRY.counter( + "http_requests_total", + "Total HTTP requests processed.", + ("method", "endpoint", "status"), +) +LLM_TOKENS = REGISTRY.counter( + "llm_tokens_total", + "Total LLM tokens consumed.", + ("provider", "model", "direction"), +) +LLM_COST = REGISTRY.counter( + "llm_cost_usd_total", + "Estimated LLM cost in USD.", + ("provider", "model"), +) + + +def record_request(method: str, endpoint: str, status_code: int) -> None: + HTTP_REQUESTS.inc( + method=method, + endpoint=endpoint, + status=f"{status_code // 100}xx", + ) + + +def record_usage( + provider: str, + model: str, + prompt_tokens: int | None = None, + completion_tokens: int | None = None, + cost_usd: float | None = None, +) -> None: + """Record LLM usage. Never raises on missing values.""" + if prompt_tokens and prompt_tokens > 0: + LLM_TOKENS.inc( + prompt_tokens, provider=provider, model=model, direction="in" + ) + if completion_tokens and completion_tokens > 0: + LLM_TOKENS.inc( + completion_tokens, provider=provider, model=model, direction="out" + ) + if cost_usd and cost_usd > 0: + LLM_COST.inc(cost_usd, provider=provider, model=model) + + +def render_metrics() -> str: + return REGISTRY.render() \ No newline at end of file diff --git a/app/observability/routes.py b/app/observability/routes.py new file mode 100644 index 0000000..9e7fcfa --- /dev/null +++ b/app/observability/routes.py @@ -0,0 +1,18 @@ +"""The /metrics endpoint.""" + +from __future__ import annotations + +from flask import Blueprint, Response, current_app + +from .metrics import CONTENT_TYPE, render_metrics + +bp = Blueprint("observability", __name__) + + +@bp.get("/metrics") +def metrics() -> Response: + if not current_app.config.get("METRICS_ENABLED", True): + return Response("not found\n", status=404, mimetype="text/plain") + response = Response(render_metrics(), content_type=CONTENT_TYPE) + response.headers["Cache-Control"] = "no-store" + return response \ No newline at end of file diff --git a/docs/observability.md b/docs/observability.md index 1d30e63..2350d3f 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -1,7 +1,7 @@ # Observability design > **Implementation status:** The ledger, metrics registry, endpoints, settings -> and module/test paths below are proposed architecture, not implemented +> and module/test paths below are proposed architecture, have been implemented in app/observability/ > interfaces in this revision. In particular, `LEDGER_MAX_RECORDS` is not an > active retention control. For what is stored and accessible today, see > [Usage and billing data privacy](usage-privacy.md). diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..bd51673 --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,126 @@ +import re + +import pytest +from flask import Flask + +from app.observability import init_observability, record_usage +from app.observability.metrics import ( + MAX_SERIES_PER_METRIC, + REGISTRY, + Counter, +) + +SAMPLE_LINE = re.compile( + r"^[a-zA-Z_:][a-zA-Z0-9_:]*(\{[^}]*\})? -?\d+(\.\d+)?([eE][+-]?\d+)?$" +) + + +def make_app(**config): + app = Flask(__name__) + app.config["TESTING"] = True + app.config.update(config) + init_observability(app) + + @app.get("/items/") + def item(item_id): + return {"id": item_id} + + return app + + +@pytest.fixture() +def client(): + REGISTRY.reset() + yield make_app().test_client() + REGISTRY.reset() + + +def test_counter_renders_help_type_and_sample(): + counter = Counter("demo_total", "A demo.", ("kind",)) + counter.inc(2, kind="a") + lines = counter.render() + assert lines[0] == "# HELP demo_total A demo." + assert lines[1] == "# TYPE demo_total counter" + assert 'demo_total{kind="a"} 2' in lines + + +def test_counter_escapes_label_values(): + counter = Counter("esc_total", "Escaping.", ("kind",)) + counter.inc(kind='a"b\\c\nd') + assert 'esc_total{kind="a\\"b\\\\c\\nd"} 1' in counter.render() + + +def test_counter_rejects_negative_and_wrong_labels(): + counter = Counter("neg_total", "Negative.", ("kind",)) + with pytest.raises(ValueError): + counter.inc(-1, kind="a") + with pytest.raises(ValueError): + counter.inc(1, other="a") + + +def test_counter_cardinality_is_bounded(): + counter = Counter("cap_total", "Cap.", ("kind",)) + for i in range(MAX_SERIES_PER_METRIC + 50): + counter.inc(kind=f"k{i}") + samples = [line for line in counter.render() if not line.startswith("#")] + assert len(samples) == MAX_SERIES_PER_METRIC + 1 + assert 'cap_total{kind="other"} 50' in samples + + +def test_metrics_endpoint_returns_valid_exposition(client): + client.get("/items/1") + response = client.get("/metrics") + assert response.status_code == 200 + assert response.content_type.startswith("text/plain") + assert "version=0.0.4" in response.content_type + body = response.get_data(as_text=True) + assert body.endswith("\n") + for line in body.strip().splitlines(): + assert line.startswith("#") or SAMPLE_LINE.match(line), line + + +def test_metrics_declares_all_families(client): + body = client.get("/metrics").get_data(as_text=True) + for name in ("http_requests_total", "llm_tokens_total", "llm_cost_usd_total"): + assert f"# TYPE {name} counter" in body + + +def test_request_counter_uses_route_pattern_not_raw_path(client): + client.get("/items/1") + client.get("/items/2") + body = client.get("/metrics").get_data(as_text=True) + assert ( + 'http_requests_total{method="GET",endpoint="/items/",' + 'status="2xx"} 2' + ) in body + assert "/items/1" not in body + + +def test_unknown_paths_share_one_series(client): + client.get("/nope-1") + client.get("/nope-2") + body = client.get("/metrics").get_data(as_text=True) + assert ( + 'http_requests_total{method="GET",endpoint="unmatched",status="4xx"} 2' + ) in body + + +def test_token_and_cost_counters_are_published(client): + record_usage("openai", "gpt-x", 10, 5, 0.25) + record_usage("openai", "gpt-x", 2, 1, 0.5) + body = client.get("/metrics").get_data(as_text=True) + assert 'llm_tokens_total{provider="openai",model="gpt-x",direction="in"} 12' in body + assert 'llm_tokens_total{provider="openai",model="gpt-x",direction="out"} 6' in body + assert 'llm_cost_usd_total{provider="openai",model="gpt-x"} 0.75' in body + + +def test_record_usage_ignores_missing_values(client): + record_usage("openai", "gpt-x", None, None, None) + body = client.get("/metrics").get_data(as_text=True) + assert 'provider="openai"' not in body + + +def test_metrics_endpoint_can_be_disabled(): + REGISTRY.reset() + response = make_app(METRICS_ENABLED=False).test_client().get("/metrics") + assert response.status_code == 404 \ No newline at end of file