From b7543cb0d02f488f2f7c4fddd84f7200dc536749 Mon Sep 17 00:00:00 2001 From: Subhransu De Date: Sun, 11 Oct 2026 11:44:16 +0530 Subject: [PATCH 1/2] Use FastAPI's native OpenTelemetry for HTTP instrumentation FastAPI 0.142 added native OpenTelemetry. This repo already passed a telemetry config, but logfire.instrument_fastapi added the contrib ASGI middleware, so FastAPI kept its own telemetry dormant. Drop the Logfire FastAPI instrumentation and let the native integration run on the providers that logfire.configure installs. - Request spans now use current semantic conventions and get fastapi.* child spans for dependencies, endpoint and serialization, plus http.server.request.duration and active-request metrics, and log events for unhandled exceptions and validation failures. - ServerSpanMiddleware sets client.address on the request span and keeps that span reachable, so record_auth_attributes tags the request rather than the fastapi.dependencies span. - Require fastapi>=0.143.0: 0.142.4 isolates excluded requests, and 0.143.0 makes exporter auto-configuration opt-in. --- AGENTS.md | 4 +- README.md | 2 +- app/main.py | 2 +- app/telemetry.py | 61 ++++++++-------- pyproject.toml | 2 +- tests/integration/test_otel_api.py | 8 +-- tests/unit/test_telemetry.py | 109 +++++++++++++++-------------- uv.lock | 8 +-- 8 files changed, 102 insertions(+), 94 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c4122a2..91f83c4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,14 +42,14 @@ app.settings pydantic-settings models and the Settings root - No `getattr`, `setattr`, `hasattr`, `delattr`, `cast` or `Any` in `app/`. `tests/architecture/test_type_discipline.py` fails otherwise. The only allowlisted exception is the OpenAPI document post-processing in `app/auth/openapi.py`. - `ty` runs with `--error-on-warning`. A warning is a failure. - Nothing happens at import time. `Settings.from_env()` is called only in `app.main.main()` and `app.main.app_from_env()`. Tests build `Settings` explicitly with `tests.support.build_settings`. -- Resources live in the lifespan. The engine, the readiness probe's single-connection engine, the session factories and the token validator are created in `app.main.lifespan` and read back through `get_sessionmaker`, `get_probe_sessionmaker` and `get_token_validator`, which are the only places that touch `request.state`. Request-scoped telemetry goes on the active span through `telemetry.record_auth_attributes`, never on `request.state`. +- Resources live in the lifespan. The engine, the readiness probe's single-connection engine, the session factories and the token validator are created in `app.main.lifespan` and read back through `get_sessionmaker`, `get_probe_sessionmaker` and `get_token_validator`, which are the only places that touch `request.state`. Request-scoped telemetry goes on the request span through `telemetry.record_auth_attributes`, never on `request.state`. - Migrations that change column types say how existing values convert. Timestamp conversions use `AT TIME ZONE 'UTC'` so the session time zone cannot shift stored instants. - Errors are RFC 9457 Problem Details. Raise a `BaseError` subclass from `app.exceptions`; never build a `JSONResponse` in a route. Declare the error statuses a route can return with `problem_responses(...)` so they appear in OpenAPI. - The OpenAPI document is a contract. `tests/contract/openapi.json` must match the served document. Change the API, review the diff, run `make openapi-snapshot`. - Models and migrations agree. `tests/integration/test_migrations.py` runs `alembic check` and a downgrade round trip. - Timestamps are timezone-aware end to end: `DateTime(timezone=True)` in models, `AwareDatetime` in schemas. - Coverage stays at or above the floor in `pyproject.toml`. -- Logfire owns the OpenTelemetry providers and the FastAPI instrumentation. FastAPI's native telemetry is constructed by `telemetry.native_telemetry_config()` with `auto_configure` off, so platform `OTEL_*` variables cannot attach a second exporter. It stays dormant while the Logfire middleware is present and takes over automatically if that middleware is removed. +- Logfire owns the OpenTelemetry providers and exporters. FastAPI's native telemetry instruments HTTP requests: request spans, `fastapi.*` operation spans, request metrics, and exception and validation logs. `telemetry.native_telemetry_config()` turns `auto_configure` off, so platform `OTEL_*` variables cannot attach a second exporter, and excludes the health probes. `telemetry.ServerSpanMiddleware` adds `client.address` to the request span and keeps that span reachable from dependencies. Do not call `logfire.instrument_fastapi`: its middleware makes FastAPI switch its native telemetry off. ## Adding a resource diff --git a/README.md b/README.md index 8af5acb..ee07f85 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ A production-minded FastAPI starter that gives you a clean async API, a real dat | Database | PostgreSQL, SQLAlchemy async sessions, Psycopg, and Alembic migrations run as a one-shot startup task before the API starts. | | Authentication | OAuth2 authorization-code flow, JWT bearer validation into typed `AuthClaims`, role checks with `require_role`, and a Keycloak-backed Docker setup. | | Validation and settings | Pydantic v2 schemas and `pydantic-settings` based application, database, and auth configuration. | -| Observability | Structured logging plus Logfire/OpenTelemetry instrumentation for FastAPI and SQLAlchemy. | +| Observability | Structured logging, FastAPI's native OpenTelemetry request spans, metrics and logs, and SQLAlchemy instrumentation, exported through Logfire. | | Local runtime | Docker Compose stack for the API, PostgreSQL, and Keycloak. | | Quality gates | Ruff linting and formatting, ty type checks with warnings as errors, import-linter architecture checks, an AST gate against dynamic attribute access, an OpenAPI contract snapshot, `alembic check`, coverage enforcement, SonarCloud analysis, and Snyk security status. | | Dependency upkeep | Dependabot is configured for Python, Docker, Docker Compose, and GitHub Actions updates. | diff --git a/app/main.py b/app/main.py index 6cb3f63..c434ddc 100644 --- a/app/main.py +++ b/app/main.py @@ -72,7 +72,7 @@ def create_app( lifespan=lifespan, validation_status=error_handling.validation_status, ) - telemetry.instrument_fastapi(app) + app.add_middleware(telemetry.ServerSpanMiddleware) telemetry.instrument_sqlalchemy() for exception_type in ( diff --git a/app/telemetry.py b/app/telemetry.py index 3641dfb..7ba3c88 100644 --- a/app/telemetry.py +++ b/app/telemetry.py @@ -1,20 +1,14 @@ -import re from dataclasses import dataclass from functools import cache import logfire -from fastapi import FastAPI, Request, WebSocket from fastapi.telemetry import TelemetryConfig -from opentelemetry import trace -from starlette.types import Scope +from opentelemetry import context, trace +from starlette.types import ASGIApp, Receive, Scope, Send HEALTH_ENDPOINT_PATHS: tuple[str, ...] = ("/health", "/health/ready") -_FASTAPI_EXCLUDED_URLS = ( - ".*(?:" - + "|".join(re.escape(path) for path in HEALTH_ENDPOINT_PATHS) - + r")(?:\?.*)?$" -) +_SERVER_SPAN_KEY = context.create_key("app.server_span") @dataclass(frozen=True, slots=True) @@ -33,8 +27,13 @@ def configure_otel(service_name: str) -> None: ) +def _server_span() -> trace.Span: + span = context.get_value(_SERVER_SPAN_KEY) + return span if isinstance(span, trace.Span) else trace.get_current_span() + + def record_auth_attributes(attributes: AuthAttributes) -> None: - span = trace.get_current_span() + span = _server_span() if not span.is_recording(): return if attributes.client_id: @@ -48,25 +47,29 @@ def record_auth_attributes(attributes: AuthAttributes) -> None: span.set_attribute("oidc.issuer", attributes.issuer) -def _request_attributes_mapper( - request: Request | WebSocket, - attributes: dict[str, object], -) -> dict[str, object]: - mapped_attributes: dict[str, object] = {} - if errors := attributes.get("errors"): - mapped_attributes["errors"] = errors - - if request.client is not None: - mapped_attributes["client.ip"] = request.client.host - return mapped_attributes - - -def instrument_fastapi(app: FastAPI) -> None: - logfire.instrument_fastapi( - app, - request_attributes_mapper=_request_attributes_mapper, - excluded_urls=_FASTAPI_EXCLUDED_URLS, - ) +class ServerSpanMiddleware: + """Tag FastAPI's request span and keep it reachable from dependencies. + + FastAPI starts the server span outside the middleware stack and runs + dependencies inside a `fastapi.dependencies` child span, so request-level + attributes must be set on the span that is current here. + """ + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + span = trace.get_current_span() + if scope["type"] not in ("http", "websocket") or not span.is_recording(): + await self.app(scope, receive, send) + return + if client := scope.get("client"): + span.set_attribute("client.address", client[0]) + token = context.attach(context.set_value(_SERVER_SPAN_KEY, span)) + try: + await self.app(scope, receive, send) + finally: + context.detach(token) @cache diff --git a/pyproject.toml b/pyproject.toml index 5471661..f458cb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "An opinionated FastAPI template with async SQLAlchemy 2.0, Alembi readme = "README.md" requires-python = ">=3.13" dependencies = [ - "fastapi>=0.142.2", + "fastapi>=0.143.0", "httpx>=0.28.1", "logfire[fastapi,psycopg,sqlalchemy]>=5.1.1", "psycopg[binary]>=3.3.6", diff --git a/tests/integration/test_otel_api.py b/tests/integration/test_otel_api.py index e7fa914..9b5ec03 100644 --- a/tests/integration/test_otel_api.py +++ b/tests/integration/test_otel_api.py @@ -37,9 +37,9 @@ async def test_entity_api_requests_emit_otel_records( ] actual_requests = [ ( - span["attributes"].get("http.method"), + span["attributes"].get("http.request.method"), span["attributes"].get("http.route"), - span["attributes"].get("http.status_code"), + span["attributes"].get("http.response.status_code"), ) for span in request_records ] @@ -51,10 +51,10 @@ async def test_entity_api_requests_emit_otel_records( post_record = next( span for span in request_records - if span["attributes"].get("http.method") == "POST" + if span["attributes"].get("http.request.method") == "POST" and span["attributes"].get("http.route") == "/entities/" ) - assert post_record["attributes"].get("client.ip") == "127.0.0.1" + assert post_record["attributes"].get("client.address") == "127.0.0.1" assert post_record["attributes"].get("oidc.client_id") == TEST_CLIENT_ID assert post_record["attributes"].get("oidc.audience") == TEST_CLIENT_ID assert post_record["attributes"].get("oidc.issuer") == TEST_ISSUER diff --git a/tests/unit/test_telemetry.py b/tests/unit/test_telemetry.py index 09a0358..a77bb37 100644 --- a/tests/unit/test_telemetry.py +++ b/tests/unit/test_telemetry.py @@ -1,8 +1,7 @@ -import re - +import httpx import logfire import pytest -from fastapi import Request +from fastapi import Depends, FastAPI from logfire.testing import CaptureLogfire from app import telemetry @@ -10,17 +9,6 @@ pytestmark = pytest.mark.unit -def _request(**state: object) -> Request: - return Request( - { - "type": "http", - "client": ("203.0.113.42", 4242), - "headers": [], - "state": state, - } - ) - - def test_configure_otel_configures_logfire_once_per_service( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -39,29 +27,6 @@ def configure(**kwargs: object) -> None: assert calls[0]["send_to_logfire"] == "if-token-present" -def test_request_attributes_mapper_keeps_errors_and_client_ip_only() -> None: - attributes: dict[str, object] = { - "values": {"payload": {"description": "private"}}, - "errors": [{"loc": ["body", "name"], "msg": "missing"}], - } - - mapped = telemetry._request_attributes_mapper(_request(), attributes) - - assert mapped == { - "errors": [{"loc": ["body", "name"], "msg": "missing"}], - "client.ip": "203.0.113.42", - } - - -def test_request_attributes_mapper_ignores_values_left_on_request_state() -> None: - mapped = telemetry._request_attributes_mapper( - _request(auth_attributes={"client_id": "spoofed"}), - {}, - ) - - assert mapped == {"client.ip": "203.0.113.42"} - - def test_record_auth_attributes_tags_the_active_span(capfire: CaptureLogfire) -> None: with logfire.span("request"): telemetry.record_auth_attributes( @@ -90,19 +55,59 @@ def test_record_auth_attributes_is_a_no_op_without_an_active_span( assert capfire.exporter.exported_spans_as_dict() == [] -@pytest.mark.parametrize( - ("url", "excluded"), - [ - ("http://api/health", True), - ("http://api/health?probe=1", True), - ("http://api/health/ready", True), - ("http://api/healthz", False), - ("http://api/entities/", False), - ], -) -def test_excluded_urls_pattern_matches_only_health_endpoints( - url: str, - *, - excluded: bool, +def _authenticated() -> None: + telemetry.record_auth_attributes( + telemetry.AuthAttributes( + client_id="api-client", audience="api", issuer="https://idp.example" + ) + ) + + +def _instrumented_app() -> FastAPI: + app = FastAPI(telemetry=telemetry.native_telemetry_config()) + app.add_middleware(telemetry.ServerSpanMiddleware) + + @app.get("/health") + def health() -> dict[str, str]: + return {"status": "up"} + + @app.get("/items/{item_id}", dependencies=[Depends(_authenticated)]) + def item(item_id: int) -> dict[str, int]: + return {"id": item_id} + + return app + + +async def _get(app: FastAPI, path: str) -> httpx.Response: + transport = httpx.ASGITransport(app=app, client=("203.0.113.42", 4242)) + async with httpx.AsyncClient(transport=transport, base_url="http://api") as client: + return await client.get(path) + + +async def test_native_request_span_carries_client_and_auth_attributes( + capfire: CaptureLogfire, ) -> None: - assert bool(re.match(telemetry._FASTAPI_EXCLUDED_URLS, url)) is excluded + response = await _get(_instrumented_app(), "/items/7") + + assert response.status_code == 200 + spans = capfire.exporter.exported_spans_as_dict() + (server,) = [span for span in spans if span["name"] == "GET /items/{item_id}"] + assert server["attributes"]["http.route"] == "/items/{item_id}" + assert server["attributes"]["http.response.status_code"] == 200 + assert server["attributes"]["client.address"] == "203.0.113.42" + assert server["attributes"]["oidc.client_id"] == "api-client" + assert server["attributes"]["oidc.audience"] == "api" + assert server["attributes"]["oidc.issuer"] == "https://idp.example" + (dependencies,) = [s for s in spans if s["name"] == "fastapi.dependencies"] + assert "oidc.client_id" not in dependencies["attributes"] + + +async def test_native_telemetry_skips_health_probes(capfire: CaptureLogfire) -> None: + response = await _get(_instrumented_app(), "/health") + + assert response.status_code == 200 + assert not [ + span + for span in capfire.exporter.exported_spans_as_dict() + if span["attributes"].get("logfire.span_type") == "span" + ] diff --git a/uv.lock b/uv.lock index 5e2fd04..08fa40c 100644 --- a/uv.lock +++ b/uv.lock @@ -427,7 +427,7 @@ wheels = [ [[package]] name = "fastapi" -version = "0.142.2" +version = "0.143.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc" }, @@ -437,9 +437,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/56/4f/f7c30a73127e0a8bbffe788369b8359e530b01ae06e2757936fa35bc5e6d/fastapi-0.142.2.tar.gz", hash = "sha256:06366626f2e70576367714d9ab2fe8472e6c8456dba69b399f9f797ab5e92570", size = 466721, upload-time = "2026-09-30T09:51:11.277Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/d7/6a8753ab6c1d432dc53703c3e1b92974a94531b7d047c32bbaae461ea844/fastapi-0.143.0.tar.gz", hash = "sha256:1acffe48206a80917cf7dac21992b5c44b25384e8902bf745c1fd9dabcf6c51f", size = 468391, upload-time = "2026-10-08T12:29:46.54Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/b6/78aaf9141fb46742928c113f3cf6ef2259d538cb02b604b7656c1dc9883c/fastapi-0.142.2-py3-none-any.whl", hash = "sha256:bd5f4d81f1e93a88bcd77caf4dfe3c2dbffc3805407a0007e9a114c18b3a670b", size = 144409, upload-time = "2026-09-30T09:51:09.8Z" }, + { url = "https://files.pythonhosted.org/packages/bd/f4/27e386913417ad32aae42bba48b0c0cce40e9ff2fba1a871ca2702c37324/fastapi-0.143.0-py3-none-any.whl", hash = "sha256:3e9395fd35276425b61b516a31fdd7c77fe2af83e41b4da22e30696fb1304c5d", size = 144665, upload-time = "2026-10-08T12:29:44.853Z" }, ] [[package]] @@ -486,7 +486,7 @@ test = [ [package.metadata] requires-dist = [ - { name = "fastapi", specifier = ">=0.142.2" }, + { name = "fastapi", specifier = ">=0.143.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "logfire", extras = ["fastapi", "psycopg", "sqlalchemy"], specifier = ">=5.1.1" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.3.6" }, From bd62cf174b7ed6945cc3ae3e5b8225e26ef19c44 Mon Sep 17 00:00:00 2001 From: Subhransu De Date: Sun, 11 Oct 2026 11:49:56 +0530 Subject: [PATCH 2/2] Skip health probes in ServerSpanMiddleware and use a literal context key FastAPI creates no request span for excluded probes, so an in-process probe would tag whatever span its caller had open. Apply the same health-probe check the native telemetry config uses. Name the OpenTelemetry context key with a string literal instead of calling create_key at import time. --- app/telemetry.py | 8 ++++++-- tests/unit/test_telemetry.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/app/telemetry.py b/app/telemetry.py index 7ba3c88..adf60b7 100644 --- a/app/telemetry.py +++ b/app/telemetry.py @@ -8,7 +8,7 @@ HEALTH_ENDPOINT_PATHS: tuple[str, ...] = ("/health", "/health/ready") -_SERVER_SPAN_KEY = context.create_key("app.server_span") +_SERVER_SPAN_KEY = "app.telemetry.server_span" @dataclass(frozen=True, slots=True) @@ -60,7 +60,11 @@ def __init__(self, app: ASGIApp) -> None: async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: span = trace.get_current_span() - if scope["type"] not in ("http", "websocket") or not span.is_recording(): + if ( + scope["type"] not in ("http", "websocket") + or _is_health_probe(scope) + or not span.is_recording() + ): await self.app(scope, receive, send) return if client := scope.get("client"): diff --git a/tests/unit/test_telemetry.py b/tests/unit/test_telemetry.py index a77bb37..f91d4d1 100644 --- a/tests/unit/test_telemetry.py +++ b/tests/unit/test_telemetry.py @@ -111,3 +111,18 @@ async def test_native_telemetry_skips_health_probes(capfire: CaptureLogfire) -> for span in capfire.exporter.exported_spans_as_dict() if span["attributes"].get("logfire.span_type") == "span" ] + + +async def test_excluded_probe_leaves_the_callers_span_untouched( + capfire: CaptureLogfire, +) -> None: + with logfire.span("caller"): + response = await _get(_instrumented_app(), "/health") + + assert response.status_code == 200 + (caller,) = [ + span + for span in capfire.exporter.exported_spans_as_dict() + if span["name"] == "caller" + ] + assert "client.address" not in caller["attributes"]