feat(postgres): Implement Database Instrumentation Suppression

backend/app/celery/tasks/job_execution.py

@@ -23,6 +23,7 @@
 from app.celery.celery_app import celery_app
 from app.celery.utils import gevent_timeout
 from app.core.config import settings
+from app.core.telemetry import suppress_db_instrumentation
 
 if TYPE_CHECKING:
     from app.services.notifications.eval_completion import (
@@ -88,16 +89,19 @@ def run_llm_job(self, project_id: int, job_id: str, trace_id: str, **kwargs):
     from app.services.llm.jobs import execute_job
 
     _set_trace(trace_id)
-    return _run_with_otel_parent(
-        self,
-        lambda: execute_job(
-            project_id=project_id,
-            job_id=job_id,
-            task_id=current_task.request.id,
-            task_instance=self,
-            **kwargs,
-        ),
-    )
+    # DB spans suppressed job-wide so LLM waterfalls stay clean (drops these queries
+    # from the Sentry Queries page too — accepted trade-off).
+    with suppress_db_instrumentation():
+        return _run_with_otel_parent(
+            self,
+            lambda: execute_job(
+                project_id=project_id,
+                job_id=job_id,
+                task_id=current_task.request.id,
+                task_instance=self,
+                **kwargs,
+            ),
+        )
 
 
 @celery_app.task(bind=True, queue="default", priority=9)
@@ -106,16 +110,17 @@ def run_llm_chain_job(self, project_id: int, job_id: str, trace_id: str, **kwarg
     from app.services.llm.jobs import execute_chain_job
 
     _set_trace(trace_id)
-    return _run_with_otel_parent(
-        self,
-        lambda: execute_chain_job(
-            project_id=project_id,
-            job_id=job_id,
-            task_id=current_task.request.id,
-            task_instance=self,
-            **kwargs,
-        ),
-    )
+    with suppress_db_instrumentation():
+        return _run_with_otel_parent(
+            self,
+            lambda: execute_chain_job(
+                project_id=project_id,
+                job_id=job_id,
+                task_id=current_task.request.id,
+                task_instance=self,
+                **kwargs,
+            ),
+        )
 
 
 @celery_app.task(bind=True, queue="default", priority=9)
@@ -124,16 +129,17 @@ def run_response_job(self, project_id: int, job_id: str, trace_id: str, **kwargs
     from app.services.response.jobs import execute_job
 
     _set_trace(trace_id)
-    return _run_with_otel_parent(
-        self,
-        lambda: execute_job(
-            project_id=project_id,
-            job_id=job_id,
-            task_id=current_task.request.id,
-            task_instance=self,
-            **kwargs,
-        ),
-    )
+    with suppress_db_instrumentation():
+        return _run_with_otel_parent(
+            self,
+            lambda: execute_job(
+                project_id=project_id,
+                job_id=job_id,
+                task_id=current_task.request.id,
+                task_instance=self,
+                **kwargs,
+            ),
+        )
 
 
 @celery_app.task(bind=True, queue="default", priority=9)

backend/app/core/config.py

@@ -153,6 +153,8 @@ def AWS_S3_BUCKET(self) -> str:
     OTEL_SERVICE_NAME: str = "kaapi-backend"
     BACKEND_SERVICE_NAME: str = "kaapi-backend"
     CRON_SERVICE_NAME: str = "kaapi-cron"
+    # Queries at/above this duration increment the db.query.slow counter.
+    DB_SLOW_QUERY_MS: int = 500
 
     # Threshold Request Rate per minute
     THRESHOLD_LLM_CALL_RATE: int = 15

backend/app/core/middleware.py

@@ -19,6 +19,11 @@
     }
 )
 
+CRON_PATH_PREFIX: str = f"{settings.API_V1_STR}/cron/"
+
+# Excluded from traces only; logs/metrics and spans inside the handler still emit.
+TRACE_EXCLUDED_PATH_PREFIXES: frozenset[str] = frozenset({CRON_PATH_PREFIX})
+
 
 class StripTrailingSlashMiddleware:
     """
@@ -50,8 +55,33 @@ def _resolve_http_route(request: Request) -> str:
     return templated or "unmatched"
 
 
+def _emit_http_metrics(
+    *, method: str, http_route: str, status: int, duration_ms: float
+) -> None:
+    """Emit HTTP traffic/latency/error counters to Sentry. No-op if the SDK is inactive."""
+    try:
+        if not sentry_sdk.get_client().is_active():
+            return
+        attrs = {
+            "http.method": method,
+            "http.route": http_route,
+            "http.status_code": str(status),
+        }
+        sentry_sdk.metrics.count("http.server.request.count", 1, attributes=attrs)
+        sentry_sdk.metrics.distribution(
+            "http.server.request.duration",
+            duration_ms,
+            unit="millisecond",
+            attributes=attrs,
+        )
+        if status >= 400:
+            sentry_sdk.metrics.count("http.server.request.error", 1, attributes=attrs)
+    except Exception:
+        logger.debug("[_emit_http_metrics] Sentry metric emit failed")
+
+
 async def http_request_logger(request: Request, call_next) -> Response:
-    if request.url.path.startswith(f"{settings.API_V1_STR}/cron/"):
+    if request.url.path.startswith(CRON_PATH_PREFIX):
         with log_service_name(settings.CRON_SERVICE_NAME):
             return await _log_http_request(request, call_next)
 
@@ -62,6 +92,8 @@ async def _log_http_request(request: Request, call_next) -> Response:
     start_time = time.time()
     method = request.method
     raw_path = request.url.path
+    # Health/utility paths excluded so they don't skew platform traffic metrics.
+    metrics_enabled = raw_path not in SILENT_LOG_PATHS
 
     span = trace.get_current_span()
     if span.is_recording():
@@ -78,6 +110,7 @@ async def _log_http_request(request: Request, call_next) -> Response:
     try:
         response = await call_next(request)
     except Exception:
+        duration_ms = (time.time() - start_time) * 1000
         status = 500
         http_route = _resolve_http_route(request)
         if span.is_recording():
@@ -88,6 +121,13 @@ async def _log_http_request(request: Request, call_next) -> Response:
             sentry_sdk.set_tag("http.route", http_route)
             sentry_sdk.set_tag("http.status_code", str(status))
             sentry_sdk.set_tag("http.response.status_code", str(status))
+        if metrics_enabled:
+            _emit_http_metrics(
+                method=method,
+                http_route=http_route,
+                status=status,
+                duration_ms=duration_ms,
+            )
         logger.exception("Unhandled exception during request")
         raise
 
@@ -101,32 +141,18 @@ async def _log_http_request(request: Request, call_next) -> Response:
         span.set_attribute("http.response.status_code", status)
         span.set_attribute("http.request.duration_ms", round(duration_ms, 2))
 
-    if raw_path not in SILENT_LOG_PATHS:
-        logger.info(f"{method} {raw_path} - {status} [{duration_ms:.2f}ms]")
-
-    try:
-        if sentry_sdk.get_client().is_active():
-            sentry_sdk.set_tag("http.route", http_route)
-            sentry_sdk.set_tag("http.status_code", str(status))
-            sentry_sdk.set_tag("http.response.status_code", str(status))
+    if sentry_sdk.get_client().is_active():
+        sentry_sdk.set_tag("http.route", http_route)
+        sentry_sdk.set_tag("http.status_code", str(status))
+        sentry_sdk.set_tag("http.response.status_code", str(status))
 
-            attrs = {
-                "http.method": method,
-                "http.route": http_route,
-                "http.status_code": str(status),
-            }
-            sentry_sdk.metrics.count("http.server.request.count", 1, attributes=attrs)
-            sentry_sdk.metrics.distribution(
-                "http.server.request.duration",
-                duration_ms,
-                unit="millisecond",
-                attributes=attrs,
-            )
-            if status >= 400:
-                sentry_sdk.metrics.count(
-                    "http.server.request.error", 1, attributes=attrs
-                )
-    except Exception:
-        logger.debug("[http_request_logger] Sentry metric emit failed")
+    if metrics_enabled:
+        logger.info(f"{method} {raw_path} - {status} [{duration_ms:.2f}ms]")
+        _emit_http_metrics(
+            method=method,
+            http_route=http_route,
+            status=status,
+            duration_ms=duration_ms,
+        )
 
     return response