fix(evaluation): Add detailed error logging

backend/app/services/evaluations/prompt_improvement.py

@@ -532,6 +532,25 @@ def _target_config_from_params(config_params: dict) -> dict:
     }
 
 
+_UPSTREAM_ERROR_DETAIL_MAX_LENGTH = 500
+
+
+def _anthropic_error_detail(exc: Exception) -> str:
+    """Return what Anthropic actually sent back, for the operator-facing message."""
+    detail = ""
+    response = getattr(exc, "response", None)
+    if response is not None:
+        try:
+            detail = (response.text or "").strip()
+        except Exception:  # httpx raises if the body was never read
+            detail = ""
+    if not detail:
+        detail = str(exc).strip()
+    if len(detail) > _UPSTREAM_ERROR_DETAIL_MAX_LENGTH:
+        detail = detail[:_UPSTREAM_ERROR_DETAIL_MAX_LENGTH] + "..."
+    return detail or type(exc).__name__
+
+
 def _call_prompt_drafting_llm(*, user_message_text: str) -> tuple[str, str]:
     """Run the Anthropic structured-output call shared by both draft variants and
     return (improved_instructions, rationale).
@@ -565,44 +584,50 @@ def _call_prompt_drafting_llm(*, user_message_text: str) -> tuple[str, str]:
         log_anthropic_error(exc, fn_name="_call_prompt_drafting_llm")
         raise RuntimeError(
             "prompt_generation_failed: Anthropic authentication failed — "
-            "verify the platform API key is valid and not expired"
-        )
+            "verify the platform API key is valid and not expired | "
+            f"anthropic_response: {_anthropic_error_detail(exc)}"
+        ) from exc
 
     except anthropic.RateLimitError as exc:
         log_anthropic_error(exc, fn_name="_call_prompt_drafting_llm")
         raise RuntimeError(
             "prompt_generation_failed: Anthropic rate limit exceeded — "
-            "wait at least 1 minute and retry"
-        )
+            "wait at least 1 minute and retry | "
+            f"anthropic_response: {_anthropic_error_detail(exc)}"
+        ) from exc
 
     except anthropic.APITimeoutError as exc:
         # Must come before APIConnectionError — APITimeoutError is a subclass.
         log_anthropic_error(exc, fn_name="_call_prompt_drafting_llm")
         raise RuntimeError(
             "prompt_generation_failed: Anthropic request timed out — "
-            "retry. If persistent, contact Kaapi"
-        )
+            "retry. If persistent, contact Kaapi | "
+            f"anthropic_response: {_anthropic_error_detail(exc)}"
+        ) from exc
 
     except anthropic.APIConnectionError as exc:
         log_anthropic_error(exc, fn_name="_call_prompt_drafting_llm")
         raise RuntimeError(
             "prompt_generation_failed: network error reaching Anthropic — "
-            "check connectivity. If persistent, contact Kaapi"
-        )
+            "check connectivity. If persistent, contact Kaapi | "
+            f"anthropic_response: {_anthropic_error_detail(exc)}"
+        ) from exc
 
     except anthropic.APIStatusError as exc:
         log_anthropic_error(exc, fn_name="_call_prompt_drafting_llm")
         raise RuntimeError(
             f"prompt_generation_failed: Anthropic returned HTTP {exc.status_code} — "
-            "retry or contact Kaapi if persistent"
-        )
+            "retry or contact Kaapi if persistent | "
+            f"anthropic_response: {_anthropic_error_detail(exc)}"
+        ) from exc
 
     except Exception as exc:
         log_anthropic_error(exc, fn_name="_call_prompt_drafting_llm")
         raise RuntimeError(
             "prompt_generation_failed: unexpected error during prompt generation — "
-            "contact Kaapi if persistent"
-        )
+            "contact Kaapi if persistent | "
+            f"anthropic_response: {_anthropic_error_detail(exc)}"
+        ) from exc
 
 
 def _draft_improved_prompt(

backend/app/tests/api/routes/test_improve_prompt.py

@@ -25,12 +25,15 @@
 
 import contextlib
 import json
+from collections.abc import Iterator
 from contextlib import ExitStack
 from dataclasses import dataclass
-from typing import Any, Iterator
+from typing import Any
 from unittest.mock import MagicMock, patch
 from uuid import uuid4
 
+import anthropic
+import httpx
 import pytest
 from celery.exceptions import SoftTimeLimitExceeded
 from fastapi import HTTPException
@@ -44,8 +47,11 @@
 from app.models.config.config import ConfigTag
 from app.models.job import Job, JobStatus, JobType
 from app.services.evaluations.prompt_improvement import (
+    _UPSTREAM_ERROR_DETAIL_MAX_LENGTH,
     AI_GENERATED_MARKER,
     COMMIT_MESSAGE_MAX_LENGTH,
+    _anthropic_error_detail,
+    _call_prompt_drafting_llm,
     execute_prompt_improvement,
     start_prompt_improvement_job,
     validate_improve_prompt,
@@ -54,7 +60,6 @@
 from app.tests.utils.test_data import create_test_evaluation_dataset
 from app.tests.utils.utils import random_lower_string
 
-
 _SERVICE = "app.services.evaluations.prompt_improvement"
 _ROUTE_VALIDATE = "app.api.routes.evaluations.evaluation.validate_callback_url"
 POST_URL = "/api/v1/evaluations/{evaluation_id}/improve-prompt"
@@ -719,6 +724,53 @@ def test_llm_runtime_error_marks_failed_and_fires_failure_callback(
         ).read_one(version_number=2)
         assert v2 is None
 
+    def test_anthropic_error_body_reaches_job_error_message(
+        self,
+        db: Session,
+        auth: TestAuthContext,
+        completed_run: EvaluationRun,
+    ) -> None:
+        """The provider's own response body must survive to Job.error_message and
+        the callback — that string is all an operator gets to triage the failure."""
+        job = self._pending_job(db, auth.project_id)
+        upstream_body = {
+            "type": "error",
+            "error": {
+                "type": "invalid_request_error",
+                "message": "max_tokens: 16384 > 8192, which is the maximum for this model",
+            },
+        }
+        claude_client = MagicMock()
+        claude_client.messages.create.side_effect = anthropic.BadRequestError(
+            "Error code: 400",
+            response=httpx.Response(
+                400,
+                request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"),
+                json=upstream_body,
+            ),
+            body=upstream_body,
+        )
+
+        with _worker_env(db, claude_client=claude_client) as env:
+            with pytest.raises(RuntimeError):
+                execute_prompt_improvement(
+                    project_id=auth.project_id,
+                    job_id=str(job.id),
+                    organization_id=auth.organization_id,
+                    evaluation_id=completed_run.id,
+                    callback_url=_CALLBACK_URL,
+                )
+
+        refreshed = db.get(Job, job.id)
+        assert refreshed.status == JobStatus.FAILED
+        assert "prompt_generation_failed" in refreshed.error_message
+        assert "Anthropic returned HTTP 400" in refreshed.error_message
+        assert "which is the maximum for this model" in refreshed.error_message
+
+        payload = _callback_payload(env.callback)
+        assert payload["success"] is False
+        assert "which is the maximum for this model" in payload["data"]["error_message"]
+
     def test_trace_download_failure_fires_failure_callback(
         self,
         db: Session,
@@ -839,3 +891,151 @@ def test_old_poll_path_no_longer_routed(
             headers=headers,
         )
         assert resp.status_code == 404, resp.text
+
+
+def _anthropic_response(status_code: int, body: dict) -> httpx.Response:
+    return httpx.Response(
+        status_code,
+        request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"),
+        json=body,
+    )
+
+
+class TestAnthropicErrorDetail:
+    """`_anthropic_error_detail` is what puts the provider's own reason into
+    Job.error_message, so each fallback rung has to hold on its own."""
+
+    def test_prefers_raw_response_body(self) -> None:
+        body = {"type": "error", "error": {"message": "overloaded"}}
+        exc = anthropic.APIStatusError(
+            "Error code: 529", response=_anthropic_response(529, body), body=body
+        )
+
+        assert "overloaded" in _anthropic_error_detail(exc)
+
+    def test_falls_back_to_str_when_there_is_no_response(self) -> None:
+        """Connection and timeout faults never reach a response."""
+        exc = anthropic.APIConnectionError(
+            message="Connection refused",
+            request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"),
+        )
+
+        assert _anthropic_error_detail(exc) == "Connection refused"
+
+    def test_falls_back_to_str_when_the_body_cannot_be_read(self) -> None:
+        """httpx raises on `.text` if the body was never read off the wire."""
+
+        class _UnreadableResponse:
+            @property
+            def text(self) -> str:
+                raise RuntimeError("Attempted to access streaming response content")
+
+        class _StreamingError(Exception):
+            response = _UnreadableResponse()
+
+        assert (
+            _anthropic_error_detail(_StreamingError("stream broke")) == "stream broke"
+        )
+
+    def test_truncates_an_oversized_body(self) -> None:
+        body = {"type": "error", "error": {"message": "x" * 2000}}
+        exc = anthropic.APIStatusError(
+            "Error code: 400", response=_anthropic_response(400, body), body=body
+        )
+
+        detail = _anthropic_error_detail(exc)
+
+        assert len(detail) == _UPSTREAM_ERROR_DETAIL_MAX_LENGTH + len("...")
+        assert detail.endswith("...")
+
+    def test_falls_back_to_the_exception_type_when_nothing_is_readable(self) -> None:
+        class _SilentError(Exception):
+            pass
+
+        assert _anthropic_error_detail(_SilentError()) == "_SilentError"
+
+
+class TestPromptDraftingErrorLadder:
+    """Every rung of the `_call_prompt_drafting_llm` except-ladder must tag the
+    failure and carry the upstream detail into the raised message."""
+
+    @pytest.mark.parametrize(
+        "exc, expected_hint, expected_detail",
+        [
+            (
+                anthropic.AuthenticationError(
+                    "Error code: 401",
+                    response=_anthropic_response(
+                        401, {"error": {"message": "bad key"}}
+                    ),
+                    body=None,
+                ),
+                "authentication failed",
+                "bad key",
+            ),
+            (
+                anthropic.RateLimitError(
+                    "Error code: 429",
+                    response=_anthropic_response(
+                        429, {"error": {"message": "slow down"}}
+                    ),
+                    body=None,
+                ),
+                "rate limit exceeded",
+                "slow down",
+            ),
+            (
+                anthropic.APITimeoutError(
+                    httpx.Request("POST", "https://api.anthropic.com/v1/messages")
+                ),
+                "timed out",
+                "timed out",
+            ),
+            (
+                anthropic.APIConnectionError(
+                    message="Name resolution failed",
+                    request=httpx.Request(
+                        "POST", "https://api.anthropic.com/v1/messages"
+                    ),
+                ),
+                "network error reaching Anthropic",
+                "Name resolution failed",
+            ),
+            (ValueError("something unmodelled"), "unexpected error", "unmodelled"),
+        ],
+        ids=["auth", "rate_limit", "timeout", "connection", "unexpected"],
+    )
+    def test_each_failure_carries_the_upstream_detail(
+        self,
+        anthropic_creds: None,
+        exc: Exception,
+        expected_hint: str,
+        expected_detail: str,
+    ) -> None:
+        claude_client = MagicMock()
+        claude_client.messages.create.side_effect = exc
+
+        with patch(
+            f"{_SERVICE}.ClaudeProvider.create_client", return_value=claude_client
+        ):
+            with pytest.raises(RuntimeError) as raised:
+                _call_prompt_drafting_llm(user_message_text="rewrite this")
+
+        message = str(raised.value)
+        assert message.startswith("prompt_generation_failed:")
+        assert expected_hint in message
+        assert "anthropic_response: " in message
+        assert expected_detail in message
+        assert raised.value.__cause__ is exc
+
+    def test_missing_platform_key_is_flagged_as_misconfiguration(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        """A missing key never reaches Anthropic, so it carries no upstream detail."""
+        monkeypatch.setattr(settings, "ANTHROPIC_API_KEY", "")
+
+        with pytest.raises(RuntimeError) as raised:
+            _call_prompt_drafting_llm(user_message_text="rewrite this")
+
+        assert "ANTHROPIC_API_KEY" in str(raised.value)
+        assert "anthropic_response" not in str(raised.value)

docs/wiki/modules/evaluations.md

@@ -38,7 +38,7 @@ v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native jud
 ## Async
 - Provider batches polled by cron (`crud/evaluations/cron.py`); no long-lived Celery task per run.
 - Fast text runs (v1 cosine + v2 judge) fan out `ceil(total_items / EVAL_FAST_CHUNK_SIZE)` `run_evaluation_fast_chunk` tasks (responses only), then a cron barrier (`dispatch_fast_evaluation_barriers`) enqueues one `run_evaluation_fast_aggregate` once every chunk has a `raw_output_url`. The aggregate merges chunks, then (v2) judges **every** row in that single task — so the judge pool is sized by its own `EVAL_JUDGE_CONCURRENCY` (not the response stage's `EVAL_FAST_API_CONCURRENCY`) to clear the max dataset (`EVAL_FAST_MAX_UNIQUE_ROWS` × `duplication_factor`) under the aggregate's `CELERY_TASK_SOFT_TIME_LIMIT`. No judge fan-out / second barrier.
-- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure. The `ConfigVersion` is persisted regardless of callback outcome. Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM.
+- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM.
 - v2 prompt iteration reuses the same job/Celery/config-version machinery (`services/evaluations/prompt_improvement.py`) and branches on `run.is_judge_run`: the v2 route calls `start_prompt_improvement_job(..., require_judge_run=True)` (a non-judge run → `422 not_a_judge_run`), and the worker drafts from the three-metric judge trace (`_draft_improved_prompt(is_judge_run=True)`, using each metric's score + reasoning) and delivers a `PromptRecommendationJobPublic` callback carrying `recommendation_type` (`Literal["prompt"]`, `models/evaluation.py`; widens to a union when knowledge-base / model recommendations land). Non-judge (v1) runs keep the default `_draft_improved_prompt` brief + `PromptImprovementJobPublic` path unchanged.
 - Evaluation iteration loop (`POST /evaluations/iterations`) chains fast-eval + v2 prompt improvement into a self-driving cycle via LangGraph (`services/evaluations/iteration_graph.py`): `start_eval_node` → `wait_eval_node` → (conditional) `start_improve_node` → `wait_improve_node` → loops back to `start_eval_node`, or → `finalize_node`. Stop-score = mean(`Adherence to Ground Truth`, `Adherence to Prompt`) from `EvaluationRun.score["summary_scores"]` (`compute_round_scores`); `Adherence to Knowledge Base` is recorded per round for visibility only, never gates stopping. Stops on 3 consecutive rounds with `<EVAL_ITERATION_CEILING_DELTA_THRESHOLD` (0.05) improvement (`ceiling_reached`), a `max_rounds` cap (`max_rounds_reached`), or a hard round failure (`round_failed`). No orchestrator polls a provider directly: each `wait_*_node` reads `EvaluationRun.status`/`Job.status` (both already maintained by the existing fast-eval and prompt-improvement machinery) and calls LangGraph's `interrupt()` if not yet terminal — a Postgres checkpointer (`get_evaluation_iteration_checkpointer`) persists state across the pause. Resumption is driven by the existing `/cron/evaluations` tick, not a new scheduler: `dispatch_pending_evaluation_iteration_resumes` (`crud/evaluations/cron.py`) re-dispatches `run_evaluation_iteration_graph_step` (Celery, `celery/tasks/job_execution.py`, priority 6) for every `EvaluationIterationRun` with `status=processing`. Every node opens/closes its own DB session — no session is held open across an `interrupt()`. `finalize_node` persists the thin row's terminal `status`/`stop_reason` and delivers the round history + best round as an `EvaluationIterationReportPublic` callback, same best-effort delivery convention as prompt improvement above.