From f0700a778c96a25dcf4a2d8eb95c681ede9b8b72 Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:42:20 +0530 Subject: [PATCH 1/2] fix(*): update the antropic error logs --- .../evaluations/prompt_improvement.py | 49 ++++++++++++++----- .../tests/api/routes/test_improve_prompt.py | 49 +++++++++++++++++++ docs/wiki/modules/evaluations.md | 2 +- 3 files changed, 87 insertions(+), 13 deletions(-) diff --git a/backend/app/services/evaluations/prompt_improvement.py b/backend/app/services/evaluations/prompt_improvement.py index 3c64463a6..bf4519ed9 100644 --- a/backend/app/services/evaluations/prompt_improvement.py +++ b/backend/app/services/evaluations/prompt_improvement.py @@ -532,6 +532,25 @@ def _target_config_from_params(config_params: dict) -> dict: } +_UPSTREAM_ERROR_DETAIL_MAX_LENGTH = 500 + + +def _anthropic_error_detail(exc: Exception) -> str: + """Return what Anthropic actually sent back, for the operator-facing message.""" + detail = "" + response = getattr(exc, "response", None) + if response is not None: + try: + detail = (response.text or "").strip() + except Exception: # httpx raises if the body was never read + detail = "" + if not detail: + detail = str(exc).strip() + if len(detail) > _UPSTREAM_ERROR_DETAIL_MAX_LENGTH: + detail = detail[:_UPSTREAM_ERROR_DETAIL_MAX_LENGTH] + "..." + return detail or type(exc).__name__ + + def _call_prompt_drafting_llm(*, user_message_text: str) -> tuple[str, str]: """Run the Anthropic structured-output call shared by both draft variants and return (improved_instructions, rationale). @@ -565,44 +584,50 @@ def _call_prompt_drafting_llm(*, user_message_text: str) -> tuple[str, str]: log_anthropic_error(exc, fn_name="_call_prompt_drafting_llm") raise RuntimeError( "prompt_generation_failed: Anthropic authentication failed — " - "verify the platform API key is valid and not expired" - ) + "verify the platform API key is valid and not expired | " + f"anthropic_response: {_anthropic_error_detail(exc)}" + ) from exc except anthropic.RateLimitError as exc: log_anthropic_error(exc, fn_name="_call_prompt_drafting_llm") raise RuntimeError( "prompt_generation_failed: Anthropic rate limit exceeded — " - "wait at least 1 minute and retry" - ) + "wait at least 1 minute and retry | " + f"anthropic_response: {_anthropic_error_detail(exc)}" + ) from exc except anthropic.APITimeoutError as exc: # Must come before APIConnectionError — APITimeoutError is a subclass. log_anthropic_error(exc, fn_name="_call_prompt_drafting_llm") raise RuntimeError( "prompt_generation_failed: Anthropic request timed out — " - "retry. If persistent, contact Kaapi" - ) + "retry. If persistent, contact Kaapi | " + f"anthropic_response: {_anthropic_error_detail(exc)}" + ) from exc except anthropic.APIConnectionError as exc: log_anthropic_error(exc, fn_name="_call_prompt_drafting_llm") raise RuntimeError( "prompt_generation_failed: network error reaching Anthropic — " - "check connectivity. If persistent, contact Kaapi" - ) + "check connectivity. If persistent, contact Kaapi | " + f"anthropic_response: {_anthropic_error_detail(exc)}" + ) from exc except anthropic.APIStatusError as exc: log_anthropic_error(exc, fn_name="_call_prompt_drafting_llm") raise RuntimeError( f"prompt_generation_failed: Anthropic returned HTTP {exc.status_code} — " - "retry or contact Kaapi if persistent" - ) + "retry or contact Kaapi if persistent | " + f"anthropic_response: {_anthropic_error_detail(exc)}" + ) from exc except Exception as exc: log_anthropic_error(exc, fn_name="_call_prompt_drafting_llm") raise RuntimeError( "prompt_generation_failed: unexpected error during prompt generation — " - "contact Kaapi if persistent" - ) + "contact Kaapi if persistent | " + f"anthropic_response: {_anthropic_error_detail(exc)}" + ) from exc def _draft_improved_prompt( diff --git a/backend/app/tests/api/routes/test_improve_prompt.py b/backend/app/tests/api/routes/test_improve_prompt.py index f87448ea1..dcb01abc8 100644 --- a/backend/app/tests/api/routes/test_improve_prompt.py +++ b/backend/app/tests/api/routes/test_improve_prompt.py @@ -31,6 +31,8 @@ from unittest.mock import MagicMock, patch from uuid import uuid4 +import anthropic +import httpx import pytest from celery.exceptions import SoftTimeLimitExceeded from fastapi import HTTPException @@ -719,6 +721,53 @@ def test_llm_runtime_error_marks_failed_and_fires_failure_callback( ).read_one(version_number=2) assert v2 is None + def test_anthropic_error_body_reaches_job_error_message( + self, + db: Session, + auth: TestAuthContext, + completed_run: EvaluationRun, + ) -> None: + """The provider's own response body must survive to Job.error_message and + the callback — that string is all an operator gets to triage the failure.""" + job = self._pending_job(db, auth.project_id) + upstream_body = { + "type": "error", + "error": { + "type": "invalid_request_error", + "message": "max_tokens: 16384 > 8192, which is the maximum for this model", + }, + } + claude_client = MagicMock() + claude_client.messages.create.side_effect = anthropic.BadRequestError( + "Error code: 400", + response=httpx.Response( + 400, + request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"), + json=upstream_body, + ), + body=upstream_body, + ) + + with _worker_env(db, claude_client=claude_client) as env: + with pytest.raises(RuntimeError): + execute_prompt_improvement( + project_id=auth.project_id, + job_id=str(job.id), + organization_id=auth.organization_id, + evaluation_id=completed_run.id, + callback_url=_CALLBACK_URL, + ) + + refreshed = db.get(Job, job.id) + assert refreshed.status == JobStatus.FAILED + assert "prompt_generation_failed" in refreshed.error_message + assert "Anthropic returned HTTP 400" in refreshed.error_message + assert "which is the maximum for this model" in refreshed.error_message + + payload = _callback_payload(env.callback) + assert payload["success"] is False + assert "which is the maximum for this model" in payload["data"]["error_message"] + def test_trace_download_failure_fires_failure_callback( self, db: Session, diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index 1e9593fa5..3e6f72c1d 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -38,7 +38,7 @@ v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native jud ## Async - Provider batches polled by cron (`crud/evaluations/cron.py`); no long-lived Celery task per run. - Fast text runs (v1 cosine + v2 judge) fan out `ceil(total_items / EVAL_FAST_CHUNK_SIZE)` `run_evaluation_fast_chunk` tasks (responses only), then a cron barrier (`dispatch_fast_evaluation_barriers`) enqueues one `run_evaluation_fast_aggregate` once every chunk has a `raw_output_url`. The aggregate merges chunks, then (v2) judges **every** row in that single task — so the judge pool is sized by its own `EVAL_JUDGE_CONCURRENCY` (not the response stage's `EVAL_FAST_API_CONCURRENCY`) to clear the max dataset (`EVAL_FAST_MAX_UNIQUE_ROWS` × `duplication_factor`) under the aggregate's `CELERY_TASK_SOFT_TIME_LIMIT`. No judge fan-out / second barrier. -- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure. The `ConfigVersion` is persisted regardless of callback outcome. Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. +- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. - v2 prompt iteration reuses the same job/Celery/config-version machinery (`services/evaluations/prompt_improvement.py`) and branches on `run.is_judge_run`: the v2 route calls `start_prompt_improvement_job(..., require_judge_run=True)` (a non-judge run → `422 not_a_judge_run`), and the worker drafts from the three-metric judge trace (`_draft_improved_prompt(is_judge_run=True)`, using each metric's score + reasoning) and delivers a `PromptRecommendationJobPublic` callback carrying `recommendation_type` (`Literal["prompt"]`, `models/evaluation.py`; widens to a union when knowledge-base / model recommendations land). Non-judge (v1) runs keep the default `_draft_improved_prompt` brief + `PromptImprovementJobPublic` path unchanged. - Evaluation iteration loop (`POST /evaluations/iterations`) chains fast-eval + v2 prompt improvement into a self-driving cycle via LangGraph (`services/evaluations/iteration_graph.py`): `start_eval_node` → `wait_eval_node` → (conditional) `start_improve_node` → `wait_improve_node` → loops back to `start_eval_node`, or → `finalize_node`. Stop-score = mean(`Adherence to Ground Truth`, `Adherence to Prompt`) from `EvaluationRun.score["summary_scores"]` (`compute_round_scores`); `Adherence to Knowledge Base` is recorded per round for visibility only, never gates stopping. Stops on 3 consecutive rounds with ` Date: Tue, 8 Sep 2026 18:32:43 +0530 Subject: [PATCH 2/2] fix(*): update the codecov report --- .../tests/api/routes/test_improve_prompt.py | 155 +++++++++++++++++- 1 file changed, 153 insertions(+), 2 deletions(-) diff --git a/backend/app/tests/api/routes/test_improve_prompt.py b/backend/app/tests/api/routes/test_improve_prompt.py index dcb01abc8..1f9b91723 100644 --- a/backend/app/tests/api/routes/test_improve_prompt.py +++ b/backend/app/tests/api/routes/test_improve_prompt.py @@ -25,9 +25,10 @@ import contextlib import json +from collections.abc import Iterator from contextlib import ExitStack from dataclasses import dataclass -from typing import Any, Iterator +from typing import Any from unittest.mock import MagicMock, patch from uuid import uuid4 @@ -46,8 +47,11 @@ from app.models.config.config import ConfigTag from app.models.job import Job, JobStatus, JobType from app.services.evaluations.prompt_improvement import ( + _UPSTREAM_ERROR_DETAIL_MAX_LENGTH, AI_GENERATED_MARKER, COMMIT_MESSAGE_MAX_LENGTH, + _anthropic_error_detail, + _call_prompt_drafting_llm, execute_prompt_improvement, start_prompt_improvement_job, validate_improve_prompt, @@ -56,7 +60,6 @@ from app.tests.utils.test_data import create_test_evaluation_dataset from app.tests.utils.utils import random_lower_string - _SERVICE = "app.services.evaluations.prompt_improvement" _ROUTE_VALIDATE = "app.api.routes.evaluations.evaluation.validate_callback_url" POST_URL = "/api/v1/evaluations/{evaluation_id}/improve-prompt" @@ -888,3 +891,151 @@ def test_old_poll_path_no_longer_routed( headers=headers, ) assert resp.status_code == 404, resp.text + + +def _anthropic_response(status_code: int, body: dict) -> httpx.Response: + return httpx.Response( + status_code, + request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"), + json=body, + ) + + +class TestAnthropicErrorDetail: + """`_anthropic_error_detail` is what puts the provider's own reason into + Job.error_message, so each fallback rung has to hold on its own.""" + + def test_prefers_raw_response_body(self) -> None: + body = {"type": "error", "error": {"message": "overloaded"}} + exc = anthropic.APIStatusError( + "Error code: 529", response=_anthropic_response(529, body), body=body + ) + + assert "overloaded" in _anthropic_error_detail(exc) + + def test_falls_back_to_str_when_there_is_no_response(self) -> None: + """Connection and timeout faults never reach a response.""" + exc = anthropic.APIConnectionError( + message="Connection refused", + request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"), + ) + + assert _anthropic_error_detail(exc) == "Connection refused" + + def test_falls_back_to_str_when_the_body_cannot_be_read(self) -> None: + """httpx raises on `.text` if the body was never read off the wire.""" + + class _UnreadableResponse: + @property + def text(self) -> str: + raise RuntimeError("Attempted to access streaming response content") + + class _StreamingError(Exception): + response = _UnreadableResponse() + + assert ( + _anthropic_error_detail(_StreamingError("stream broke")) == "stream broke" + ) + + def test_truncates_an_oversized_body(self) -> None: + body = {"type": "error", "error": {"message": "x" * 2000}} + exc = anthropic.APIStatusError( + "Error code: 400", response=_anthropic_response(400, body), body=body + ) + + detail = _anthropic_error_detail(exc) + + assert len(detail) == _UPSTREAM_ERROR_DETAIL_MAX_LENGTH + len("...") + assert detail.endswith("...") + + def test_falls_back_to_the_exception_type_when_nothing_is_readable(self) -> None: + class _SilentError(Exception): + pass + + assert _anthropic_error_detail(_SilentError()) == "_SilentError" + + +class TestPromptDraftingErrorLadder: + """Every rung of the `_call_prompt_drafting_llm` except-ladder must tag the + failure and carry the upstream detail into the raised message.""" + + @pytest.mark.parametrize( + "exc, expected_hint, expected_detail", + [ + ( + anthropic.AuthenticationError( + "Error code: 401", + response=_anthropic_response( + 401, {"error": {"message": "bad key"}} + ), + body=None, + ), + "authentication failed", + "bad key", + ), + ( + anthropic.RateLimitError( + "Error code: 429", + response=_anthropic_response( + 429, {"error": {"message": "slow down"}} + ), + body=None, + ), + "rate limit exceeded", + "slow down", + ), + ( + anthropic.APITimeoutError( + httpx.Request("POST", "https://api.anthropic.com/v1/messages") + ), + "timed out", + "timed out", + ), + ( + anthropic.APIConnectionError( + message="Name resolution failed", + request=httpx.Request( + "POST", "https://api.anthropic.com/v1/messages" + ), + ), + "network error reaching Anthropic", + "Name resolution failed", + ), + (ValueError("something unmodelled"), "unexpected error", "unmodelled"), + ], + ids=["auth", "rate_limit", "timeout", "connection", "unexpected"], + ) + def test_each_failure_carries_the_upstream_detail( + self, + anthropic_creds: None, + exc: Exception, + expected_hint: str, + expected_detail: str, + ) -> None: + claude_client = MagicMock() + claude_client.messages.create.side_effect = exc + + with patch( + f"{_SERVICE}.ClaudeProvider.create_client", return_value=claude_client + ): + with pytest.raises(RuntimeError) as raised: + _call_prompt_drafting_llm(user_message_text="rewrite this") + + message = str(raised.value) + assert message.startswith("prompt_generation_failed:") + assert expected_hint in message + assert "anthropic_response: " in message + assert expected_detail in message + assert raised.value.__cause__ is exc + + def test_missing_platform_key_is_flagged_as_misconfiguration( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A missing key never reaches Anthropic, so it carries no upstream detail.""" + monkeypatch.setattr(settings, "ANTHROPIC_API_KEY", "") + + with pytest.raises(RuntimeError) as raised: + _call_prompt_drafting_llm(user_message_text="rewrite this") + + assert "ANTHROPIC_API_KEY" in str(raised.value) + assert "anthropic_response" not in str(raised.value)