From cb75d0bc1fffc46ec0ca63c0cd610160c5ef6647 Mon Sep 17 00:00:00 2001 From: AkhileshNegi Date: Tue, 8 Sep 2026 15:55:02 +0530 Subject: [PATCH 1/3] fix(evaluation): AI summaries no longer truncate mid-sentence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `generate_run_ai_summary` requested Anthropic structured output for what is a single prose field. Under constrained decoding a bare `"` inside the string value of `{"summary": "..."}` is a legal string terminator, so when the summary reached a quoted phrase and the sampler emitted the raw quote instead of `\"`, the object closed early. The result was valid JSON with an amputated sentence, `stop_reason == "end_turn"`, and nothing to log — 11 of 74 stored summaries were affected. Drop the schema: a one-field JSON wrapper bought nothing and also hid two other defects, since `next(b.text ...)` discarded every text block after the first and any parse failure degraded the whole summary to `None`. Text blocks are now joined and a non-`end_turn` stop is logged as a truncation warning. `_call_prompt_drafting_llm` keeps its schema — two fields plus a `maxLength` cap on the rationale earn it — but now raises `prompt_generation_failed` on a non-`end_turn` stop, so a `max_tokens`-truncated draft fails the job instead of being persisted as a live system prompt. Closes #1188 Co-Authored-By: Claude Opus 5 (1M context) --- backend/app/crud/evaluations/summary.py | 36 +++++++++---------- .../evaluations/prompt_improvement.py | 13 +++++++ .../tests/api/routes/test_improve_prompt.py | 35 +++++++++++++++++- .../api/routes/test_improve_prompt_v2.py | 1 + .../tests/crud/evaluations/test_fast_judge.py | 3 +- .../crud/evaluations/test_run_ai_summary.py | 35 ++++++++++-------- docs/wiki/modules/evaluations.md | 4 +-- 7 files changed, 91 insertions(+), 36 deletions(-) diff --git a/backend/app/crud/evaluations/summary.py b/backend/app/crud/evaluations/summary.py index 0fd377dc6..be408a9fb 100644 --- a/backend/app/crud/evaluations/summary.py +++ b/backend/app/crud/evaluations/summary.py @@ -3,7 +3,6 @@ import json import logging -from typing import Any from app.core.config import settings from app.crud.evaluations.score import TraceData @@ -11,17 +10,11 @@ logger = logging.getLogger(__name__) -# Headroom for the overall read + up to 3 flagged items + closing line; too low -# truncates into invalid JSON. +# Headroom for the overall read + up to 3 flagged items + closing line. _SUMMARY_MAX_TOKENS: int = 3000 -_LLM_KEY_SUMMARY: str = "summary" -_OUTPUT_SCHEMA: dict[str, Any] = { - "type": "object", - "properties": {_LLM_KEY_SUMMARY: {"type": "string"}}, - "required": [_LLM_KEY_SUMMARY], - "additionalProperties": False, -} +# Anything else (max_tokens, refusal) means the text is partial. +_STOP_REASON_COMPLETE: str = "end_turn" _NO_CONFIG_PROMPT: str = "(no instructions configured)" @@ -166,15 +159,15 @@ def generate_run_ai_summary( max_tokens=_SUMMARY_MAX_TOKENS, system=_SUMMARY_SYSTEM_PROMPT, messages=[{"role": "user", "content": user_message}], - output_config={"format": {"type": "json_schema", "schema": _OUTPUT_SCHEMA}}, ) - text = next(b.text for b in response.content if b.type == "text") - data: dict[str, str] = json.loads(text) - summary: str = data[_LLM_KEY_SUMMARY].strip() - - # Deliberately broad: a summary failure (typed Anthropic error, bad JSON, - # unexpected shape) must never fail the run, so it degrades to a None - # result regardless of cause. + # No json_schema: a quote in the prose reads as the JSON string + # terminator and silently amputates the rest. + summary: str = "".join( + block.text for block in response.content if block.type == "text" + ).strip() + stop_reason: str | None = response.stop_reason + + # Deliberately broad: a summary failure must never fail the run. except Exception as exc: log_anthropic_error( exc, @@ -190,4 +183,11 @@ def generate_run_ai_summary( ) return None + # A partial summary still reads usefully; operators just need to know. + if stop_reason != _STOP_REASON_COMPLETE: + logger.warning( + f"[generate_run_ai_summary] Summary may be truncated | " + f"stop_reason={stop_reason} | model={model} | run_name={run_name}" + ) + return summary diff --git a/backend/app/services/evaluations/prompt_improvement.py b/backend/app/services/evaluations/prompt_improvement.py index 3c64463a6..15f8216e9 100644 --- a/backend/app/services/evaluations/prompt_improvement.py +++ b/backend/app/services/evaluations/prompt_improvement.py @@ -68,6 +68,8 @@ "additionalProperties": False, } +_STOP_REASON_COMPLETE = "end_turn" + COMMIT_MESSAGE_MAX_LENGTH = 512 # Prefix that marks a commit_message as AI-generated; used as a search token for @@ -557,6 +559,13 @@ def _call_prompt_drafting_llm(*, user_message_text: str) -> tuple[str, str]: messages=[{"role": "user", "content": user_message_text}], output_config={"format": {"type": "json_schema", "schema": _OUTPUT_SCHEMA}}, ) + # A max_tokens cut-off still parses under the schema, so stop_reason is + # the only signal — never persist a half-written prompt as a version. + if response.stop_reason != _STOP_REASON_COMPLETE: + raise RuntimeError( + "prompt_generation_failed: the model stopped before finishing " + f"the draft (stop_reason={response.stop_reason}) — retry" + ) text = next(b.text for b in response.content if b.type == "text") data = json.loads(text) return data[_LLM_KEY_INSTRUCTIONS], data[_LLM_KEY_RATIONALE] @@ -597,6 +606,10 @@ def _call_prompt_drafting_llm(*, user_message_text: str) -> tuple[str, str]: "retry or contact Kaapi if persistent" ) + # The guard's message is precise; the generic handler would flatten it. + except RuntimeError: + raise + except Exception as exc: log_anthropic_error(exc, fn_name="_call_prompt_drafting_llm") raise RuntimeError( diff --git a/backend/app/tests/api/routes/test_improve_prompt.py b/backend/app/tests/api/routes/test_improve_prompt.py index f87448ea1..20b746d04 100644 --- a/backend/app/tests/api/routes/test_improve_prompt.py +++ b/backend/app/tests/api/routes/test_improve_prompt.py @@ -46,6 +46,7 @@ from app.services.evaluations.prompt_improvement import ( AI_GENERATED_MARKER, COMMIT_MESSAGE_MAX_LENGTH, + _call_prompt_drafting_llm, execute_prompt_improvement, start_prompt_improvement_job, validate_improve_prompt, @@ -54,7 +55,6 @@ from app.tests.utils.test_data import create_test_evaluation_dataset from app.tests.utils.utils import random_lower_string - _SERVICE = "app.services.evaluations.prompt_improvement" _ROUTE_VALIDATE = "app.api.routes.evaluations.evaluation.validate_callback_url" POST_URL = "/api/v1/evaluations/{evaluation_id}/improve-prompt" @@ -101,6 +101,7 @@ def _make_fake_claude_client(text_content: str | None = None) -> MagicMock: response = MagicMock() response.content = [content_block] response.id = "msg_test_id" + response.stop_reason = "end_turn" client = MagicMock() client.messages.create.return_value = response @@ -839,3 +840,35 @@ def test_old_poll_path_no_longer_routed( headers=headers, ) assert resp.status_code == 404, resp.text + + +class TestDraftingLlmTruncation: + """A max_tokens cut-off still parses under the schema; stop_reason is the guard.""" + + def _run(self, client: MagicMock) -> tuple[str, str]: + with ( + patch.object(settings, "ANTHROPIC_API_KEY", "sk-test"), + patch( + "app.services.evaluations.prompt_improvement." + "ClaudeProvider.create_client", + return_value=client, + ), + ): + return _call_prompt_drafting_llm(user_message_text="brief") + + def test_complete_response_returns_instructions_and_rationale(self) -> None: + instructions, rationale = self._run(_make_fake_claude_client()) + assert instructions == _IMPROVED_INSTRUCTIONS + assert rationale == _RATIONALE + + def test_truncated_response_raises_instead_of_returning_a_partial_prompt( + self, + ) -> None: + client = _make_fake_claude_client() + client.messages.create.return_value.stop_reason = "max_tokens" + + with pytest.raises(RuntimeError) as exc: + self._run(client) + + assert "prompt_generation_failed" in str(exc.value) + assert "max_tokens" in str(exc.value) diff --git a/backend/app/tests/api/routes/test_improve_prompt_v2.py b/backend/app/tests/api/routes/test_improve_prompt_v2.py index ed9cef8ad..e42c1dcd8 100644 --- a/backend/app/tests/api/routes/test_improve_prompt_v2.py +++ b/backend/app/tests/api/routes/test_improve_prompt_v2.py @@ -120,6 +120,7 @@ def _make_fake_claude_client(text_content: str | None = None) -> MagicMock: response = MagicMock() response.content = [content_block] response.id = "msg_test_id" + response.stop_reason = "end_turn" client = MagicMock() client.messages.create.return_value = response diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index 2beedef77..76dbd53a3 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -202,7 +202,8 @@ def _raw_judge_response(text: str, *, usage=(12, 6, 18)): def _summary_response(text: str) -> SimpleNamespace: return SimpleNamespace( - content=[SimpleNamespace(type="text", text=json.dumps({"summary": text}))] + content=[SimpleNamespace(type="text", text=text)], + stop_reason="end_turn", ) diff --git a/backend/app/tests/crud/evaluations/test_run_ai_summary.py b/backend/app/tests/crud/evaluations/test_run_ai_summary.py index 44d62bba9..4b7f5c27a 100644 --- a/backend/app/tests/crud/evaluations/test_run_ai_summary.py +++ b/backend/app/tests/crud/evaluations/test_run_ai_summary.py @@ -6,8 +6,8 @@ factor, the evaluated AI config, and the per-question judge traces as JSON — question, golden answer, generated answer, and each judge score with its rationale. Trace bookkeeping (`data_type`, `verdict`, `unscoreable`) stays out of -the payload. Every failure mode (provider error, generic error, unparseable -payload, empty output) must resolve to None WITHOUT raising, leaving the +the payload. Every failure mode (provider error, generic error, no text +block, empty output) must resolve to None WITHOUT raising, leaving the deterministic overall to persist. """ @@ -74,11 +74,10 @@ def _traces() -> list[TraceData]: ] -def _message(summary: str) -> SimpleNamespace: +def _message(summary: str, *, stop_reason: str = "end_turn") -> SimpleNamespace: return SimpleNamespace( - content=[ - SimpleNamespace(type="text", text=json.dumps({"summary": summary})), - ] + content=[SimpleNamespace(type="text", text=summary)], + stop_reason=stop_reason, ) @@ -134,7 +133,7 @@ def test_returns_the_summary_field_stripped(self) -> None: ) assert _call(client) == "Consistently grounded and on-tone across the set." - def test_call_params_carry_the_model_token_cap_and_json_schema(self) -> None: + def test_call_params_carry_the_model_and_token_cap(self) -> None: client = MagicMock() client.messages.create.return_value = _message("A note.") _call(client) @@ -143,7 +142,8 @@ def test_call_params_carry_the_model_token_cap_and_json_schema(self) -> None: assert params["model"] == _MODEL assert params["max_tokens"] == 3000 assert params["messages"][0]["role"] == "user" - assert params["output_config"]["format"]["type"] == "json_schema" + # A quote in the prose would close a schema-constrained string early. + assert "output_config" not in params class TestTraceBrief: @@ -285,16 +285,23 @@ def test_empty_or_whitespace_summary_returns_none(self, summary: str) -> None: client.messages.create.return_value = _message(summary) assert _call(client) is None - def test_non_json_text_block_returns_none_without_raising(self) -> None: + def test_response_without_a_text_block_returns_none_without_raising(self) -> None: client = MagicMock() client.messages.create.return_value = SimpleNamespace( - content=[SimpleNamespace(type="text", text="not json at all")] + content=[SimpleNamespace(type="tool_use", text=None)], + stop_reason="end_turn", ) assert _call(client) is None - def test_response_without_a_text_block_returns_none_without_raising(self) -> None: + def test_quotes_in_the_prose_survive_intact(self) -> None: client = MagicMock() - client.messages.create.return_value = SimpleNamespace( - content=[SimpleNamespace(type="tool_use", text=None)] + prose = 'Question 3 omits the required "wellness message" line.' + client.messages.create.return_value = _message(prose) + assert _call(client) == prose + + def test_truncated_response_is_still_returned(self) -> None: + client = MagicMock() + client.messages.create.return_value = _message( + "Question 10 omits the required", stop_reason="max_tokens" ) - assert _call(client) is None + assert _call(client) == "Question 10 omits the required" diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index 1e9593fa5..750fac9e8 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -32,13 +32,13 @@ v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native jud ## Services / CRUD - `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py` (`validate_fast_evaluation_inputs` extracted for reuse by both the direct eval-start path and the iteration loop), `batch_job.py`, `validators.py`, `prompt_improvement.py`, `iteration.py` (`validate_and_start_evaluation_iteration`, `compute_round_scores`), `iteration_graph.py` (the LangGraph `StateGraph`: nodes, checkpointer) - `services/stt_evaluations/`, `services/tts_evaluations/` -- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Anthropic `messages.create` call via `ClaudeProvider`, structured JSON output; prompt carries every trace's raw per-question scores + judge rationale, golden/generated answers, and the evaluated config, each trace keyed on `question_id` (the 1-based dataset row number from `merge.py`, cited back as "Question N") rather than the Langfuse `trace_id`, and returns a severity-ranked diagnostic note, not just a qualitative band summary), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py`, `iteration.py` (thin-row CRUD for the iteration loop) +- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Anthropic `messages.create` call via `ClaudeProvider`, plain-text output — deliberately NOT a `json_schema` `output_config`, since a quotation mark in the prose reads as the JSON string terminator and silently truncates the note; `stop_reason != end_turn` is logged as a truncation warning; prompt carries every trace's raw per-question scores + judge rationale, golden/generated answers, and the evaluated config, each trace keyed on `question_id` (the 1-based dataset row number from `merge.py`, cited back as "Question N") rather than the Langfuse `trace_id`, and returns a severity-ranked diagnostic note, not just a qualitative band summary), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py`, `iteration.py` (thin-row CRUD for the iteration loop) - `core/batch/` — shared provider batch clients: `openai.py`, `gemini.py`, `anthropic.py`, `polling.py`, `operations.py` ## Async - Provider batches polled by cron (`crud/evaluations/cron.py`); no long-lived Celery task per run. - Fast text runs (v1 cosine + v2 judge) fan out `ceil(total_items / EVAL_FAST_CHUNK_SIZE)` `run_evaluation_fast_chunk` tasks (responses only), then a cron barrier (`dispatch_fast_evaluation_barriers`) enqueues one `run_evaluation_fast_aggregate` once every chunk has a `raw_output_url`. The aggregate merges chunks, then (v2) judges **every** row in that single task — so the judge pool is sized by its own `EVAL_JUDGE_CONCURRENCY` (not the response stage's `EVAL_FAST_API_CONCURRENCY`) to clear the max dataset (`EVAL_FAST_MAX_UNIQUE_ROWS` × `duplication_factor`) under the aggregate's `CELERY_TASK_SOFT_TIME_LIMIT`. No judge fan-out / second barrier. -- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure. The `ConfigVersion` is persisted regardless of callback outcome. Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. +- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure. The `ConfigVersion` is persisted regardless of callback outcome. `_call_prompt_drafting_llm` keeps `json_schema` structured output (two fields, `rationale` length-capped) but raises `prompt_generation_failed` when `stop_reason != end_turn`, so a `max_tokens`-truncated draft fails the job instead of minting a half-written system prompt. Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. - v2 prompt iteration reuses the same job/Celery/config-version machinery (`services/evaluations/prompt_improvement.py`) and branches on `run.is_judge_run`: the v2 route calls `start_prompt_improvement_job(..., require_judge_run=True)` (a non-judge run → `422 not_a_judge_run`), and the worker drafts from the three-metric judge trace (`_draft_improved_prompt(is_judge_run=True)`, using each metric's score + reasoning) and delivers a `PromptRecommendationJobPublic` callback carrying `recommendation_type` (`Literal["prompt"]`, `models/evaluation.py`; widens to a union when knowledge-base / model recommendations land). Non-judge (v1) runs keep the default `_draft_improved_prompt` brief + `PromptImprovementJobPublic` path unchanged. - Evaluation iteration loop (`POST /evaluations/iterations`) chains fast-eval + v2 prompt improvement into a self-driving cycle via LangGraph (`services/evaluations/iteration_graph.py`): `start_eval_node` → `wait_eval_node` → (conditional) `start_improve_node` → `wait_improve_node` → loops back to `start_eval_node`, or → `finalize_node`. Stop-score = mean(`Adherence to Ground Truth`, `Adherence to Prompt`) from `EvaluationRun.score["summary_scores"]` (`compute_round_scores`); `Adherence to Knowledge Base` is recorded per round for visibility only, never gates stopping. Stops on 3 consecutive rounds with ` Date: Tue, 8 Sep 2026 22:16:02 +0530 Subject: [PATCH 2/3] refactor(evaluation): share STOP_REASON_COMPLETE, trim comments Address PR review: the "end_turn" stop-reason literal was defined twice (summary.py, prompt_improvement.py) and repeated across four test modules. Hoist it to services/llm/providers/claude.py, already the shared home for one-shot Anthropic call-site helpers. Also drop the "what" comments the review flagged in summary.py. Co-Authored-By: Claude Opus 5 (1M context) --- backend/app/crud/evaluations/summary.py | 14 +++++++------- .../app/services/evaluations/prompt_improvement.py | 13 ++++++++----- backend/app/services/llm/providers/claude.py | 3 +++ .../app/tests/api/routes/test_improve_prompt.py | 5 +++-- .../app/tests/api/routes/test_improve_prompt_v2.py | 3 ++- .../app/tests/crud/evaluations/test_fast_judge.py | 3 ++- .../tests/crud/evaluations/test_run_ai_summary.py | 7 +++++-- docs/wiki/modules/evaluations.md | 4 ++-- 8 files changed, 32 insertions(+), 20 deletions(-) diff --git a/backend/app/crud/evaluations/summary.py b/backend/app/crud/evaluations/summary.py index be408a9fb..eacc5fc84 100644 --- a/backend/app/crud/evaluations/summary.py +++ b/backend/app/crud/evaluations/summary.py @@ -6,16 +6,17 @@ from app.core.config import settings from app.crud.evaluations.score import TraceData -from app.services.llm.providers.claude import ClaudeProvider, log_anthropic_error +from app.services.llm.providers.claude import ( + STOP_REASON_COMPLETE, + ClaudeProvider, + log_anthropic_error, +) logger = logging.getLogger(__name__) # Headroom for the overall read + up to 3 flagged items + closing line. _SUMMARY_MAX_TOKENS: int = 3000 -# Anything else (max_tokens, refusal) means the text is partial. -_STOP_REASON_COMPLETE: str = "end_turn" - _NO_CONFIG_PROMPT: str = "(no instructions configured)" _SUMMARY_SYSTEM_PROMPT: str = """You are the summarizer for Glific's AI evaluation runs. After an evaluation you @@ -167,7 +168,7 @@ def generate_run_ai_summary( ).strip() stop_reason: str | None = response.stop_reason - # Deliberately broad: a summary failure must never fail the run. + # A summary failure must never fail the run. except Exception as exc: log_anthropic_error( exc, @@ -183,8 +184,7 @@ def generate_run_ai_summary( ) return None - # A partial summary still reads usefully; operators just need to know. - if stop_reason != _STOP_REASON_COMPLETE: + if stop_reason != STOP_REASON_COMPLETE: logger.warning( f"[generate_run_ai_summary] Summary may be truncated | " f"stop_reason={stop_reason} | model={model} | run_name={run_name}" diff --git a/backend/app/services/evaluations/prompt_improvement.py b/backend/app/services/evaluations/prompt_improvement.py index a9c772f14..9b92d5c22 100644 --- a/backend/app/services/evaluations/prompt_improvement.py +++ b/backend/app/services/evaluations/prompt_improvement.py @@ -42,7 +42,11 @@ PromptRecommendationJobPublic, ) from app.models.job import Job, JobStatus, JobType, JobUpdate -from app.services.llm.providers.claude import ClaudeProvider, log_anthropic_error +from app.services.llm.providers.claude import ( + STOP_REASON_COMPLETE, + ClaudeProvider, + log_anthropic_error, +) from app.utils import APIResponse, get_webhook_secret, send_callback logger = logging.getLogger(__name__) @@ -68,8 +72,6 @@ "additionalProperties": False, } -_STOP_REASON_COMPLETE = "end_turn" - COMMIT_MESSAGE_MAX_LENGTH = 512 # Prefix that marks a commit_message as AI-generated; used as a search token for @@ -580,7 +582,7 @@ def _call_prompt_drafting_llm(*, user_message_text: str) -> tuple[str, str]: ) # A max_tokens cut-off still parses under the schema, so stop_reason is # the only signal — never persist a half-written prompt as a version. - if response.stop_reason != _STOP_REASON_COMPLETE: + if response.stop_reason != STOP_REASON_COMPLETE: raise RuntimeError( "prompt_generation_failed: the model stopped before finishing " f"the draft (stop_reason={response.stop_reason}) — retry" @@ -630,7 +632,8 @@ def _call_prompt_drafting_llm(*, user_message_text: str) -> tuple[str, str]: f"anthropic_response: {_anthropic_error_detail(exc)}" ) from exc - # The guard's message is precise; the generic handler would flatten it. + # The truncation guard above already raised a precise message; without this + # re-raise the generic handler below flattens it. except RuntimeError: raise diff --git a/backend/app/services/llm/providers/claude.py b/backend/app/services/llm/providers/claude.py index ad8002425..5ae51461f 100644 --- a/backend/app/services/llm/providers/claude.py +++ b/backend/app/services/llm/providers/claude.py @@ -29,6 +29,9 @@ FILES_API_BETA = "files-api-2025-04-14" +# Any other stop_reason (max_tokens, refusal) means the text is partial. +STOP_REASON_COMPLETE = "end_turn" + def log_anthropic_error(exc: Exception, *, fn_name: str, context: str = "") -> None: """Log an Anthropic `messages.create` failure at the right level/tag. diff --git a/backend/app/tests/api/routes/test_improve_prompt.py b/backend/app/tests/api/routes/test_improve_prompt.py index f99fe5efa..17027b399 100644 --- a/backend/app/tests/api/routes/test_improve_prompt.py +++ b/backend/app/tests/api/routes/test_improve_prompt.py @@ -56,6 +56,7 @@ start_prompt_improvement_job, validate_improve_prompt, ) +from app.services.llm.providers.claude import STOP_REASON_COMPLETE from app.tests.utils.auth import TestAuthContext from app.tests.utils.test_data import create_test_evaluation_dataset from app.tests.utils.utils import random_lower_string @@ -106,7 +107,7 @@ def _make_fake_claude_client(text_content: str | None = None) -> MagicMock: response = MagicMock() response.content = [content_block] response.id = "msg_test_id" - response.stop_reason = "end_turn" + response.stop_reason = STOP_REASON_COMPLETE client = MagicMock() client.messages.create.return_value = response @@ -926,7 +927,7 @@ def test_truncated_response_raises_instead_of_returning_a_partial_prompt( assert "max_tokens" in str(exc.value) -def _anthropic_response(status_code: int, body: dict) -> httpx.Response: +def _anthropic_response(status_code: int, body: dict[str, object]) -> httpx.Response: return httpx.Response( status_code, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"), diff --git a/backend/app/tests/api/routes/test_improve_prompt_v2.py b/backend/app/tests/api/routes/test_improve_prompt_v2.py index e42c1dcd8..47963eae4 100644 --- a/backend/app/tests/api/routes/test_improve_prompt_v2.py +++ b/backend/app/tests/api/routes/test_improve_prompt_v2.py @@ -44,6 +44,7 @@ AI_GENERATED_MARKER, execute_prompt_improvement, ) +from app.services.llm.providers.claude import STOP_REASON_COMPLETE from app.tests.utils.auth import TestAuthContext from app.tests.utils.test_data import create_test_evaluation_dataset from app.tests.utils.utils import random_lower_string @@ -120,7 +121,7 @@ def _make_fake_claude_client(text_content: str | None = None) -> MagicMock: response = MagicMock() response.content = [content_block] response.id = "msg_test_id" - response.stop_reason = "end_turn" + response.stop_reason = STOP_REASON_COMPLETE client = MagicMock() client.messages.create.return_value = response diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index 76dbd53a3..f2f210b6e 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -56,6 +56,7 @@ build_kaapi_completion_config, ) from app.models.response import FileResultChunk +from app.services.llm.providers.claude import STOP_REASON_COMPLETE from app.tests.utils.auth import TestAuthContext from app.tests.utils.test_data import ( create_test_config, @@ -203,7 +204,7 @@ def _raw_judge_response(text: str, *, usage=(12, 6, 18)): def _summary_response(text: str) -> SimpleNamespace: return SimpleNamespace( content=[SimpleNamespace(type="text", text=text)], - stop_reason="end_turn", + stop_reason=STOP_REASON_COMPLETE, ) diff --git a/backend/app/tests/crud/evaluations/test_run_ai_summary.py b/backend/app/tests/crud/evaluations/test_run_ai_summary.py index 4b7f5c27a..bd00a2be7 100644 --- a/backend/app/tests/crud/evaluations/test_run_ai_summary.py +++ b/backend/app/tests/crud/evaluations/test_run_ai_summary.py @@ -21,6 +21,7 @@ from app.core.config import settings from app.crud.evaluations.score import TraceData from app.crud.evaluations.summary import generate_run_ai_summary +from app.services.llm.providers.claude import STOP_REASON_COMPLETE _MODEL = "claude-sonnet-4-6" _RUN_NAME = "run-x" @@ -74,7 +75,9 @@ def _traces() -> list[TraceData]: ] -def _message(summary: str, *, stop_reason: str = "end_turn") -> SimpleNamespace: +def _message( + summary: str, *, stop_reason: str = STOP_REASON_COMPLETE +) -> SimpleNamespace: return SimpleNamespace( content=[SimpleNamespace(type="text", text=summary)], stop_reason=stop_reason, @@ -289,7 +292,7 @@ def test_response_without_a_text_block_returns_none_without_raising(self) -> Non client = MagicMock() client.messages.create.return_value = SimpleNamespace( content=[SimpleNamespace(type="tool_use", text=None)], - stop_reason="end_turn", + stop_reason=STOP_REASON_COMPLETE, ) assert _call(client) is None diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index 695b99423..8e27bf992 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -32,13 +32,13 @@ v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native jud ## Services / CRUD - `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py` (`validate_fast_evaluation_inputs` extracted for reuse by both the direct eval-start path and the iteration loop), `batch_job.py`, `validators.py`, `prompt_improvement.py`, `iteration.py` (`validate_and_start_evaluation_iteration`, `compute_round_scores`), `iteration_graph.py` (the LangGraph `StateGraph`: nodes, checkpointer) - `services/stt_evaluations/`, `services/tts_evaluations/` -- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Anthropic `messages.create` call via `ClaudeProvider`, plain-text output — deliberately NOT a `json_schema` `output_config`, since a quotation mark in the prose reads as the JSON string terminator and silently truncates the note; `stop_reason != end_turn` is logged as a truncation warning; prompt carries every trace's raw per-question scores + judge rationale, golden/generated answers, and the evaluated config, each trace keyed on `question_id` (the 1-based dataset row number from `merge.py`, cited back as "Question N") rather than the Langfuse `trace_id`, and returns a severity-ranked diagnostic note, not just a qualitative band summary), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py`, `iteration.py` (thin-row CRUD for the iteration loop) +- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Anthropic `messages.create` call via `ClaudeProvider`, plain-text output — deliberately NOT a `json_schema` `output_config`, since a quotation mark in the prose reads as the JSON string terminator and silently truncates the note; `stop_reason != STOP_REASON_COMPLETE` (the shared `end_turn` constant in `services/llm/providers/claude.py`) is logged as a truncation warning; prompt carries every trace's raw per-question scores + judge rationale, golden/generated answers, and the evaluated config, each trace keyed on `question_id` (the 1-based dataset row number from `merge.py`, cited back as "Question N") rather than the Langfuse `trace_id`, and returns a severity-ranked diagnostic note, not just a qualitative band summary), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py`, `iteration.py` (thin-row CRUD for the iteration loop) - `core/batch/` — shared provider batch clients: `openai.py`, `gemini.py`, `anthropic.py`, `polling.py`, `operations.py` ## Async - Provider batches polled by cron (`crud/evaluations/cron.py`); no long-lived Celery task per run. - Fast text runs (v1 cosine + v2 judge) fan out `ceil(total_items / EVAL_FAST_CHUNK_SIZE)` `run_evaluation_fast_chunk` tasks (responses only), then a cron barrier (`dispatch_fast_evaluation_barriers`) enqueues one `run_evaluation_fast_aggregate` once every chunk has a `raw_output_url`. The aggregate merges chunks, then (v2) judges **every** row in that single task — so the judge pool is sized by its own `EVAL_JUDGE_CONCURRENCY` (not the response stage's `EVAL_FAST_API_CONCURRENCY`) to clear the max dataset (`EVAL_FAST_MAX_UNIQUE_ROWS` × `duplication_factor`) under the aggregate's `CELERY_TASK_SOFT_TIME_LIMIT`. No judge fan-out / second barrier. -- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. `_call_prompt_drafting_llm` keeps `json_schema` structured output (two fields, `rationale` length-capped) but raises `prompt_generation_failed` when `stop_reason != end_turn`, so a `max_tokens`-truncated draft fails the job instead of minting a half-written system prompt. Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. +- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. `_call_prompt_drafting_llm` keeps `json_schema` structured output (two fields, `rationale` length-capped) but raises `prompt_generation_failed` when `stop_reason != STOP_REASON_COMPLETE`, so a `max_tokens`-truncated draft fails the job instead of minting a half-written system prompt. Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. - v2 prompt iteration reuses the same job/Celery/config-version machinery (`services/evaluations/prompt_improvement.py`) and branches on `run.is_judge_run`: the v2 route calls `start_prompt_improvement_job(..., require_judge_run=True)` (a non-judge run → `422 not_a_judge_run`), and the worker drafts from the three-metric judge trace (`_draft_improved_prompt(is_judge_run=True)`, using each metric's score + reasoning) and delivers a `PromptRecommendationJobPublic` callback carrying `recommendation_type` (`Literal["prompt"]`, `models/evaluation.py`; widens to a union when knowledge-base / model recommendations land). Non-judge (v1) runs keep the default `_draft_improved_prompt` brief + `PromptImprovementJobPublic` path unchanged. - Evaluation iteration loop (`POST /evaluations/iterations`) chains fast-eval + v2 prompt improvement into a self-driving cycle via LangGraph (`services/evaluations/iteration_graph.py`): `start_eval_node` → `wait_eval_node` → (conditional) `start_improve_node` → `wait_improve_node` → loops back to `start_eval_node`, or → `finalize_node`. Stop-score = mean(`Adherence to Ground Truth`, `Adherence to Prompt`) from `EvaluationRun.score["summary_scores"]` (`compute_round_scores`); `Adherence to Knowledge Base` is recorded per round for visibility only, never gates stopping. Stops on 3 consecutive rounds with ` Date: Tue, 8 Sep 2026 22:22:13 +0530 Subject: [PATCH 3/3] refactor(evaluation): drop flagged comments in summary.py Per PR review. The json_schema rationale stays documented in docs/wiki/modules/evaluations.md. Co-Authored-By: Claude Opus 5 (1M context) --- backend/app/crud/evaluations/summary.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/backend/app/crud/evaluations/summary.py b/backend/app/crud/evaluations/summary.py index eacc5fc84..2afedb194 100644 --- a/backend/app/crud/evaluations/summary.py +++ b/backend/app/crud/evaluations/summary.py @@ -14,7 +14,6 @@ logger = logging.getLogger(__name__) -# Headroom for the overall read + up to 3 flagged items + closing line. _SUMMARY_MAX_TOKENS: int = 3000 _NO_CONFIG_PROMPT: str = "(no instructions configured)" @@ -161,8 +160,6 @@ def generate_run_ai_summary( system=_SUMMARY_SYSTEM_PROMPT, messages=[{"role": "user", "content": user_message}], ) - # No json_schema: a quote in the prose reads as the JSON string - # terminator and silently amputates the rest. summary: str = "".join( block.text for block in response.content if block.type == "text" ).strip()