fix(evaluation): Prevent summary truncation

backend/app/crud/evaluations/summary.py

@@ -3,26 +3,19 @@
 
 import json
 import logging
-from typing import Any
 
 from app.core.config import settings
 from app.crud.evaluations.score import TraceData
-from app.services.llm.providers.claude import ClaudeProvider, log_anthropic_error
+from app.services.llm.providers.claude import (
+    STOP_REASON_COMPLETE,
+    ClaudeProvider,
+    log_anthropic_error,
+)
 
 logger = logging.getLogger(__name__)
 
-# Headroom for the overall read + up to 3 flagged items + closing line; too low
-# truncates into invalid JSON.
 _SUMMARY_MAX_TOKENS: int = 3000
 
-_LLM_KEY_SUMMARY: str = "summary"
-_OUTPUT_SCHEMA: dict[str, Any] = {
-    "type": "object",
-    "properties": {_LLM_KEY_SUMMARY: {"type": "string"}},
-    "required": [_LLM_KEY_SUMMARY],
-    "additionalProperties": False,
-}
-
 _NO_CONFIG_PROMPT: str = "(no instructions configured)"
 
 _SUMMARY_SYSTEM_PROMPT: str = """You are the summarizer for Glific's AI evaluation runs. After an evaluation you
@@ -166,15 +159,13 @@ def generate_run_ai_summary(
             max_tokens=_SUMMARY_MAX_TOKENS,
             system=_SUMMARY_SYSTEM_PROMPT,
             messages=[{"role": "user", "content": user_message}],
-            output_config={"format": {"type": "json_schema", "schema": _OUTPUT_SCHEMA}},
         )
-        text = next(b.text for b in response.content if b.type == "text")
-        data: dict[str, str] = json.loads(text)
-        summary: str = data[_LLM_KEY_SUMMARY].strip()
+        summary: str = "".join(
+            block.text for block in response.content if block.type == "text"
+        ).strip()
+        stop_reason: str | None = response.stop_reason
 
-    # Deliberately broad: a summary failure (typed Anthropic error, bad JSON,
-    # unexpected shape) must never fail the run, so it degrades to a None
-    # result regardless of cause.
+    # A summary failure must never fail the run.
     except Exception as exc:
         log_anthropic_error(
             exc,
@@ -190,4 +181,10 @@ def generate_run_ai_summary(
         )
         return None
 
+    if stop_reason != STOP_REASON_COMPLETE:
+        logger.warning(
+            f"[generate_run_ai_summary] Summary may be truncated | "
+            f"stop_reason={stop_reason} | model={model} | run_name={run_name}"
+        )
+
     return summary

backend/app/services/evaluations/prompt_improvement.py

@@ -42,7 +42,11 @@
     PromptRecommendationJobPublic,
 )
 from app.models.job import Job, JobStatus, JobType, JobUpdate
-from app.services.llm.providers.claude import ClaudeProvider, log_anthropic_error
+from app.services.llm.providers.claude import (
+    STOP_REASON_COMPLETE,
+    ClaudeProvider,
+    log_anthropic_error,
+)
 from app.utils import APIResponse, get_webhook_secret, send_callback
 
 logger = logging.getLogger(__name__)
@@ -576,6 +580,13 @@ def _call_prompt_drafting_llm(*, user_message_text: str) -> tuple[str, str]:
             messages=[{"role": "user", "content": user_message_text}],
             output_config={"format": {"type": "json_schema", "schema": _OUTPUT_SCHEMA}},
         )
+        # A max_tokens cut-off still parses under the schema, so stop_reason is
+        # the only signal — never persist a half-written prompt as a version.
+        if response.stop_reason != STOP_REASON_COMPLETE:
+            raise RuntimeError(
+                "prompt_generation_failed: the model stopped before finishing "
+                f"the draft (stop_reason={response.stop_reason}) — retry"
+            )
         text = next(b.text for b in response.content if b.type == "text")
         data = json.loads(text)
         return data[_LLM_KEY_INSTRUCTIONS], data[_LLM_KEY_RATIONALE]
@@ -621,6 +632,11 @@ def _call_prompt_drafting_llm(*, user_message_text: str) -> tuple[str, str]:
             f"anthropic_response: {_anthropic_error_detail(exc)}"
         ) from exc
 
+    # The truncation guard above already raised a precise message; without this
+    # re-raise the generic handler below flattens it.
+    except RuntimeError:
+        raise
+
     except Exception as exc:
         log_anthropic_error(exc, fn_name="_call_prompt_drafting_llm")
         raise RuntimeError(

backend/app/services/llm/providers/claude.py

@@ -29,6 +29,9 @@
 
 FILES_API_BETA = "files-api-2025-04-14"
 
+# Any other stop_reason (max_tokens, refusal) means the text is partial.
+STOP_REASON_COMPLETE = "end_turn"
+
 
 def log_anthropic_error(exc: Exception, *, fn_name: str, context: str = "") -> None:
     """Log an Anthropic `messages.create` failure at the right level/tag.

backend/app/tests/api/routes/test_improve_prompt.py

@@ -56,6 +56,7 @@
     start_prompt_improvement_job,
     validate_improve_prompt,
 )
+from app.services.llm.providers.claude import STOP_REASON_COMPLETE
 from app.tests.utils.auth import TestAuthContext
 from app.tests.utils.test_data import create_test_evaluation_dataset
 from app.tests.utils.utils import random_lower_string
@@ -106,6 +107,7 @@ def _make_fake_claude_client(text_content: str | None = None) -> MagicMock:
     response = MagicMock()
     response.content = [content_block]
     response.id = "msg_test_id"
+    response.stop_reason = STOP_REASON_COMPLETE
 
     client = MagicMock()
     client.messages.create.return_value = response
@@ -893,7 +895,39 @@ def test_old_poll_path_no_longer_routed(
         assert resp.status_code == 404, resp.text
 
 
-def _anthropic_response(status_code: int, body: dict) -> httpx.Response:
+class TestDraftingLlmTruncation:
+    """A max_tokens cut-off still parses under the schema; stop_reason is the guard."""
+
+    def _run(self, client: MagicMock) -> tuple[str, str]:
+        with (
+            patch.object(settings, "ANTHROPIC_API_KEY", "sk-test"),
+            patch(
+                "app.services.evaluations.prompt_improvement."
+                "ClaudeProvider.create_client",
+                return_value=client,
+            ),
+        ):
+            return _call_prompt_drafting_llm(user_message_text="brief")
+
+    def test_complete_response_returns_instructions_and_rationale(self) -> None:
+        instructions, rationale = self._run(_make_fake_claude_client())
+        assert instructions == _IMPROVED_INSTRUCTIONS
+        assert rationale == _RATIONALE
+
+    def test_truncated_response_raises_instead_of_returning_a_partial_prompt(
+        self,
+    ) -> None:
+        client = _make_fake_claude_client()
+        client.messages.create.return_value.stop_reason = "max_tokens"
+
+        with pytest.raises(RuntimeError) as exc:
+            self._run(client)
+
+        assert "prompt_generation_failed" in str(exc.value)
+        assert "max_tokens" in str(exc.value)
+
+
+def _anthropic_response(status_code: int, body: dict[str, object]) -> httpx.Response:
     return httpx.Response(
         status_code,
         request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"),

backend/app/tests/api/routes/test_improve_prompt_v2.py

@@ -44,6 +44,7 @@
     AI_GENERATED_MARKER,
     execute_prompt_improvement,
 )
+from app.services.llm.providers.claude import STOP_REASON_COMPLETE
 from app.tests.utils.auth import TestAuthContext
 from app.tests.utils.test_data import create_test_evaluation_dataset
 from app.tests.utils.utils import random_lower_string
@@ -120,6 +121,7 @@ def _make_fake_claude_client(text_content: str | None = None) -> MagicMock:
     response = MagicMock()
     response.content = [content_block]
     response.id = "msg_test_id"
+    response.stop_reason = STOP_REASON_COMPLETE
 
     client = MagicMock()
     client.messages.create.return_value = response

backend/app/tests/crud/evaluations/test_fast_judge.py

@@ -56,6 +56,7 @@
     build_kaapi_completion_config,
 )
 from app.models.response import FileResultChunk
+from app.services.llm.providers.claude import STOP_REASON_COMPLETE
 from app.tests.utils.auth import TestAuthContext
 from app.tests.utils.test_data import (
     create_test_config,
@@ -202,7 +203,8 @@ def _raw_judge_response(text: str, *, usage=(12, 6, 18)):
 
 def _summary_response(text: str) -> SimpleNamespace:
     return SimpleNamespace(
-        content=[SimpleNamespace(type="text", text=json.dumps({"summary": text}))]
+        content=[SimpleNamespace(type="text", text=text)],
+        stop_reason=STOP_REASON_COMPLETE,
     )
 
 

backend/app/tests/crud/evaluations/test_run_ai_summary.py

@@ -6,8 +6,8 @@
 factor, the evaluated AI config, and the per-question judge traces as JSON —
 question, golden answer, generated answer, and each judge score with its
 rationale. Trace bookkeeping (`data_type`, `verdict`, `unscoreable`) stays out of
-the payload. Every failure mode (provider error, generic error, unparseable
-payload, empty output) must resolve to None WITHOUT raising, leaving the
+the payload. Every failure mode (provider error, generic error, no text
+block, empty output) must resolve to None WITHOUT raising, leaving the
 deterministic overall to persist.
 """
 
@@ -21,6 +21,7 @@
 from app.core.config import settings
 from app.crud.evaluations.score import TraceData
 from app.crud.evaluations.summary import generate_run_ai_summary
+from app.services.llm.providers.claude import STOP_REASON_COMPLETE
 
 _MODEL = "claude-sonnet-4-6"
 _RUN_NAME = "run-x"
@@ -74,11 +75,12 @@ def _traces() -> list[TraceData]:
     ]
 
 
-def _message(summary: str) -> SimpleNamespace:
+def _message(
+    summary: str, *, stop_reason: str = STOP_REASON_COMPLETE
+) -> SimpleNamespace:
     return SimpleNamespace(
-        content=[
-            SimpleNamespace(type="text", text=json.dumps({"summary": summary})),
-        ]
+        content=[SimpleNamespace(type="text", text=summary)],
+        stop_reason=stop_reason,
     )
 
 
@@ -134,7 +136,7 @@ def test_returns_the_summary_field_stripped(self) -> None:
         )
         assert _call(client) == "Consistently grounded and on-tone across the set."
 
-    def test_call_params_carry_the_model_token_cap_and_json_schema(self) -> None:
+    def test_call_params_carry_the_model_and_token_cap(self) -> None:
         client = MagicMock()
         client.messages.create.return_value = _message("A note.")
         _call(client)
@@ -143,7 +145,8 @@ def test_call_params_carry_the_model_token_cap_and_json_schema(self) -> None:
         assert params["model"] == _MODEL
         assert params["max_tokens"] == 3000
         assert params["messages"][0]["role"] == "user"
-        assert params["output_config"]["format"]["type"] == "json_schema"
+        # A quote in the prose would close a schema-constrained string early.
+        assert "output_config" not in params
 
 
 class TestTraceBrief:
@@ -285,16 +288,23 @@ def test_empty_or_whitespace_summary_returns_none(self, summary: str) -> None:
         client.messages.create.return_value = _message(summary)
         assert _call(client) is None
 
-    def test_non_json_text_block_returns_none_without_raising(self) -> None:
+    def test_response_without_a_text_block_returns_none_without_raising(self) -> None:
         client = MagicMock()
         client.messages.create.return_value = SimpleNamespace(
-            content=[SimpleNamespace(type="text", text="not json at all")]
+            content=[SimpleNamespace(type="tool_use", text=None)],
+            stop_reason=STOP_REASON_COMPLETE,
         )
         assert _call(client) is None
 
-    def test_response_without_a_text_block_returns_none_without_raising(self) -> None:
+    def test_quotes_in_the_prose_survive_intact(self) -> None:
         client = MagicMock()
-        client.messages.create.return_value = SimpleNamespace(
-            content=[SimpleNamespace(type="tool_use", text=None)]
+        prose = 'Question 3 omits the required "wellness message" line.'
+        client.messages.create.return_value = _message(prose)
+        assert _call(client) == prose
+
+    def test_truncated_response_is_still_returned(self) -> None:
+        client = MagicMock()
+        client.messages.create.return_value = _message(
+            "Question 10 omits the required", stop_reason="max_tokens"
         )
-        assert _call(client) is None
+        assert _call(client) == "Question 10 omits the required"

docs/wiki/modules/evaluations.md

@@ -32,13 +32,13 @@ v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native jud
 ## Services / CRUD
 - `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py` (`validate_fast_evaluation_inputs` extracted for reuse by both the direct eval-start path and the iteration loop), `batch_job.py`, `validators.py`, `prompt_improvement.py`, `iteration.py` (`validate_and_start_evaluation_iteration`, `compute_round_scores`), `iteration_graph.py` (the LangGraph `StateGraph`: nodes, checkpointer)
 - `services/stt_evaluations/`, `services/tts_evaluations/`
-- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Anthropic `messages.create` call via `ClaudeProvider`, structured JSON output; prompt carries every trace's raw per-question scores + judge rationale, golden/generated answers, and the evaluated config, each trace keyed on `question_id` (the 1-based dataset row number from `merge.py`, cited back as "Question N") rather than the Langfuse `trace_id`, and returns a severity-ranked diagnostic note, not just a qualitative band summary), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py`, `iteration.py` (thin-row CRUD for the iteration loop)
+- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Anthropic `messages.create` call via `ClaudeProvider`, plain-text output — deliberately NOT a `json_schema` `output_config`, since a quotation mark in the prose reads as the JSON string terminator and silently truncates the note; `stop_reason != STOP_REASON_COMPLETE` (the shared `end_turn` constant in `services/llm/providers/claude.py`) is logged as a truncation warning; prompt carries every trace's raw per-question scores + judge rationale, golden/generated answers, and the evaluated config, each trace keyed on `question_id` (the 1-based dataset row number from `merge.py`, cited back as "Question N") rather than the Langfuse `trace_id`, and returns a severity-ranked diagnostic note, not just a qualitative band summary), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py`, `iteration.py` (thin-row CRUD for the iteration loop)
 - `core/batch/` — shared provider batch clients: `openai.py`, `gemini.py`, `anthropic.py`, `polling.py`, `operations.py`
 
 ## Async
 - Provider batches polled by cron (`crud/evaluations/cron.py`); no long-lived Celery task per run.
 - Fast text runs (v1 cosine + v2 judge) fan out `ceil(total_items / EVAL_FAST_CHUNK_SIZE)` `run_evaluation_fast_chunk` tasks (responses only), then a cron barrier (`dispatch_fast_evaluation_barriers`) enqueues one `run_evaluation_fast_aggregate` once every chunk has a `raw_output_url`. The aggregate merges chunks, then (v2) judges **every** row in that single task — so the judge pool is sized by its own `EVAL_JUDGE_CONCURRENCY` (not the response stage's `EVAL_FAST_API_CONCURRENCY`) to clear the max dataset (`EVAL_FAST_MAX_UNIQUE_ROWS` × `duplication_factor`) under the aggregate's `CELERY_TASK_SOFT_TIME_LIMIT`. No judge fan-out / second barrier.
-- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM.
+- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. `_call_prompt_drafting_llm` keeps `json_schema` structured output (two fields, `rationale` length-capped) but raises `prompt_generation_failed` when `stop_reason != STOP_REASON_COMPLETE`, so a `max_tokens`-truncated draft fails the job instead of minting a half-written system prompt. Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM.
 - v2 prompt iteration reuses the same job/Celery/config-version machinery (`services/evaluations/prompt_improvement.py`) and branches on `run.is_judge_run`: the v2 route calls `start_prompt_improvement_job(..., require_judge_run=True)` (a non-judge run → `422 not_a_judge_run`), and the worker drafts from the three-metric judge trace (`_draft_improved_prompt(is_judge_run=True)`, using each metric's score + reasoning) and delivers a `PromptRecommendationJobPublic` callback carrying `recommendation_type` (`Literal["prompt"]`, `models/evaluation.py`; widens to a union when knowledge-base / model recommendations land). Non-judge (v1) runs keep the default `_draft_improved_prompt` brief + `PromptImprovementJobPublic` path unchanged.
 - Evaluation iteration loop (`POST /evaluations/iterations`) chains fast-eval + v2 prompt improvement into a self-driving cycle via LangGraph (`services/evaluations/iteration_graph.py`): `start_eval_node` → `wait_eval_node` → (conditional) `start_improve_node` → `wait_improve_node` → loops back to `start_eval_node`, or → `finalize_node`. Stop-score = mean(`Adherence to Ground Truth`, `Adherence to Prompt`) from `EvaluationRun.score["summary_scores"]` (`compute_round_scores`); `Adherence to Knowledge Base` is recorded per round for visibility only, never gates stopping. Stops on 3 consecutive rounds with `<EVAL_ITERATION_CEILING_DELTA_THRESHOLD` (0.05) improvement (`ceiling_reached`), a `max_rounds` cap (`max_rounds_reached`), or a hard round failure (`round_failed`). No orchestrator polls a provider directly: each `wait_*_node` reads `EvaluationRun.status`/`Job.status` (both already maintained by the existing fast-eval and prompt-improvement machinery) and calls LangGraph's `interrupt()` if not yet terminal — a Postgres checkpointer (`get_evaluation_iteration_checkpointer`) persists state across the pause. Resumption is driven by the existing `/cron/evaluations` tick, not a new scheduler: `dispatch_pending_evaluation_iteration_resumes` (`crud/evaluations/cron.py`) re-dispatches `run_evaluation_iteration_graph_step` (Celery, `celery/tasks/job_execution.py`, priority 6) for every `EvaluationIterationRun` with `status=processing`. Every node opens/closes its own DB session — no session is held open across an `interrupt()`. `finalize_node` persists the thin row's terminal `status`/`stop_reason` and delivers the round history + best round as an `EvaluationIterationReportPublic` callback, same best-effort delivery convention as prompt improvement above.