diff --git a/backend/app/crud/evaluations/summary.py b/backend/app/crud/evaluations/summary.py index 0fd377dc6..2afedb194 100644 --- a/backend/app/crud/evaluations/summary.py +++ b/backend/app/crud/evaluations/summary.py @@ -3,26 +3,19 @@ import json import logging -from typing import Any from app.core.config import settings from app.crud.evaluations.score import TraceData -from app.services.llm.providers.claude import ClaudeProvider, log_anthropic_error +from app.services.llm.providers.claude import ( + STOP_REASON_COMPLETE, + ClaudeProvider, + log_anthropic_error, +) logger = logging.getLogger(__name__) -# Headroom for the overall read + up to 3 flagged items + closing line; too low -# truncates into invalid JSON. _SUMMARY_MAX_TOKENS: int = 3000 -_LLM_KEY_SUMMARY: str = "summary" -_OUTPUT_SCHEMA: dict[str, Any] = { - "type": "object", - "properties": {_LLM_KEY_SUMMARY: {"type": "string"}}, - "required": [_LLM_KEY_SUMMARY], - "additionalProperties": False, -} - _NO_CONFIG_PROMPT: str = "(no instructions configured)" _SUMMARY_SYSTEM_PROMPT: str = """You are the summarizer for Glific's AI evaluation runs. After an evaluation you @@ -166,15 +159,13 @@ def generate_run_ai_summary( max_tokens=_SUMMARY_MAX_TOKENS, system=_SUMMARY_SYSTEM_PROMPT, messages=[{"role": "user", "content": user_message}], - output_config={"format": {"type": "json_schema", "schema": _OUTPUT_SCHEMA}}, ) - text = next(b.text for b in response.content if b.type == "text") - data: dict[str, str] = json.loads(text) - summary: str = data[_LLM_KEY_SUMMARY].strip() + summary: str = "".join( + block.text for block in response.content if block.type == "text" + ).strip() + stop_reason: str | None = response.stop_reason - # Deliberately broad: a summary failure (typed Anthropic error, bad JSON, - # unexpected shape) must never fail the run, so it degrades to a None - # result regardless of cause. + # A summary failure must never fail the run. except Exception as exc: log_anthropic_error( exc, @@ -190,4 +181,10 @@ def generate_run_ai_summary( ) return None + if stop_reason != STOP_REASON_COMPLETE: + logger.warning( + f"[generate_run_ai_summary] Summary may be truncated | " + f"stop_reason={stop_reason} | model={model} | run_name={run_name}" + ) + return summary diff --git a/backend/app/services/evaluations/prompt_improvement.py b/backend/app/services/evaluations/prompt_improvement.py index bf4519ed9..9b92d5c22 100644 --- a/backend/app/services/evaluations/prompt_improvement.py +++ b/backend/app/services/evaluations/prompt_improvement.py @@ -42,7 +42,11 @@ PromptRecommendationJobPublic, ) from app.models.job import Job, JobStatus, JobType, JobUpdate -from app.services.llm.providers.claude import ClaudeProvider, log_anthropic_error +from app.services.llm.providers.claude import ( + STOP_REASON_COMPLETE, + ClaudeProvider, + log_anthropic_error, +) from app.utils import APIResponse, get_webhook_secret, send_callback logger = logging.getLogger(__name__) @@ -576,6 +580,13 @@ def _call_prompt_drafting_llm(*, user_message_text: str) -> tuple[str, str]: messages=[{"role": "user", "content": user_message_text}], output_config={"format": {"type": "json_schema", "schema": _OUTPUT_SCHEMA}}, ) + # A max_tokens cut-off still parses under the schema, so stop_reason is + # the only signal — never persist a half-written prompt as a version. + if response.stop_reason != STOP_REASON_COMPLETE: + raise RuntimeError( + "prompt_generation_failed: the model stopped before finishing " + f"the draft (stop_reason={response.stop_reason}) — retry" + ) text = next(b.text for b in response.content if b.type == "text") data = json.loads(text) return data[_LLM_KEY_INSTRUCTIONS], data[_LLM_KEY_RATIONALE] @@ -621,6 +632,11 @@ def _call_prompt_drafting_llm(*, user_message_text: str) -> tuple[str, str]: f"anthropic_response: {_anthropic_error_detail(exc)}" ) from exc + # The truncation guard above already raised a precise message; without this + # re-raise the generic handler below flattens it. + except RuntimeError: + raise + except Exception as exc: log_anthropic_error(exc, fn_name="_call_prompt_drafting_llm") raise RuntimeError( diff --git a/backend/app/services/llm/providers/claude.py b/backend/app/services/llm/providers/claude.py index ad8002425..5ae51461f 100644 --- a/backend/app/services/llm/providers/claude.py +++ b/backend/app/services/llm/providers/claude.py @@ -29,6 +29,9 @@ FILES_API_BETA = "files-api-2025-04-14" +# Any other stop_reason (max_tokens, refusal) means the text is partial. +STOP_REASON_COMPLETE = "end_turn" + def log_anthropic_error(exc: Exception, *, fn_name: str, context: str = "") -> None: """Log an Anthropic `messages.create` failure at the right level/tag. diff --git a/backend/app/tests/api/routes/test_improve_prompt.py b/backend/app/tests/api/routes/test_improve_prompt.py index 1f9b91723..17027b399 100644 --- a/backend/app/tests/api/routes/test_improve_prompt.py +++ b/backend/app/tests/api/routes/test_improve_prompt.py @@ -56,6 +56,7 @@ start_prompt_improvement_job, validate_improve_prompt, ) +from app.services.llm.providers.claude import STOP_REASON_COMPLETE from app.tests.utils.auth import TestAuthContext from app.tests.utils.test_data import create_test_evaluation_dataset from app.tests.utils.utils import random_lower_string @@ -106,6 +107,7 @@ def _make_fake_claude_client(text_content: str | None = None) -> MagicMock: response = MagicMock() response.content = [content_block] response.id = "msg_test_id" + response.stop_reason = STOP_REASON_COMPLETE client = MagicMock() client.messages.create.return_value = response @@ -893,7 +895,39 @@ def test_old_poll_path_no_longer_routed( assert resp.status_code == 404, resp.text -def _anthropic_response(status_code: int, body: dict) -> httpx.Response: +class TestDraftingLlmTruncation: + """A max_tokens cut-off still parses under the schema; stop_reason is the guard.""" + + def _run(self, client: MagicMock) -> tuple[str, str]: + with ( + patch.object(settings, "ANTHROPIC_API_KEY", "sk-test"), + patch( + "app.services.evaluations.prompt_improvement." + "ClaudeProvider.create_client", + return_value=client, + ), + ): + return _call_prompt_drafting_llm(user_message_text="brief") + + def test_complete_response_returns_instructions_and_rationale(self) -> None: + instructions, rationale = self._run(_make_fake_claude_client()) + assert instructions == _IMPROVED_INSTRUCTIONS + assert rationale == _RATIONALE + + def test_truncated_response_raises_instead_of_returning_a_partial_prompt( + self, + ) -> None: + client = _make_fake_claude_client() + client.messages.create.return_value.stop_reason = "max_tokens" + + with pytest.raises(RuntimeError) as exc: + self._run(client) + + assert "prompt_generation_failed" in str(exc.value) + assert "max_tokens" in str(exc.value) + + +def _anthropic_response(status_code: int, body: dict[str, object]) -> httpx.Response: return httpx.Response( status_code, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"), diff --git a/backend/app/tests/api/routes/test_improve_prompt_v2.py b/backend/app/tests/api/routes/test_improve_prompt_v2.py index ed9cef8ad..47963eae4 100644 --- a/backend/app/tests/api/routes/test_improve_prompt_v2.py +++ b/backend/app/tests/api/routes/test_improve_prompt_v2.py @@ -44,6 +44,7 @@ AI_GENERATED_MARKER, execute_prompt_improvement, ) +from app.services.llm.providers.claude import STOP_REASON_COMPLETE from app.tests.utils.auth import TestAuthContext from app.tests.utils.test_data import create_test_evaluation_dataset from app.tests.utils.utils import random_lower_string @@ -120,6 +121,7 @@ def _make_fake_claude_client(text_content: str | None = None) -> MagicMock: response = MagicMock() response.content = [content_block] response.id = "msg_test_id" + response.stop_reason = STOP_REASON_COMPLETE client = MagicMock() client.messages.create.return_value = response diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index 2beedef77..f2f210b6e 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -56,6 +56,7 @@ build_kaapi_completion_config, ) from app.models.response import FileResultChunk +from app.services.llm.providers.claude import STOP_REASON_COMPLETE from app.tests.utils.auth import TestAuthContext from app.tests.utils.test_data import ( create_test_config, @@ -202,7 +203,8 @@ def _raw_judge_response(text: str, *, usage=(12, 6, 18)): def _summary_response(text: str) -> SimpleNamespace: return SimpleNamespace( - content=[SimpleNamespace(type="text", text=json.dumps({"summary": text}))] + content=[SimpleNamespace(type="text", text=text)], + stop_reason=STOP_REASON_COMPLETE, ) diff --git a/backend/app/tests/crud/evaluations/test_run_ai_summary.py b/backend/app/tests/crud/evaluations/test_run_ai_summary.py index 44d62bba9..bd00a2be7 100644 --- a/backend/app/tests/crud/evaluations/test_run_ai_summary.py +++ b/backend/app/tests/crud/evaluations/test_run_ai_summary.py @@ -6,8 +6,8 @@ factor, the evaluated AI config, and the per-question judge traces as JSON — question, golden answer, generated answer, and each judge score with its rationale. Trace bookkeeping (`data_type`, `verdict`, `unscoreable`) stays out of -the payload. Every failure mode (provider error, generic error, unparseable -payload, empty output) must resolve to None WITHOUT raising, leaving the +the payload. Every failure mode (provider error, generic error, no text +block, empty output) must resolve to None WITHOUT raising, leaving the deterministic overall to persist. """ @@ -21,6 +21,7 @@ from app.core.config import settings from app.crud.evaluations.score import TraceData from app.crud.evaluations.summary import generate_run_ai_summary +from app.services.llm.providers.claude import STOP_REASON_COMPLETE _MODEL = "claude-sonnet-4-6" _RUN_NAME = "run-x" @@ -74,11 +75,12 @@ def _traces() -> list[TraceData]: ] -def _message(summary: str) -> SimpleNamespace: +def _message( + summary: str, *, stop_reason: str = STOP_REASON_COMPLETE +) -> SimpleNamespace: return SimpleNamespace( - content=[ - SimpleNamespace(type="text", text=json.dumps({"summary": summary})), - ] + content=[SimpleNamespace(type="text", text=summary)], + stop_reason=stop_reason, ) @@ -134,7 +136,7 @@ def test_returns_the_summary_field_stripped(self) -> None: ) assert _call(client) == "Consistently grounded and on-tone across the set." - def test_call_params_carry_the_model_token_cap_and_json_schema(self) -> None: + def test_call_params_carry_the_model_and_token_cap(self) -> None: client = MagicMock() client.messages.create.return_value = _message("A note.") _call(client) @@ -143,7 +145,8 @@ def test_call_params_carry_the_model_token_cap_and_json_schema(self) -> None: assert params["model"] == _MODEL assert params["max_tokens"] == 3000 assert params["messages"][0]["role"] == "user" - assert params["output_config"]["format"]["type"] == "json_schema" + # A quote in the prose would close a schema-constrained string early. + assert "output_config" not in params class TestTraceBrief: @@ -285,16 +288,23 @@ def test_empty_or_whitespace_summary_returns_none(self, summary: str) -> None: client.messages.create.return_value = _message(summary) assert _call(client) is None - def test_non_json_text_block_returns_none_without_raising(self) -> None: + def test_response_without_a_text_block_returns_none_without_raising(self) -> None: client = MagicMock() client.messages.create.return_value = SimpleNamespace( - content=[SimpleNamespace(type="text", text="not json at all")] + content=[SimpleNamespace(type="tool_use", text=None)], + stop_reason=STOP_REASON_COMPLETE, ) assert _call(client) is None - def test_response_without_a_text_block_returns_none_without_raising(self) -> None: + def test_quotes_in_the_prose_survive_intact(self) -> None: client = MagicMock() - client.messages.create.return_value = SimpleNamespace( - content=[SimpleNamespace(type="tool_use", text=None)] + prose = 'Question 3 omits the required "wellness message" line.' + client.messages.create.return_value = _message(prose) + assert _call(client) == prose + + def test_truncated_response_is_still_returned(self) -> None: + client = MagicMock() + client.messages.create.return_value = _message( + "Question 10 omits the required", stop_reason="max_tokens" ) - assert _call(client) is None + assert _call(client) == "Question 10 omits the required" diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index 3e6f72c1d..8e27bf992 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -32,13 +32,13 @@ v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native jud ## Services / CRUD - `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py` (`validate_fast_evaluation_inputs` extracted for reuse by both the direct eval-start path and the iteration loop), `batch_job.py`, `validators.py`, `prompt_improvement.py`, `iteration.py` (`validate_and_start_evaluation_iteration`, `compute_round_scores`), `iteration_graph.py` (the LangGraph `StateGraph`: nodes, checkpointer) - `services/stt_evaluations/`, `services/tts_evaluations/` -- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Anthropic `messages.create` call via `ClaudeProvider`, structured JSON output; prompt carries every trace's raw per-question scores + judge rationale, golden/generated answers, and the evaluated config, each trace keyed on `question_id` (the 1-based dataset row number from `merge.py`, cited back as "Question N") rather than the Langfuse `trace_id`, and returns a severity-ranked diagnostic note, not just a qualitative band summary), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py`, `iteration.py` (thin-row CRUD for the iteration loop) +- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Anthropic `messages.create` call via `ClaudeProvider`, plain-text output — deliberately NOT a `json_schema` `output_config`, since a quotation mark in the prose reads as the JSON string terminator and silently truncates the note; `stop_reason != STOP_REASON_COMPLETE` (the shared `end_turn` constant in `services/llm/providers/claude.py`) is logged as a truncation warning; prompt carries every trace's raw per-question scores + judge rationale, golden/generated answers, and the evaluated config, each trace keyed on `question_id` (the 1-based dataset row number from `merge.py`, cited back as "Question N") rather than the Langfuse `trace_id`, and returns a severity-ranked diagnostic note, not just a qualitative band summary), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py`, `iteration.py` (thin-row CRUD for the iteration loop) - `core/batch/` — shared provider batch clients: `openai.py`, `gemini.py`, `anthropic.py`, `polling.py`, `operations.py` ## Async - Provider batches polled by cron (`crud/evaluations/cron.py`); no long-lived Celery task per run. - Fast text runs (v1 cosine + v2 judge) fan out `ceil(total_items / EVAL_FAST_CHUNK_SIZE)` `run_evaluation_fast_chunk` tasks (responses only), then a cron barrier (`dispatch_fast_evaluation_barriers`) enqueues one `run_evaluation_fast_aggregate` once every chunk has a `raw_output_url`. The aggregate merges chunks, then (v2) judges **every** row in that single task — so the judge pool is sized by its own `EVAL_JUDGE_CONCURRENCY` (not the response stage's `EVAL_FAST_API_CONCURRENCY`) to clear the max dataset (`EVAL_FAST_MAX_UNIQUE_ROWS` × `duplication_factor`) under the aggregate's `CELERY_TASK_SOFT_TIME_LIMIT`. No judge fan-out / second barrier. -- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. +- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. `_call_prompt_drafting_llm` keeps `json_schema` structured output (two fields, `rationale` length-capped) but raises `prompt_generation_failed` when `stop_reason != STOP_REASON_COMPLETE`, so a `max_tokens`-truncated draft fails the job instead of minting a half-written system prompt. Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. - v2 prompt iteration reuses the same job/Celery/config-version machinery (`services/evaluations/prompt_improvement.py`) and branches on `run.is_judge_run`: the v2 route calls `start_prompt_improvement_job(..., require_judge_run=True)` (a non-judge run → `422 not_a_judge_run`), and the worker drafts from the three-metric judge trace (`_draft_improved_prompt(is_judge_run=True)`, using each metric's score + reasoning) and delivers a `PromptRecommendationJobPublic` callback carrying `recommendation_type` (`Literal["prompt"]`, `models/evaluation.py`; widens to a union when knowledge-base / model recommendations land). Non-judge (v1) runs keep the default `_draft_improved_prompt` brief + `PromptImprovementJobPublic` path unchanged. - Evaluation iteration loop (`POST /evaluations/iterations`) chains fast-eval + v2 prompt improvement into a self-driving cycle via LangGraph (`services/evaluations/iteration_graph.py`): `start_eval_node` → `wait_eval_node` → (conditional) `start_improve_node` → `wait_improve_node` → loops back to `start_eval_node`, or → `finalize_node`. Stop-score = mean(`Adherence to Ground Truth`, `Adherence to Prompt`) from `EvaluationRun.score["summary_scores"]` (`compute_round_scores`); `Adherence to Knowledge Base` is recorded per round for visibility only, never gates stopping. Stops on 3 consecutive rounds with `