From 2d367dc223e247ae8cbe8e4c7ec2f35661123239 Mon Sep 17 00:00:00 2001 From: AkhileshNegi Date: Mon, 19 Oct 2026 15:53:03 +0530 Subject: [PATCH 1/5] fix(evaluations): Compact traces in prompt-improvement brief A 500-row Hindi judge run failed prompt improvement with Anthropic's "prompt is too long: 1121367 tokens > 1000000 maximum". The drafting brief shipped the raw score-trace file: json.dumps' default \uXXXX escaping inflated every Devanagari character, each duplication-factor repeat re-sent question + ground truth, and trace_id / data_type / verdict / guardrail bookkeeping rode along unused. _compact_traces_for_brief now projects traces to what the brief reads: one entry per question with repeats nested as attempts, scores reduced to name/value/comment/unscoreable, and guardrail-rewritten rows substituting input_to_llm / output_from_llm for the user-facing text. The JSON is dumped with ensure_ascii=False. Both v1 and v2 briefs describe the new shape and tell the model that score spread across attempts signals inconsistency. Refs glific/glific#5751 Co-Authored-By: Claude Fable 5.1 --- .../evaluations/prompt_improvement.py | 69 +++++- .../api/routes/test_improve_prompt_v2.py | 207 ++++++++++++++++++ docs/wiki/modules/evaluations.md | 2 +- 3 files changed, 269 insertions(+), 9 deletions(-) diff --git a/backend/app/services/evaluations/prompt_improvement.py b/backend/app/services/evaluations/prompt_improvement.py index 9b92d5c22..f2f58258f 100644 --- a/backend/app/services/evaluations/prompt_improvement.py +++ b/backend/app/services/evaluations/prompt_improvement.py @@ -8,6 +8,7 @@ import copy import json import logging +from typing import Any from uuid import UUID import anthropic @@ -646,11 +647,54 @@ def _call_prompt_drafting_llm(*, user_message_text: str) -> tuple[str, str]: ) from exc +# Score keys the drafting brief reads; data_type/verdict are UI bookkeeping. +_BRIEF_SCORE_KEYS = ("name", "value", "comment", "unscoreable") + + +def _compact_traces_for_brief(traces: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Group repeated attempts per question and keep only what the brief reads. + + A 500-row Hindi judge run blew past Claude's 1M-token context: every repeat + (duplication factor) re-shipped question + ground truth, and trace_id / + data_type / verdict / guardrail bookkeeping rode along for nothing. Rows the + guardrails rewrote substitute `input_to_llm` / `output_from_llm`, since the + prompt under repair acts on what the LLM actually saw and produced, not on + the user-facing text. + """ + grouped: dict[Any, dict[str, Any]] = {} + for trace in traces: + question = trace.get("input_to_llm") or trace.get("question", "") + ground_truth = trace.get("ground_truth_answer", "") + question_id = trace.get("question_id") + # v1 traces may lack question_id; the text pair is the next-best identity. + key = question_id if question_id is not None else (question, ground_truth) + entry = grouped.setdefault( + key, + { + "question_id": question_id, + "question": question, + "ground_truth_answer": ground_truth, + "attempts": [], + }, + ) + entry["attempts"].append( + { + "llm_answer": trace.get("output_from_llm") + or trace.get("llm_answer", ""), + "scores": [ + {k: score[k] for k in _BRIEF_SCORE_KEYS if k in score} + for score in trace.get("scores", []) + ], + } + ) + return list(grouped.values()) + + def _draft_improved_prompt( *, current_instructions: str, config_params: dict, - traces: list | dict, + traces: list[dict[str, Any]], is_judge_run: bool = False, ) -> tuple[str, str]: """Rewrite the prompt from a run's score traces. @@ -662,9 +706,13 @@ def _draft_improved_prompt( """ if is_judge_run: trace_description = ( - " from an LLM-as-judge run. Each trace has the fields: `question`, " - "`ground_truth_answer`, `llm_answer`, and `scores`. `scores` is a list of " - "metric objects; each has `name`, `value`, and `comment`, where:\n" + " from an LLM-as-judge run. Each entry is one dataset question with the " + "fields: `question_id`, `question`, `ground_truth_answer`, and `attempts`. " + "`attempts` is a list, one per time the question was run, each with " + "`llm_answer` and `scores`; a wide spread of scores across a question's " + "attempts means the prompt yields inconsistent behaviour on it. `scores` " + "is a list of metric objects; each has `name`, `value`, and `comment`, " + "where:\n" f"- `name` is one of `{GROUND_TRUTH_SCORE_NAME}`, `{PROMPT_SCORE_NAME}`, " f"or `{KNOWLEDGE_BASE_SCORE_NAME}`.\n" "- `value` is the metric's score, an integer from 0 (worst) to 5 (best). Metrics that " @@ -706,9 +754,10 @@ def _draft_improved_prompt( ] else: trace_description = ( - ". Each trace has the fields: `question`, `ground_truth_answer`, " - "`llm_answer`, `category`, and `scores` (a list of scoring objects with " - "`name`, `value`, and `unscoreable`)." + ". Each entry is one dataset question with the fields: `question`, " + "`ground_truth_answer`, and `attempts`, a list, one per time the " + "question was run, each with `llm_answer` and `scores` (a list of " + "scoring objects with `name`, `value`, and `unscoreable`)." ) task_steps = [ "1. Identify the answers that performed poorly — those with low scores or " @@ -730,11 +779,15 @@ def _draft_improved_prompt( f"sentence (≤ {_RATIONALE_MAX_LENGTH} characters): what you changed and why." ) target_config = _target_config_from_params(config_params) + # ensure_ascii=False: \uXXXX-escaping Devanagari inflates its token count several-fold. + traces_json = json.dumps( + _compact_traces_for_brief(traces), ensure_ascii=False, separators=(",", ":") + ) user_message_text = ( "You are a prompt engineer. Below is a JSON array of evaluation traces" f"{trace_description}\n\n" - f"## Evaluation traces\n```\n{json.dumps(traces)}\n```\n\n" + f"## Evaluation traces\n```\n{traces_json}\n```\n\n" f"## Current system prompt\n```\n{current_instructions}\n```\n\n" "## Target configuration (read-only — do NOT change any of these)\n" f"```\n{json.dumps(target_config)}\n```\n\n" diff --git a/backend/app/tests/api/routes/test_improve_prompt_v2.py b/backend/app/tests/api/routes/test_improve_prompt_v2.py index c83c5d9d4..48a232b4d 100644 --- a/backend/app/tests/api/routes/test_improve_prompt_v2.py +++ b/backend/app/tests/api/routes/test_improve_prompt_v2.py @@ -42,6 +42,7 @@ from app.models.job import Job, JobStatus, JobType from app.services.evaluations.prompt_improvement import ( AI_GENERATED_MARKER, + _compact_traces_for_brief, execute_prompt_improvement, ) from app.services.llm.providers.claude import STOP_REASON_COMPLETE @@ -671,3 +672,209 @@ def test_redelivery_of_success_judge_job_resends_v2_callback_without_new_version assert payload["success"] is True assert payload["data"]["recommendation_type"] == "prompt" assert payload["data"]["config_version"]["version"] == first_version + + +def _score(name: str, value: int | str, comment: str) -> dict: + return { + "name": name, + "value": value, + "data_type": "NUMERIC", + "comment": comment, + "verdict": "Good", + } + + +class TestCompactTracesForBrief: + """Pure-function checks on the trace projection fed to the drafting LLM.""" + + def test_groups_repeats_per_question_and_drops_bookkeeping(self) -> None: + traces = [ + { + "trace_id": "t1", + "question": "Q", + "llm_answer": "A1", + "ground_truth_answer": "G", + "question_id": 7, + "category": "default", + "guardrail": None, + "input_to_llm": None, + "output_from_llm": None, + "scores": [_score(GROUND_TRUTH_SCORE_NAME, 1, "wrong")], + }, + { + "trace_id": "t2", + "question": "Q", + "llm_answer": "A2", + "ground_truth_answer": "G", + "question_id": 7, + "guardrail": None, + "input_to_llm": None, + "output_from_llm": None, + "scores": [ + _score(GROUND_TRUTH_SCORE_NAME, 5, "right"), + { + "name": KNOWLEDGE_BASE_SCORE_NAME, + "value": "N/A", + "data_type": "CATEGORICAL", + "comment": "no KB", + "unscoreable": True, + }, + ], + }, + ] + + compact = _compact_traces_for_brief(traces) + + assert compact == [ + { + "question_id": 7, + "question": "Q", + "ground_truth_answer": "G", + "attempts": [ + { + "llm_answer": "A1", + "scores": [ + { + "name": GROUND_TRUTH_SCORE_NAME, + "value": 1, + "comment": "wrong", + } + ], + }, + { + "llm_answer": "A2", + "scores": [ + { + "name": GROUND_TRUTH_SCORE_NAME, + "value": 5, + "comment": "right", + }, + { + "name": KNOWLEDGE_BASE_SCORE_NAME, + "value": "N/A", + "comment": "no KB", + "unscoreable": True, + }, + ], + }, + ], + } + ] + + def test_guardrail_rewritten_rows_use_what_the_llm_saw(self) -> None: + traces = [ + { + "trace_id": "t1", + "question": "user text with PII", + "llm_answer": "answer shown to user", + "ground_truth_answer": "G", + "question_id": 1, + "guardrail": "applied", + "input_to_llm": "user text masked", + "output_from_llm": "raw model answer", + "scores": [], + }, + { + "trace_id": "t2", + "question": "Q2", + "llm_answer": "A2", + "ground_truth_answer": "G2", + "question_id": 2, + "guardrail": "applied", + "input_to_llm": "Q2 masked", + "output_from_llm": None, + "scores": [], + }, + ] + + compact = _compact_traces_for_brief(traces) + + assert compact[0]["question"] == "user text masked" + assert compact[0]["attempts"][0]["llm_answer"] == "raw model answer" + # Only the input side was rewritten; the answer falls back to llm_answer. + assert compact[1]["question"] == "Q2 masked" + assert compact[1]["attempts"][0]["llm_answer"] == "A2" + for entry in compact: + assert "guardrail" not in entry + assert "input_to_llm" not in entry + assert "output_from_llm" not in entry + assert "trace_id" not in entry + + def test_v1_traces_without_question_id_group_by_text(self) -> None: + traces = [ + { + "trace_id": "t1", + "question": "Q", + "llm_answer": "A1", + "ground_truth_answer": "G", + "category": "Geo", + "scores": [{"name": "cosine_similarity", "value": 0.3}], + }, + { + "trace_id": "t2", + "question": "Q", + "llm_answer": "A2", + "ground_truth_answer": "G", + "category": "Geo", + "scores": [{"name": "cosine_similarity", "value": 0.9}], + }, + { + "trace_id": "t3", + "question": "Other", + "llm_answer": "A3", + "ground_truth_answer": "G3", + "scores": [], + }, + ] + + compact = _compact_traces_for_brief(traces) + + assert [entry["question"] for entry in compact] == ["Q", "Other"] + assert compact[0]["question_id"] is None + assert [a["llm_answer"] for a in compact[0]["attempts"]] == ["A1", "A2"] + assert "category" not in compact[0] + + +class TestV2BriefEncoding: + """The drafting message must ship Indic text raw, not \\uXXXX-escaped.""" + + def _pending_job(self, db: Session, project_id: int) -> Job: + return JobCrud(session=db).create( + job_type=JobType.PROMPT_IMPROVEMENT, + trace_id="N/A", + project_id=project_id, + ) + + def test_indic_text_is_not_ascii_escaped_in_brief( + self, + db: Session, + auth: TestAuthContext, + judge_run: EvaluationRun, + ) -> None: + hindi_question = "भारत की राजधानी क्या है?" + hindi_answer = "नई दिल्ली" + traces = [ + { + **_JUDGE_TRACES[0], + "question": hindi_question, + "llm_answer": hindi_answer, + "ground_truth_answer": hindi_answer, + } + ] + job = self._pending_job(db, auth.project_id) + + with _worker_env(db, traces=traces) as env: + execute_prompt_improvement( + project_id=auth.project_id, + job_id=str(job.id), + organization_id=auth.organization_id, + evaluation_id=judge_run.id, + callback_url=_CALLBACK_URL, + ) + + message = _llm_user_message(env.llm) + assert hindi_question in message + assert hindi_answer in message + assert "\\u0" not in message + assert '"attempts":[{"llm_answer":' in message + assert '"trace_id"' not in message diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index c39f05de9..b3ef2c40d 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -38,7 +38,7 @@ v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native jud ## Async - Provider batches polled by cron (`crud/evaluations/cron.py`); no long-lived Celery task per run. - Fast text runs (v1 cosine + v2 judge) fan out `ceil(total_items / EVAL_FAST_CHUNK_SIZE)` `run_evaluation_fast_chunk` tasks (responses only), then a cron barrier (`dispatch_fast_evaluation_barriers`) enqueues one `run_evaluation_fast_aggregate` once every chunk has a `raw_output_url`. The aggregate merges chunks, then (v2) judges **every** row in that single task — so the judge pool is sized by its own `EVAL_JUDGE_CONCURRENCY` (not the response stage's `EVAL_FAST_API_CONCURRENCY`) to clear the max dataset (`EVAL_FAST_MAX_UNIQUE_ROWS` × `duplication_factor`) under the aggregate's `CELERY_TASK_SOFT_TIME_LIMIT`. No judge fan-out / second barrier. The aggregate persists the score unit (`save_score` → traces to S3 behind `score_trace_url`, summary + overall to the DB) **before** flipping the run to `completed`: on v2 that unit is the only copy of the per-row judge scores, so a crash between the two must leave the run `processing`, never `completed` with its scores gone. Chunk-artifact cleanup and the v1 Langfuse score sync run after completion as a best-effort tail that may never fail the run. -- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. `_call_prompt_drafting_llm` keeps `json_schema` structured output (two fields, `rationale` length-capped) but raises `prompt_generation_failed` when `stop_reason != STOP_REASON_COMPLETE`, so a `max_tokens`-truncated draft fails the job instead of minting a half-written system prompt. Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. +- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. `_call_prompt_drafting_llm` keeps `json_schema` structured output (two fields, `rationale` length-capped) but raises `prompt_generation_failed` when `stop_reason != STOP_REASON_COMPLETE`, so a `max_tokens`-truncated draft fails the job instead of minting a half-written system prompt. Before the call, `_compact_traces_for_brief` projects the score traces to what the brief actually reads — one entry per question (`question_id`, `question`, `ground_truth_answer`) with repeated runs nested as `attempts[{llm_answer, scores[{name, value, comment, unscoreable}]}]`, dropping `trace_id`/`data_type`/`verdict`/`category`/guardrail bookkeeping; a guardrail-rewritten row substitutes `input_to_llm`/`output_from_llm` for `question`/`llm_answer` — and the JSON is dumped with `ensure_ascii=False` (escaping Devanagari inflates its token count several-fold; a 500-row Hindi run previously exceeded Claude's 1M-token context). Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. - v2 prompt iteration reuses the same job/Celery/config-version machinery (`services/evaluations/prompt_improvement.py`) and branches on `run.is_judge_run`: the v2 route calls `start_prompt_improvement_job(..., require_judge_run=True)` (a non-judge run → `422 not_a_judge_run`), and the worker drafts from the three-metric judge trace (`_draft_improved_prompt(is_judge_run=True)`, using each metric's score + reasoning) and delivers a `PromptRecommendationJobPublic` callback carrying `recommendation_type` (`Literal["prompt"]`, `models/evaluation.py`; widens to a union when knowledge-base / model recommendations land). Non-judge (v1) runs keep the default `_draft_improved_prompt` brief + `PromptImprovementJobPublic` path unchanged. - Evaluation iteration loop (`POST /evaluations/iterations`) chains fast-eval + v2 prompt improvement into a self-driving cycle via LangGraph (`services/evaluations/iteration_graph.py`): `start_eval_node` → `wait_eval_node` → (conditional) `start_improve_node` → `wait_improve_node` → loops back to `start_eval_node`, or → `finalize_node`. Stop-score = mean(`Adherence to Ground Truth`, `Adherence to Prompt`) from `EvaluationRun.score["summary_scores"]` (`compute_round_scores`); `Adherence to Knowledge Base` is recorded per round for visibility only, never gates stopping. Stops on 3 consecutive rounds with ` Date: Mon, 19 Oct 2026 16:02:46 +0530 Subject: [PATCH 2/5] refactor(evaluations): Rename trace compaction to _format_traces_for_brief Mirrors summary.py::_format_traces_for_prompt, the sibling that shapes the same traces for the other Claude call. Docstring now says why group_traces_by_question_id (the export shape) is not reused. Co-Authored-By: Claude Fable 5.1 --- backend/app/services/evaluations/prompt_improvement.py | 8 +++++--- backend/app/tests/api/routes/test_improve_prompt_v2.py | 10 +++++----- docs/wiki/modules/evaluations.md | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/backend/app/services/evaluations/prompt_improvement.py b/backend/app/services/evaluations/prompt_improvement.py index f2f58258f..3c8588c54 100644 --- a/backend/app/services/evaluations/prompt_improvement.py +++ b/backend/app/services/evaluations/prompt_improvement.py @@ -651,7 +651,7 @@ def _call_prompt_drafting_llm(*, user_message_text: str) -> tuple[str, str]: _BRIEF_SCORE_KEYS = ("name", "value", "comment", "unscoreable") -def _compact_traces_for_brief(traces: list[dict[str, Any]]) -> list[dict[str, Any]]: +def _format_traces_for_brief(traces: list[dict[str, Any]]) -> list[dict[str, Any]]: """Group repeated attempts per question and keep only what the brief reads. A 500-row Hindi judge run blew past Claude's 1M-token context: every repeat @@ -659,7 +659,9 @@ def _compact_traces_for_brief(traces: list[dict[str, Any]]) -> list[dict[str, An data_type / verdict / guardrail bookkeeping rode along for nothing. Rows the guardrails rewrote substitute `input_to_llm` / `output_from_llm`, since the prompt under repair acts on what the LLM actually saw and produced, not on - the user-facing text. + the user-facing text. Not `crud.evaluations.core.group_traces_by_question_id`: + that is the export shape (keeps trace_ids/category, scores as parallel lists) + and rejects v1 traces that have no question_id. """ grouped: dict[Any, dict[str, Any]] = {} for trace in traces: @@ -781,7 +783,7 @@ def _draft_improved_prompt( target_config = _target_config_from_params(config_params) # ensure_ascii=False: \uXXXX-escaping Devanagari inflates its token count several-fold. traces_json = json.dumps( - _compact_traces_for_brief(traces), ensure_ascii=False, separators=(",", ":") + _format_traces_for_brief(traces), ensure_ascii=False, separators=(",", ":") ) user_message_text = ( diff --git a/backend/app/tests/api/routes/test_improve_prompt_v2.py b/backend/app/tests/api/routes/test_improve_prompt_v2.py index 48a232b4d..a4fc3f08b 100644 --- a/backend/app/tests/api/routes/test_improve_prompt_v2.py +++ b/backend/app/tests/api/routes/test_improve_prompt_v2.py @@ -42,7 +42,7 @@ from app.models.job import Job, JobStatus, JobType from app.services.evaluations.prompt_improvement import ( AI_GENERATED_MARKER, - _compact_traces_for_brief, + _format_traces_for_brief, execute_prompt_improvement, ) from app.services.llm.providers.claude import STOP_REASON_COMPLETE @@ -684,7 +684,7 @@ def _score(name: str, value: int | str, comment: str) -> dict: } -class TestCompactTracesForBrief: +class TestFormatTracesForBrief: """Pure-function checks on the trace projection fed to the drafting LLM.""" def test_groups_repeats_per_question_and_drops_bookkeeping(self) -> None: @@ -723,7 +723,7 @@ def test_groups_repeats_per_question_and_drops_bookkeeping(self) -> None: }, ] - compact = _compact_traces_for_brief(traces) + compact = _format_traces_for_brief(traces) assert compact == [ { @@ -787,7 +787,7 @@ def test_guardrail_rewritten_rows_use_what_the_llm_saw(self) -> None: }, ] - compact = _compact_traces_for_brief(traces) + compact = _format_traces_for_brief(traces) assert compact[0]["question"] == "user text masked" assert compact[0]["attempts"][0]["llm_answer"] == "raw model answer" @@ -827,7 +827,7 @@ def test_v1_traces_without_question_id_group_by_text(self) -> None: }, ] - compact = _compact_traces_for_brief(traces) + compact = _format_traces_for_brief(traces) assert [entry["question"] for entry in compact] == ["Q", "Other"] assert compact[0]["question_id"] is None diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index b3ef2c40d..7fe7292e8 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -38,7 +38,7 @@ v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native jud ## Async - Provider batches polled by cron (`crud/evaluations/cron.py`); no long-lived Celery task per run. - Fast text runs (v1 cosine + v2 judge) fan out `ceil(total_items / EVAL_FAST_CHUNK_SIZE)` `run_evaluation_fast_chunk` tasks (responses only), then a cron barrier (`dispatch_fast_evaluation_barriers`) enqueues one `run_evaluation_fast_aggregate` once every chunk has a `raw_output_url`. The aggregate merges chunks, then (v2) judges **every** row in that single task — so the judge pool is sized by its own `EVAL_JUDGE_CONCURRENCY` (not the response stage's `EVAL_FAST_API_CONCURRENCY`) to clear the max dataset (`EVAL_FAST_MAX_UNIQUE_ROWS` × `duplication_factor`) under the aggregate's `CELERY_TASK_SOFT_TIME_LIMIT`. No judge fan-out / second barrier. The aggregate persists the score unit (`save_score` → traces to S3 behind `score_trace_url`, summary + overall to the DB) **before** flipping the run to `completed`: on v2 that unit is the only copy of the per-row judge scores, so a crash between the two must leave the run `processing`, never `completed` with its scores gone. Chunk-artifact cleanup and the v1 Langfuse score sync run after completion as a best-effort tail that may never fail the run. -- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. `_call_prompt_drafting_llm` keeps `json_schema` structured output (two fields, `rationale` length-capped) but raises `prompt_generation_failed` when `stop_reason != STOP_REASON_COMPLETE`, so a `max_tokens`-truncated draft fails the job instead of minting a half-written system prompt. Before the call, `_compact_traces_for_brief` projects the score traces to what the brief actually reads — one entry per question (`question_id`, `question`, `ground_truth_answer`) with repeated runs nested as `attempts[{llm_answer, scores[{name, value, comment, unscoreable}]}]`, dropping `trace_id`/`data_type`/`verdict`/`category`/guardrail bookkeeping; a guardrail-rewritten row substitutes `input_to_llm`/`output_from_llm` for `question`/`llm_answer` — and the JSON is dumped with `ensure_ascii=False` (escaping Devanagari inflates its token count several-fold; a 500-row Hindi run previously exceeded Claude's 1M-token context). Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. +- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. `_call_prompt_drafting_llm` keeps `json_schema` structured output (two fields, `rationale` length-capped) but raises `prompt_generation_failed` when `stop_reason != STOP_REASON_COMPLETE`, so a `max_tokens`-truncated draft fails the job instead of minting a half-written system prompt. Before the call, `_format_traces_for_brief` projects the score traces to what the brief actually reads — one entry per question (`question_id`, `question`, `ground_truth_answer`) with repeated runs nested as `attempts[{llm_answer, scores[{name, value, comment, unscoreable}]}]`, dropping `trace_id`/`data_type`/`verdict`/`category`/guardrail bookkeeping; a guardrail-rewritten row substitutes `input_to_llm`/`output_from_llm` for `question`/`llm_answer` — and the JSON is dumped with `ensure_ascii=False` (escaping Devanagari inflates its token count several-fold; a 500-row Hindi run previously exceeded Claude's 1M-token context). Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. - v2 prompt iteration reuses the same job/Celery/config-version machinery (`services/evaluations/prompt_improvement.py`) and branches on `run.is_judge_run`: the v2 route calls `start_prompt_improvement_job(..., require_judge_run=True)` (a non-judge run → `422 not_a_judge_run`), and the worker drafts from the three-metric judge trace (`_draft_improved_prompt(is_judge_run=True)`, using each metric's score + reasoning) and delivers a `PromptRecommendationJobPublic` callback carrying `recommendation_type` (`Literal["prompt"]`, `models/evaluation.py`; widens to a union when knowledge-base / model recommendations land). Non-judge (v1) runs keep the default `_draft_improved_prompt` brief + `PromptImprovementJobPublic` path unchanged. - Evaluation iteration loop (`POST /evaluations/iterations`) chains fast-eval + v2 prompt improvement into a self-driving cycle via LangGraph (`services/evaluations/iteration_graph.py`): `start_eval_node` → `wait_eval_node` → (conditional) `start_improve_node` → `wait_improve_node` → loops back to `start_eval_node`, or → `finalize_node`. Stop-score = mean(`Adherence to Ground Truth`, `Adherence to Prompt`) from `EvaluationRun.score["summary_scores"]` (`compute_round_scores`); `Adherence to Knowledge Base` is recorded per round for visibility only, never gates stopping. Stops on 3 consecutive rounds with ` Date: Mon, 19 Oct 2026 16:05:49 +0530 Subject: [PATCH 3/5] refactor(evaluations): Type the prompt-improvement brief as TypedDicts BriefQuestion / BriefAttempt / BriefScore spell out the shape handed to the drafting LLM; _format_traces_for_brief now takes list[TraceData] and returns list[BriefQuestion] instead of untyped dicts. Co-Authored-By: Claude Fable 5.1 --- .../evaluations/prompt_improvement.py | 49 +++++++++++++++---- docs/wiki/modules/evaluations.md | 2 +- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/backend/app/services/evaluations/prompt_improvement.py b/backend/app/services/evaluations/prompt_improvement.py index 3c8588c54..85f23ffdc 100644 --- a/backend/app/services/evaluations/prompt_improvement.py +++ b/backend/app/services/evaluations/prompt_improvement.py @@ -8,7 +8,7 @@ import copy import json import logging -from typing import Any +from typing import Any, NotRequired, TypedDict from uuid import UUID import anthropic @@ -29,6 +29,8 @@ GROUND_TRUTH_SCORE_NAME, KNOWLEDGE_BASE_SCORE_NAME, PROMPT_SCORE_NAME, + TraceData, + TraceScore, ) from app.crud.jobs import JobCrud from app.models.config.config import ConfigTag @@ -647,11 +649,41 @@ def _call_prompt_drafting_llm(*, user_message_text: str) -> tuple[str, str]: ) from exc -# Score keys the drafting brief reads; data_type/verdict are UI bookkeeping. -_BRIEF_SCORE_KEYS = ("name", "value", "comment", "unscoreable") +class BriefScore(TypedDict): + """One metric as the drafting brief sees it; data_type/verdict are UI-only.""" + name: str + value: float | str + comment: NotRequired[str] + unscoreable: NotRequired[bool] -def _format_traces_for_brief(traces: list[dict[str, Any]]) -> list[dict[str, Any]]: + +class BriefAttempt(TypedDict): + """One run of a question: the answer it produced and how the judge scored it.""" + + llm_answer: str + scores: list[BriefScore] + + +class BriefQuestion(TypedDict): + """One dataset question with every repeat (duplication factor) nested under it.""" + + question_id: int | str | None + question: str + ground_truth_answer: str + attempts: list[BriefAttempt] + + +def _brief_score(score: TraceScore) -> BriefScore: + brief: BriefScore = {"name": score["name"], "value": score["value"]} + if "comment" in score: + brief["comment"] = score["comment"] + if "unscoreable" in score: + brief["unscoreable"] = score["unscoreable"] + return brief + + +def _format_traces_for_brief(traces: list[TraceData]) -> list[BriefQuestion]: """Group repeated attempts per question and keep only what the brief reads. A 500-row Hindi judge run blew past Claude's 1M-token context: every repeat @@ -663,7 +695,7 @@ def _format_traces_for_brief(traces: list[dict[str, Any]]) -> list[dict[str, Any that is the export shape (keeps trace_ids/category, scores as parallel lists) and rejects v1 traces that have no question_id. """ - grouped: dict[Any, dict[str, Any]] = {} + grouped: dict[Any, BriefQuestion] = {} for trace in traces: question = trace.get("input_to_llm") or trace.get("question", "") ground_truth = trace.get("ground_truth_answer", "") @@ -683,10 +715,7 @@ def _format_traces_for_brief(traces: list[dict[str, Any]]) -> list[dict[str, Any { "llm_answer": trace.get("output_from_llm") or trace.get("llm_answer", ""), - "scores": [ - {k: score[k] for k in _BRIEF_SCORE_KEYS if k in score} - for score in trace.get("scores", []) - ], + "scores": [_brief_score(score) for score in trace.get("scores", [])], } ) return list(grouped.values()) @@ -696,7 +725,7 @@ def _draft_improved_prompt( *, current_instructions: str, config_params: dict, - traces: list[dict[str, Any]], + traces: list[TraceData], is_judge_run: bool = False, ) -> tuple[str, str]: """Rewrite the prompt from a run's score traces. diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index 7fe7292e8..10c5357d8 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -38,7 +38,7 @@ v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native jud ## Async - Provider batches polled by cron (`crud/evaluations/cron.py`); no long-lived Celery task per run. - Fast text runs (v1 cosine + v2 judge) fan out `ceil(total_items / EVAL_FAST_CHUNK_SIZE)` `run_evaluation_fast_chunk` tasks (responses only), then a cron barrier (`dispatch_fast_evaluation_barriers`) enqueues one `run_evaluation_fast_aggregate` once every chunk has a `raw_output_url`. The aggregate merges chunks, then (v2) judges **every** row in that single task — so the judge pool is sized by its own `EVAL_JUDGE_CONCURRENCY` (not the response stage's `EVAL_FAST_API_CONCURRENCY`) to clear the max dataset (`EVAL_FAST_MAX_UNIQUE_ROWS` × `duplication_factor`) under the aggregate's `CELERY_TASK_SOFT_TIME_LIMIT`. No judge fan-out / second barrier. The aggregate persists the score unit (`save_score` → traces to S3 behind `score_trace_url`, summary + overall to the DB) **before** flipping the run to `completed`: on v2 that unit is the only copy of the per-row judge scores, so a crash between the two must leave the run `processing`, never `completed` with its scores gone. Chunk-artifact cleanup and the v1 Langfuse score sync run after completion as a best-effort tail that may never fail the run. -- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. `_call_prompt_drafting_llm` keeps `json_schema` structured output (two fields, `rationale` length-capped) but raises `prompt_generation_failed` when `stop_reason != STOP_REASON_COMPLETE`, so a `max_tokens`-truncated draft fails the job instead of minting a half-written system prompt. Before the call, `_format_traces_for_brief` projects the score traces to what the brief actually reads — one entry per question (`question_id`, `question`, `ground_truth_answer`) with repeated runs nested as `attempts[{llm_answer, scores[{name, value, comment, unscoreable}]}]`, dropping `trace_id`/`data_type`/`verdict`/`category`/guardrail bookkeeping; a guardrail-rewritten row substitutes `input_to_llm`/`output_from_llm` for `question`/`llm_answer` — and the JSON is dumped with `ensure_ascii=False` (escaping Devanagari inflates its token count several-fold; a 500-row Hindi run previously exceeded Claude's 1M-token context). Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. +- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. `_call_prompt_drafting_llm` keeps `json_schema` structured output (two fields, `rationale` length-capped) but raises `prompt_generation_failed` when `stop_reason != STOP_REASON_COMPLETE`, so a `max_tokens`-truncated draft fails the job instead of minting a half-written system prompt. Before the call, `_format_traces_for_brief` projects the score traces to what the brief actually reads (`BriefQuestion` / `BriefAttempt` / `BriefScore` TypedDicts in the same module) — one entry per question (`question_id`, `question`, `ground_truth_answer`) with repeated runs nested as `attempts[{llm_answer, scores[{name, value, comment, unscoreable}]}]`, dropping `trace_id`/`data_type`/`verdict`/`category`/guardrail bookkeeping; a guardrail-rewritten row substitutes `input_to_llm`/`output_from_llm` for `question`/`llm_answer` — and the JSON is dumped with `ensure_ascii=False` (escaping Devanagari inflates its token count several-fold; a 500-row Hindi run previously exceeded Claude's 1M-token context). Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. - v2 prompt iteration reuses the same job/Celery/config-version machinery (`services/evaluations/prompt_improvement.py`) and branches on `run.is_judge_run`: the v2 route calls `start_prompt_improvement_job(..., require_judge_run=True)` (a non-judge run → `422 not_a_judge_run`), and the worker drafts from the three-metric judge trace (`_draft_improved_prompt(is_judge_run=True)`, using each metric's score + reasoning) and delivers a `PromptRecommendationJobPublic` callback carrying `recommendation_type` (`Literal["prompt"]`, `models/evaluation.py`; widens to a union when knowledge-base / model recommendations land). Non-judge (v1) runs keep the default `_draft_improved_prompt` brief + `PromptImprovementJobPublic` path unchanged. - Evaluation iteration loop (`POST /evaluations/iterations`) chains fast-eval + v2 prompt improvement into a self-driving cycle via LangGraph (`services/evaluations/iteration_graph.py`): `start_eval_node` → `wait_eval_node` → (conditional) `start_improve_node` → `wait_improve_node` → loops back to `start_eval_node`, or → `finalize_node`. Stop-score = mean(`Adherence to Ground Truth`, `Adherence to Prompt`) from `EvaluationRun.score["summary_scores"]` (`compute_round_scores`); `Adherence to Knowledge Base` is recorded per round for visibility only, never gates stopping. Stops on 3 consecutive rounds with ` Date: Mon, 19 Oct 2026 16:09:46 +0530 Subject: [PATCH 4/5] refactor(evaluations): Rename brief TypedDicts to Trace* TraceQuestion / TraceAttempt / TraceScoreSummary sit next to the TraceData / TraceScore vocabulary they are derived from. Co-Authored-By: Claude Fable 5.1 --- .../evaluations/prompt_improvement.py | 28 ++++++++++--------- docs/wiki/modules/evaluations.md | 2 +- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/backend/app/services/evaluations/prompt_improvement.py b/backend/app/services/evaluations/prompt_improvement.py index 85f23ffdc..d182766dd 100644 --- a/backend/app/services/evaluations/prompt_improvement.py +++ b/backend/app/services/evaluations/prompt_improvement.py @@ -649,7 +649,7 @@ def _call_prompt_drafting_llm(*, user_message_text: str) -> tuple[str, str]: ) from exc -class BriefScore(TypedDict): +class TraceScoreSummary(TypedDict): """One metric as the drafting brief sees it; data_type/verdict are UI-only.""" name: str @@ -658,32 +658,32 @@ class BriefScore(TypedDict): unscoreable: NotRequired[bool] -class BriefAttempt(TypedDict): +class TraceAttempt(TypedDict): """One run of a question: the answer it produced and how the judge scored it.""" llm_answer: str - scores: list[BriefScore] + scores: list[TraceScoreSummary] -class BriefQuestion(TypedDict): +class TraceQuestion(TypedDict): """One dataset question with every repeat (duplication factor) nested under it.""" question_id: int | str | None question: str ground_truth_answer: str - attempts: list[BriefAttempt] + attempts: list[TraceAttempt] -def _brief_score(score: TraceScore) -> BriefScore: - brief: BriefScore = {"name": score["name"], "value": score["value"]} +def _trace_score_summary(score: TraceScore) -> TraceScoreSummary: + summary: TraceScoreSummary = {"name": score["name"], "value": score["value"]} if "comment" in score: - brief["comment"] = score["comment"] + summary["comment"] = score["comment"] if "unscoreable" in score: - brief["unscoreable"] = score["unscoreable"] - return brief + summary["unscoreable"] = score["unscoreable"] + return summary -def _format_traces_for_brief(traces: list[TraceData]) -> list[BriefQuestion]: +def _format_traces_for_brief(traces: list[TraceData]) -> list[TraceQuestion]: """Group repeated attempts per question and keep only what the brief reads. A 500-row Hindi judge run blew past Claude's 1M-token context: every repeat @@ -695,7 +695,7 @@ def _format_traces_for_brief(traces: list[TraceData]) -> list[BriefQuestion]: that is the export shape (keeps trace_ids/category, scores as parallel lists) and rejects v1 traces that have no question_id. """ - grouped: dict[Any, BriefQuestion] = {} + grouped: dict[Any, TraceQuestion] = {} for trace in traces: question = trace.get("input_to_llm") or trace.get("question", "") ground_truth = trace.get("ground_truth_answer", "") @@ -715,7 +715,9 @@ def _format_traces_for_brief(traces: list[TraceData]) -> list[BriefQuestion]: { "llm_answer": trace.get("output_from_llm") or trace.get("llm_answer", ""), - "scores": [_brief_score(score) for score in trace.get("scores", [])], + "scores": [ + _trace_score_summary(score) for score in trace.get("scores", []) + ], } ) return list(grouped.values()) diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index 10c5357d8..630c34134 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -38,7 +38,7 @@ v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native jud ## Async - Provider batches polled by cron (`crud/evaluations/cron.py`); no long-lived Celery task per run. - Fast text runs (v1 cosine + v2 judge) fan out `ceil(total_items / EVAL_FAST_CHUNK_SIZE)` `run_evaluation_fast_chunk` tasks (responses only), then a cron barrier (`dispatch_fast_evaluation_barriers`) enqueues one `run_evaluation_fast_aggregate` once every chunk has a `raw_output_url`. The aggregate merges chunks, then (v2) judges **every** row in that single task — so the judge pool is sized by its own `EVAL_JUDGE_CONCURRENCY` (not the response stage's `EVAL_FAST_API_CONCURRENCY`) to clear the max dataset (`EVAL_FAST_MAX_UNIQUE_ROWS` × `duplication_factor`) under the aggregate's `CELERY_TASK_SOFT_TIME_LIMIT`. No judge fan-out / second barrier. The aggregate persists the score unit (`save_score` → traces to S3 behind `score_trace_url`, summary + overall to the DB) **before** flipping the run to `completed`: on v2 that unit is the only copy of the per-row judge scores, so a crash between the two must leave the run `processing`, never `completed` with its scores gone. Chunk-artifact cleanup and the v1 Langfuse score sync run after completion as a best-effort tail that may never fail the run. -- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. `_call_prompt_drafting_llm` keeps `json_schema` structured output (two fields, `rationale` length-capped) but raises `prompt_generation_failed` when `stop_reason != STOP_REASON_COMPLETE`, so a `max_tokens`-truncated draft fails the job instead of minting a half-written system prompt. Before the call, `_format_traces_for_brief` projects the score traces to what the brief actually reads (`BriefQuestion` / `BriefAttempt` / `BriefScore` TypedDicts in the same module) — one entry per question (`question_id`, `question`, `ground_truth_answer`) with repeated runs nested as `attempts[{llm_answer, scores[{name, value, comment, unscoreable}]}]`, dropping `trace_id`/`data_type`/`verdict`/`category`/guardrail bookkeeping; a guardrail-rewritten row substitutes `input_to_llm`/`output_from_llm` for `question`/`llm_answer` — and the JSON is dumped with `ensure_ascii=False` (escaping Devanagari inflates its token count several-fold; a 500-row Hindi run previously exceeded Claude's 1M-token context). Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. +- Prompt improvement is job-based with callback delivery: `POST /evaluations/{id}/improve-prompt` validates preconditions, enqueues a `Job` (`JobType.PROMPT_IMPROVEMENT`, `models/job.py`) run by Celery task `run_prompt_improvement` (`celery/tasks/job_execution.py`), and returns `202` with an `LLMJobImmediatePublic` handle. On finish the worker POSTs a single best-effort callback to the caller-supplied `callback_url` (SSRF-guarded via `validate_callback_url`): an `APIResponse[PromptImprovementJobPublic]` (`models/evaluation.py`) carrying the new `ConfigVersion` on success or `error_message` on failure (for an Anthropic fault `error_message` carries the provider's own response body, appended by `_anthropic_error_detail` in `services/evaluations/prompt_improvement.py`). The `ConfigVersion` is persisted regardless of callback outcome. `_call_prompt_drafting_llm` keeps `json_schema` structured output (two fields, `rationale` length-capped) but raises `prompt_generation_failed` when `stop_reason != STOP_REASON_COMPLETE`, so a `max_tokens`-truncated draft fails the job instead of minting a half-written system prompt. Before the call, `_format_traces_for_brief` projects the score traces to what the brief actually reads (`TraceQuestion` / `TraceAttempt` / `TraceScoreSummary` TypedDicts in the same module) — one entry per question (`question_id`, `question`, `ground_truth_answer`) with repeated runs nested as `attempts[{llm_answer, scores[{name, value, comment, unscoreable}]}]`, dropping `trace_id`/`data_type`/`verdict`/`category`/guardrail bookkeeping; a guardrail-rewritten row substitutes `input_to_llm`/`output_from_llm` for `question`/`llm_answer` — and the JSON is dumped with `ensure_ascii=False` (escaping Devanagari inflates its token count several-fold; a 500-row Hindi run previously exceeded Claude's 1M-token context). Celery redelivery of a `SUCCESS` job re-sends the callback without re-running the LLM. - v2 prompt iteration reuses the same job/Celery/config-version machinery (`services/evaluations/prompt_improvement.py`) and branches on `run.is_judge_run`: the v2 route calls `start_prompt_improvement_job(..., require_judge_run=True)` (a non-judge run → `422 not_a_judge_run`), and the worker drafts from the three-metric judge trace (`_draft_improved_prompt(is_judge_run=True)`, using each metric's score + reasoning) and delivers a `PromptRecommendationJobPublic` callback carrying `recommendation_type` (`Literal["prompt"]`, `models/evaluation.py`; widens to a union when knowledge-base / model recommendations land). Non-judge (v1) runs keep the default `_draft_improved_prompt` brief + `PromptImprovementJobPublic` path unchanged. - Evaluation iteration loop (`POST /evaluations/iterations`) chains fast-eval + v2 prompt improvement into a self-driving cycle via LangGraph (`services/evaluations/iteration_graph.py`): `start_eval_node` → `wait_eval_node` → (conditional) `start_improve_node` → `wait_improve_node` → loops back to `start_eval_node`, or → `finalize_node`. Stop-score = mean(`Adherence to Ground Truth`, `Adherence to Prompt`) from `EvaluationRun.score["summary_scores"]` (`compute_round_scores`); `Adherence to Knowledge Base` is recorded per round for visibility only, never gates stopping. Stops on 3 consecutive rounds with ` Date: Mon, 19 Oct 2026 18:13:34 +0530 Subject: [PATCH 5/5] refactor(evaluations): Tidy prompt-improvement brief helpers Collapse the Trace* TypedDict docstrings to one line to match TraceScore / TraceData, drop comments that restate the code, and reduce _trace_score_summary to a single key filter. Replace the `is None` guard on the downloaded trace blob with `isinstance(..., list)`, which both validates the shape and narrows the type pyright flagged at the _draft_improved_prompt call. Co-Authored-By: Claude Fable 5.1 --- .../evaluations/prompt_improvement.py | 32 +++++++------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/backend/app/services/evaluations/prompt_improvement.py b/backend/app/services/evaluations/prompt_improvement.py index d182766dd..be8d93d8e 100644 --- a/backend/app/services/evaluations/prompt_improvement.py +++ b/backend/app/services/evaluations/prompt_improvement.py @@ -8,7 +8,7 @@ import copy import json import logging -from typing import Any, NotRequired, TypedDict +from typing import NotRequired, TypedDict, cast from uuid import UUID import anthropic @@ -57,14 +57,12 @@ # Headroom for a full prompt rewrite + JSON wrapper; too low truncates into invalid JSON. _LLM_MAX_TOKENS = 16384 -# JSON keys expected in the LLM's structured response. _LLM_KEY_INSTRUCTIONS = "improved_instructions" _LLM_KEY_RATIONALE = "rationale" # Cap the rationale so it stays a one-line summary in the commit_message. _RATIONALE_MAX_LENGTH = 200 -# Structured-output schema: guarantees the response is valid JSON with these fields. _OUTPUT_SCHEMA = { "type": "object", "properties": { @@ -137,7 +135,6 @@ def validate_improve_prompt( detail=f"evaluation_not_completed: run status is '{run.status}', must be '{_COMPLETED_STATUS}'", ) if not run.score_trace_url: - # Run exists but a precondition is unmet, so 422 rather than 404. raise HTTPException( status_code=422, detail=( @@ -146,7 +143,6 @@ def validate_improve_prompt( ), ) if run.config_id is None or run.config_version is None: - # Both FKs are nullable; without them there's no version to improve. raise HTTPException( status_code=422, detail="source_config_unavailable: run has no config_id/config_version reference", @@ -409,9 +405,9 @@ def execute_prompt_improvement( traces = load_json_from_object_store( storage=storage, url=run.score_trace_url ) - if traces is None: + if not isinstance(traces, list): raise RuntimeError( - "trace_download_failed: could not retrieve trace file from storage" + "trace_download_failed: could not retrieve trace list from storage" ) improved_instructions, rationale = _draft_improved_prompt( @@ -487,8 +483,6 @@ def execute_prompt_improvement( ) raise except Exception as exc: - # HTTPException (from re-validation) carries the clean detail; everything - # else falls back to str(exc). error_message = getattr(exc, "detail", None) or str(exc) logger.error( f"[execute_prompt_improvement] Failed | job_id={job_id} " @@ -568,7 +562,6 @@ def _call_prompt_drafting_llm(*, user_message_text: str) -> tuple[str, str]: per-cause log lines are kept so operators can still triage the fault. """ if not settings.ANTHROPIC_API_KEY: - # Missing platform key is a server-side misconfiguration, not an upstream fault. raise RuntimeError( "prompt_generation_failed: the platform Anthropic key " "(ANTHROPIC_API_KEY) is not configured" @@ -650,7 +643,7 @@ def _call_prompt_drafting_llm(*, user_message_text: str) -> tuple[str, str]: class TraceScoreSummary(TypedDict): - """One metric as the drafting brief sees it; data_type/verdict are UI-only.""" + """A TraceScore without the UI-only fields (data_type, verdict).""" name: str value: float | str @@ -659,14 +652,14 @@ class TraceScoreSummary(TypedDict): class TraceAttempt(TypedDict): - """One run of a question: the answer it produced and how the judge scored it.""" + """One run of a question: its answer and scores.""" llm_answer: str scores: list[TraceScoreSummary] class TraceQuestion(TypedDict): - """One dataset question with every repeat (duplication factor) nested under it.""" + """A dataset question with one attempt per duplication-factor repeat.""" question_id: int | str | None question: str @@ -675,12 +668,10 @@ class TraceQuestion(TypedDict): def _trace_score_summary(score: TraceScore) -> TraceScoreSummary: - summary: TraceScoreSummary = {"name": score["name"], "value": score["value"]} - if "comment" in score: - summary["comment"] = score["comment"] - if "unscoreable" in score: - summary["unscoreable"] = score["unscoreable"] - return summary + return cast( + TraceScoreSummary, + {k: v for k, v in score.items() if k not in ("data_type", "verdict")}, + ) def _format_traces_for_brief(traces: list[TraceData]) -> list[TraceQuestion]: @@ -695,7 +686,7 @@ def _format_traces_for_brief(traces: list[TraceData]) -> list[TraceQuestion]: that is the export shape (keeps trace_ids/category, scores as parallel lists) and rejects v1 traces that have no question_id. """ - grouped: dict[Any, TraceQuestion] = {} + grouped: dict[int | str | tuple[str, str], TraceQuestion] = {} for trace in traces: question = trace.get("input_to_llm") or trace.get("question", "") ground_truth = trace.get("ground_truth_answer", "") @@ -812,7 +803,6 @@ def _draft_improved_prompt( f"sentence (≤ {_RATIONALE_MAX_LENGTH} characters): what you changed and why." ) target_config = _target_config_from_params(config_params) - # ensure_ascii=False: \uXXXX-escaping Devanagari inflates its token count several-fold. traces_json = json.dumps( _format_traces_for_brief(traces), ensure_ascii=False, separators=(",", ":") )