From 0fdd3a176a015798babab2945365e3d42b29bdf5 Mon Sep 17 00:00:00 2001 From: RainieLLM Date: Sat, 17 Oct 2026 22:33:58 -0700 Subject: [PATCH] Preserve verifier errors during output normalization --- evaluation/run_task.py | 2 +- evaluation/tests/test_verifier_error_status.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/evaluation/run_task.py b/evaluation/run_task.py index 4130d1f3..c88acdcc 100644 --- a/evaluation/run_task.py +++ b/evaluation/run_task.py @@ -44,7 +44,7 @@ # task-specific data never mix at the top level. _UNIVERSAL_KEYS = {"reward", "score", "detail", "kind", "passed", "persona", "criteria", "checks", "verdict", "passed_count", "total_checks", - "points", "max_points"} + "points", "max_points", "error", "status", "error_stage", "stage"} def _normalize_output(findings: dict, evaluator: str | None) -> dict: diff --git a/evaluation/tests/test_verifier_error_status.py b/evaluation/tests/test_verifier_error_status.py index 38b53ea5..0a1e9364 100644 --- a/evaluation/tests/test_verifier_error_status.py +++ b/evaluation/tests/test_verifier_error_status.py @@ -38,6 +38,8 @@ def test_scored_results_are_not_errors(self) -> None: "checks": [{"verdict": "HELD"}, {"verdict": "VIOLATED"}, {"verdict": "NOT_ADDRESSED"}]}, ): self.assertIsNone(run_task._verifier_reported_error(findings), findings) + normalized = run_task._normalize_output(findings, "rule-based") + self.assertIsNone(run_task._verifier_reported_error(normalized), normalized) def test_error_shapes_verifiers_write_are_errors(self) -> None: for findings in ( @@ -52,6 +54,20 @@ def test_error_shapes_verifiers_write_are_errors(self) -> None: ): self.assertIsNotNone(run_task._verifier_reported_error(findings), findings) + def test_normalization_preserves_verifier_errors(self) -> None: + for findings in ( + {"reward": 0.0, "stage": "verify", "error": "invalid app output"}, + {"reward": 0.0, "status": "judge_error", "detail": "judge failed"}, + {"reward": 0.0, "status": "configuration_error"}, + {"reward": 0.0, "error_stage": "judge_call", "detail": "judge failed"}, + ): + with self.subTest(findings=findings): + normalized = run_task._normalize_output(findings, "rule-based") + self.assertEqual(run_task._verifier_reported_error(normalized), + run_task._verifier_reported_error(findings)) + if "stage" in findings: + self.assertEqual(normalized.get("stage"), findings["stage"]) + class ReportMaxPointsTest(unittest.TestCase): def test_missing_envelope_takes_denominator_from_score(self) -> None: