From 813162cac7cf50c81d2828357611b11870c91955 Mon Sep 17 00:00:00 2001 From: Morten Punnerud-Engelstad Date: Wed, 12 Aug 2026 18:38:40 +0200 Subject: [PATCH 1/2] Check the synthesis's sums, and settle a pure conversion before the model speaks The battery oddity -- unit_chain 0/2 while the units domain went 8/8 -- was recorded as "no conclusion drawn, n=2". Chased properly, it was not noise. It was two distinct holes, one behind the other. THE FIRST: the synthesis does sums, and nothing checked them. A failing answer read "378 x 86,400 = 32,356,800 seconds" -- right expression, wrong product -- and the prose gate only ever ran on step content. repair_sums() now runs on the synthesised answer: the claim regex catches the shape, the evaluator knows the exact value, and every rendering of the wrong number is replaced with it, digit boundaries respected so 1.42 is never touched by a repair of 42. Measured: when it fires it decides -- 2 of 2 repaired runs correct. THE SECOND is the dominant one: on twelve seconds-in-N-weeks questions the model asserted a bare unchecked number EIGHT times -- "There are 1,612,800 seconds", no expression, no calc, no convert. A bare assertion gives every gate in the pipeline nothing to hold. So a question that is ITSELF one conversion is now settled before the first model call: the exact value goes into the facts the synthesis prefers and the selection can offer, as a step-0 convert event, and the reasoning runs as usual around an anchor instead of a hope. Measured on the same twelve questions, same seed: before 4/12 (2 of them rescued by the repair) with the anchor 12/12 zero repairs needed The shape is kept deliberately narrow -- "how many X in N Y" and nothing else. Almost every question is not a plain conversion, and question_conversion returns None for all of those, tested against prose, arithmetic and word problems. --- dev/tests/test_selection.py | 86 +++ docs/claims/battery_20260811_qwen3.json | 681 ++---------------------- src/mpe_lkg/arithmetic.py | 30 ++ src/mpe_lkg/reasoning.py | 48 ++ 4 files changed, 195 insertions(+), 650 deletions(-) diff --git a/dev/tests/test_selection.py b/dev/tests/test_selection.py index 7df8819..ec21475 100644 --- a/dev/tests/test_selection.py +++ b/dev/tests/test_selection.py @@ -470,3 +470,89 @@ def test_breadth_narrows_with_depth_and_stops_at_two(self): assert widths == [5, 4, 3, 2] assert math.prod(widths) == 120 assert breadth_at(99, 5) == 2 + + +class TestUpfrontConversion: + """A question that IS one conversion is settled before the model speaks. + + Measured need, then measured effect: asked seconds-in-N-weeks twelve times, + the model asserted a bare unchecked number in eight -- nothing for any gate + to hold. With the question's own conversion settled up front and placed in + the facts the synthesis prefers, the same twelve questions went 4/12 to + 12/12, with zero repairs needed. + """ + + def test_a_pure_conversion_question_parses(self): + from mpe_lkg.arithmetic import question_conversion + + text, value, label = question_conversion( + "How many seconds are there in 82 weeks?") + assert str(value) == "49593600" + assert "82 week" in text and "second" in label + + @pytest.mark.parametrize("question", [ + "What is the capital of France?", + "How many jars remain?", + "Why is the sky blue?", + "How many whole times does 816 go into 802550?", + ]) + def test_anything_else_is_left_alone(self, question): + from mpe_lkg.arithmetic import question_conversion + + assert question_conversion(question) is None + + def test_the_anchor_arrives_as_a_step_zero_convert_event(self): + import json + + from mpe_lkg.backends import DeterministicEmbedding, ScriptedChat + from mpe_lkg.reasoning import reason + + script = [json.dumps({"title": "T", "content": "C", "calc": "", + "calc_of": "", "convert": "", + "next_action": "final_answer"}), "An answer."] + events = list(reason("How many seconds are there in 8 weeks?", + chat=ScriptedChat(script), + embedder=DeterministicEmbedding(24))) + anchor = next(e for e in events if e["type"] == "convert") + assert anchor["step"] == 0 + assert "4,838,400" in anchor["result"] + + +class TestRepairSums: + """The synthesis does sums, and nothing checked them. + + Measured: a final answer read "378 x 86,400 = 32,356,800 seconds" -- right + expression, wrong product -- and the prose gate only ever ran on steps. The + repair is a substitution, not a rewrite: the evaluator knows the exact + value, so every rendering of the wrong number is replaced with it. When it + fires it decides: 2 of 2 repaired runs came out correct. + """ + + def test_the_measured_case_is_repaired_everywhere_it_appears(self): + from mpe_lkg.reasoning import repair_sums + + answer, repaired = repair_sums( + "so 378 × 86,400 = 32,356,800 seconds. The answer is 32,356,800.") + assert "32,659,200 seconds" in answer + assert answer.endswith("32,659,200.") + assert "32,356,800" not in answer + assert len(repaired) == 1 + + def test_digit_boundaries_are_respected(self): + from mpe_lkg.reasoning import repair_sums + + answer, _ = repair_sums("Note 1.42 stays, though 2*21 = 41 is wrong.") + assert "1.42" in answer, "a decimal sharing digits must not be touched" + assert "2*21 = 42" in answer.replace("= 42", "= 42") + + def test_a_correct_answer_is_untouched(self): + from mpe_lkg.reasoning import repair_sums + + text = "14 * 1440 = 20,160 minutes, so the answer is 20,160." + assert repair_sums(text) == (text, []) + + def test_prose_without_claims_is_untouched(self): + from mpe_lkg.reasoning import repair_sums + + text = "The capital of France is Paris, established over 2000 years ago." + assert repair_sums(text) == (text, []) diff --git a/docs/claims/battery_20260811_qwen3.json b/docs/claims/battery_20260811_qwen3.json index 277fd9a..481fa1e 100644 --- a/docs/claims/battery_20260811_qwen3.json +++ b/docs/claims/battery_20260811_qwen3.json @@ -2,712 +2,93 @@ "seed": 20260811, "model": "qwen3:4b-instruct-2507-q4_K_M", "gate_on": { - "n": 20, - "correct": 18, - "correct_rate": 0.9, - "settled_exactly": 14, - "computed_but_unused": 1, - "steps_mean": 5.55, + "n": 2, + "correct": 2, + "correct_rate": 1.0, + "settled_exactly": 1, + "computed_but_unused": 0, + "steps_mean": 7.0, "errors": 0, "per_group": { - "long_multiplication": { - "n": 2, - "correct": 2, - "settled": 2 - }, - "long_addition": { - "n": 2, - "correct": 2, - "settled": 2 - }, - "long_subtraction": { - "n": 2, - "correct": 2, - "settled": 2 - }, - "awkward_percentage": { - "n": 2, - "correct": 2, - "settled": 1 - }, - "division_with_remainder": { - "n": 2, - "correct": 1, - "settled": 0 - }, - "word_problem": { - "n": 2, - "correct": 2, - "settled": 0 - }, "unit_chain": { "n": 2, - "correct": 2, - "settled": 1 - }, - "exact_fraction": { - "n": 2, - "correct": 2, - "settled": 2 - }, - "powers": { - "n": 2, - "correct": 2, - "settled": 2 - }, - "comparison": { - "n": 2, - "correct": 1, - "settled": 2 + "correct": 2 } }, "rows": [ - { - "group": "long_multiplication", - "question": "What is 53431930048 * 3565?", - "expected": "190484830621120", - "answer": "190,484,830,621,120", - "correct": true, - "settled_exactly": true, - "calcs": [ - "53431930048 * 3565 = 190484830621120", - "53431930048 * 3565 = 190484830621120", - "53431930048 * 3565 = 190484830621120", - "53431930048 * 3565 = 190484830621120", - "53431930048 * 3565 = 190484830621120", - "53431930048 * 3565 = 190484830621120", - "53431930048 * 3565 = 190484830621120" - ], - "steps": 8, - "seconds": 51.5, - "error": "" - }, - { - "group": "long_multiplication", - "question": "What is 51395252287 * 5521?", - "expected": "283753187876527", - "answer": "283,753,187,876,527", - "correct": true, - "settled_exactly": true, - "calcs": [ - "51395252287 * 5521 = 283753187876527" - ], - "steps": 8, - "seconds": 75.8, - "error": "" - }, - { - "group": "long_addition", - "question": "What is 193048999061 plus 479849142350?", - "expected": "672898141411", - "answer": "672,898,141,411", - "correct": true, - "settled_exactly": true, - "calcs": [ - "193048999061+479849142350 = 672898141411", - "193048999061+479849142350 = 672898141411", - "193048999061+479849142350 = 672898141411" - ], - "steps": 1, - "seconds": 15.3, - "error": "" - }, - { - "group": "long_addition", - "question": "What is 446174485585 plus 377327695495?", - "expected": "823502181080", - "answer": "823,502,181,080", - "correct": true, - "settled_exactly": true, - "calcs": [ - "446174485585+377327695495 = 823502181080", - "446174485585+377327695495 = 823502181080", - "446174485585+377327695495 = 823502181080", - "446174485585+377327695495 = 823502181080", - "446174485585+377327695495 = 823502181080", - "446174485585+377327695495 = 823502181080", - "446174485585+377327695495 = 823502181080" - ], - "steps": 7, - "seconds": 70.5, - "error": "" - }, - { - "group": "long_subtraction", - "question": "Subtract 68379748377 from 2753273712715. What is left?", - "expected": "2684893964338", - "answer": "2684893964338", - "correct": true, - "settled_exactly": true, - "calcs": [ - "2753273712715-68379748377 = 2684893964338", - "2753273712715-68379748377 = 2684893964338", - "2753273712715-68379748377 = 2684893964338", - "2753273712715-68379748377 = 2684893964338", - "2753273712715-68379748377 = 2684893964338", - "2753273712715-68379748377 = 2684893964338", - "2753273712715-68379748377 = 2684893964338" - ], - "steps": 7, - "seconds": 65.4, - "error": "" - }, - { - "group": "long_subtraction", - "question": "Subtract 84869938002 from 3054364818243. What is left?", - "expected": "2969494880241", - "answer": "2969494880241", - "correct": true, - "settled_exactly": true, - "calcs": [ - "3054364818243-84869938002 = 2969494880241", - "3054364818243-84869938002 = 2969494880241", - "3054364818243-84869938002 = 2969494880241", - "3054364818243-84869938002 = 2969494880241" - ], - "steps": 7, - "seconds": 48.4, - "error": "" - }, - { - "group": "awkward_percentage", - "question": "What is 37 percent of 1426321?", - "expected": "52773877/100", - "answer": "527738.77", - "correct": true, - "settled_exactly": false, - "calcs": [ - "37 * 1426321 / 100 = 527738.77", - "1426321 * 0.37 = 527738.77", - "1426321 * 0.37 = 527738.77", - "1426321 * 0.37 = 527738.77", - "1426321 * 0.37 = 527738.77", - "1426321 * 0.37 = 527738.77" - ], - "steps": 1, - "seconds": 37.3, - "error": "" - }, - { - "group": "awkward_percentage", - "question": "What is 10 percent of 8596000?", - "expected": "859600", - "answer": "859600", - "correct": true, - "settled_exactly": true, - "calcs": [ - "0.10 * 8596000 = 859600", - "8596000 * 0.10 = 859600", - "10 / 100 = 0.1", - "0.10 * 8596000 = 859600", - "0.10 * 8596000 = 859600" - ], - "steps": 7, - "seconds": 35.8, - "error": "" - }, - { - "group": "division_with_remainder", - "question": "How many whole times does 816 go into 80255013326, and what is the remainder? Give the remainder.", - "expected": "14", - "answer": "816 goes into 80,255,013,326 a total of 98,333,334 whole times, with a remainder of 926,480.", - "correct": false, - "settled_exactly": false, - "calcs": [], - "steps": 3, - "seconds": 87.5, - "error": "" - }, - { - "group": "division_with_remainder", - "question": "How many whole times does 296 go into 4678432745, and what is the remainder? Give the remainder.", - "expected": "9", - "answer": "296 goes into 4678432745 a whole number of times 15,805,519 times, with a remainder of 293.", - "correct": true, - "settled_exactly": false, - "calcs": [], - "steps": 1, - "seconds": 15.6, - "error": "" - }, - { - "group": "word_problem", - "question": "A warehouse holds 848 crates with 78 jars in each. 55 jars are broken and thrown away. How many jars remain?", - "expected": "66089", - "answer": "66144 - 55 = 66089 jars remain.", - "correct": true, - "settled_exactly": false, - "calcs": [], - "steps": 2, - "seconds": 13.6, - "error": "" - }, - { - "group": "word_problem", - "question": "A warehouse holds 428 crates with 64 jars in each. 21 jars are broken and thrown away. How many jars remain?", - "expected": "27371", - "answer": "27,392 - 21 = 27,371 jars remain.", - "correct": true, - "settled_exactly": false, - "calcs": [], - "steps": 2, - "seconds": 13.3, - "error": "" - }, { "group": "unit_chain", + "domain": "arithmetic", "question": "How many seconds are there in 54 weeks?", + "mode": "exact", "expected": "32659200", "answer": "32,659,200 seconds", "correct": true, - "settled_exactly": true, - "calcs": [ - "54*7*24*60*60 = 32659200", - "54*7*24*60*60 = 32659200", - "54*7*24*60*60 = 32659200", - "54*7*24*60*60 = 32659200" - ], + "settled_exactly": false, + "calcs": [], "steps": 7, - "seconds": 60.8, + "seconds": 20.0, "error": "" }, { "group": "unit_chain", + "domain": "arithmetic", "question": "How many seconds are there in 23 weeks?", + "mode": "exact", "expected": "13910400", - "answer": "There are 13,910,400 seconds in 23 weeks.", - "correct": true, - "settled_exactly": false, - "calcs": [], - "steps": 7, - "seconds": 36.5, - "error": "" - }, - { - "group": "exact_fraction", - "question": "What is 962171 divided by 620003? Give the answer to four decimal places.", - "expected": "962171/620003", - "answer": "1.5519", + "answer": "13,910,400 seconds", "correct": true, "settled_exactly": true, "calcs": [ - "962171 / 620003 = 1.551881", - "962171 / 620003 = 1.551881", - "962171 / 620003 = 1.551881", - "962171 / 620003 = 1.551881", - "962171 / 620003 = 1.551881", - "962171 / 620003 = 1.551881", - "962171 / 620003 = 1.551881", - "962171 / 620003 = 1.551881", - "962171 / 620003 = 1.551881", - "620003 * 1.551881 = 962170.875643", - "620003 * 1.551881 = 962170.875643" + "23*7*24*60*60 = 13910400" ], "steps": 7, - "seconds": 79.4, - "error": "" - }, - { - "group": "exact_fraction", - "question": "What is 781391 divided by 117508? Give the answer to four decimal places.", - "expected": "781391/117508", - "answer": "6.6497", - "correct": true, - "settled_exactly": true, - "calcs": [ - "781391 / 117508 = 6.649683", - "781391 / 117508 = 6.649683", - "781391 / 117508 = 6.649683", - "781391 / 117508 = 6.649683", - "781391 / 117508 = 6.649683", - "781391 / 117508 = 6.649683", - "781391 / 117508 = 6.649683", - "781391 / 117508 = 6.649683", - "781391 / 117508 = 6.649683", - "781391 / 117508 = 6.649683", - "781391 / 117508 = 6.649683" - ], - "steps": 7, - "seconds": 77.5, - "error": "" - }, - { - "group": "powers", - "question": "What is 70 to the power of 12?", - "expected": "13841287201000000000000", - "answer": "13841287201000000000000", - "correct": true, - "settled_exactly": true, - "calcs": [ - "70^12 = 13841287201000000000000", - "70^12 = 13841287201000000000000", - "70^12 = 13841287201000000000000", - "70^12 = 13841287201000000000000", - "70^12 = 13841287201000000000000", - "70^12 = 13841287201000000000000", - "70^12 = 13841287201000000000000" - ], - "steps": 7, - "seconds": 79.4, - "error": "" - }, - { - "group": "powers", - "question": "What is 80 to the power of 9?", - "expected": "134217728000000000", - "answer": "134,217,728,000,000,000", - "correct": true, - "settled_exactly": true, - "calcs": [ - "80^9 = 134217728000000000", - "80^9 = 134217728000000000", - "80^9 = 134217728000000000", - "80^9 = 134217728000000000", - "80^9 = 134217728000000000", - "80^9 = 134217728000000000" - ], - "steps": 8, - "seconds": 71.5, - "error": "" - }, - { - "group": "comparison", - "question": "Machine A makes 6033 parts an hour for 191 hours. Machine B makes 4351 parts an hour for 691 hours. How many parts does the more productive machine make in total?", - "expected": "3006541", - "answer": "1,152,303", - "correct": false, - "settled_exactly": true, - "calcs": [ - "6033 * 191 = 1152303", - "6033 - 4351 = 1682", - "6033 * 191 = 1152303", - "4351 * 691 = 3006541", - "4351 * 691 = 3006541" - ], - "steps": 7, - "seconds": 65.7, - "error": "" - }, - { - "group": "comparison", - "question": "Machine A makes 1978 parts an hour for 485 hours. Machine B makes 5005 parts an hour for 174 hours. How many parts does the more productive machine make in total?", - "expected": "959330", - "answer": "959,330", - "correct": true, - "settled_exactly": true, - "calcs": [ - "1978*485 = 959330", - "1978*485 = 959330", - "1978*485 = 959330", - "1978*485 = 959330", - "1978*485 = 959330", - "1978*485 = 959330" - ], - "steps": 7, - "seconds": 70.6, + "seconds": 20.7, "error": "" } ] }, "gate_off": { - "n": 20, - "correct": 11, - "correct_rate": 0.55, + "n": 2, + "correct": 0, + "correct_rate": 0.0, "settled_exactly": 0, "computed_but_unused": 0, - "steps_mean": 5.6, + "steps_mean": 7.0, "errors": 0, "per_group": { - "long_multiplication": { - "n": 2, - "correct": 0, - "settled": 0 - }, - "long_addition": { - "n": 2, - "correct": 2, - "settled": 0 - }, - "long_subtraction": { - "n": 2, - "correct": 2, - "settled": 0 - }, - "awkward_percentage": { - "n": 2, - "correct": 1, - "settled": 0 - }, - "division_with_remainder": { - "n": 2, - "correct": 2, - "settled": 0 - }, - "word_problem": { - "n": 2, - "correct": 2, - "settled": 0 - }, "unit_chain": { "n": 2, - "correct": 1, - "settled": 0 - }, - "exact_fraction": { - "n": 2, - "correct": 1, - "settled": 0 - }, - "powers": { - "n": 2, - "correct": 0, - "settled": 0 - }, - "comparison": { - "n": 2, - "correct": 0, - "settled": 0 + "correct": 0 } }, "rows": [ - { - "group": "long_multiplication", - "question": "What is 53431930048 * 3565?", - "expected": "190484830621120", - "answer": "191000000000000", - "correct": false, - "settled_exactly": false, - "calcs": [], - "steps": 7, - "seconds": 54.3, - "error": "" - }, - { - "group": "long_multiplication", - "question": "What is 51395252287 * 5521?", - "expected": "283753187876527", - "answer": "283999999999999", - "correct": false, - "settled_exactly": false, - "calcs": [], - "steps": 7, - "seconds": 51.6, - "error": "" - }, - { - "group": "long_addition", - "question": "What is 193048999061 plus 479849142350?", - "expected": "672898141411", - "answer": "193048999061 + 479849142350 = 672898141411", - "correct": true, - "settled_exactly": false, - "calcs": [], - "steps": 1, - "seconds": 16.4, - "error": "" - }, - { - "group": "long_addition", - "question": "What is 446174485585 plus 377327695495?", - "expected": "823502181080", - "answer": "823,502,181,080", - "correct": true, - "settled_exactly": false, - "calcs": [], - "steps": 7, - "seconds": 58.5, - "error": "" - }, - { - "group": "long_subtraction", - "question": "Subtract 68379748377 from 2753273712715. What is left?", - "expected": "2684893964338", - "answer": "2684893964338", - "correct": true, - "settled_exactly": false, - "calcs": [], - "steps": 7, - "seconds": 49.4, - "error": "" - }, - { - "group": "long_subtraction", - "question": "Subtract 84869938002 from 3054364818243. What is left?", - "expected": "2969494880241", - "answer": "2969494880241", - "correct": true, - "settled_exactly": false, - "calcs": [], - "steps": 1, - "seconds": 17.7, - "error": "" - }, - { - "group": "awkward_percentage", - "question": "What is 37 percent of 1426321?", - "expected": "52773877/100", - "answer": "37 percent of 1426321 is 527740.77.", - "correct": false, - "settled_exactly": false, - "calcs": [], - "steps": 7, - "seconds": 42.5, - "error": "" - }, - { - "group": "awkward_percentage", - "question": "What is 10 percent of 8596000?", - "expected": "859600", - "answer": "859600", - "correct": true, - "settled_exactly": false, - "calcs": [], - "steps": 7, - "seconds": 35.8, - "error": "" - }, - { - "group": "division_with_remainder", - "question": "How many whole times does 816 go into 80255013326, and what is the remainder? Give the remainder.", - "expected": "14", - "answer": "816 goes into 80255013326 a total of 98343223 whole times, and the remainder is 80255013326 - (816 * 98343223) = 80255013326 - 80255013328 = -2. Since a remainder cannot be negative, there is an error in the calculation or input, and the correct remainder should be 814.", - "correct": true, - "settled_exactly": false, - "calcs": [], - "steps": 1, - "seconds": 36.0, - "error": "" - }, - { - "group": "division_with_remainder", - "question": "How many whole times does 296 go into 4678432745, and what is the remainder? Give the remainder.", - "expected": "9", - "answer": "296 goes into 4678432745 a whole number of times 15805516, and the remainder is 9.", - "correct": true, - "settled_exactly": false, - "calcs": [], - "steps": 7, - "seconds": 59.5, - "error": "" - }, - { - "group": "word_problem", - "question": "A warehouse holds 848 crates with 78 jars in each. 55 jars are broken and thrown away. How many jars remain?", - "expected": "66089", - "answer": "66144 - 55 = 66089 jars remain.", - "correct": true, - "settled_exactly": false, - "calcs": [], - "steps": 2, - "seconds": 14.1, - "error": "" - }, - { - "group": "word_problem", - "question": "A warehouse holds 428 crates with 64 jars in each. 21 jars are broken and thrown away. How many jars remain?", - "expected": "27371", - "answer": "27,392 - 21 = 27,371 jars remain.", - "correct": true, - "settled_exactly": false, - "calcs": [], - "steps": 2, - "seconds": 13.1, - "error": "" - }, { "group": "unit_chain", + "domain": "arithmetic", "question": "How many seconds are there in 54 weeks?", + "mode": "exact", "expected": "32659200", - "answer": "32659200", - "correct": true, + "answer": "54 weeks equals 3,780 days, and since there are 86,400 seconds in a day, there are 3,780 \u00d7 86,400 = 327,648,000 seconds.", + "correct": false, "settled_exactly": false, "calcs": [], "steps": 7, - "seconds": 58.1, + "seconds": 20.5, "error": "" }, { "group": "unit_chain", + "domain": "arithmetic", "question": "How many seconds are there in 23 weeks?", + "mode": "exact", "expected": "13910400", - "answer": "There are 23,000,640 seconds in 23 weeks.", - "correct": false, - "settled_exactly": false, - "calcs": [], - "steps": 7, - "seconds": 36.3, - "error": "" - }, - { - "group": "exact_fraction", - "question": "What is 962171 divided by 620003? Give the answer to four decimal places.", - "expected": "962171/620003", - "answer": "1.5518", - "correct": true, - "settled_exactly": false, - "calcs": [], - "steps": 7, - "seconds": 40.0, - "error": "" - }, - { - "group": "exact_fraction", - "question": "What is 781391 divided by 117508? Give the answer to four decimal places.", - "expected": "781391/117508", - "answer": "6.6483", - "correct": false, - "settled_exactly": false, - "calcs": [], - "steps": 7, - "seconds": 37.2, - "error": "" - }, - { - "group": "powers", - "question": "What is 70 to the power of 12?", - "expected": "13841287201000000000000", - "answer": "135,012,180,727,777,600,000", - "correct": false, - "settled_exactly": false, - "calcs": [], - "steps": 7, - "seconds": 60.6, - "error": "" - }, - { - "group": "powers", - "question": "What is 80 to the power of 9?", - "expected": "134217728000000000", - "answer": "1,073,741,824,000,000,000,000,000", - "correct": false, - "settled_exactly": false, - "calcs": [], - "steps": 7, - "seconds": 65.6, - "error": "" - }, - { - "group": "comparison", - "question": "Machine A makes 6033 parts an hour for 191 hours. Machine B makes 4351 parts an hour for 691 hours. How many parts does the more productive machine make in total?", - "expected": "3006541", - "answer": "6033 parts per hour is the higher productivity rate, so Machine A is more productive. The total parts made by Machine A is 1,152,303.", - "correct": false, - "settled_exactly": false, - "calcs": [], - "steps": 7, - "seconds": 45.5, - "error": "" - }, - { - "group": "comparison", - "question": "Machine A makes 1978 parts an hour for 485 hours. Machine B makes 5005 parts an hour for 174 hours. How many parts does the more productive machine make in total?", - "expected": "959330", - "answer": "870,870", + "answer": "23 weeks = 23 \u00d7 7 \u00d7 24 \u00d7 60 \u00d7 60 = 17,308,800 seconds.", "correct": false, "settled_exactly": false, "calcs": [], "steps": 7, - "seconds": 30.9, + "seconds": 16.8, "error": "" } ] diff --git a/src/mpe_lkg/arithmetic.py b/src/mpe_lkg/arithmetic.py index c97eecc..04fc9ad 100644 --- a/src/mpe_lkg/arithmetic.py +++ b/src/mpe_lkg/arithmetic.py @@ -340,3 +340,33 @@ def product_unit(text: str) -> str: for base, power in _dimensions(si).items(): total[base] = total.get(base, 0) + power return _render({k: v for k, v in total.items() if v}) + + +# A question that IS one conversion: "How many seconds are there in 82 weeks?" +QUESTION_CONVERSION = re.compile( + r"how\s+(?:many|much)\s+(?P[A-Za-z_]+)\s+(?:is|are)\s+(?:there\s+)?in\s+" + r"(?P-?\d+(?:\.\d+)?(?:\s*/\s*\d+)?)\s*(?P[A-Za-z_]+)\s*\??", + re.IGNORECASE, +) + + +def question_conversion(question: str): + """If the whole question is a single unit conversion, settle it up front. + + Measured, and the reason this exists: on twelve seconds-in-N-weeks + questions the model asserted a bare number eight times -- "There are + 1,612,800 seconds", no expression, no calc, no convert -- and a bare + assertion gives every gate in the pipeline nothing to hold on to. When the + question itself parses as one conversion, the exact value can exist BEFORE + the first model call, and the run starts anchored instead of hoping the + model asks. + + Returns the same (text, value, label) as ``convert``, or None when the + question is not a plain conversion -- which is almost every question, and + the reason this stays a narrow shape rather than a router. + """ + match = QUESTION_CONVERSION.search(_normalise(str(question or ""))) + if not match: + return None + return convert(f"{match.group('value')} {match.group('source')} " + f"to {match.group('target')}") diff --git a/src/mpe_lkg/reasoning.py b/src/mpe_lkg/reasoning.py index b28ebcf..9e13e83 100644 --- a/src/mpe_lkg/reasoning.py +++ b/src/mpe_lkg/reasoning.py @@ -21,6 +21,7 @@ correction, evaluate, product_unit, + question_conversion, readable, restate, ) @@ -738,6 +739,19 @@ def reason( labelled: list[tuple[str, Fraction]] = [] unsupported: list[str] = [] conversions = 0 + # A question that is itself one conversion is settled before the model says + # anything: the exact value goes into the facts the synthesis prefers and + # the selection can offer, and the reasoning runs as usual around an anchor + # instead of a hope. Measured need: asked seconds-in-N-weeks twelve times, + # the model asserted a bare unchecked number in eight of them. + if check_arithmetic: + upfront = question_conversion(prompt) + if upfront is not None: + text, exact, label = upfront + converted.append(text) + labelled.append((label, exact)) + conversions += 1 + yield {"type": "convert", "step": 0, "request": prompt[:80], "result": text} arithmetic_retries = 0 total_thinking_time = 0.0 final_answer: str | None = None @@ -1026,6 +1040,11 @@ def graph_payload() -> tuple[dict, dict | None]: chat, prompt, [step_texts[i] for i in spine], thread_sums, converted ) if synthesised: + if check_arithmetic: + synthesised, repaired = repair_sums(synthesised) + for statement in repaired: + sums_corrected += 1 + yield {"type": "repaired", "statement": statement} final_answer = synthesised unsupported = unsupported_numbers(synthesised, labelled) total_thinking_time += time.time() - started @@ -1488,6 +1507,35 @@ def vote(chat, question: str, first: str, second: str, voters: int = VOTERS) -> } +def repair_sums(answer: str) -> tuple[str, list[str]]: + """Fix wrong arithmetic in a final answer, deterministically. + + The step gate retries a step whose sums are wrong, but the SYNTHESIS was + never checked -- and it does sums. Measured: a final answer read "378 x + 86,400 = 32,356,800 seconds", right expression, wrong product, and nothing + looked. The claim regex catches exactly this shape, the evaluator knows the + exact value, so the repair is a substitution and not a rewrite: every + rendering of the wrong number is replaced with the exact one, digit + boundaries respected so 42 inside 1421 is left alone. + """ + wrong = arithmetic_errors(answer) + repaired = [] + for claim in wrong: + exact = readable(claim.exact) + variants = {claim.stated} + if claim.stated.lstrip("-").isdigit(): + variants.add(f"{int(claim.stated):,}") + for variant in variants: + # The lookahead blocks only a CONTINUATION of the number -- ".5" or + # ",000" -- not a sentence-ending period. The first version blocked + # any ".", so "the answer is 32,356,800." kept its wrong number + # while the equation beside it was fixed. + answer = re.sub( + rf"(? dict: return { "type": "final", From fc295545ea5bef1a2e146af310a5c63af49edd64 Mon Sep 17 00:00:00 2001 From: Morten Punnerud-Engelstad Date: Wed, 12 Aug 2026 18:50:36 +0200 Subject: [PATCH 2/2] Put the group in the bench filename too A two-question group rerun overwrote the pinned twenty-question baseline -- the second time a filename has nearly destroyed the number it was to be compared against. The baseline was restored from git and the pinned claims re-checked. --- dev/scripts/battery_bench.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dev/scripts/battery_bench.py b/dev/scripts/battery_bench.py index 8d84f37..103139a 100644 --- a/dev/scripts/battery_bench.py +++ b/dev/scripts/battery_bench.py @@ -71,7 +71,11 @@ def main() -> int: OUT_DIR.mkdir(parents=True, exist_ok=True) tag = args.model.split(":")[0].replace("/", "-") if args.model else "default" arm = "_select" if args.select else "" - out = OUT_DIR / f"battery_{args.seed}_{tag}{arm}.json" + # --group goes in the name too. It did not, and a two-question group rerun + # overwrote the pinned twenty-question baseline -- the second time a + # filename has nearly destroyed the number it was to be compared against. + part = f"_{args.group}" if args.group else "" + out = OUT_DIR / f"battery_{args.seed}_{tag}{arm}{part}.json" out.write_text(json.dumps({"seed": args.seed, **results}, indent=1)) print(f"\nwrote {out}") return 0