From d4c8b2200d9b21fb7631d8d57f1b51f248a36cdf Mon Sep 17 00:00:00 2001 From: Morten Punnerud-Engelstad Date: Wed, 12 Aug 2026 19:11:12 +0200 Subject: [PATCH] A physics domain: light, wind and speed, with every constant stated The battery's rule is that the truth is computed by the same code that renders the question, and physics threatens the rule because it leans on constants. The resolution: every constant is IN the question text. The model is never asked to remember the speed of light or the distance to Mars -- it is asked to compose figures it has been handed, across units, which is the failure mode this project has measured models into and built machinery for. Four groups: light travel time to the Moon, the Sun, Mars and Jupiter; wind distance in mixed units; time around the equator; mph to metres per second. Answers are exact Fractions; most do not terminate as decimals, and matches() grades those at the places a sensible answer states. The tests re-derive every answer from the question's own text, and that discipline caught the author twice before any model was graded: both mile-based answers were a factor of a thousand off, because MILE_KM is already kilometres per mile and the generator multiplied by 1000 again. Two test assumptions were also wrong where the code was right -- "One mile" is spelled out, not a digit, and a test that waits for the RNG to draw the Moon tests the RNG. Measured live against qwen3: 8/8, pinned. --- dev/scripts/check_numbers.py | 11 ++ dev/tests/test_battery_domains.py | 72 +++++++ .../battery_physics_20260811_qwen3.json | 184 ++++++++++++++++++ src/mpe_lkg/battery/__init__.py | 2 +- src/mpe_lkg/battery/physics.py | 101 ++++++++++ 5 files changed, 369 insertions(+), 1 deletion(-) create mode 100644 docs/claims/battery_physics_20260811_qwen3.json create mode 100644 src/mpe_lkg/battery/physics.py diff --git a/dev/scripts/check_numbers.py b/dev/scripts/check_numbers.py index 6c3d4d2..eed556c 100644 --- a/dev/scripts/check_numbers.py +++ b/dev/scripts/check_numbers.py @@ -45,6 +45,7 @@ QWEN_REPLICATION = "docs/claims/battery_555_qwen3.json" LLAMA_REPLICATION = "docs/claims/battery_555_llama3.2.json" FULL_BATTERY = "docs/claims/battery_full_20260811_qwen3.json" +PHYSICS = "docs/claims/battery_physics_20260811_qwen3.json" # (file, extractor, expected, tolerance, label) # @@ -484,6 +485,16 @@ 0.001, "the consistency caveat is attached to the data itself", ), + ( + PHYSICS, + # Constants stated in the question, so the task is composition across + # units rather than recall -- the exact failure mode this project has + # built machinery for, and the machinery carries it. + lambda d: d["correct_rate"], + 1.0, + 0.25, + "physics composition with stated constants is answered nearly clean", + ), ( QWEN_SELECT, # And it does the job it exists for: a value computed exactly and then not diff --git a/dev/tests/test_battery_domains.py b/dev/tests/test_battery_domains.py index aa3f8d2..dcf5268 100644 --- a/dev/tests/test_battery_domains.py +++ b/dev/tests/test_battery_domains.py @@ -146,3 +146,75 @@ def test_text_questions_grade_by_expectation(self): expect=("neither", "the same", "equal")) assert q.matches("They weigh the same.") is True assert q.matches("The lead, obviously.") is False + + +class TestPhysics: + """Every answer re-derived from the question's own text. + + The physics domain leans on constants, and the rule that makes it safe is + that every constant is STATED in the question -- so these tests parse the + figures back out of the text and recompute the answer from nothing else. + The very first version of this domain failed this test twice: both + mile-based answers were a factor of a thousand off, because MILE_KM is + already kilometres per mile and the generator multiplied by 1000 again. + A constant embedded twice is exactly the mistake this style of test exists + to catch before a model is ever graded against it. + """ + + def _numbers(self, text): + return [Fraction(n.replace(",", "")) + for n in re.findall(r"\d[\d,]*(?:\.\d+)?", text)] + + @pytest.mark.parametrize("question", + [q for q in build(per_group=3, domains=["physics"]) + if q.group == "light_travel"], + ids=lambda q: q.text[60:90]) + def test_light_travel_from_the_stated_figures(self, question): + speed, distance = self._numbers(question.text)[:2] + assert question.answer == distance * 1000 / speed + + @pytest.mark.parametrize("question", + [q for q in build(per_group=3, domains=["physics"]) + if q.group == "wind_distance"], + ids=lambda q: q.text[30:55]) + def test_wind_distance_from_the_stated_figures(self, question): + # "One mile" is spelled out, so the digits in the text are exactly + # speed, the mile factor and the hours. + speed, mile_km, hours = self._numbers(question.text)[:3] + assert question.answer == speed * hours * mile_km + + @pytest.mark.parametrize("question", + [q for q in build(per_group=3, domains=["physics"]) + if q.group == "around_earth"], + ids=lambda q: q.text[50:70]) + def test_around_earth_from_the_stated_figures(self, question): + circumference, speed = self._numbers(question.text)[:2] + assert question.answer == circumference / speed + + @pytest.mark.parametrize("question", + [q for q in build(per_group=3, domains=["physics"]) + if q.group == "speed_conversion"], + ids=lambda q: q.text[17:35]) + def test_speed_conversion_from_the_stated_figures(self, question): + speed, mile_km = self._numbers(question.text)[:2] + assert question.answer == speed * mile_km * 1000 / 3600 + + def test_no_question_asks_the_model_to_remember_a_constant(self): + """The rule that keeps physics inside the battery's discipline.""" + for question in build(per_group=2, domains=["physics"]): + numbers = self._numbers(question.text) + assert len(numbers) >= 2, \ + f"a constant is missing from the text: {question.text}" + + def test_the_light_constants_give_plausible_magnitudes(self): + """Sanity anchors on the constants themselves, not on an RNG draw. + + The first version waited for the Moon to be drawn and it never was at + that seed -- a test hoping the RNG cooperates tests the RNG. + """ + from mpe_lkg.battery.physics import BODIES, LIGHT + + seconds = {name: float(d * 1000 / LIGHT) for name, d in BODIES.items()} + assert 1 < seconds["the Moon"] < 2 + assert 480 < seconds["the Sun"] < 520 # about eight minutes + assert 150 < seconds["Mars at its closest"] < 220 diff --git a/docs/claims/battery_physics_20260811_qwen3.json b/docs/claims/battery_physics_20260811_qwen3.json new file mode 100644 index 0000000..aac8af3 --- /dev/null +++ b/docs/claims/battery_physics_20260811_qwen3.json @@ -0,0 +1,184 @@ +{ + "n": 8, + "correct": 8, + "correct_rate": 1.0, + "settled_exactly": 0, + "computed_but_unused": 0, + "steps_mean": 6.75, + "errors": 0, + "per_group": { + "light_travel": { + "n": 2, + "correct": 2 + }, + "wind_distance": { + "n": 2, + "correct": 2 + }, + "around_earth": { + "n": 2, + "correct": 2 + }, + "speed_conversion": { + "n": 2, + "correct": 2 + } + }, + "rows": [ + { + "group": "light_travel", + "domain": "physics", + "question": "Light travels at 299,792,458 metres per second. The average distance to the Sun is 149,597,870 kilometres. How many seconds does light take to cover that distance?", + "mode": "exact", + "expected": "74798935000/149896229", + "answer": "499.004782 seconds", + "correct": true, + "settled_exactly": false, + "calcs": [ + "149597870000 / 299792458 = 499.004782", + "149597870000 / 299792458 = 499.004782", + "149597870 * 1000 = 149597870000", + "149597870000 / 299792458 = 499.004782" + ], + "steps": 7, + "seconds": 35.3, + "error": "" + }, + { + "group": "light_travel", + "domain": "physics", + "question": "Light travels at 299,792,458 metres per second. The average distance to Mars at its closest is 54,600,000 kilometres. How many seconds does light take to cover that distance?", + "mode": "exact", + "expected": "3900000000/21413747", + "answer": "182.126 seconds", + "correct": true, + "settled_exactly": false, + "calcs": [ + "54600000 * 1000 / 299792458 = 182.125996", + "54600000 * 1000 / 299792458 = 182.125996", + "54600000 * 1000 / 299792458 = 182.125996", + "54600000 * 1000 / 299792458 = 182.125996", + "54600000 * 1000 / 299792458 = 182.125996", + "54600000 * 1000 / 299792458 = 182.125996", + "54600000 * 1000 / 299792458 = 182.125996" + ], + "steps": 7, + "seconds": 31.8, + "error": "" + }, + { + "group": "wind_distance", + "domain": "physics", + "question": "A storm wind blows steadily at 45 miles per hour. One mile is exactly 1.609344 kilometres. How many kilometres does the wind travel in 40 hours?", + "mode": "exact", + "expected": "1810512/625", + "answer": "2896.8192 kilometres", + "correct": true, + "settled_exactly": false, + "calcs": [ + "45 * 40 = 1800", + "45 * 40 = 1800", + "45 * 40 = 1800", + "45 * 40 = 1800", + "1.609344 * (45 * 40) = 2896.8192", + "1.609344 * 45 * 40 = 2896.8192", + "1.609344 * 45 * 40 = 2896.8192", + "1.609344 * 45 * 40 = 2896.8192" + ], + "steps": 7, + "seconds": 31.3, + "error": "" + }, + { + "group": "wind_distance", + "domain": "physics", + "question": "A storm wind blows steadily at 102 miles per hour. One mile is exactly 1.609344 kilometres. How many kilometres does the wind travel in 35 hours?", + "mode": "exact", + "expected": "17954244/3125", + "answer": "5745.35808 kilometres", + "correct": true, + "settled_exactly": false, + "calcs": [ + "102 * 35 = 3570", + "102 * 35 = 3570", + "102 * 35 * 1.609344 = 5745.35808", + "102 * 35 * 1.609344 = 5745.35808" + ], + "steps": 7, + "seconds": 22.9, + "error": "" + }, + { + "group": "around_earth", + "domain": "physics", + "question": "The Earth's equator is 40,075 kilometres around. Travelling at a constant 886 kilometres per hour, how many hours does one full circuit take?", + "mode": "exact", + "expected": "40075/886", + "answer": "45.231377 hours", + "correct": true, + "settled_exactly": false, + "calcs": [ + "40075 / 886 = 45.231377", + "40075 / 886 = 45.231377" + ], + "steps": 7, + "seconds": 19.7, + "error": "" + }, + { + "group": "around_earth", + "domain": "physics", + "question": "The Earth's equator is 40,075 kilometres around. Travelling at a constant 499 kilometres per hour, how many hours does one full circuit take?", + "mode": "exact", + "expected": "40075/499", + "answer": "80.310621", + "correct": true, + "settled_exactly": false, + "calcs": [ + "40075 / 499 = 80.310621", + "40075 / 499 = 80.310621", + "40075 / 499 = 80.310621" + ], + "steps": 5, + "seconds": 16.7, + "error": "" + }, + { + "group": "speed_conversion", + "domain": "physics", + "question": "A vehicle moves at 88 miles per hour. One mile is exactly 1.609344 kilometres. What is its speed in metres per second?", + "mode": "exact", + "expected": "122936/3125", + "answer": "39.33952 metres per second", + "correct": true, + "settled_exactly": false, + "calcs": [ + "88 * 1.609344 * 1000 / 3600 = 39.33952", + "88 * 1.609344 * 1000 / 3600 = 39.33952" + ], + "steps": 7, + "seconds": 21.1, + "error": "" + }, + { + "group": "speed_conversion", + "domain": "physics", + "question": "A vehicle moves at 45 miles per hour. One mile is exactly 1.609344 kilometres. What is its speed in metres per second?", + "mode": "exact", + "expected": "12573/625", + "answer": "20.1168 metres per second.", + "correct": true, + "settled_exactly": false, + "calcs": [ + "45 * 1.609344 * 1000 / 3600 = 20.1168", + "45 * 1.609344 * 1000 / 3600 = 20.1168", + "45 * 1.609344 * 1000 / 3600 = 20.1168" + ], + "steps": 7, + "seconds": 31.7, + "error": "" + } + ], + "seed": 20260811, + "model": "qwen3:4b-instruct-2507-q4_K_M" +} \ No newline at end of file diff --git a/src/mpe_lkg/battery/__init__.py b/src/mpe_lkg/battery/__init__.py index b0026c1..3f871a5 100644 --- a/src/mpe_lkg/battery/__init__.py +++ b/src/mpe_lkg/battery/__init__.py @@ -128,4 +128,4 @@ def truth_table(questions: list[Question]) -> str: # Importing the domains registers them. At the bottom so the decorator exists. -from . import arithmetic, consistency, logic, units # noqa: E402, F401 +from . import arithmetic, consistency, logic, physics, units # noqa: E402, F401 diff --git a/src/mpe_lkg/battery/physics.py b/src/mpe_lkg/battery/physics.py new file mode 100644 index 0000000..2467022 --- /dev/null +++ b/src/mpe_lkg/battery/physics.py @@ -0,0 +1,101 @@ +"""Physical composition: light, wind, speed and distance, graded exactly. + +The battery's rule is that the truth is computed by the same code that renders +the question -- and physics questions threaten that rule, because they lean on +constants. The resolution: EVERY constant is stated in the question text. The +model is never asked to remember the speed of light or the distance to Mars; it +is asked to compose figures it has been handed, across units, which is exactly +the failure mode this project has measured models into and built machinery for. + +The constants are reference values, fixed here so the battery is stable, and +carried into the question verbatim so the tests can re-derive every answer from +the question's own text -- a question whose stated figures disagreed with its +graded answer would fail its own test. + +Answers are exact Fractions; most do not terminate as decimals (the speed of +light has ugly prime factors), and ``matches`` grades those at the 2-4 decimal +places a sensible answer would state. +""" + +from __future__ import annotations + +import random +from fractions import Fraction + +from . import Question, generator + +# Metres per second, exact by definition of the metre. +LIGHT = Fraction(299_792_458) + +# Average distances in kilometres, fixed reference values. Stated in every +# question that uses them, so nothing depends on the model or the reader +# agreeing with the almanac. +BODIES = { + "the Moon": Fraction(384_400), + "the Sun": Fraction(149_597_870), + "Mars at its closest": Fraction(54_600_000), + "Mars on average": Fraction(225_000_000), + "Jupiter on average": Fraction(778_500_000), +} + +# One mile is exactly 201168/125 metres; the factor to km/h from mph. +MILE_KM = Fraction(201_168, 125_000) + +# Equatorial circumference, km. +EQUATOR = Fraction(40_075) + + +@generator("physics") +def build(seed: int, per_group: int) -> list[Question]: + rng = random.Random(seed) + out: list[Question] = [] + + for _ in range(per_group): + body = rng.choice(sorted(BODIES)) + distance = BODIES[body] + out.append(Question( + group="light_travel", + text=(f"Light travels at 299,792,458 metres per second. The average " + f"distance to {body} is {int(distance):,} kilometres. How many " + f"seconds does light take to cover that distance?"), + answer=distance * 1000 / LIGHT, + expression=f"{distance}*1000/299792458", + )) + + for _ in range(per_group): + # Wind speed held in one unit, distance asked in another: the mixed-unit + # composition the assembly used to guess exponents on. + speed = rng.randrange(8, 130) + hours = rng.randrange(2, 48) + out.append(Question( + group="wind_distance", + text=(f"A storm wind blows steadily at {speed} miles per hour. One " + f"mile is exactly 1.609344 kilometres. How many kilometres " + f"does the wind travel in {hours} hours?"), + answer=Fraction(speed) * hours * MILE_KM, + expression=f"{speed}*{hours}*1.609344", + )) + + for _ in range(per_group): + speed = rng.randrange(15, 900) + out.append(Question( + group="around_earth", + text=(f"The Earth's equator is 40,075 kilometres around. Travelling " + f"at a constant {speed} kilometres per hour, how many hours " + f"does one full circuit take?"), + answer=EQUATOR / speed, + expression=f"40075/{speed}", + )) + + for _ in range(per_group): + speed = rng.randrange(10, 200) + out.append(Question( + group="speed_conversion", + text=(f"A vehicle moves at {speed} miles per hour. One mile is " + f"exactly 1.609344 kilometres. What is its speed in metres " + f"per second?"), + answer=Fraction(speed) * MILE_KM * 1000 / 3600, + expression=f"{speed}*1609.344/3600", + )) + + return out