diff --git a/dev/scripts/check_numbers.py b/dev/scripts/check_numbers.py index 6c3d4d2..eed556c 100644 --- a/dev/scripts/check_numbers.py +++ b/dev/scripts/check_numbers.py @@ -45,6 +45,7 @@ QWEN_REPLICATION = "docs/claims/battery_555_qwen3.json" LLAMA_REPLICATION = "docs/claims/battery_555_llama3.2.json" FULL_BATTERY = "docs/claims/battery_full_20260811_qwen3.json" +PHYSICS = "docs/claims/battery_physics_20260811_qwen3.json" # (file, extractor, expected, tolerance, label) # @@ -484,6 +485,16 @@ 0.001, "the consistency caveat is attached to the data itself", ), + ( + PHYSICS, + # Constants stated in the question, so the task is composition across + # units rather than recall -- the exact failure mode this project has + # built machinery for, and the machinery carries it. + lambda d: d["correct_rate"], + 1.0, + 0.25, + "physics composition with stated constants is answered nearly clean", + ), ( QWEN_SELECT, # And it does the job it exists for: a value computed exactly and then not diff --git a/dev/tests/test_battery_domains.py b/dev/tests/test_battery_domains.py index aa3f8d2..dcf5268 100644 --- a/dev/tests/test_battery_domains.py +++ b/dev/tests/test_battery_domains.py @@ -146,3 +146,75 @@ def test_text_questions_grade_by_expectation(self): expect=("neither", "the same", "equal")) assert q.matches("They weigh the same.") is True assert q.matches("The lead, obviously.") is False + + +class TestPhysics: + """Every answer re-derived from the question's own text. + + The physics domain leans on constants, and the rule that makes it safe is + that every constant is STATED in the question -- so these tests parse the + figures back out of the text and recompute the answer from nothing else. + The very first version of this domain failed this test twice: both + mile-based answers were a factor of a thousand off, because MILE_KM is + already kilometres per mile and the generator multiplied by 1000 again. + A constant embedded twice is exactly the mistake this style of test exists + to catch before a model is ever graded against it. + """ + + def _numbers(self, text): + return [Fraction(n.replace(",", "")) + for n in re.findall(r"\d[\d,]*(?:\.\d+)?", text)] + + @pytest.mark.parametrize("question", + [q for q in build(per_group=3, domains=["physics"]) + if q.group == "light_travel"], + ids=lambda q: q.text[60:90]) + def test_light_travel_from_the_stated_figures(self, question): + speed, distance = self._numbers(question.text)[:2] + assert question.answer == distance * 1000 / speed + + @pytest.mark.parametrize("question", + [q for q in build(per_group=3, domains=["physics"]) + if q.group == "wind_distance"], + ids=lambda q: q.text[30:55]) + def test_wind_distance_from_the_stated_figures(self, question): + # "One mile" is spelled out, so the digits in the text are exactly + # speed, the mile factor and the hours. + speed, mile_km, hours = self._numbers(question.text)[:3] + assert question.answer == speed * hours * mile_km + + @pytest.mark.parametrize("question", + [q for q in build(per_group=3, domains=["physics"]) + if q.group == "around_earth"], + ids=lambda q: q.text[50:70]) + def test_around_earth_from_the_stated_figures(self, question): + circumference, speed = self._numbers(question.text)[:2] + assert question.answer == circumference / speed + + @pytest.mark.parametrize("question", + [q for q in build(per_group=3, domains=["physics"]) + if q.group == "speed_conversion"], + ids=lambda q: q.text[17:35]) + def test_speed_conversion_from_the_stated_figures(self, question): + speed, mile_km = self._numbers(question.text)[:2] + assert question.answer == speed * mile_km * 1000 / 3600 + + def test_no_question_asks_the_model_to_remember_a_constant(self): + """The rule that keeps physics inside the battery's discipline.""" + for question in build(per_group=2, domains=["physics"]): + numbers = self._numbers(question.text) + assert len(numbers) >= 2, \ + f"a constant is missing from the text: {question.text}" + + def test_the_light_constants_give_plausible_magnitudes(self): + """Sanity anchors on the constants themselves, not on an RNG draw. + + The first version waited for the Moon to be drawn and it never was at + that seed -- a test hoping the RNG cooperates tests the RNG. + """ + from mpe_lkg.battery.physics import BODIES, LIGHT + + seconds = {name: float(d * 1000 / LIGHT) for name, d in BODIES.items()} + assert 1 < seconds["the Moon"] < 2 + assert 480 < seconds["the Sun"] < 520 # about eight minutes + assert 150 < seconds["Mars at its closest"] < 220 diff --git a/docs/claims/battery_physics_20260811_qwen3.json b/docs/claims/battery_physics_20260811_qwen3.json new file mode 100644 index 0000000..aac8af3 --- /dev/null +++ b/docs/claims/battery_physics_20260811_qwen3.json @@ -0,0 +1,184 @@ +{ + "n": 8, + "correct": 8, + "correct_rate": 1.0, + "settled_exactly": 0, + "computed_but_unused": 0, + "steps_mean": 6.75, + "errors": 0, + "per_group": { + "light_travel": { + "n": 2, + "correct": 2 + }, + "wind_distance": { + "n": 2, + "correct": 2 + }, + "around_earth": { + "n": 2, + "correct": 2 + }, + "speed_conversion": { + "n": 2, + "correct": 2 + } + }, + "rows": [ + { + "group": "light_travel", + "domain": "physics", + "question": "Light travels at 299,792,458 metres per second. The average distance to the Sun is 149,597,870 kilometres. How many seconds does light take to cover that distance?", + "mode": "exact", + "expected": "74798935000/149896229", + "answer": "499.004782 seconds", + "correct": true, + "settled_exactly": false, + "calcs": [ + "149597870000 / 299792458 = 499.004782", + "149597870000 / 299792458 = 499.004782", + "149597870 * 1000 = 149597870000", + "149597870000 / 299792458 = 499.004782" + ], + "steps": 7, + "seconds": 35.3, + "error": "" + }, + { + "group": "light_travel", + "domain": "physics", + "question": "Light travels at 299,792,458 metres per second. The average distance to Mars at its closest is 54,600,000 kilometres. How many seconds does light take to cover that distance?", + "mode": "exact", + "expected": "3900000000/21413747", + "answer": "182.126 seconds", + "correct": true, + "settled_exactly": false, + "calcs": [ + "54600000 * 1000 / 299792458 = 182.125996", + "54600000 * 1000 / 299792458 = 182.125996", + "54600000 * 1000 / 299792458 = 182.125996", + "54600000 * 1000 / 299792458 = 182.125996", + "54600000 * 1000 / 299792458 = 182.125996", + "54600000 * 1000 / 299792458 = 182.125996", + "54600000 * 1000 / 299792458 = 182.125996" + ], + "steps": 7, + "seconds": 31.8, + "error": "" + }, + { + "group": "wind_distance", + "domain": "physics", + "question": "A storm wind blows steadily at 45 miles per hour. One mile is exactly 1.609344 kilometres. How many kilometres does the wind travel in 40 hours?", + "mode": "exact", + "expected": "1810512/625", + "answer": "2896.8192 kilometres", + "correct": true, + "settled_exactly": false, + "calcs": [ + "45 * 40 = 1800", + "45 * 40 = 1800", + "45 * 40 = 1800", + "45 * 40 = 1800", + "1.609344 * (45 * 40) = 2896.8192", + "1.609344 * 45 * 40 = 2896.8192", + "1.609344 * 45 * 40 = 2896.8192", + "1.609344 * 45 * 40 = 2896.8192" + ], + "steps": 7, + "seconds": 31.3, + "error": "" + }, + { + "group": "wind_distance", + "domain": "physics", + "question": "A storm wind blows steadily at 102 miles per hour. One mile is exactly 1.609344 kilometres. How many kilometres does the wind travel in 35 hours?", + "mode": "exact", + "expected": "17954244/3125", + "answer": "5745.35808 kilometres", + "correct": true, + "settled_exactly": false, + "calcs": [ + "102 * 35 = 3570", + "102 * 35 = 3570", + "102 * 35 * 1.609344 = 5745.35808", + "102 * 35 * 1.609344 = 5745.35808" + ], + "steps": 7, + "seconds": 22.9, + "error": "" + }, + { + "group": "around_earth", + "domain": "physics", + "question": "The Earth's equator is 40,075 kilometres around. Travelling at a constant 886 kilometres per hour, how many hours does one full circuit take?", + "mode": "exact", + "expected": "40075/886", + "answer": "45.231377 hours", + "correct": true, + "settled_exactly": false, + "calcs": [ + "40075 / 886 = 45.231377", + "40075 / 886 = 45.231377" + ], + "steps": 7, + "seconds": 19.7, + "error": "" + }, + { + "group": "around_earth", + "domain": "physics", + "question": "The Earth's equator is 40,075 kilometres around. Travelling at a constant 499 kilometres per hour, how many hours does one full circuit take?", + "mode": "exact", + "expected": "40075/499", + "answer": "80.310621", + "correct": true, + "settled_exactly": false, + "calcs": [ + "40075 / 499 = 80.310621", + "40075 / 499 = 80.310621", + "40075 / 499 = 80.310621" + ], + "steps": 5, + "seconds": 16.7, + "error": "" + }, + { + "group": "speed_conversion", + "domain": "physics", + "question": "A vehicle moves at 88 miles per hour. One mile is exactly 1.609344 kilometres. What is its speed in metres per second?", + "mode": "exact", + "expected": "122936/3125", + "answer": "39.33952 metres per second", + "correct": true, + "settled_exactly": false, + "calcs": [ + "88 * 1.609344 * 1000 / 3600 = 39.33952", + "88 * 1.609344 * 1000 / 3600 = 39.33952" + ], + "steps": 7, + "seconds": 21.1, + "error": "" + }, + { + "group": "speed_conversion", + "domain": "physics", + "question": "A vehicle moves at 45 miles per hour. One mile is exactly 1.609344 kilometres. What is its speed in metres per second?", + "mode": "exact", + "expected": "12573/625", + "answer": "20.1168 metres per second.", + "correct": true, + "settled_exactly": false, + "calcs": [ + "45 * 1.609344 * 1000 / 3600 = 20.1168", + "45 * 1.609344 * 1000 / 3600 = 20.1168", + "45 * 1.609344 * 1000 / 3600 = 20.1168" + ], + "steps": 7, + "seconds": 31.7, + "error": "" + } + ], + "seed": 20260811, + "model": "qwen3:4b-instruct-2507-q4_K_M" +} \ No newline at end of file diff --git a/src/mpe_lkg/battery/__init__.py b/src/mpe_lkg/battery/__init__.py index b0026c1..3f871a5 100644 --- a/src/mpe_lkg/battery/__init__.py +++ b/src/mpe_lkg/battery/__init__.py @@ -128,4 +128,4 @@ def truth_table(questions: list[Question]) -> str: # Importing the domains registers them. At the bottom so the decorator exists. -from . import arithmetic, consistency, logic, units # noqa: E402, F401 +from . import arithmetic, consistency, logic, physics, units # noqa: E402, F401 diff --git a/src/mpe_lkg/battery/physics.py b/src/mpe_lkg/battery/physics.py new file mode 100644 index 0000000..2467022 --- /dev/null +++ b/src/mpe_lkg/battery/physics.py @@ -0,0 +1,101 @@ +"""Physical composition: light, wind, speed and distance, graded exactly. + +The battery's rule is that the truth is computed by the same code that renders +the question -- and physics questions threaten that rule, because they lean on +constants. The resolution: EVERY constant is stated in the question text. The +model is never asked to remember the speed of light or the distance to Mars; it +is asked to compose figures it has been handed, across units, which is exactly +the failure mode this project has measured models into and built machinery for. + +The constants are reference values, fixed here so the battery is stable, and +carried into the question verbatim so the tests can re-derive every answer from +the question's own text -- a question whose stated figures disagreed with its +graded answer would fail its own test. + +Answers are exact Fractions; most do not terminate as decimals (the speed of +light has ugly prime factors), and ``matches`` grades those at the 2-4 decimal +places a sensible answer would state. +""" + +from __future__ import annotations + +import random +from fractions import Fraction + +from . import Question, generator + +# Metres per second, exact by definition of the metre. +LIGHT = Fraction(299_792_458) + +# Average distances in kilometres, fixed reference values. Stated in every +# question that uses them, so nothing depends on the model or the reader +# agreeing with the almanac. +BODIES = { + "the Moon": Fraction(384_400), + "the Sun": Fraction(149_597_870), + "Mars at its closest": Fraction(54_600_000), + "Mars on average": Fraction(225_000_000), + "Jupiter on average": Fraction(778_500_000), +} + +# One mile is exactly 201168/125 metres; the factor to km/h from mph. +MILE_KM = Fraction(201_168, 125_000) + +# Equatorial circumference, km. +EQUATOR = Fraction(40_075) + + +@generator("physics") +def build(seed: int, per_group: int) -> list[Question]: + rng = random.Random(seed) + out: list[Question] = [] + + for _ in range(per_group): + body = rng.choice(sorted(BODIES)) + distance = BODIES[body] + out.append(Question( + group="light_travel", + text=(f"Light travels at 299,792,458 metres per second. The average " + f"distance to {body} is {int(distance):,} kilometres. How many " + f"seconds does light take to cover that distance?"), + answer=distance * 1000 / LIGHT, + expression=f"{distance}*1000/299792458", + )) + + for _ in range(per_group): + # Wind speed held in one unit, distance asked in another: the mixed-unit + # composition the assembly used to guess exponents on. + speed = rng.randrange(8, 130) + hours = rng.randrange(2, 48) + out.append(Question( + group="wind_distance", + text=(f"A storm wind blows steadily at {speed} miles per hour. One " + f"mile is exactly 1.609344 kilometres. How many kilometres " + f"does the wind travel in {hours} hours?"), + answer=Fraction(speed) * hours * MILE_KM, + expression=f"{speed}*{hours}*1.609344", + )) + + for _ in range(per_group): + speed = rng.randrange(15, 900) + out.append(Question( + group="around_earth", + text=(f"The Earth's equator is 40,075 kilometres around. Travelling " + f"at a constant {speed} kilometres per hour, how many hours " + f"does one full circuit take?"), + answer=EQUATOR / speed, + expression=f"40075/{speed}", + )) + + for _ in range(per_group): + speed = rng.randrange(10, 200) + out.append(Question( + group="speed_conversion", + text=(f"A vehicle moves at {speed} miles per hour. One mile is " + f"exactly 1.609344 kilometres. What is its speed in metres " + f"per second?"), + answer=Fraction(speed) * MILE_KM * 1000 / 3600, + expression=f"{speed}*1609.344/3600", + )) + + return out