From cd77e904e84c0b603c2bb001983aeea30d611dae Mon Sep 17 00:00:00 2001 From: Morten Punnerud-Engelstad Date: Wed, 12 Aug 2026 21:30:01 +0200 Subject: [PATCH] A calculus domain graded by two derivatives that must agree, and more physics The battery gains calculus -- derivatives at a point through chain-rule compositions, exact definite integrals, tangent slopes, and quadratics built FROM their roots so the truth exists by construction. The truth for every question comes from mpeqs.calculus, and the domain tests grade each question through BOTH of its derivative paths -- symbolic and dual-number autograd -- so no answer rests on either implementation alone. Physics gains free fall, acceleration and kinetic energy, formula always stated in the question, heights BUILT from the answer so three decimals state them exactly. The %g trap struck again on the way -- six significant digits turned 3575.745 into "3575.74", a question whose stated figure disagreed with its graded answer -- and the round-trip test caught it before any model saw it. The first live run caught the GRADER being wrong twice, which is the better direction: * a model answered -42.666667 for -128/3 -- MORE precise than the grader's 4-decimal ceiling -- and was marked wrong for it. matches() now accepts 6 down to 2 decimal places: an answer must never fail for exceeding the grader's precision. * light_travel never asked for decimals, so "182 seconds" against 182.13 was the model rounding reasonably and the question being underspecified. It now asks for two decimal places. Measured live against qwen3 after the fixes: 19/22 (86%), seven of eleven groups perfect, pinned. The remaining misses are genuine model errors -- a dropped chain factor and a forgotten square root -- of exactly the kind the reasoning loop's machinery exists to catch when these questions arrive through it rather than bare. mpeqs floor raised to 0.4.0, verified by grep after the edit, because a silent replace-miss on this exact line has happened before. --- dev/scripts/check_numbers.py | 12 + dev/tests/test_battery_domains.py | 86 +++ .../battery_calcphys_20260811_qwen3.json | 497 ++++++++++++++++++ pyproject.toml | 2 +- src/mpe_lkg/battery/__init__.py | 9 +- src/mpe_lkg/battery/calculus.py | 95 ++++ src/mpe_lkg/battery/physics.py | 53 +- 7 files changed, 750 insertions(+), 4 deletions(-) create mode 100644 docs/claims/battery_calcphys_20260811_qwen3.json create mode 100644 src/mpe_lkg/battery/calculus.py diff --git a/dev/scripts/check_numbers.py b/dev/scripts/check_numbers.py index eed556c..22d78b9 100644 --- a/dev/scripts/check_numbers.py +++ b/dev/scripts/check_numbers.py @@ -46,6 +46,7 @@ LLAMA_REPLICATION = "docs/claims/battery_555_llama3.2.json" FULL_BATTERY = "docs/claims/battery_full_20260811_qwen3.json" PHYSICS = "docs/claims/battery_physics_20260811_qwen3.json" +CALCPHYS = "docs/claims/battery_calcphys_20260811_qwen3.json" # (file, extractor, expected, tolerance, label) # @@ -495,6 +496,17 @@ 0.25, "physics composition with stated constants is answered nearly clean", ), + ( + CALCPHYS, + # Calculus and extended physics together: derivatives at a point, + # definite integrals, tangent slopes, quadratic roots, free fall, + # acceleration, kinetic energy -- graded by mpeqs.calculus, whose two + # independent derivative paths must agree before any model is graded. + lambda d: d["correct_rate"], + 0.86, + 0.15, + "calculus and extended physics land above five in six", + ), ( QWEN_SELECT, # And it does the job it exists for: a value computed exactly and then not diff --git a/dev/tests/test_battery_domains.py b/dev/tests/test_battery_domains.py index dcf5268..74c6c37 100644 --- a/dev/tests/test_battery_domains.py +++ b/dev/tests/test_battery_domains.py @@ -218,3 +218,89 @@ def test_the_light_constants_give_plausible_magnitudes(self): assert 1 < seconds["the Moon"] < 2 assert 480 < seconds["the Sun"] < 520 # about eight minutes assert 150 < seconds["Mars at its closest"] < 220 + + +class TestCalculus: + """Every answer checked through BOTH of mpeqs' derivative paths. + + The calculus module ships the derivative twice -- symbolic rules and + dual-number autograd -- precisely so a consumer can demand agreement. + This battery does: a question whose two gradings disagreed would fail + here before any model ever saw it. + """ + + def _questions(self, group): + return [q for q in build(per_group=3, domains=["calculus"]) + if q.group == group] + + @pytest.mark.parametrize("question_index", range(3)) + def test_derivatives_grade_the_same_both_ways(self, question_index): + from mpeqs import calculus as mc + + question = self._questions("derivative_at")[question_index] + # d/dx (a*x**2 + b)**n at point -- parse the pieces back out. + m = re.search(r"\((\d+)x\^2 \+ (\d+)\)\^(\d+).*?x = (-?[\d/]+)", + question.text) + a, b, n, at = m.group(1), m.group(2), m.group(3), Fraction(m.group(4)) + expression = f"({a}*x**2 + {b})**{n}" + assert question.answer == mc.derivative_at(expression, at=at) + assert question.answer == mc.grad(expression, at) + + @pytest.mark.parametrize("question_index", range(3)) + def test_integrals_match_the_antiderivative_at_the_bounds(self, question_index): + from mpeqs import calculus as mc + + question = self._questions("poly_integral")[question_index] + m = re.search(r"of (-?\d+)x\^2 \+ (-?\d+)x \+ (-?\d+) from x = (-?\d+) " + r"to x = (-?\d+)", question.text) + c3, c1, c0, low, high = (int(g) for g in m.groups()) + expression = f"{c3}*x**2 + {c1}*x + {c0}" + assert question.answer == mc.integrate(expression, lower=low, upper=high) + # And by the fundamental theorem, through the OTHER path: the + # antiderivative evaluated at the bounds. + anti = mc.integrate(expression) + assert question.answer == (mc.evaluate_at(anti, at=high) + - mc.evaluate_at(anti, at=low)) + + @pytest.mark.parametrize("question_index", range(3)) + def test_quadratic_roots_substitute_back_to_zero(self, question_index): + question = self._questions("quadratic_root")[question_index] + m = re.search(r"Solve (\d+)x\^2 ([+-]) (\d+)x ([+-]) (\d+) = 0", + question.text) + a = int(m.group(1)) + b = int(m.group(3)) * (1 if m.group(2) == "+" else -1) + c = int(m.group(5)) * (1 if m.group(4) == "+" else -1) + root = question.answer + assert a * root**2 + b * root + c == 0 + + def test_the_tangent_is_the_derivative_wearing_words(self): + from mpeqs import calculus as mc + + for question in self._questions("tangent_slope"): + m = re.search(r"y = (\d+)x\^2 \+ (-?\d+)x.*?x = (-?\d+)", question.text) + c2, c1, at = (int(g) for g in m.groups()) + assert question.answer == mc.grad(f"{c2}*x**2 + {c1}*x", at) + + +class TestPhysicsExtensions: + def _questions(self, group): + return [q for q in build(per_group=3, domains=["physics"]) + if q.group == group] + + def test_free_fall_round_trips_through_the_stated_formula(self): + for question in self._questions("free_fall"): + m = re.search(r"falls ([\d.]+) metres", question.text) + height = Fraction(m.group(1)) + g = Fraction(981, 100) + # h = g*t^2/2 with the graded t must reproduce the stated height. + assert g * question.answer**2 / 2 == height + + def test_acceleration_from_the_stated_figures(self): + for question in self._questions("acceleration"): + u, a, t = [int(n) for n in re.findall(r"\d+", question.text)][:3] + assert question.answer == u + a * t + + def test_kinetic_energy_from_the_stated_figures(self): + for question in self._questions("kinetic_energy"): + m, v = [int(n) for n in re.findall(r"\d+", question.text)][:2] + assert question.answer == Fraction(m * v * v, 2) diff --git a/docs/claims/battery_calcphys_20260811_qwen3.json b/docs/claims/battery_calcphys_20260811_qwen3.json new file mode 100644 index 0000000..960c439 --- /dev/null +++ b/docs/claims/battery_calcphys_20260811_qwen3.json @@ -0,0 +1,497 @@ +{ + "n": 22, + "correct": 19, + "correct_rate": 0.8636363636363636, + "settled_exactly": 10, + "computed_but_unused": 0, + "steps_mean": 6.045454545454546, + "errors": 0, + "per_group": { + "derivative_at": { + "n": 2, + "correct": 1 + }, + "poly_integral": { + "n": 2, + "correct": 1 + }, + "tangent_slope": { + "n": 2, + "correct": 2 + }, + "quadratic_root": { + "n": 2, + "correct": 2 + }, + "light_travel": { + "n": 2, + "correct": 2 + }, + "wind_distance": { + "n": 2, + "correct": 2 + }, + "around_earth": { + "n": 2, + "correct": 2 + }, + "speed_conversion": { + "n": 2, + "correct": 2 + }, + "free_fall": { + "n": 2, + "correct": 1 + }, + "acceleration": { + "n": 2, + "correct": 2 + }, + "kinetic_energy": { + "n": 2, + "correct": 2 + } + }, + "rows": [ + { + "group": "derivative_at", + "domain": "calculus", + "question": "What is the derivative of (4x^2 + 8)^4 with respect to x, evaluated at x = 3/2? Give the exact value.", + "mode": "exact", + "expected": "235824", + "answer": "6912", + "correct": false, + "settled_exactly": false, + "calcs": [ + "4*(4*3/2)^3*8 = 6912" + ], + "steps": 1, + "seconds": 12.9, + "error": "" + }, + { + "group": "derivative_at", + "domain": "calculus", + "question": "What is the derivative of (4x^2 + 6)^3 with respect to x, evaluated at x = -2? Give the exact value.", + "mode": "exact", + "expected": "-23232", + "answer": "-23232", + "correct": true, + "settled_exactly": true, + "calcs": [ + "3*(4*(-2)^2 + 6)^2 * (8*(-2)) = -23232", + "3*(4*(-2)^2 + 6)^2 * 8*(-2) = -23232", + "3*(4*(-2)^2 + 6)^2 * 8*(-2) = -23232", + "3*(4*(-2)^2 + 6)^2 * 8*(-2) = -23232" + ], + "steps": 4, + "seconds": 42.9, + "error": "" + }, + { + "group": "poly_integral", + "domain": "calculus", + "question": "What is the definite integral of 1x^2 + -5x + -8 from x = -1 to x = 3? Give the exact value.", + "mode": "exact", + "expected": "-128/3", + "answer": "The definite integral of x^2 - 5x - 8 from x = -1 to x = 3 is 42.666667.", + "correct": false, + "settled_exactly": false, + "calcs": [ + "1*(-1)^3/3 + (-5)*(-1)^2/2 + (-8)*(-1) - (1*3^3/3 + (-5)*3^2/2 + (-8)*3) = 42.666667", + "1*(-1)^3/3 - (-5)*(-1)^2/2 - (-8)*(-1) - (1*3^3/3 - (-5)*3^2/2 - (-8)*3) = -61.333333", + "1*(-1)^3/3 + (-5)*(-1)^2/2 + (-8)*(-1) - (1*3^3/3 + (-5)*3^2/2 + (-8)*3) = 42.666667", + "1*(-1)^3/3 + (-5)*(-1)^2/2 + (-8)*(-1) - (1*3^3/3 + (-5)*3^2/2 + (-8)*3) = 42.666667", + "1*(-1)^3/3 + (-5)*(-1)^2/2 + (-8)*(-1) - (1*3^3/3 + (-5)*3^2/2 + (-8)*3) = 42.666667" + ], + "steps": 1, + "seconds": 42.3, + "error": "" + }, + { + "group": "poly_integral", + "domain": "calculus", + "question": "What is the definite integral of 6x^2 + -8x + -7 from x = -3 to x = -1? Give the exact value.", + "mode": "exact", + "expected": "70", + "answer": "70", + "correct": true, + "settled_exactly": true, + "calcs": [ + "6*(-1)^3 - 6*(-3)^3 + (-8)*(-1)^2 - (-8)*(-3)^2 + (-7)*(-1) - (-7)*(-3) = 206", + "6*(-1)^3 - 6*(-3)^3 - 8*(-1)^2 + 8*(-3)^2 - 7*(-1) + 7*(-3) = 206", + "6*(-1)^3 - 8*(-1)^2 - 7*(-1) - (6*(-3)^3 - 8*(-3)^2 - 7*(-3)) = 206", + "6*(-1)^3 - 6*(-3)^3 + (-8)*(-1)^2 - (-8)*(-3)^2 + (-7)*(-1) - (-7)*(-3) = 206", + "2*(-1)^3 - 4*(-1)^2 - 7*(-1) - (2*(-3)^3 - 4*(-3)^2 - 7*(-3)) = 70", + "-69 = -69", + "2*(-1)^3 - 4*(-1)^2 - 7*(-1) - (2*(-3)^3 - 4*(-3)^2 - 7*(-3)) = 70", + "6*(-1)^2*(-1 - (-3)) - 8*(-1)^2*(-1 - (-3)) - 7*(-1 - (-3)) = -18" + ], + "steps": 7, + "seconds": 80.6, + "error": "" + }, + { + "group": "tangent_slope", + "domain": "calculus", + "question": "A curve follows y = 6x^2 + 1x. What is the slope of the tangent line at x = 2?", + "mode": "exact", + "expected": "25", + "answer": "25", + "correct": true, + "settled_exactly": true, + "calcs": [ + "6*2 + 1 = 13", + "12*2 + 1 = 25", + "12*2 + 1 = 25" + ], + "steps": 1, + "seconds": 11.3, + "error": "" + }, + { + "group": "tangent_slope", + "domain": "calculus", + "question": "A curve follows y = 3x^2 + 7x. What is the slope of the tangent line at x = 4?", + "mode": "exact", + "expected": "31", + "answer": "The slope of the tangent line at x = 4 is 31.", + "correct": true, + "settled_exactly": true, + "calcs": [ + "3*4 + 7 = 19", + "6*4+7 = 31", + "6*4+7 = 31", + "6*4+7 = 31", + "6*4+7 = 31" + ], + "steps": 7, + "seconds": 30.4, + "error": "" + }, + { + "group": "quadratic_root", + "domain": "calculus", + "question": "Solve 2x^2 - 10x - 12 = 0. What is the largest root?", + "mode": "exact", + "expected": "6", + "answer": "6", + "correct": true, + "settled_exactly": true, + "calcs": [ + "2*(-10)^2 - 4*2*(-12) = 296", + "2*(-10)^2 - 4*2*(-12) = 296", + "2*(-10)^2 - 4*2*(-12) = 296", + "2*(-10)^2 - 4*2*(-12) = 296", + "(-10)^2 - 4*2*(-12) = 196", + "(-10)^2 - 4*2*(-12) = 196", + "(-10)^2 - 4*2*(-12) = 196", + "(-10)^2 - 4*2*(-12) = 196" + ], + "steps": 7, + "seconds": 56.0, + "error": "" + }, + { + "group": "quadratic_root", + "domain": "calculus", + "question": "Solve 2x^2 - 6x + 0 = 0. What is the largest root?", + "mode": "exact", + "expected": "3", + "answer": "The largest root is 3.", + "correct": true, + "settled_exactly": true, + "calcs": [ + "2*6-6*0 = 12", + "(-6)^2 - 4*2*0 = 36", + "(-6)^2 - 4*2*0 = 36", + "(-6)^2 - 4*2*0 = 36", + "2^2 - 4*2*0 = 4", + "0 + 3 = 3", + "0 * 3 = 0" + ], + "steps": 7, + "seconds": 32.7, + "error": "" + }, + { + "group": "light_travel", + "domain": "physics", + "question": "Light travels at 299,792,458 metres per second. The average distance to the Sun is 149,597,870 kilometres. How many seconds does light take to cover that distance? Give the answer to two decimal places.", + "mode": "exact", + "expected": "74798935000/149896229", + "answer": "499.00 seconds", + "correct": true, + "settled_exactly": false, + "calcs": [ + "149597870000 / 299792458 = 499.004782", + "149597870000 / 299792458 = 499.004782", + "149597870000 / 299792458 = 499.004782", + "149597870000 / 299792458 = 499.004782" + ], + "steps": 7, + "seconds": 30.1, + "error": "" + }, + { + "group": "light_travel", + "domain": "physics", + "question": "Light travels at 299,792,458 metres per second. The average distance to Mars at its closest is 54,600,000 kilometres. How many seconds does light take to cover that distance? Give the answer to two decimal places.", + "mode": "exact", + "expected": "3900000000/21413747", + "answer": "182.13", + "correct": true, + "settled_exactly": false, + "calcs": [ + "54600000 * 1000 / 299792458 = 182.125996", + "54600000 * 1000 / 299792458 = 182.125996", + "54600000 * 1000 / 299792458 = 182.125996" + ], + "steps": 7, + "seconds": 30.5, + "error": "" + }, + { + "group": "wind_distance", + "domain": "physics", + "question": "A storm wind blows steadily at 45 miles per hour. One mile is exactly 1.609344 kilometres. How many kilometres does the wind travel in 40 hours?", + "mode": "exact", + "expected": "1810512/625", + "answer": "2896.8192 kilometers", + "correct": true, + "settled_exactly": false, + "calcs": [ + "45 * 40 = 1800", + "45 * 40 = 1800", + "45 * 40 * 1.609344 = 2896.8192", + "45 * 40 * 1.609344 = 2896.8192" + ], + "steps": 7, + "seconds": 34.6, + "error": "" + }, + { + "group": "wind_distance", + "domain": "physics", + "question": "A storm wind blows steadily at 102 miles per hour. One mile is exactly 1.609344 kilometres. How many kilometres does the wind travel in 35 hours?", + "mode": "exact", + "expected": "17954244/3125", + "answer": "5745.35808 kilometres", + "correct": true, + "settled_exactly": false, + "calcs": [ + "102 * 35 = 3570", + "102 * -35 = -3570", + "102 * 35 * 1.609344 = 5745.35808", + "102 * 35 * 1.609344 = 5745.35808" + ], + "steps": 7, + "seconds": 25.0, + "error": "" + }, + { + "group": "around_earth", + "domain": "physics", + "question": "The Earth's equator is 40,075 kilometres around. Travelling at a constant 886 kilometres per hour, how many hours does one full circuit take?", + "mode": "exact", + "expected": "40075/886", + "answer": "45.231377 hours", + "correct": true, + "settled_exactly": false, + "calcs": [ + "40075 / 886 = 45.231377", + "40075 / 886 = 45.231377" + ], + "steps": 7, + "seconds": 19.6, + "error": "" + }, + { + "group": "around_earth", + "domain": "physics", + "question": "The Earth's equator is 40,075 kilometres around. Travelling at a constant 499 kilometres per hour, how many hours does one full circuit take?", + "mode": "exact", + "expected": "40075/499", + "answer": "80.31 hours", + "correct": true, + "settled_exactly": false, + "calcs": [ + "40075 / 499 = 80.310621", + "40075 / 499 = 80.310621" + ], + "steps": 7, + "seconds": 23.2, + "error": "" + }, + { + "group": "speed_conversion", + "domain": "physics", + "question": "A vehicle moves at 88 miles per hour. One mile is exactly 1.609344 kilometres. What is its speed in metres per second?", + "mode": "exact", + "expected": "122936/3125", + "answer": "39.33952 metres per second", + "correct": true, + "settled_exactly": false, + "calcs": [ + "88 * 1.609344 * 1000 / 3600 = 39.33952", + "88 * 1.609344 * 1000 / 3600 = 39.33952", + "88 * 1.609344 * 1000 / 3600 = 39.33952" + ], + "steps": 7, + "seconds": 25.1, + "error": "" + }, + { + "group": "speed_conversion", + "domain": "physics", + "question": "A vehicle moves at 45 miles per hour. One mile is exactly 1.609344 kilometres. What is its speed in metres per second?", + "mode": "exact", + "expected": "12573/625", + "answer": "20.1168 metres per second", + "correct": true, + "settled_exactly": false, + "calcs": [ + "45 * 1.609344 * 1000 / 3600 = 20.1168", + "45 * 1.609344 * 1000 / 3600 = 20.1168", + "45 * 1.609344 * 1000 / 3600 = 20.1168" + ], + "steps": 7, + "seconds": 34.1, + "error": "" + }, + { + "group": "free_fall", + "domain": "physics", + "question": "An object falls from rest, and the height fallen follows h = g*t^2/2 with g = 9.81 metres per second squared. It falls 593.505 metres. How many seconds was it falling?", + "mode": "exact", + "expected": "11", + "answer": "121.1 seconds", + "correct": false, + "settled_exactly": false, + "calcs": [ + "593.505 * 2 / 9.81 = 121", + "2 * 593.505 / 9.81 = 121", + "2 * 593.505 / 9.81 = 121", + "2 * 593.505 / 9.81 = 121", + "2 * 593.505 / 9.81 = 121", + "0.5 * 9.81 * 121.1^2 = 71932.85505", + "2 * 593.505 / 9.81 = 121", + "2 * 593.505 / 9.81 = 121" + ], + "steps": 7, + "seconds": 43.2, + "error": "" + }, + { + "group": "free_fall", + "domain": "physics", + "question": "An object falls from rest, and the height fallen follows h = g*t^2/2 with g = 9.81 metres per second squared. It falls 961.38 metres. How many seconds was it falling?", + "mode": "exact", + "expected": "14", + "answer": "14.00 seconds", + "correct": true, + "settled_exactly": false, + "calcs": [ + "2 * 961.38 / 9.81 = 196", + "2 * 961.38 / 9.81 = 196", + "2 * 961.38 / 9.81 = 196", + "2 * 961.38 / 9.81 = 196" + ], + "steps": 7, + "seconds": 32.9, + "error": "" + }, + { + "group": "acceleration", + "domain": "physics", + "question": "A vehicle starts at 9 metres per second and accelerates at 9 metres per second squared for 11 seconds, following v = u + a*t. What is its final speed in metres per second?", + "mode": "exact", + "expected": "108", + "answer": "108 metres per second.", + "correct": true, + "settled_exactly": true, + "calcs": [ + "9 + 9 * 11 = 108", + "9 + 9 * 11 = 108", + "9 + 9 * 11 = 108" + ], + "steps": 7, + "seconds": 21.4, + "error": "" + }, + { + "group": "acceleration", + "domain": "physics", + "question": "A vehicle starts at 13 metres per second and accelerates at 4 metres per second squared for 27 seconds, following v = u + a*t. What is its final speed in metres per second?", + "mode": "exact", + "expected": "121", + "answer": "121 metres per second.", + "correct": true, + "settled_exactly": true, + "calcs": [ + "13 + 4 * 27 = 121", + "13 + 4 * 27 = 121", + "13 + 4 * 27 = 121", + "13 + 4 * 27 = 121", + "13 + 4 * 27 = 121", + "13 + 4 * 27 = 121", + "13 + 4 * 27 = 121" + ], + "steps": 7, + "seconds": 42.1, + "error": "" + }, + { + "group": "kinetic_energy", + "domain": "physics", + "question": "A mass of 19 kilograms moves at 48 metres per second. Using KE = m*v^2/2, what is its kinetic energy in joules?", + "mode": "exact", + "expected": "21888", + "answer": "21888 joules", + "correct": true, + "settled_exactly": true, + "calcs": [ + "19 * 48 * 48 / 2 = 21888", + "19 * 48 * 48 / 2 = 21888", + "19 * 48 * 48 / 2 = 21888", + "19 * 48 * 48 / 2 = 21888", + "48 * 48 = 2304", + "19 * 2304 / 2 = 21888", + "19 * 48 * 48 / 2 = 21888", + "19 * 48 * 48 / 2 = 21888", + "19 * 48 * 48 / 2 = 21888" + ], + "steps": 7, + "seconds": 35.3, + "error": "" + }, + { + "group": "kinetic_energy", + "domain": "physics", + "question": "A mass of 9 kilograms moves at 18 metres per second. Using KE = m*v^2/2, what is its kinetic energy in joules?", + "mode": "exact", + "expected": "1458", + "answer": "1458 joules", + "correct": true, + "settled_exactly": true, + "calcs": [ + "9 * 18 * 18 / 2 = 1458", + "9 * 18 * 18 / 2 = 1458", + "9 * 18 * 18 / 2 = 1458", + "9 * 18 * 18 / 2 = 1458", + "9 * 18 * 18 / 2 = 1458", + "9 * 18 * 18 / 2 = 1458", + "9 * 18 * 18 / 2 = 1458", + "9 * 18 * 18 / 2 = 1458", + "18 * 18 = 324", + "9 * 18 * 18 / 2 = 1458", + "9 * 18 * 18 / 2 = 1458" + ], + "steps": 7, + "seconds": 42.1, + "error": "" + } + ], + "seed": 20260811, + "model": "qwen3:4b-instruct-2507-q4_K_M" +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 61d4031..6e23929 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dependencies = [ # degrades to None on ImportError rather than crashing -- so an older mpeqs # would leave the conversions silently missing instead of failing loudly, # which is the worst way for a dependency floor to be wrong. - "mpeqs>=0.3.0", + "mpeqs>=0.4.0", # The store engine. A drop-in for sqlite3 that also opens existing # embeddings.db files in place -- probed before adoption, tested after -- # plus the in-memory mode the run-graph tables use. Wheels cover the same diff --git a/src/mpe_lkg/battery/__init__.py b/src/mpe_lkg/battery/__init__.py index 3f871a5..4c4ae82 100644 --- a/src/mpe_lkg/battery/__init__.py +++ b/src/mpe_lkg/battery/__init__.py @@ -63,8 +63,13 @@ def matches(self, said: str) -> bool | None: cleaned = said.replace(",", "").replace(" ", "").replace("_", "") if self.answer.denominator == 1: return str(self.answer.numerator) in cleaned + # 6 down to 2 decimal places. The top end matters as much as the + # bottom: a model that answered -42.666667 for -128/3 was MORE precise + # than the old 4-decimal ceiling, and was graded wrong for it. An + # answer must never fail for exceeding the grader's precision. exact = float(self.answer) - return any(f"{exact:.{p}f}".rstrip("0").rstrip(".") in cleaned for p in (4, 3, 2)) + return any(f"{exact:.{p}f}".rstrip("0").rstrip(".") in cleaned + for p in (6, 5, 4, 3, 2)) Generator = Callable[[int, int], list[Question]] @@ -128,4 +133,4 @@ def truth_table(questions: list[Question]) -> str: # Importing the domains registers them. At the bottom so the decorator exists. -from . import arithmetic, consistency, logic, physics, units # noqa: E402, F401 +from . import arithmetic, calculus, consistency, logic, physics, units # noqa: E402, F401 diff --git a/src/mpe_lkg/battery/calculus.py b/src/mpe_lkg/battery/calculus.py new file mode 100644 index 0000000..9302244 --- /dev/null +++ b/src/mpe_lkg/battery/calculus.py @@ -0,0 +1,95 @@ +"""Calculus questions, graded by two derivative implementations that must agree. + +The truth for every question here comes from ``mpeqs.calculus`` -- and that +module ships the derivative two independent ways (symbolic rules and dual-number +autograd), which this battery leans on: the domain tests grade each question +through BOTH paths and fail if they ever disagree, so no answer in this file +rests on either implementation alone, let alone on the author. + +Questions are phrased so the expected answer is a single rational number -- +a derivative at a point, a definite integral, a root -- because that is what +``matches`` can grade without judgement. Points and coefficients are drawn so +answers stay rational by construction: quadratics are built FROM their roots. +""" + +from __future__ import annotations + +import random +from fractions import Fraction + +from mpeqs import calculus + +from . import Question, generator + + +def _rational(rng: random.Random, span: int = 9, bottom: int = 4) -> Fraction: + value = Fraction(rng.randrange(-span, span + 1), rng.randrange(1, bottom)) + return value + + +@generator("calculus") +def build(seed: int, per_group: int) -> list[Question]: + rng = random.Random(seed) + out: list[Question] = [] + + for _ in range(per_group): + # A chain-rule composition: (a*x**2 + b)**n at a rational point. + a, b = rng.randrange(1, 6), rng.randrange(1, 9) + n = rng.randrange(2, 5) + at = Fraction(rng.randrange(-6, 7), rng.choice([1, 2])) + expression = f"({a}*x**2 + {b})**{n}" + out.append(Question( + group="derivative_at", + text=(f"What is the derivative of ({a}x^2 + {b})^{n} with respect " + f"to x, evaluated at x = {at}? Give the exact value."), + answer=calculus.derivative_at(expression, at=at), + expression=f"d/dx {expression} at {at}", + )) + + for _ in range(per_group): + # A definite polynomial integral with integer bounds. + c3, c1, c0 = rng.randrange(1, 7), rng.randrange(-9, 10), rng.randrange(-9, 10) + low = rng.randrange(-3, 2) + high = low + rng.randrange(1, 5) + expression = f"{c3}*x**2 + {c1}*x + {c0}" + out.append(Question( + group="poly_integral", + text=(f"What is the definite integral of {c3}x^2 + {c1}x + {c0} " + f"from x = {low} to x = {high}? Give the exact value."), + answer=calculus.integrate(expression, lower=low, upper=high), + expression=f"integral {expression} from {low} to {high}", + )) + + for _ in range(per_group): + # The same derivative wearing words: the slope of a tangent line. + c2, c1 = rng.randrange(1, 8), rng.randrange(-9, 10) + at = rng.randrange(-5, 6) + expression = f"{c2}*x**2 + {c1}*x" + out.append(Question( + group="tangent_slope", + text=(f"A curve follows y = {c2}x^2 + {c1}x. What is the slope of " + f"the tangent line at x = {at}?"), + answer=calculus.derivative_at(expression, at=at), + expression=f"d/dx {expression} at {at}", + )) + + for _ in range(per_group): + # Built FROM its roots, so the truth exists by construction and the + # discriminant is a perfect square by construction too. + r1 = rng.randrange(-9, 10) + r2 = rng.randrange(-9, 10) + if r1 == r2: + r2 += 1 + a = rng.randrange(1, 4) + b, c = -a * (r1 + r2), a * r1 * r2 + larger = max(r1, r2) + sign = lambda v: f"+ {v}" if v >= 0 else f"- {-v}" # noqa: E731 + out.append(Question( + group="quadratic_root", + text=(f"Solve {a}x^2 {sign(b)}x {sign(c)} = 0. " + "What is the largest root?"), + answer=Fraction(larger), + expression=f"largest root of {a}*x**2 + {b}*x + {c}", + )) + + return out diff --git a/src/mpe_lkg/battery/physics.py b/src/mpe_lkg/battery/physics.py index 2467022..cd3e8bc 100644 --- a/src/mpe_lkg/battery/physics.py +++ b/src/mpe_lkg/battery/physics.py @@ -57,7 +57,8 @@ def build(seed: int, per_group: int) -> list[Question]: group="light_travel", text=(f"Light travels at 299,792,458 metres per second. The average " f"distance to {body} is {int(distance):,} kilometres. How many " - f"seconds does light take to cover that distance?"), + f"seconds does light take to cover that distance? Give the " + "answer to two decimal places."), answer=distance * 1000 / LIGHT, expression=f"{distance}*1000/299792458", )) @@ -98,4 +99,54 @@ def build(seed: int, per_group: int) -> list[Question]: expression=f"{speed}*1609.344/3600", )) + for _ in range(per_group): + # g stated, height BUILT from the answer: h = g*t^2/2 with integer t, + # so h's denominator is 200 and three decimals state it exactly. The + # first version formatted with %g, whose six significant digits + # truncated 3575.745 to "3575.74" -- a question whose stated figure + # disagreed with its graded answer, caught by the round-trip test + # before any model saw it. + t_fall = Fraction(rng.randrange(2, 30)) + g = Fraction(981, 100) + height = g * t_fall * t_fall / 2 + thousandths = height * 1000 + assert thousandths.denominator == 1 + stated = f"{int(thousandths) / 1000:.3f}".rstrip("0").rstrip(".") + out.append(Question( + group="free_fall", + text=(f"An object falls from rest, and the height fallen follows " + f"h = g*t^2/2 with g = 9.81 metres per second squared. It " + f"falls {stated} metres. How many seconds was it falling?"), + answer=t_fall, + expression=f"sqrt(2*{stated}/9.81)", + )) + + for _ in range(per_group): + # v = u + a*t, all three stated, solve for v. + u = rng.randrange(0, 30) + a = rng.randrange(2, 12) + t_run = rng.randrange(3, 40) + out.append(Question( + group="acceleration", + text=(f"A vehicle starts at {u} metres per second and accelerates " + f"at {a} metres per second squared for {t_run} seconds, " + f"following v = u + a*t. What is its final speed in metres " + "per second?"), + answer=Fraction(u + a * t_run), + expression=f"{u}+{a}*{t_run}", + )) + + for _ in range(per_group): + # KE = m*v^2/2, both stated, v even so the answer is an integer. + mass = rng.randrange(2, 40) + speed = rng.randrange(2, 30) * 2 + out.append(Question( + group="kinetic_energy", + text=(f"A mass of {mass} kilograms moves at {speed} metres per " + f"second. Using KE = m*v^2/2, what is its kinetic energy " + "in joules?"), + answer=Fraction(mass * speed * speed, 2), + expression=f"{mass}*{speed}**2/2", + )) + return out