diff --git a/dev/scripts/check_numbers.py b/dev/scripts/check_numbers.py index 266567c..6c3d4d2 100644 --- a/dev/scripts/check_numbers.py +++ b/dev/scripts/check_numbers.py @@ -44,6 +44,7 @@ QWEN_SELECT = "docs/claims/battery_555_qwen3_select.json" QWEN_REPLICATION = "docs/claims/battery_555_qwen3.json" LLAMA_REPLICATION = "docs/claims/battery_555_llama3.2.json" +FULL_BATTERY = "docs/claims/battery_full_20260811_qwen3.json" # (file, extractor, expected, tolerance, label) # @@ -449,6 +450,40 @@ 0.15, "answering by selection costs no accuracy on qwen3", ), + # --- The full battery: all four domains, all three grading modes --- + ( + FULL_BATTERY, + lambda d: d["correct_rate"], + 0.89, + 0.12, + "qwen3 answers about nine in ten of the full battery's gradeable questions", + ), + ( + FULL_BATTERY, + # The knowledge-edge probe as a battery, and it behaved exactly as + # designed: steady on facts broad enough that any grounded model holds + # them, UNSTEADY on places assembled from an RNG -- where a steady + # answer would be manufactured confidence. + lambda d: float(d["consistency"]["per_group"]["anchored"]["steady"]), + 2.0, + 1.0, + "anchored facts are held steadily", + ), + ( + FULL_BATTERY, + lambda d: float(d["consistency"]["per_group"]["invented"]["steady"]), + 0.0, + 1.0, + "and invented places are not -- the honest signal, pinned so it stays", + ), + ( + FULL_BATTERY, + # Never mixed into correctness: the note travels with the data. + lambda d: 1.0 if "invisible" in d["consistency"]["note"] else 0.0, + 1.0, + 0.001, + "the consistency caveat is attached to the data itself", + ), ( QWEN_SELECT, # And it does the job it exists for: a value computed exactly and then not diff --git a/docs/claims/battery_full_20260811_qwen3.json b/docs/claims/battery_full_20260811_qwen3.json new file mode 100644 index 0000000..0c31fb7 --- /dev/null +++ b/docs/claims/battery_full_20260811_qwen3.json @@ -0,0 +1,803 @@ +{ + "n": 36, + "correct": 32, + "correct_rate": 0.8888888888888888, + "settled_exactly": 24, + "computed_but_unused": 1, + "steps_mean": 5.083333333333333, + "errors": 0, + "per_group": { + "long_multiplication": { + "n": 2, + "correct": 2 + }, + "long_addition": { + "n": 2, + "correct": 2 + }, + "long_subtraction": { + "n": 2, + "correct": 2 + }, + "awkward_percentage": { + "n": 2, + "correct": 2 + }, + "division_with_remainder": { + "n": 2, + "correct": 1 + }, + "word_problem": { + "n": 2, + "correct": 2 + }, + "unit_chain": { + "n": 2, + "correct": 0 + }, + "exact_fraction": { + "n": 2, + "correct": 2 + }, + "powers": { + "n": 2, + "correct": 2 + }, + "comparison": { + "n": 2, + "correct": 1 + }, + "pigeonhole_pair": { + "n": 2, + "correct": 2 + }, + "pigeonhole_k": { + "n": 2, + "correct": 2 + }, + "inclusion_exclusion": { + "n": 2, + "correct": 2 + }, + "handshakes": { + "n": 2, + "correct": 2 + }, + "convert_mass": { + "n": 2, + "correct": 2 + }, + "convert_length": { + "n": 2, + "correct": 2 + }, + "convert_time": { + "n": 2, + "correct": 2 + }, + "convert_digital": { + "n": 2, + "correct": 2 + } + }, + "consistency": { + "n": 4, + "steady": 2, + "note": "self-agreement only; a consistently wrong answer is invisible to this grading", + "per_group": { + "anchored": { + "n": 2, + "steady": 2 + }, + "invented": { + "n": 2, + "steady": 0 + } + } + }, + "rows": [ + { + "group": "long_multiplication", + "domain": "arithmetic", + "question": "What is 53431930048 * 3565?", + "mode": "exact", + "expected": "190484830621120", + "answer": "190484830621120", + "correct": true, + "settled_exactly": true, + "calcs": [ + "53431930048 * 3565 = 190484830621120", + "53431930048 * 3565 = 190484830621120", + "53431930048 * 3565 = 190484830621120", + "53431930048 * 3565 = 190484830621120", + "53431930048 * 3565 = 190484830621120", + "53431930048 * 3565 = 190484830621120", + "53431930048 * 3565 = 190484830621120" + ], + "steps": 8, + "seconds": 54.8, + "error": "" + }, + { + "group": "long_multiplication", + "domain": "arithmetic", + "question": "What is 51395252287 * 5521?", + "mode": "exact", + "expected": "283753187876527", + "answer": "283753187876527", + "correct": true, + "settled_exactly": true, + "calcs": [ + "51395252287 * 5521 = 283753187876527", + "51395252287 * 5521 = 283753187876527", + "51395252287 * 5521 = 283753187876527", + "51395252287 * 5521 = 283753187876527", + "51395252287 * 5521 = 283753187876527", + "51395252287 * 5521 = 283753187876527", + "51395252287 * 5521 = 283753187876527" + ], + "steps": 7, + "seconds": 40.1, + "error": "" + }, + { + "group": "long_addition", + "domain": "arithmetic", + "question": "What is 193048999061 plus 479849142350?", + "mode": "exact", + "expected": "672898141411", + "answer": "672,898,141,411", + "correct": true, + "settled_exactly": true, + "calcs": [ + "193048999061 + 479849142350 = 672898141411", + "193048999061 + 479849142350 = 672898141411", + "193048999061 + 479849142350 = 672898141411" + ], + "steps": 1, + "seconds": 11.1, + "error": "" + }, + { + "group": "long_addition", + "domain": "arithmetic", + "question": "What is 446174485585 plus 377327695495?", + "mode": "exact", + "expected": "823502181080", + "answer": "823,502,181,080", + "correct": true, + "settled_exactly": true, + "calcs": [ + "446174485585 + 377327695495 = 823502181080", + "446174485585 + 377327695495 = 823502181080", + "446174485585 + 377327695495 = 823502181080" + ], + "steps": 1, + "seconds": 11.9, + "error": "" + }, + { + "group": "long_subtraction", + "domain": "arithmetic", + "question": "Subtract 68379748377 from 2753273712715. What is left?", + "mode": "exact", + "expected": "2684893964338", + "answer": "2684893964338", + "correct": true, + "settled_exactly": true, + "calcs": [ + "2753273712715 - 68379748377 = 2684893964338", + "2753273712715 - 68379748377 = 2684893964338", + "2753273712715 - 68379748377 = 2684893964338" + ], + "steps": 1, + "seconds": 10.7, + "error": "" + }, + { + "group": "long_subtraction", + "domain": "arithmetic", + "question": "Subtract 84869938002 from 3054364818243. What is left?", + "mode": "exact", + "expected": "2969494880241", + "answer": "2969494880241", + "correct": true, + "settled_exactly": true, + "calcs": [ + "3054364818243 - 84869938002 = 2969494880241", + "3054364818243 - 84869938002 = 2969494880241", + "3054364818243 - 84869938002 = 2969494880241" + ], + "steps": 1, + "seconds": 10.4, + "error": "" + }, + { + "group": "awkward_percentage", + "domain": "arithmetic", + "question": "What is 37 percent of 1426321?", + "mode": "exact", + "expected": "52773877/100", + "answer": "527,738.77", + "correct": true, + "settled_exactly": false, + "calcs": [ + "1426321 * 0.37 = 527738.77", + "1426321 * 0.37 = 527738.77", + "1426321 * 0.37 = 527738.77" + ], + "steps": 1, + "seconds": 8.0, + "error": "" + }, + { + "group": "awkward_percentage", + "domain": "arithmetic", + "question": "What is 10 percent of 8596000?", + "mode": "exact", + "expected": "859600", + "answer": "859,600", + "correct": true, + "settled_exactly": true, + "calcs": [ + "8596000 * 0.10 = 859600", + "8596000 * 0.10 = 859600", + "8596000 * 0.10 = 859600" + ], + "steps": 3, + "seconds": 12.0, + "error": "" + }, + { + "group": "division_with_remainder", + "domain": "arithmetic", + "question": "How many whole times does 816 go into 80255013326, and what is the remainder? Give the remainder.", + "mode": "exact", + "expected": "14", + "answer": "816 goes into 80255013326 exactly 98,351,732 whole times, with a remainder of 80255013326 % 816 = 80255013326.", + "correct": false, + "settled_exactly": false, + "calcs": [ + "80255013326 // 816 = 98351732", + "816 * 98351732 + 80255013326 % 816 = 80255013326" + ], + "steps": 7, + "seconds": 33.2, + "error": "" + }, + { + "group": "division_with_remainder", + "domain": "arithmetic", + "question": "How many whole times does 296 go into 4678432745, and what is the remainder? Give the remainder.", + "mode": "exact", + "expected": "9", + "answer": "296 goes into 4678432745 whole times 15,771,735 times, with a remainder of 161843.", + "correct": true, + "settled_exactly": false, + "calcs": [], + "steps": 1, + "seconds": 10.1, + "error": "" + }, + { + "group": "word_problem", + "domain": "arithmetic", + "question": "A warehouse holds 848 crates with 78 jars in each. 55 jars are broken and thrown away. How many jars remain?", + "mode": "exact", + "expected": "66089", + "answer": "66,144 - 55 = 66,089 jars remain.", + "correct": true, + "settled_exactly": false, + "calcs": [ + "848 * 78 = 66144", + "848 * 78 = 66144", + "848 * 78 = 66144" + ], + "steps": 4, + "seconds": 13.7, + "error": "" + }, + { + "group": "word_problem", + "domain": "arithmetic", + "question": "A warehouse holds 428 crates with 64 jars in each. 21 jars are broken and thrown away. How many jars remain?", + "mode": "exact", + "expected": "27371", + "answer": "27371 jars remain.", + "correct": true, + "settled_exactly": true, + "calcs": [ + "428 * 64 = 27392", + "428 * 64 - 21 = 27371" + ], + "steps": 7, + "seconds": 21.2, + "error": "" + }, + { + "group": "unit_chain", + "domain": "arithmetic", + "question": "How many seconds are there in 54 weeks?", + "mode": "exact", + "expected": "32659200", + "answer": "54 weeks is 378 days. There are 86,400 seconds in a day, so 378 days equals 378 \u00d7 86,400 = 32,356,800 seconds.", + "correct": false, + "settled_exactly": false, + "calcs": [], + "steps": 7, + "seconds": 21.4, + "error": "" + }, + { + "group": "unit_chain", + "domain": "arithmetic", + "question": "How many seconds are there in 23 weeks?", + "mode": "exact", + "expected": "13910400", + "answer": "There are 1,584,000 seconds in 23 weeks.", + "correct": false, + "settled_exactly": false, + "calcs": [], + "steps": 7, + "seconds": 20.2, + "error": "" + }, + { + "group": "exact_fraction", + "domain": "arithmetic", + "question": "What is 962171 divided by 620003? Give the answer to four decimal places.", + "mode": "exact", + "expected": "962171/620003", + "answer": "1.5519", + "correct": true, + "settled_exactly": false, + "calcs": [ + "962171 / 620003 = 1.551881", + "962171 / 620003 = 1.551881", + "962171 / 620003 = 1.551881", + "962171 / 620003 = 1.551881" + ], + "steps": 2, + "seconds": 10.9, + "error": "" + }, + { + "group": "exact_fraction", + "domain": "arithmetic", + "question": "What is 781391 divided by 117508? Give the answer to four decimal places.", + "mode": "exact", + "expected": "781391/117508", + "answer": "6.6497", + "correct": true, + "settled_exactly": false, + "calcs": [ + "781391 / 117508 = 6.649683", + "781391 / 117508 = 6.649683", + "781391 / 117508 = 6.649683" + ], + "steps": 1, + "seconds": 8.5, + "error": "" + }, + { + "group": "powers", + "domain": "arithmetic", + "question": "What is 70 to the power of 12?", + "mode": "exact", + "expected": "13841287201000000000000", + "answer": "13841287201000000000000", + "correct": true, + "settled_exactly": true, + "calcs": [ + "70^12 = 13841287201000000000000", + "70^12 = 13841287201000000000000", + "70^12 = 13841287201000000000000", + "70^12 = 13841287201000000000000", + "70^12 = 13841287201000000000000", + "70^12 = 13841287201000000000000" + ], + "steps": 7, + "seconds": 32.9, + "error": "" + }, + { + "group": "powers", + "domain": "arithmetic", + "question": "What is 80 to the power of 9?", + "mode": "exact", + "expected": "134217728000000000", + "answer": "134,217,728,000,000,000", + "correct": true, + "settled_exactly": true, + "calcs": [ + "80^9 = 134217728000000000", + "80^9 = 134217728000000000", + "80^9 = 134217728000000000", + "80^9 = 134217728000000000", + "80^9 = 134217728000000000", + "80^9 = 134217728000000000" + ], + "steps": 7, + "seconds": 33.5, + "error": "" + }, + { + "group": "comparison", + "domain": "arithmetic", + "question": "Machine A makes 6033 parts an hour for 191 hours. Machine B makes 4351 parts an hour for 691 hours. How many parts does the more productive machine make in total?", + "mode": "exact", + "expected": "3006541", + "answer": "Machine B makes more total parts, with a total of 3,006,541 parts.", + "correct": true, + "settled_exactly": true, + "calcs": [ + "6033 * 191 = 1152303", + "4351 * 691 = 3006541", + "6033 * 191 = 1152303" + ], + "steps": 7, + "seconds": 29.4, + "error": "" + }, + { + "group": "comparison", + "domain": "arithmetic", + "question": "Machine A makes 1978 parts an hour for 485 hours. Machine B makes 5005 parts an hour for 174 hours. How many parts does the more productive machine make in total?", + "mode": "exact", + "expected": "959330", + "answer": "870870", + "correct": false, + "settled_exactly": true, + "calcs": [ + "1978 * 485 = 959330", + "1978 * 485 = 959330" + ], + "steps": 7, + "seconds": 26.6, + "error": "" + }, + { + "group": "anchored", + "domain": "consistency", + "question": "On which continent is Canada?", + "mode": "consistency", + "steady": true, + "said": "North America", + "seconds": 0.6 + }, + { + "group": "anchored", + "domain": "consistency", + "question": "On which continent is Thailand?", + "mode": "consistency", + "steady": true, + "said": "Asia", + "seconds": 0.6 + }, + { + "group": "invented", + "domain": "consistency", + "question": "What is the postal code of Brennveien 51 in Fjellmo?", + "mode": "consistency", + "steady": false, + "said": "I do not know.", + "seconds": 0.8 + }, + { + "group": "invented", + "domain": "consistency", + "question": "What is the postal code of Solhaalle 7 in Ostero?", + "mode": "consistency", + "steady": false, + "said": "I do not know.", + "seconds": 0.8 + }, + { + "group": "pigeonhole_pair", + "domain": "logic", + "question": "A drawer holds 14 grey, 11 blue, 26 black, 20 red socks. How many socks must be drawn in the dark to be certain of a matching pair?", + "mode": "exact", + "expected": "5", + "answer": "5 socks must be drawn to be certain of a matching pair.", + "correct": true, + "settled_exactly": true, + "calcs": [ + "14 + 11 + 26 + 20 = 71", + "4 + 1 = 5" + ], + "steps": 7, + "seconds": 34.8, + "error": "" + }, + { + "group": "pigeonhole_pair", + "domain": "logic", + "question": "A drawer holds 14 grey, 7 black, 28 white, 20 blue socks. How many socks must be drawn in the dark to be certain of a matching pair?", + "mode": "exact", + "expected": "5", + "answer": "5 socks must be drawn to be certain of a matching pair.", + "correct": true, + "settled_exactly": true, + "calcs": [ + "14 + 7 + 28 + 20 = 69", + "4 + 1 = 5", + "4 + 1 = 5" + ], + "steps": 7, + "seconds": 39.8, + "error": "" + }, + { + "group": "pigeonhole_k", + "domain": "logic", + "question": "A drawer holds 5 grey, 8 green socks. How many socks must be drawn in the dark to be certain of 3 socks of the same colour?", + "mode": "exact", + "expected": "5", + "answer": "5 socks must be drawn to be certain of 3 socks of the same colour.", + "correct": true, + "settled_exactly": true, + "calcs": [ + "5 + 8 = 13", + "2 + 2 + 1 = 5", + "2 + 2 + 1 = 5", + "2 + 2 + 1 = 5" + ], + "steps": 2, + "seconds": 16.1, + "error": "" + }, + { + "group": "pigeonhole_k", + "domain": "logic", + "question": "A drawer holds 11 green, 13 red, 23 grey socks. How many socks must be drawn in the dark to be certain of 4 socks of the same colour?", + "mode": "exact", + "expected": "10", + "answer": "10 socks must be drawn to be certain of having 4 socks of the same color.", + "correct": true, + "settled_exactly": true, + "calcs": [ + "11 + 13 + 23 = 47", + "3 + 3 + 3 + 1 = 10", + "3 + 3 + 3 + 1 = 10" + ], + "steps": 1, + "seconds": 13.3, + "error": "" + }, + { + "group": "inclusion_exclusion", + "domain": "logic", + "question": "In a class, 37 students study French, 27 study German, and 16 study both. How many students study at least one of the two languages?", + "mode": "exact", + "expected": "48", + "answer": "48 students study at least one of the two languages.", + "correct": true, + "settled_exactly": true, + "calcs": [ + "37 + 27 - 16 = 48", + "37 - 16 = 21", + "27 - 16 = 11", + "27 - 16 = 11", + "27 - 16 = 11" + ], + "steps": 7, + "seconds": 29.4, + "error": "" + }, + { + "group": "inclusion_exclusion", + "domain": "logic", + "question": "In a class, 49 students study French, 60 study German, and 43 study both. How many students study at least one of the two languages?", + "mode": "exact", + "expected": "66", + "answer": "66 students study at least one of the two languages.", + "correct": true, + "settled_exactly": true, + "calcs": [ + "49 + 60 - 43 = 66", + "49 - 43 = 6", + "60 - 43 = 17", + "60 - 43 = 17", + "60 - 43 = 17", + "60 - 43 = 17" + ], + "steps": 2, + "seconds": 19.4, + "error": "" + }, + { + "group": "handshakes", + "domain": "logic", + "question": "12 people are at a meeting and every pair shakes hands exactly once. How many handshakes happen?", + "mode": "exact", + "expected": "66", + "answer": "66 handshakes happen.", + "correct": true, + "settled_exactly": true, + "calcs": [ + "12 * 11 / 2 = 66", + "12 * 11 / 2 = 66", + "12 * 11 / 2 = 66" + ], + "steps": 7, + "seconds": 27.9, + "error": "" + }, + { + "group": "handshakes", + "domain": "logic", + "question": "33 people are at a meeting and every pair shakes hands exactly once. How many handshakes happen?", + "mode": "exact", + "expected": "528", + "answer": "528 handshakes happen.", + "correct": true, + "settled_exactly": true, + "calcs": [ + "33 * 32 / 2 = 528", + "33 * 32 / 2 = 528", + "33 * 32 / 2 = 528", + "33 * 32 / 2 = 528", + "33 * 32 / 2 = 528", + "33 * 32 / 2 = 528", + "33 * 32 / 2 = 528", + "33 * 32 / 2 = 528" + ], + "steps": 7, + "seconds": 30.9, + "error": "" + }, + { + "group": "convert_mass", + "domain": "units", + "question": "How many kilograms are there in 530 tonnes?", + "mode": "exact", + "expected": "530000", + "answer": "530,000 kilograms.", + "correct": true, + "settled_exactly": true, + "calcs": [ + "530 * 1000 = 530000", + "530 * 1000 = 530000", + "530 * 1000 = 530000", + "530 * 1000 = 530000", + "530 * 1000 = 530000", + "530 * 1000 = 530000", + "530 * 1000 = 530000", + "530 * 1000 = 530000" + ], + "steps": 7, + "seconds": 28.5, + "error": "" + }, + { + "group": "convert_mass", + "domain": "units", + "question": "How many kilograms are there in 706 tonnes?", + "mode": "exact", + "expected": "706000", + "answer": "706,000 kilograms.", + "correct": true, + "settled_exactly": true, + "calcs": [ + "706 * 1000 = 706000" + ], + "steps": 7, + "seconds": 19.0, + "error": "" + }, + { + "group": "convert_length", + "domain": "units", + "question": "How many millimetres are there in 711 inchs?", + "mode": "exact", + "expected": "90297/5", + "answer": "18,059.4 millimetres", + "correct": true, + "settled_exactly": false, + "calcs": [ + "711 * 25.4 = 18059.4", + "711 * 25.4 = 18059.4", + "711 * 25.4 = 18059.4", + "711 * 25.4 = 18059.4", + "711 * 25.4 = 18059.4", + "711 * 25.4 = 18059.4", + "711 * 25.4 = 18059.4", + "711 * 25.4 = 18059.4" + ], + "steps": 7, + "seconds": 34.6, + "error": "" + }, + { + "group": "convert_length", + "domain": "units", + "question": "How many millimetres are there in 466 foots?", + "mode": "exact", + "expected": "710184/5", + "answer": "142,036.8 millimetres", + "correct": true, + "settled_exactly": false, + "calcs": [ + "466 * 304.8 = 142036.8", + "466 * 304.8 = 142036.8", + "466 * 304.8 = 142036.8" + ], + "steps": 7, + "seconds": 35.9, + "error": "" + }, + { + "group": "convert_time", + "domain": "units", + "question": "How many hours are there in 82 fortnights?", + "mode": "exact", + "expected": "27552", + "answer": "27,552 hours", + "correct": true, + "settled_exactly": true, + "calcs": [ + "82 * 14 = 1148", + "82 * 14 = 1148", + "1148 * 24 = 27552" + ], + "steps": 7, + "seconds": 17.0, + "error": "" + }, + { + "group": "convert_time", + "domain": "units", + "question": "How many hours are there in 49 fortnights?", + "mode": "exact", + "expected": "16464", + "answer": "16,464 hours", + "correct": true, + "settled_exactly": true, + "calcs": [ + "49 * 14 = 686", + "49 * 14 = 686", + "686 * 24 = 16464" + ], + "steps": 7, + "seconds": 16.4, + "error": "" + }, + { + "group": "convert_digital", + "domain": "units", + "question": "How many bytes are there in 5 gibibytes?", + "mode": "exact", + "expected": "5368709120", + "answer": "5 gibibytes equal 5,368,709,120 bytes.", + "correct": true, + "settled_exactly": false, + "calcs": [ + "1024^3 = 1073741824", + "1024^3 = 1073741824", + "1024^3 = 1073741824" + ], + "steps": 7, + "seconds": 27.9, + "error": "" + }, + { + "group": "convert_digital", + "domain": "units", + "question": "How many bytes are there in 44 mebibytes?", + "mode": "exact", + "expected": "46137344", + "answer": "44 mebibytes equals 44 \u00d7 1,048,576 = 46,137,344 bytes.", + "correct": true, + "settled_exactly": false, + "calcs": [ + "2^20 = 1048576", + "1024^2 = 1048576" + ], + "steps": 7, + "seconds": 34.1, + "error": "" + } + ], + "seed": 20260811, + "model": "qwen3:4b-instruct-2507-q4_K_M" +} \ No newline at end of file