From 898cba344dc8e42cb8524787719794147efc91f4 Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Thu, 15 Oct 2026 01:19:04 -0500 Subject: [PATCH] feat(meter-eval): report top-three tap outcomes beside OCR counters Preserve historical OCR summary counts and report separate offered-number, unnecessary-retake and accepted-miss counts with explicit denominators. Add per-photo top-three availability and document the outcome definitions. Decision: D12 option A, recommended pending Sam selection. Thread: https://github.com/SamGu-NRX/BaseScanning/pull/16#discussion_r4114894516 --- experiments/meter-closeup/FIELD_TEST.md | 2 + .../meter-closeup/src/meter_eval/fieldtest.py | 23 +++++-- .../meter-closeup/tests/test_fieldtest.py | 67 +++++++++++++++++++ 3 files changed, 88 insertions(+), 4 deletions(-) diff --git a/experiments/meter-closeup/FIELD_TEST.md b/experiments/meter-closeup/FIELD_TEST.md index 1087400e2..ec59d9dba 100644 --- a/experiments/meter-closeup/FIELD_TEST.md +++ b/experiments/meter-closeup/FIELD_TEST.md @@ -12,6 +12,8 @@ The thresholds in [PORTING.md](PORTING.md) come from processed Commons JPEGs, no The command prints each photo's read result, the candidate's rank and the check values. It then counts retakes asked for photos that read, and photos accepted that did not. Ranking and the top candidate used for retake checks depend only on blind heuristic features. The supplied number scores the read and its position in that order; it never changes the selected candidate. The number is never written to disk. +The photo table also reports whether the true number appears among the three offered taps. The historical OCR retake counts are unchanged. A separate top-three summary reports offered-number photos over all photos, retakes with the number offered over all retakes, and accepted photos without the number offered over all accepted photos. A zero denominator means there were no photos in that group, not a measured success rate. + | Threshold | Photos to take | The threshold holds if | |---|---|---| | Out of focus | three with focus locked on a distant background, one from 5 cm, three sharp | blurred photos score 6.68 or less and fail to read, and sharp photos score far above | diff --git a/experiments/meter-closeup/src/meter_eval/fieldtest.py b/experiments/meter-closeup/src/meter_eval/fieldtest.py index 8c8909e87..b8b610a01 100644 --- a/experiments/meter-closeup/src/meter_eval/fieldtest.py +++ b/experiments/meter-closeup/src/meter_eval/fieldtest.py @@ -7,7 +7,8 @@ come from the blind heuristic ranking; --number only scores the read and the number's rank. The summary counts a retake asked for a photo that read (a wasted retake) and a photo accepted that did not read (a re-request later). The meter number is taken from the command -line and is never written to disk. +line and is never written to disk. Separate top-three counters report whether the true +number was offered, with denominators for all photos, retakes and accepted photos. """ import argparse @@ -74,10 +75,12 @@ def main() -> None: print( "| Photo | Read | Rank | Top candidate is the number | Top line px | Sharpness | " - "Retake because |" + "Retake because | True number among three taps |" ) - print("|---|---|---|---|---|---|---|") + print("|---|---|---|---|---|---|---|---|") wasted = missed = 0 + offered = offered_wasted = offered_missed = 0 + retakes = accepted = 0 # Working copies live in a private temporary folder, never in the photo folder, so no # file of the user's is overwritten, deleted or scored twice. with tempfile.TemporaryDirectory() as scratch, Reader() as reader: @@ -97,16 +100,28 @@ def main() -> None: why = retake.reasons(g, box) wasted += read and bool(why) missed += not read and not why + in_top_three = target_core in order[:3] + offered += in_top_three + offered_wasted += in_top_three and bool(why) + offered_missed += not in_top_three and not why + retakes += bool(why) + accepted += not why height_px = f"{retake.line_height_px(box, g.shape[0]):.0f}" if box else "–" print( f"| {photo.name} | {'yes' if read else 'no'} | {rank or '–'} | " f"{'yes' if rank == 1 else 'no'} | {height_px} | " - f"{retake.whole_photo_sharpness(g):.1f} | {', '.join(why) or '–'} |" + f"{retake.whole_photo_sharpness(g):.1f} | {', '.join(why) or '–'} | " + f"{'yes' if in_top_three else 'no'} |" ) print( f"\n{len(photos)} photos; {wasted} retakes asked for photos that read; " f"{missed} photos accepted that did not read." ) + print( + f"Top-three taps: {offered}/{len(photos)} photos offered the true number; " + f"{offered_wasted}/{retakes} retakes asked when the true number was offered; " + f"{offered_missed}/{accepted} accepted photos did not offer the true number." + ) if __name__ == "__main__": diff --git a/experiments/meter-closeup/tests/test_fieldtest.py b/experiments/meter-closeup/tests/test_fieldtest.py index 69748e9e8..69aa2c2df 100644 --- a/experiments/meter-closeup/tests/test_fieldtest.py +++ b/experiments/meter-closeup/tests/test_fieldtest.py @@ -173,3 +173,70 @@ def test_read_target_preserves_letters_outside_a_recognised_label_prefix(serial) from meter_eval.match import read_target assert read_target(serial) == serial.casefold() + + +@pytest.mark.parametrize( + "ocr_read,target_position,retake,offered", + [ + (False, 2, False, True), + (False, 2, True, True), + (True, 4, False, False), + (True, 4, True, False), + ], +) +def test_field_summary_keeps_ocr_and_offered_tap_outcomes_separate( + tmp_path, monkeypatch, capsys, ocr_read, target_position, retake, offered +): + import sys + + from meter_eval import fieldtest + + target = "1234567" + numbers = ["7654321", "7654322", "7654323", "7654324"] + numbers[target_position - 1] = target + result = { + "lines": [ + {"text": number, "box": [0.1, 0.05 + i * 0.22, 0.4, 0.2 - i * 0.02]} + for i, number in enumerate(numbers) + ], + "barcodes": [], + } + + class Reader: + def __enter__(self): + return self + + def __exit__(self, *exc): + pass + + def read(self, path, barcodes): + return result + + # The shared matcher currently makes offered imply read. A stand-in diagnostic + # exercises the requested disagreement without changing historical OCR semantics. + if not ocr_read: + monkeypatch.setattr(fieldtest, "reads_number", lambda lines, number: False) + Image.new("RGB", (128, 128), "white").save(tmp_path / "photo.jpg") + monkeypatch.setattr(fieldtest, "Reader", Reader) + monkeypatch.setattr( + fieldtest.retake, "reasons", lambda gray, box: ["out of focus"] if retake else [] + ) + monkeypatch.setattr(sys, "argv", ["fieldtest", str(tmp_path), "--number", target]) + fieldtest.main() + output = capsys.readouterr().out + assert ( + f"1 photos; {int(ocr_read and retake)} retakes asked for photos that read; " + f"{int(not ocr_read and not retake)} photos accepted that did not read." + ) in output + assert f"Top-three taps: {int(offered)}/1 photos offered the true number" in output + assert ( + f"{int(offered and retake)}/{int(retake)} retakes asked when the true number was offered" + in output + ) + assert ( + f"{int(not offered and not retake)}/{int(not retake)} accepted photos did not offer the true number" + in output + ) + row = next(line for line in output.splitlines() if line.startswith("| photo.jpg |")) + assert row.endswith(f"| {'yes' if offered else 'no'} |") + assert "True number among three taps" in output