feat(evals): real-data evals for AR drift, wall reconstruction, coverage and the field test

experiments/evals/Makefile

@@ -68,16 +68,30 @@ pose-priors:
 # (experiments/scoring, PR #4) and scores every row in one table.
 FIELD_OUT ?= $(DATA)/evals/field/results/$(basename $(notdir $(SESSION)))
 RULES ?= field/rules.json
-# PR #4's harness, checked out read-only at a pinned commit outside the repo.
-SCORING_COMMIT := 77ce830
-SCORING_CHECKOUT := $(DATA)/evals/scoring-$(SCORING_COMMIT)
+# PR #4's harness, checked out read-only at a pinned commit outside the repo. The full SHA is
+# fetched by name first, because a shallow or single-branch clone doesn't have it.
+SCORING_COMMIT := 77ce830dd08acf0c7982edcec91cd3240e0565ea
+SCORING_CHECKOUT := $(DATA)/evals/scoring-77ce830
 SCORING ?= $(SCORING_CHECKOUT)/experiments/scoring
 STAMPED_MAP = $(FIELD_OUT)/inputs/map.json
 field:
-	@# The pinned checkout is created only at its default place; an overridden SCORING must exist.
-	test -d "$(SCORING)" || { test "$(SCORING)" = "$(SCORING_CHECKOUT)/experiments/scoring" && \
-	  git -C ../.. worktree add --detach "$(SCORING_CHECKOUT)" $(SCORING_COMMIT); } || \
-	  { echo "SCORING=$(SCORING) is not a scoring harness checkout" >&2; exit 1; }
+	@# The default checkout is created at the pinned commit and, when reused, must still be there
+	@# with the harness unmodified. An overridden SCORING is the caller's choice and must exist.
+	@if [ "$(SCORING)" = "$(SCORING_CHECKOUT)/experiments/scoring" ]; then \
+	  if [ ! -d "$(SCORING_CHECKOUT)" ]; then \
+	    { git -C ../.. cat-file -e $(SCORING_COMMIT)^{commit} 2>/dev/null || \
+	      git -C ../.. fetch --no-tags origin $(SCORING_COMMIT); } && \
+	    git -C ../.. worktree add --detach "$(SCORING_CHECKOUT)" $(SCORING_COMMIT) || exit 1; \
+	  fi; \
+	  head=$$(git -C "$(SCORING_CHECKOUT)" rev-parse HEAD) && \
+	  dirty=$$(git -C "$(SCORING_CHECKOUT)" status --porcelain -- experiments/scoring) || exit 1; \
+	  if [ "$$head" != "$(SCORING_COMMIT)" ] || [ -n "$$dirty" ]; then \
+	    echo "$(SCORING_CHECKOUT) is at $$head with changes [$$dirty]; expected $(SCORING_COMMIT)," \
+	      "unmodified. Remove it (git worktree remove) and rerun." >&2; exit 1; \
+	  fi; \
+	elif [ ! -d "$(SCORING)" ]; then \
+	  echo "SCORING=$(SCORING) is not a scoring harness checkout" >&2; exit 1; \
+	fi
 	uv sync --project models
 	$(if $(TRUTH),uv run python -m evals.field stamp "$(SESSION)" --truth "$(TRUTH)" --map "$(MAP)" --out-dir "$(FIELD_OUT)" &&) \
 	W=$$(uv run python -m evals.field prepare "$(SESSION)") && \

experiments/evals/coverage_driver/Sources/coverage-driver/main.swift

@@ -3,7 +3,7 @@
 // each keyframe's sightings so the caller can attribute errors to the frames the app credited.
 // Without a wall it writes only the config.
 //
-// Options for occlusion (README section 7b), modelled here so HouseScanKit stays unedited:
+// Options for occlusion (METHODS.md section 7b), modelled here so HouseScanKit stays unedited:
 // - `variant.coveringBaseline` and `variant.rowsPerBand` set the app's own CoverageConfig fields.
 // - `hidden` removes rows from a keyframe's sightings before the app's `record`: a depth test that
 //   `sees` (CoverageMap.swift lines 241-248) would make, computed by the caller.

experiments/evals/evals/coverage.py

@@ -1,4 +1,4 @@
-"""Does the app's coverage map claim wall and ground that no photo saw? (README section 7)
+"""Does the app's coverage map claim wall and ground that no photo saw? (METHODS.md section 7)
 
 The app exports as observed the stretches `CoverageMap.coveredIntervals` returns (`ios/HouseScanKit`
 on t3/ios-mvf), and the server passes a check only over them. This runs that code, unmodified, on
@@ -10,7 +10,7 @@
 to the app in ARKit's camera axes (+x right, +y up, looking along -z), intrinsics unrotated, with
 (0, 0) at the image's top-left corner, the convention of `CameraFrame`.
 
-The truth, the pass criteria and the cause labels are defined in README section 7. The only copy of
+The truth, the pass criteria and the cause labels are defined in METHODS.md section 7. The only copy of
 app logic here is `app_rows`, a replica of `CoverageMap.visibleRows` used to name why the app turned
 a photo down; `main` refuses to report unless it reproduces the app's own sightings exactly.
 """
@@ -37,6 +37,7 @@
     scan_points,
     up_direction,
 )
+from evals.pairs import results_json
 from evals.paths import ETH3D_DIR, EVALS_DIR
 
 KIT_COMMIT = "beede15f568b3a4d694fb275caf9eaaa882c546b"
@@ -412,7 +413,8 @@ class PhotoTruth:
 
     @property
     def saw(self) -> np.ndarray:
-        return self.framed & self.in_range & self.facing & ~self.hidden
+        # No scan point near the pixel leaves occlusion unknown, which is not seen.
+        return self.framed & self.in_range & self.facing & ~self.hidden & ~self.no_scan
 
 
 def photo_truth(
@@ -602,8 +604,10 @@ def evaluate_band(
 
     ft = COLUMN_M / FEET
     claimed_ft = claimed.sum() * ft
-    no_scan = np.stack([t.no_scan & t.saw for t in truths], axis=-1)
-    only_unknown = seen1 & ~(saw & ~no_scan).any(axis=-1)
+    # Samples no photo saw, but where some photo framed them over an empty scan pixel: unknown
+    # rather than unseen. Counted apart so a laser hole is visible in the result.
+    unknown = np.stack([t.framed & t.in_range & t.facing & t.no_scan for t in truths], axis=-1)
+    only_unknown = ~seen1 & unknown.any(axis=-1)
     return {
         "claimed_ft": claimed_ft,
         "false_observed_ft": false_cols.sum() * ft,
@@ -614,7 +618,7 @@ def evaluate_band(
         "seen_two_view_ft": seen2.all(axis=1).sum() * ft,
         "missed_ft": missed_cols.sum() * ft,
         "missed_by_cause_ft": {k: missed_causes[k] * ft for k in MISSED_CAUSES},
-        "claimed_samples_seen_only_where_scan_is_empty": float(only_unknown[claimed].mean())
+        "claimed_samples_unknown_for_lack_of_scan": float(only_unknown[claimed].mean())
         if claimed.any()
         else 0.0,
         # Nothing claimed tests nothing.
@@ -816,7 +820,7 @@ def markdown(runs: list[dict]) -> str:
         f"HouseScanKit `CoverageMap` at {KIT_COMMIT} (t3/ios-mvf), unmodified, default "
         "`CoverageConfig`. ETH3D photos with true poses, "
         "every photo a kept keyframe. Lengths in feet along the wall. Definitions and pass criteria: "
-        "README section 7.",
+        "METHODS.md section 7.",
         "",
         "| Scene | Band | Claimed | False-observed (share) | Pass (<= 0.5 ft) | Causes: occlusion / frame edge / range | False-observed, 2-position truth (share) | Seen from 2 positions | Missed | Missed causes: frame edge / grazing / range / one position |",
         "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |",
@@ -839,7 +843,7 @@ def markdown(runs: list[dict]) -> str:
         "",
         "## Setup and validity checks",
         "",
-        "| Scene | Photos (with a sighting) | Wall stretch | Of it, pilaster faces | Wall plane fit RMS | Ground fit RMS | App sightings | Replica mismatches | Claimed samples seen only where the scan is empty (wall / ground) |",
+        "| Scene | Photos (with a sighting) | Wall stretch | Of it, pilaster faces | Wall plane fit RMS | Ground fit RMS | App sightings | Replica mismatches | Claimed samples unknown for lack of scan (wall / ground) |",
         "| --- | --- | --- | --- | --- | --- | --- | --- | --- |",
     ]
     for r in runs:
@@ -854,8 +858,8 @@ def markdown(runs: list[dict]) -> str:
             f"{r['wall_length_ft']:.1f} ft | {r['pilaster_ft']:.1f} ft | "
             f"{r['wall_fit_rms_cm']:.1f} cm | {r['ground_fit_rms_cm']:.1f} cm | "
             f"{r['sightings']} | {r['replica_mismatches']} | "
-            f"{100 * b['wall']['claimed_samples_seen_only_where_scan_is_empty']:.1f}% / "
-            f"{100 * b['ground']['claimed_samples_seen_only_where_scan_is_empty']:.1f}% |"
+            f"{100 * b['wall']['claimed_samples_unknown_for_lack_of_scan']:.1f}% / "
+            f"{100 * b['ground']['claimed_samples_unknown_for_lack_of_scan']:.1f}% |"
         )
     return "\n".join(lines) + "\n"
 
@@ -865,7 +869,7 @@ def main() -> None:
     runs = [evaluate_scene(scene) for scene in RUNS]
     bad = [r["scene"] for r in runs if r["wall"] and r["replica_mismatches"]]
     RESULTS.mkdir(exist_ok=True)
-    (RESULTS / "coverage.json").write_text(json.dumps(runs, indent=1))
+    (RESULTS / "coverage.json").write_text(results_json(runs))
     if bad:
         raise SystemExit(f"cause replica disagrees with the app on {bad}; causes would be wrong")
     md = markdown(runs)

experiments/evals/evals/coverage_options.py

@@ -1,4 +1,4 @@
-"""Options for occlusion in the app's coverage map, on section 7's wall and truth (README section 7b).
+"""Options for occlusion in the app's coverage map, on section 7's wall and truth (METHODS.md section 7b).
 
 Each option runs the app's CoverageMap (HouseScanKit at `coverage.KIT_COMMIT`, unedited) through
 `coverage_driver/`, which models the rule change around it:
@@ -11,8 +11,6 @@
 
 from __future__ import annotations
 
-import json
-
 from evals.coverage import (
     FEET,
     HIDE_ABS_M,
@@ -25,6 +23,7 @@
     run_app,
     setup_scene,
 )
+from evals.pairs import results_json
 
 SCENE = "electro"
 # Pre-registered: 0-45 degrees and 0.25-2 m. 60 and 75 degrees and 5 m were added after that grid
@@ -64,7 +63,8 @@ def depth_hidden(
             setup.cfg["maxDistance"] if reach is None else reach,
             relief,
         )
-        bad = (t.hidden | ~t.in_range).any(axis=1)
+        # A depth test with no depth there (no scan return) cannot credit the row.
+        bad = (t.hidden | t.no_scan | ~t.in_range).any(axis=1)
         if bad.any():
             out.append({**x, "rows": [r for r, b in zip(x["rows"], bad, strict=True) if b]})
     return out
@@ -80,7 +80,9 @@ def _row(label: str, bands: dict, lidar: bool) -> dict:
         "missed_ft": wall["missed_ft"],
         "ground_claimed_ft": bands["ground"]["claimed_ft"],
         "ground_false_observed_ft": bands["ground"]["false_observed_ft"],
-        "passes": bool(wall["false_observed_ft"] <= PASS_FT),
+        # An option that claims no wall tested nothing, so it neither passes nor fails; the main
+        # evaluator (`evals.coverage`) records the same case as untested.
+        "passes": bool(wall["false_observed_ft"] <= PASS_FT) if wall["claimed_ft"] > 0 else None,
     }
 
 
@@ -123,6 +125,14 @@ def depth_option(label: str, n_rows: int, reach: float | None, relief: float) ->
     }
 
 
+PASS_CELL = {True: "yes", False: "no", None: "untested"}
+
+
+def false_observed_cell(row: dict) -> str:
+    """False-observed feet, or "untested" when the option claimed no wall to test."""
+    return "untested" if row["passes"] is None else f"{row['false_observed_ft']:.1f}"
+
+
 def markdown(res: dict) -> str:
     lines = [
         "# Options for occlusion in the coverage map (generated by `make coverage-options`)",
@@ -133,30 +143,30 @@ def markdown(res: dict) -> str:
         "",
         "## Position baseline b and angle diversity theta (no LiDAR): false-observed / missed",
         "",
+        "An option that claims no wall is untested, not passed.",
+        "",
         "| b \\ theta | " + " | ".join(f"{t} deg" for t in ANGLES_DEG) + " |",
         "| --- |" + " --- |" * len(ANGLES_DEG),
     ]
     for b in BASELINES_M:
         cells = [
-            f"{g['false_observed_ft']:.1f} / {g['missed_ft']:.1f}"
-            for g in res["grid"]
-            if g["b_m"] == b
+            f"{false_observed_cell(g)} / {g['missed_ft']:.1f}" for g in res["grid"] if g["b_m"] == b
         ]
         lines.append(f"| {b:g} m | " + " | ".join(cells) + " |")
     lines += [
         "",
         "## Options",
         "",
         "The last four rows allow that much wall relief in front of the tapped plane, a setting "
-        "chosen after seeing this wall; see README section 7b.",
+        "chosen after seeing this wall; see METHODS.md section 7b.",
         "",
         "| Option | Needs LiDAR | Claimed | False-observed | Pass | Missed | Ground claimed |",
         "| --- | --- | --- | --- | --- | --- | --- |",
     ]
     for r in res["options"] + res["relief"]:
         lines.append(
             f"| {r['option']} | {'yes' if r['needs_lidar'] else 'no'} | {r['claimed_ft']:.1f} | "
-            f"{r['false_observed_ft']:.1f} | {'yes' if r['passes'] else 'no'} | "
+            f"{false_observed_cell(r)} | {PASS_CELL[r['passes']]} | "
             f"{r['missed_ft']:.1f} | {r['ground_claimed_ft']:.1f} |"
         )
     return "\n".join(lines) + "\n"
@@ -166,7 +176,7 @@ def main() -> None:
     check_kit()
     res = evaluate()
     RESULTS.mkdir(exist_ok=True)
-    (RESULTS / "coverage_options.json").write_text(json.dumps(res, indent=1))
+    (RESULTS / "coverage_options.json").write_text(results_json(res))
     md = markdown(res)
     (RESULTS / "coverage_options.md").write_text(md)
     print(md)

experiments/evals/evals/datasets.py

@@ -4,7 +4,9 @@
     uv run python -m evals.datasets eth3d      # ETH3D facade and electro (about 2.4 GB of archives)
 
 Each archive is checked against its size and sha256 below, unpacked (only the files the evals use),
-then deleted. A download refuses to start with less than `MIN_FREE_GB` free, because the Mac these
+then deleted. A completion record, written only after every expected file is in place, ties the
+unpacked files to the archive's sha256; a later run skips the archive only if that record matches
+and every recorded file is still there at its recorded size. A download refuses to start with less than `MIN_FREE_GB` free, because the Mac these
 run on is shared. Both datasets are for non-commercial research: they measure accuracy here and are
 never committed or redistributed.
 """
@@ -13,6 +15,7 @@
 
 import argparse
 import hashlib
+import json
 import shutil
 import urllib.request
 import zipfile
@@ -152,6 +155,7 @@ def fetch(archive: Archive) -> None:
         raise SystemExit(
             f"{archive.name}: got {size} bytes sha256 {digest}, expected {archive.size} {archive.sha256}"
         )
+    record_path(archive).unlink(missing_ok=True)
     if archive.name.endswith(".zip"):
         with zipfile.ZipFile(tmp) as z:
             names = [
@@ -160,11 +164,48 @@ def fetch(archive: Archive) -> None:
             z.extractall(archive.dest, members=names)
     else:
         with py7zr.SevenZipFile(tmp) as z:
+            names = z.getnames()
             z.extractall(archive.dest)
     tmp.unlink()
+    write_record(archive, names)
     print(f"  verified and unpacked into {archive.dest}")
 
 
+def record_path(archive: Archive) -> Path:
+    return archive.dest / ".complete" / f"{archive.name}.json"
+
+
+def write_record(archive: Archive, names: list[str]) -> None:
+    """Record the unpacked files once every expected one is present: the marker, and at least one
+    file under each requested member."""
+    files = {n: (archive.dest / n).stat().st_size for n in names if (archive.dest / n).is_file()}
+    missing = [m for m in archive.members if not any(n.startswith(m) for n in files)]
+    if not (archive.dest / archive.marker).exists():
+        missing.append(archive.marker)
+    if missing:
+        raise SystemExit(f"{archive.name}: unpacked without {missing}; not recording it complete")
+    path = record_path(archive)
+    path.parent.mkdir(exist_ok=True)
+    path.write_text(json.dumps({"sha256": archive.sha256, "files": files}, indent=0))
+
+
+def is_complete(archive: Archive) -> bool:
+    """True when this archive's record matches its pinned sha256 and every recorded file exists at
+    its recorded size. Sizes catch a truncated or partly deleted copy without rehashing gigabytes;
+    an edit that keeps a file's size is not caught."""
+    path = record_path(archive)
+    if not path.exists():
+        return False
+    record = json.loads(path.read_text())
+    if record.get("sha256") != archive.sha256 or not record.get("files"):
+        return False
+    for name, size in record["files"].items():
+        f = archive.dest / name
+        if not f.is_file() or f.stat().st_size != size:
+            return False
+    return True
+
+
 def main() -> None:
     ap = argparse.ArgumentParser(
         description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
@@ -175,9 +216,8 @@ def main() -> None:
     )
     args = ap.parse_args()
     for a in ADVIO if args.dataset == "advio" else ETH3D:
-        marker = a.dest / a.marker
-        if marker.exists() and not args.force:
-            print(f"{a.name}: already unpacked ({marker})")
+        if is_complete(a) and not args.force:
+            print(f"{a.name}: already unpacked and recorded complete ({record_path(a)})")
             continue
         fetch(a)
 

experiments/evals/evals/drift.py

@@ -32,13 +32,13 @@
 from __future__ import annotations
 
 import argparse
-import json
 from dataclasses import asdict, dataclass
 from pathlib import Path
 
 import numpy as np
 
 from evals.advio import PoseTrack, _read_pose_csv, load_sequence
+from evals.pairs import results_json
 from evals.paths import ADVIO_DIR
 
 FEET = 0.3048
@@ -415,9 +415,7 @@ def main() -> None:
     pooled = _pooled(results)
     args.out.mkdir(parents=True, exist_ok=True)
     slim = [{k: v for k, v in r.items() if k != "errors_in"} for r in results]
-    (args.out / "advio_drift.json").write_text(
-        json.dumps({"sequences": slim, "pooled": pooled}, indent=1)
-    )
+    (args.out / "advio_drift.json").write_text(results_json({"sequences": slim, "pooled": pooled}))
     md = _markdown(results, pooled)
     (args.out / "advio_drift.md").write_text(md)
     print(md)

experiments/evals/evals/eth3d.py

@@ -138,7 +138,10 @@ def read_ply_xyz(path: Path) -> np.ndarray:
     with path.open("rb") as fh:
         header = []
         while True:
-            line = fh.readline().decode("ascii").strip()
+            raw = fh.readline()
+            if not raw:
+                raise ValueError(f"{path}: file ends before the PLY header's end_header line")
+            line = raw.decode("ascii").strip()
             header.append(line)
             if line == "end_header":
                 break