From 4b6c430956bb35edff546f140e3152ed75c56246 Mon Sep 17 00:00:00 2001 From: huntertcarver Date: Fri, 25 Sep 2026 23:28:04 -0500 Subject: [PATCH] experiments: validate camera metadata for reconstruction --- experiments/capture-contract/README.md | 61 +++++ .../capture-contract/capture_contract.py | 216 ++++++++++++++++++ .../fixtures/synthetic-session.json | 118 ++++++++++ experiments/capture-contract/test_contract.py | 148 ++++++++++++ 4 files changed, 543 insertions(+) create mode 100644 experiments/capture-contract/README.md create mode 100644 experiments/capture-contract/capture_contract.py create mode 100644 experiments/capture-contract/fixtures/synthetic-session.json create mode 100644 experiments/capture-contract/test_contract.py diff --git a/experiments/capture-contract/README.md b/experiments/capture-contract/README.md new file mode 100644 index 000000000..5925b8ec5 --- /dev/null +++ b/experiments/capture-contract/README.md @@ -0,0 +1,61 @@ +# Capture metadata contract probe + +## Question and pass criteria + +Can a recorded camera frame preserve the same pixel ray when converted from ARKit to the OpenCV camera convention used by reconstruction models? + +Before running the probe, require: +- Translated and rotated synthetic cameras produce hand-computed rays and projections. +- Crop/resize transforms preserve the original ray when intrinsics change with the image. +- Incorrect camera axes, inverted extrinsics, matrix storage, focal lengths and source units fail explicit controls. +- Keyframes retain source IDs, image references and tracking state; arrival order may differ from timestamp order. +- A conversion does not pretend that session coordinates are meter-anchor coordinates or that successful arithmetic establishes physical accuracy. + +## Scope + +This isolated experiment consumes the **measure-lab-session version 2** camera/tap subset documented in the team's [measurement-rig proposal](https://github.com/SamGu-NRX/house-scanning/pull/7), inspected at commit `ca89849224814732b389e4b6ea9e26f3d41d09a5`. It complements that rig; it does not implement or modify capture. It converts validated camera metadata into an inspectable JSON file for a future reconstruction adapter. It does not run MapAnything, load photos, validate the full session schema, inspect image dimensions, or determine installation eligibility. + +The synthetic manifest is a focused contract fixture, not a complete app export. Its image references are placeholders; there are no real photos, addresses, or customer identifiers. + +## Run + +Python 3.10+ standard library only. From this directory: + +```sh +python3 -m unittest -v +python3 capture_contract.py fixtures/synthetic-session.json --out data/converted.json +``` + +For a real permitted session, point the first argument at its session.json and keep outputs in ignored data/. Never commit real manifests or images. + +## Coordinate contract + +- Input: ARKit camera-to-world, 16 values **column by column**; right-handed camera axes with X right, Y up, looking along negative Z. +- Output: OpenCV camera-to-world, a 4×4 matrix serialized as **nested rows**; X right, Y down, Z forward. +- Conversion changes the camera basis with diag(1,-1,-1,1) on the right. It does not rotate or re-anchor the world. +- World remains the source ARKit session world, Y up. No meter anchor is available in this source contract; consumers must supply and validate one before anchor-relative placement. +- A COLMAP exporter would additionally invert the camera-to-world transform; COLMAP stores world-to-camera. This probe does not emit COLMAP files. +- Intrinsics apply to each saved sensor-native image. The crop/resize helper is tested separately; original images are not automatically resized or reoriented. +- The output's image references remain relative to the **source manifest directory**, not the converted JSON's directory. The CLI records that local directory as `image_reference_base`; keep the generated output private with the source capture. + +The source format/version defines the expected conventions. A rigid matrix alone cannot prove the producer used those conventions. Tap records provide an additional consistency check when available; absent taps mean that check was not performed. + +## Validation and evidence limits + +The rigid-rotation tolerance 1e-4 and ray arithmetic tolerance 1e-6 detect serialization and coordinate mistakes. They are numerical test tolerances, **not measured accuracy bounds**. A geometrically self-consistent but wrong pose or wrongly calibrated camera can pass these tests. + +The output preserves limited tracking; it never silently upgrades it to normal. Even normal tracking does not establish metric accuracy. Original source images, field measurements, calibration and tracking-epoch validation remain necessary. + +Duplicate frame IDs, missing tap references, incompatible versions/units, nonfinite numeric inputs, reflections and scaled rotations are rejected. The CLI rejects duplicate JSON keys and emits no new output on validation failure. It does not access the referenced image paths or execute anything from the manifest. + +## Result + +All 14 known-answer and malformed-input controls passed on macOS using Python 3.13.7. The CLI converted two synthetic cameras and checked both saved tap rays. An independent review also checked 100 generated proper rotations against a separate construction and verified the extreme-intrinsics, nonunit-ray and off-image-tap controls. This is arithmetic/metadata evidence only; real-device projection and geometric accuracy remain untested. The fixture includes a limited-tracking frame, which remains explicitly limited in the output. + +## Primary references + +- [Apple ARCamera transform](https://developer.apple.com/documentation/arkit/arcamera/transform) +- [Apple camera intrinsics](https://developer.apple.com/documentation/arkit/arcamera/intrinsics) +- [COLMAP coordinate definitions](https://colmap.github.io/format.html) +- [MapAnything inputs and conventions](https://github.com/facebookresearch/map-anything) +- [OpenCV calibration and image scaling](https://docs.opencv.org/4.13.0/dc/dbb/tutorial_py_calibration.html) diff --git a/experiments/capture-contract/capture_contract.py b/experiments/capture-contract/capture_contract.py new file mode 100644 index 000000000..9e2ea70bc --- /dev/null +++ b/experiments/capture-contract/capture_contract.py @@ -0,0 +1,216 @@ +"""Validate and adapt measure-lab v2 camera metadata. No images or models are run.""" +from __future__ import annotations + +import argparse +from hashlib import sha256 +import json +from math import hypot, isfinite +from pathlib import Path + +ROTATION_TOLERANCE = 1e-4 # Float32 serialization tolerance, not field accuracy. +RAY_TOLERANCE = 1e-6 # Arithmetic agreement, not a measurement error bound. + + +def vector(value, size, label): + if not isinstance(value, list) or len(value) != size: + raise ValueError(f"{label}: expected {size} numbers") + if any(isinstance(x, bool) or not isinstance(x, (int, float)) for x in value): + raise ValueError(f"{label}: numbers must be finite") + try: + numbers = tuple(float(x) for x in value) + except OverflowError as error: + raise ValueError(f"{label}: number exceeds floating-point range") from error + if not all(isfinite(x) for x in numbers): + raise ValueError(f"{label}: numbers must be finite") + return numbers + + +def dot(a, b): + return sum(x * y for x, y in zip(a, b)) + + +def norm(v): + return hypot(*v) + + +def cross(a, b): + return (a[1]*b[2]-a[2]*b[1], a[2]*b[0]-a[0]*b[2], + a[0]*b[1]-a[1]*b[0]) + + +def unit(v): + length = norm(v) + if not isfinite(length) or length == 0: + raise ValueError("direction must be finite and nonzero") + return tuple(x / length for x in v) + + +def distance(a, b): + return norm(tuple(x - y for x, y in zip(a, b))) + + +def camera_pose(value, label): + p = vector(value, 16, label) + if any(abs(p[i]-expected) > 1e-8 + for i, expected in zip((3, 7, 11, 15), (0, 0, 0, 1))): + raise ValueError(f"{label}: expected column-major affine camera-to-world pose") + axes = [p[i:i+3] for i in (0, 4, 8)] + if (any(abs(norm(a)-1) > ROTATION_TOLERANCE for a in axes) + or any(abs(dot(axes[i], axes[j])) > ROTATION_TOLERANCE + for i, j in ((0, 1), (0, 2), (1, 2))) + or distance(cross(axes[0], axes[1]), axes[2]) > ROTATION_TOLERANCE): + raise ValueError(f"{label}: expected a proper rigid rotation, no scale/reflection") + return p + + +def positive_size(value, label): + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{label}: expected a positive integer") + return value + + +def validate_frame(frame): + label = f"keyframe {frame.get('id', '?')}" + if not isinstance(frame.get("id"), str) or not frame["id"]: + raise ValueError(f"{label}: missing id") + width = positive_size(frame.get("w"), label + ".w") + height = positive_size(frame.get("h"), label + ".h") + intrinsics = vector(frame.get("intrinsics"), 4, label + ".intrinsics") + if intrinsics[0] <= 0 or intrinsics[1] <= 0: + raise ValueError(f"{label}: focal lengths must be positive") + pose = camera_pose(frame.get("pose"), label + ".pose") + vector([frame.get("timestamp")], 1, label + ".timestamp") + if not isinstance(frame.get("tracking"), str): + raise ValueError(f"{label}: tracking must be explicit") + if not isinstance(frame.get("img"), str) or not frame["img"]: + raise ValueError(f"{label}: image reference must be explicit") + return width, height, intrinsics, pose + + +def ray_for_pixel(frame, pixel): + width, height, (fx, fy, cx, cy), pose = validate_frame(frame) + u, v = vector(pixel, 2, "pixel") + if not (0 <= u <= width and 0 <= v <= height): + raise ValueError("tap pixel lies outside its saved image") + local = ((u-cx)/fx, -(v-cy)/fy, -1) + # ARKit camera axes are columns; camera looks along -Z. + direction = tuple(sum(pose[4*j+i]*local[j] for j in range(3)) + for i in range(3)) + return pose[12:15], unit(direction) + + +def opencv_camera_to_world(pose): + p = list(camera_pose(pose, "pose")) + # Change CAMERA basis only. The session world stays right-handed, Y up. + for column in (1, 2): + for row in range(4): + p[4*column+row] *= -1 + return [[p[4*column+row] for column in range(4)] for row in range(4)] + + +def crop_resize_intrinsics(intrinsics, crop_origin, scale): + fx, fy, cx, cy = vector(intrinsics, 4, "intrinsics") + x0, y0 = vector(crop_origin, 2, "crop origin") + sx, sy = vector(scale, 2, "scale") + if fx <= 0 or fy <= 0 or sx <= 0 or sy <= 0: + raise ValueError("focal lengths and image scale must be positive") + return [fx*sx, fy*sy, (cx-x0)*sx, (cy-y0)*sy] + + +def validate_session(session): + if session.get("format") != "measure-lab-session" or session.get("formatVersion") != 2: + raise ValueError("expected measure-lab-session formatVersion 2") + if session.get("units", {}).get("length") != "meters": + raise ValueError("expected explicit meters; no implicit unit conversion") + session_id = session.get("session", {}).get("id") + if not isinstance(session_id, str) or not session_id: + raise ValueError("session.id must be explicit") + frames = {} + for frame in session.get("keyframes", []): + validate_frame(frame) + if frame["id"] in frames: + raise ValueError(f"duplicate keyframe id: {frame['id']}") + frames[frame["id"]] = frame + if not frames: + raise ValueError("no listed keyframes") + tap_checks = [] + for tap in session.get("taps", []): + frame = frames.get(tap.get("keyframe")) + if frame is None: + raise ValueError(f"tap {tap.get('id')}: unknown keyframe") + origin, direction = ray_for_pixel(frame, tap.get("pixel")) + saved_origin = vector(tap.get("rayOrigin"), 3, "tap.rayOrigin") + saved_direction = vector(tap.get("rayDirection"), 3, "tap.rayDirection") + if abs(norm(saved_direction)-1) > RAY_TOLERANCE: + raise ValueError(f"tap {tap.get('id')}: saved direction must be a unit vector") + origin_error = distance(origin, saved_origin) + direction_error = distance(direction, saved_direction) + if origin_error > RAY_TOLERANCE or direction_error > RAY_TOLERANCE: + raise ValueError(f"tap {tap.get('id')}: saved ray disagrees with its keyframe " + f"(origin delta={origin_error:g}, direction delta={direction_error:g})") + tap_checks.append({"id": tap.get("id"), "origin_delta_m": origin_error, + "unit_direction_delta": direction_error}) + converted = [] + # Arrival order can differ from exposure order; source explicitly permits it. + for frame in sorted(frames.values(), key=lambda item: (item["timestamp"], item["id"])): + width, height, (fx, fy, cx, cy), pose = validate_frame(frame) + converted.append({ + "id": frame["id"], "image_reference": frame["img"], + "width": width, "height": height, "timestamp": frame["timestamp"], + "intrinsics": [[fx, 0, cx], [0, fy, cy], [0, 0, 1]], + "camera_to_world": opencv_camera_to_world(list(pose)), + "tracking": frame["tracking"], + "tracking_normal": frame["tracking"] == "normal", + }) + return { + "format": "capture-contract-probe-1", + "world_frame": {"id": session_id, "axes": "ARKit session: right-handed, Y up", + "units": "meters as reported by ARKit; not independently calibrated", + "meter_anchored": False}, + "camera_axes": "OpenCV: X right, Y down, Z forward", + "matrix_serialization": "nested rows; camera-to-world", + "keyframes": converted, "tap_checks": tap_checks, + "limits": ["metadata arithmetic only", "no image-size or image-content verification", + "no device accuracy, pose epoch or scale calibration established", + "tracking-normal does not establish measurement accuracy", + "no meter anchor exists in this source contract"], + } + + +def no_duplicates(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def reject_constant(value): + raise ValueError(f"invalid JSON numeric constant: {value}") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("session", type=Path) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + try: + if args.session.resolve() == args.out.resolve(): + raise ValueError("output must not overwrite the source manifest") + raw = args.session.read_bytes() + session = json.loads(raw, object_pairs_hook=no_duplicates, + parse_constant=reject_constant) + result = validate_session(session) + result["source_manifest_sha256"] = sha256(raw).hexdigest() + result["image_reference_base"] = str(args.session.resolve().parent) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(result, indent=2, allow_nan=False) + "\n") + except (ValueError, TypeError, KeyError, AttributeError, OSError) as error: + parser.exit(2, f"capture contract: {error}\n") + print(f"Checked {len(result['keyframes'])} frame(s), {len(result['tap_checks'])} tap(s). " + f"Wrote {args.out}. Metadata only; physical accuracy untested.") + + +if __name__ == "__main__": + main() diff --git a/experiments/capture-contract/fixtures/synthetic-session.json b/experiments/capture-contract/fixtures/synthetic-session.json new file mode 100644 index 000000000..b50c1ab94 --- /dev/null +++ b/experiments/capture-contract/fixtures/synthetic-session.json @@ -0,0 +1,118 @@ +{ + "format": "measure-lab-session", + "formatVersion": 2, + "units": { + "length": "meters" + }, + "session": { + "id": "synthetic-contract" + }, + "keyframes": [ + { + "id": "k00002", + "img": "keyframes/synthetic-2.jpg", + "w": 1000, + "h": 800, + "intrinsics": [ + 500, + 500, + 500, + 400 + ], + "pose": [ + 0, + 0, + -1, + 0, + 0, + 1, + 0, + 0, + 1, + 0, + 0, + 0, + 2, + 1, + 5, + 1 + ], + "timestamp": 2, + "tracking": "limited.relocalizing", + "reason": "tap", + "depth": null + }, + { + "id": "k00001", + "img": "keyframes/synthetic-1.jpg", + "w": 1000, + "h": 800, + "intrinsics": [ + 500, + 500, + 500, + 400 + ], + "pose": [ + 1, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 2, + 3, + 1 + ], + "timestamp": 1, + "tracking": "normal", + "reason": "tap", + "depth": null + } + ], + "taps": [ + { + "id": "tap-1", + "keyframe": "k00001", + "pixel": [ + 750, + 650 + ], + "rayOrigin": [ + 0, + 2, + 3 + ], + "rayDirection": [ + 0.4082482904638631, + -0.4082482904638631, + -0.8164965809277261 + ] + }, + { + "id": "tap-2", + "keyframe": "k00002", + "pixel": [ + 500, + 400 + ], + "rayOrigin": [ + 2, + 1, + 5 + ], + "rayDirection": [ + -1, + 0, + 0 + ] + } + ] +} diff --git a/experiments/capture-contract/test_contract.py b/experiments/capture-contract/test_contract.py new file mode 100644 index 000000000..38709b6a7 --- /dev/null +++ b/experiments/capture-contract/test_contract.py @@ -0,0 +1,148 @@ +"""Known-answer controls. These verify adapters, not device measurement accuracy.""" +from copy import deepcopy +import json +from pathlib import Path +import tempfile +import subprocess +import sys +import unittest + +from capture_contract import (camera_pose, crop_resize_intrinsics, + no_duplicates, ray_for_pixel, validate_session) + +ROOT = Path(__file__).parent + +class CameraContractTests(unittest.TestCase): + def setUp(self): + self.session = json.loads((ROOT/"fixtures/synthetic-session.json").read_text()) + + def test_translated_off_axis_ray_and_rotated_center_ray(self): + result = validate_session(self.session) + self.assertEqual([f["id"] for f in result["keyframes"]], ["k00001","k00002"]) + a, b = result["keyframes"] + self.assertEqual(a["camera_to_world"], + [[1,0,0,0],[0,-1,0,2],[0,0,-1,3],[0,0,0,1]]) + self.assertEqual(b["camera_to_world"], + [[0,0,-1,2],[0,-1,0,1],[-1,0,0,5],[0,0,0,1]]) + # Independently supplied world targets must project to known image pixels. + for frame, point, expected in [(a,[1,1,1],[750,650]), (b,[0,1,5],[500,400])]: + matrix = frame["camera_to_world"] + delta = [point[i]-matrix[i][3] for i in range(3)] + camera = [sum(delta[i]*matrix[i][j] for i in range(3)) for j in range(3)] + self.assertGreater(camera[2],0) + k = frame["intrinsics"] + pixel = [k[0][0]*camera[0]/camera[2]+k[0][2], + k[1][1]*camera[1]/camera[2]+k[1][2]] + self.assertEqual(pixel,expected) + self.assertFalse(b["tracking_normal"]) + self.assertFalse(result["world_frame"]["meter_anchored"]) + + def test_crop_and_resize_preserve_a_known_ray(self): + original = self.session["keyframes"][1] + processed = deepcopy(original) + processed["intrinsics"] = crop_resize_intrinsics( + original["intrinsics"], [100,50], [.5,.25]) + processed["w"], processed["h"] = 450, 175 + self.assertEqual(processed["intrinsics"],[250,125,200,87.5]) + # Original (750,650) becomes ((750-100)/2,(650-50)/4). + self.assertEqual(ray_for_pixel(original,[750,650]), + ray_for_pixel(processed,[325,150])) + + def test_wrong_intrinsics_are_detected_by_saved_tap(self): + self.session["keyframes"][1]["intrinsics"][0] *= .5 + with self.assertRaisesRegex(ValueError,"saved ray disagrees"): + validate_session(self.session) + + def test_axis_flip_or_inverted_pose_is_detected(self): + for pose in [ + [1,0,0,0,0,-1,0,0,0,0,-1,0,0,2,3,1], # CV basis mislabeled AR + [1,0,0,0,0,1,0,0,0,0,1,0,0,-2,-3,1], # world-to-camera mislabeled + ]: + session=deepcopy(self.session) + session["keyframes"][1]["pose"]=pose + with self.assertRaisesRegex(ValueError,"saved ray disagrees"): + validate_session(session) + + def test_row_major_pose_is_rejected(self): + p=[1,0,0,0,0,1,0,2,0,0,1,3,0,0,0,1] + with self.assertRaisesRegex(ValueError,"column-major"): + camera_pose(p,"pose") + + def test_reflection_and_nonrigid_scale_are_rejected(self): + for multiplier in [-1,2]: + pose=self.session["keyframes"][1]["pose"][:] + pose[0]=multiplier + with self.assertRaisesRegex(ValueError,"proper rigid"): + camera_pose(pose,"pose") + + def test_unrecognized_source_contract_and_units_are_rejected(self): + for field,value in [("formatVersion",3),("format","other")]: + session=deepcopy(self.session);session[field]=value + with self.assertRaises(ValueError):validate_session(session) + self.session["units"]["length"]="feet" + with self.assertRaisesRegex(ValueError,"explicit meters"): + validate_session(self.session) + + def test_duplicate_frames_and_missing_references_are_rejected(self): + duplicate=deepcopy(self.session) + duplicate["keyframes"].append(duplicate["keyframes"][0]) + with self.assertRaisesRegex(ValueError,"duplicate keyframe"): + validate_session(duplicate) + self.session["taps"][0]["keyframe"]="not-present" + with self.assertRaisesRegex(ValueError,"unknown keyframe"): + validate_session(self.session) + + def test_nan_boolean_and_zero_intrinsics_are_rejected(self): + for invalid in [float("nan"),float("inf"),True,0]: + session=deepcopy(self.session) + session["keyframes"][1]["intrinsics"][0]=invalid + with self.assertRaises(ValueError):validate_session(session) + + def test_duplicate_json_keys_are_rejected(self): + with self.assertRaisesRegex(ValueError,"duplicate JSON"): + json.loads('{"format": 1, "format": 2}',object_pairs_hook=no_duplicates) + + def test_taps_outside_saved_image_are_rejected(self): + for pixel in [[-100,10000],[-1,400],[500,801]]: + session=deepcopy(self.session) + session["taps"][0]["pixel"]=pixel + with self.assertRaisesRegex(ValueError,"outside its saved image"): + validate_session(session) + + def test_extreme_focal_length_cannot_turn_a_ray_into_zero(self): + self.session["keyframes"][1]["intrinsics"][0]=1e-200 + self.session["taps"][0]["rayDirection"]=[0,0,0] + with self.assertRaisesRegex(ValueError,"unit vector"): + validate_session(self.session) + self.session["keyframes"][1]["intrinsics"][0]=1e-320 + with self.assertRaisesRegex(ValueError,"finite and nonzero"): + validate_session(self.session) + + def test_integer_overflow_is_a_validation_error(self): + self.session["keyframes"][1]["intrinsics"][0]=10**400 + with self.assertRaisesRegex(ValueError,"floating-point range"): + validate_session(self.session) + + def test_cli_writes_only_after_validation(self): + with tempfile.TemporaryDirectory() as temp: + out=Path(temp)/"converted.json" + cmd=[sys.executable,str(ROOT/"capture_contract.py"), + str(ROOT/"fixtures/synthetic-session.json"),"--out",str(out)] + result=subprocess.run(cmd,capture_output=True,text=True) + self.assertEqual(result.returncode,0,result.stderr) + converted=json.loads(out.read_text()) + self.assertEqual(len(converted["source_manifest_sha256"]),64) + invalid=Path(temp)/"bad.json";invalid.write_text('{"format": 5}') + badout=Path(temp)/"not-created.json" + result=subprocess.run([sys.executable,str(ROOT/"capture_contract.py"), + str(invalid),"--out",str(badout)],capture_output=True,text=True) + self.assertEqual(result.returncode,2) + self.assertFalse(badout.exists()) + original=invalid.read_bytes() + result=subprocess.run([sys.executable,str(ROOT/"capture_contract.py"), + str(invalid),"--out",str(invalid)],capture_output=True,text=True) + self.assertEqual(result.returncode,2) + self.assertEqual(invalid.read_bytes(),original) + +if __name__ == "__main__": + unittest.main()