From f1e8a20e6b95cd4067a5c4ec1ddceb5efc1969f1 Mon Sep 17 00:00:00 2001 From: Obvious Date: Fri, 16 Oct 2026 18:14:58 +0000 Subject: [PATCH 1/3] feat(backend): deterministic app-shaped synthetic capture generator Render a 30-keyframe walk, LiDAR depth/confidence, CSV streams, meter stills, a confirmed tap and ground truth from a scene spec, with pure NumPy/PIL. The packed tap replays through the packet's own conventions onto the meter box, depth rays share the keyframe FOV with scaled intrinsics, and gzip/JPEG/ids are byte-stable for a given spec and seed. The real server.packet validator and normalizer accept both scenes. Scene specs: clear-wall (no hazards) and gas-left (gas meter 0.6 m centre-to-centre left of the meter). No rule thresholds are encoded. Co-authored-by: Sam Gu --- backend/scripts/gen_app_capture.py | 1205 +++++++++++++++++ .../app_capture_synth/clear-wall.json | 10 + .../fixtures/app_capture_synth/gas-left.json | 10 + backend/tests/test_app_capture_synth.py | 212 +++ 4 files changed, 1437 insertions(+) create mode 100644 backend/scripts/gen_app_capture.py create mode 100644 backend/tests/fixtures/app_capture_synth/clear-wall.json create mode 100644 backend/tests/fixtures/app_capture_synth/gas-left.json create mode 100644 backend/tests/test_app_capture_synth.py diff --git a/backend/scripts/gen_app_capture.py b/backend/scripts/gen_app_capture.py new file mode 100644 index 000000000..84df2df63 --- /dev/null +++ b/backend/scripts/gen_app_capture.py @@ -0,0 +1,1205 @@ +#!/usr/bin/env python3 +"""Deterministic app-shaped capture packet with known ground truth. + +The morning report's biggest risk is that the answer the phone gets back does +not yet depend on the scan: the only real-image packet in the repository +(``ios-kit/fixtures/mini-arkit-lidar``) has six frames and no ground truth for +the meter, hazards or clearances, and the ETH3D fixture store is unreachable +(``backend/docs/STATUS.md``). This script renders the packet the app would have +uploaded from a scene whose true geometry is exact, so the real CPU pipeline can +be checked against ground truth and two different walls can be shown to produce +different answers. + +The renderer is pure numpy/PIL: no Blender, no downloads, no model calls. The +output layout copies ``ios-kit/fixtures/mini-arkit-lidar/packet.json`` (packet +spec v0.4) and passes ``server.packet`` stage ``validate`` and the ios-kit +schema. Next to ``packet.json`` the script writes ``ground_truth.json``: the +wall plane, the meter centre, hazard boxes and the true clearance distances in +metres. + +Scene spec (JSON):: + + { + "name": "clear-wall", # packet/seed identity + "seed": 1401, + "wall": {"xMin": -1.0, "xMax": 5.0, "height": 2.8, "distance": 2.0}, + "camera": {"height": 1.45}, # device height above the ground + "meter": {"x": 2.0, "centreHeight": 1.2, "size": [0.45, 0.65, 0.18]}, + "gasMeter": null, # or {"x": ..., "centreHeight": ..., "size": [...]} + "windows": [{"x": 3.6, "sillHeight": 1.0, "size": [1.2, 1.4]}], + "vents": [{"x": 0.4, "centreHeight": 2.45, "size": [0.3, 0.15]}] + } + +World frame (matches the packet's own ``conventions``): ARKit ``.gravity`` — +y up, origin at the device position when the session started, the camera facing +``-z`` toward the wall. ``meter.x`` and ``centreHeight`` are metres along the +wall from the session origin and above the ground; "left of the meter" as the +homeowner sees the wall is ``-x``. + +Usage:: + + uv run python scripts/gen_app_capture.py \ + --spec tests/fixtures/app_capture_synth/clear-wall.json --out /tmp/clear-wall + +Same spec and seed in, same bytes out: ids are uuid5 of the spec name and seed, +gzip members carry ``mtime=0``, JPEG quality is fixed and every timestamp is a +constant. Camera poses are exact (no drift): the packet *is* the ground truth. + +""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import json +import math +import sys +import uuid +from pathlib import Path + +import numpy as np +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from server.common.geometry import column_major, from_column_major, pixel_ray_arkit # noqa: E402 +from server.packet.conventions import DEVICE_FROM_CAMERA, project_arkit, rotmat_to_quat_xyzw # noqa: E402 + +KEYFRAME_W, KEYFRAME_H = 640, 480 +DEPTH_W, DEPTH_H = 256, 192 +FX = FY = 554.25 # 60 deg horizontal FOV at 640 px +CX, CY = KEYFRAME_W / 2.0, KEYFRAME_H / 2.0 +INTRINSICS = [FX, FY, CX, CY] + +SESSION_START_UPTIME = 4800.0 +WALK_SECONDS = 10.0 +WALK_SPEED_MPS = 1.0 +WALK_START_X = -2.0 +N_KEYFRAMES = 30 # exactly 3 Hz, so every keyframe lands on the 30 Hz arkitPoses grid +KEYFRAME_DT = 1.0 / 3.0 +JPEG_QUALITY = 92 +FEATURES_PER_FRAME = 160 +FEATURES_STRIDE_PX = 24 + +SUN = np.array([-0.45, 0.7, 0.56]) +SUN = SUN / np.linalg.norm(SUN) + +NSYNTH_FIELDS = [ + "keyframes[].pose, intrinsics, timestamp, exposure, exif, lightEstimate (exact walk model)", + "keyframes[].depth and confidence (rays cast against the true scene geometry)", + "keyframes[].features (deterministic grid over the rendered texture)", + "streams.motion, streams.heading, streams.location, streams.imuRaw (walk model + seed noise)", + "streams.arkitPoses (trajectory sampled on the 30 Hz grid)", + "planes, taps (ray cast against the meter box)", +] + +UNITS = { + "length": "meters", + "angle": "degrees", + "time": "seconds of device uptime, the clock of ARFrame.timestamp and CMDeviceMotion.timestamp", + "image": "pixels of the saved JPEG", + "rotationRate": "radians per second", + "acceleration": "g (9.80665 m/s^2)", + "magneticField": "microtesla", +} + +CONVENTIONS = { + "world": ( + "ARKit world frame with worldAlignment .gravity: right-handed, y up (away from gravity), " + "origin at the device's position when the session (epoch) started, -z the camera's initial " + "facing projected onto the horizontal" + ), + "camera": ( + "ARKit camera frame: +x right and +y up in the unrotated landscape sensor image, the camera looks along -z" + ), + "device": "Core Motion device frame: +x right and +y toward the top with the phone upright in portrait, +z out of the screen", + "pose": "camera-to-world 4x4 matrix, 16 numbers column by column (simd_float4x4 layout), meters", + "intrinsics": "[fx, fy, cx, cy] in pixels of the saved, unrotated landscape image", + "pixel": "[u, v] continuous image coordinates: (0, 0) is the top-left corner of the top-left pixel, v grows down", + "ray": "origin is the camera position; direction is a unit vector; camera-space direction ((u - cx)/fx, -(v - cy)/fy, -1)", + "geodetic": "WGS84 degrees; altitude above mean sea level, ellipsoidalAltitude above the WGS84 ellipsoid, meters", + "heading": "degrees clockwise from north, 0 to 360", +} + +SESSION_GATES = { + "trackingStableSeconds": 1.0, + "minimumGroundLookDown": 30.0, + "minimumContactSeparation": 2.0, + "maximumAngleFromWallNormal": 60.0, + "wallValidationTolerance": 0.0508, + "minimumCameraOffsetFromWall": 0.1, + "minimumRayAngle": 15.0, + "maximumRayGap": 0.0508, + "keyframeSpacingMeters": 0.5, + "keyframeSpacingDegrees": 15.0, +} + + +# -------------------------------------------------------------------------------------- +# Scene model +# -------------------------------------------------------------------------------------- + + +class Scene: + """The wall world, in exact metres. Every box is axis-aligned.""" + + def __init__(self, spec: dict) -> None: + self.spec = spec + self.seed = int(spec["seed"]) + self.name = str(spec["name"]) + wall = spec["wall"] + self.wall_x0 = float(wall["xMin"]) + self.wall_x1 = float(wall["xMax"]) + self.wall_height = float(wall["height"]) + self.wall_z = -float(wall["distance"]) # wall plane, cameras on the +z side + self.camera_height = float(spec.get("camera", {}).get("height", 1.45)) + self.ground_y = -self.camera_height + meter = spec["meter"] + self.meter = { + "kind": "electric", + "x": float(meter["x"]), + "centre_y": self.ground_y + float(meter["centreHeight"]), + "size": [float(v) for v in meter["size"]], + } + self.meter["aabb"] = _box_aabb(self.meter["x"], self.meter["centre_y"], self.wall_z, self.meter["size"]) + gas = spec.get("gasMeter") + self.gas = None + if gas: + self.gas = { + "kind": "gas_meter", + "x": float(gas["x"]), + "centre_y": self.ground_y + float(gas["centreHeight"]), + "size": [float(v) for v in gas["size"]], + } + self.gas["aabb"] = _box_aabb(self.gas["x"], self.gas["centre_y"], self.wall_z, self.gas["size"]) + self.windows = [ + { + "x": float(w["x"]), + "sill_y": self.ground_y + float(w["sillHeight"]), + "w": float(w["size"][0]), + "h": float(w["size"][1]), + } + for w in spec.get("windows", []) + ] + self.vents = [ + { + "x": float(v["x"]), + "centre_y": self.ground_y + float(v["centreHeight"]), + "w": float(v["size"][0]), + "h": float(v["size"][1]), + } + for v in spec.get("vents", []) + ] + self.wall_top = self.ground_y + self.wall_height + + def hazards(self) -> list[dict]: + if not self.gas: + return [] + centre = [ + round(self.gas["x"], 6), + round(self.gas["centre_y"], 6), + round(self.wall_z + self.gas["size"][2] / 2.0, 6), + ] + return [{"kind": "gas_meter", "centre": centre, "size": self.gas["size"], "aabb": self.gas["aabb"]}] + + +def _box_aabb(x: float, centre_y: float, wall_z: float, size: list[float]) -> dict: + w, h, d = size + return { + "lo": [x - w / 2.0, centre_y - h / 2.0, wall_z], + "hi": [x + w / 2.0, centre_y + h / 2.0, wall_z + d], + } + + +# -------------------------------------------------------------------------------------- +# Walk trajectory and camera poses +# -------------------------------------------------------------------------------------- + + +def walk_x(t: float) -> float: + return WALK_START_X + WALK_SPEED_MPS * t + + +def walk_y(t: float) -> float: + return 0.03 * math.sin(2.0 * math.pi * t / 1.7) + + +def walk_z(t: float) -> float: + # Distance from the wall is wall.distance - z(t), i.e. 1.55 .. 2.45 m. + return 0.45 * math.sin(2.0 * math.pi * t / 6.4) + + +def aim_point(scene: Scene, t: float) -> np.ndarray: + """The point on the wall the homeowner looks at: toward the meter, with walk lookahead.""" + x = walk_x(t) + return np.array([scene.meter["x"] + 0.45 * (x - scene.meter["x"]), scene.ground_y + 1.1, scene.wall_z]) + + +def camera_pose(position: np.ndarray, target: np.ndarray) -> np.ndarray: + """ARKit camera-to-world matrix: +x right, +y up, looking along -z.""" + forward = target - position + forward = forward / np.linalg.norm(forward) + z_c = -forward + up = np.array([0.0, 1.0, 0.0]) + x_c = np.cross(up, z_c) + x_c = x_c / np.linalg.norm(x_c) + y_c = np.cross(z_c, x_c) + matrix = np.eye(4) + matrix[:3, 0] = x_c + matrix[:3, 1] = y_c + matrix[:3, 2] = z_c + matrix[:3, 3] = position + return matrix + + +def pose_at(scene: Scene, t: float) -> np.ndarray: + return camera_pose(np.array([walk_x(t), walk_y(t), walk_z(t)]), aim_point(scene, t)) + + +def keyframe_times() -> list[float]: + return [k * KEYFRAME_DT for k in range(N_KEYFRAMES)] + + +# -------------------------------------------------------------------------------------- +# Ray casting +# -------------------------------------------------------------------------------------- + + +def ray_grid(pose: np.ndarray, w: int, h: int) -> tuple[np.ndarray, np.ndarray]: + """Camera origin and one unit world-space direction per pixel centre. + + The depth grid shares the keyframe FOV with proportionally scaled + intrinsics (fx * w/640, cx = w/2), the way ARKit reports a scaled plane. + """ + fx = FX * w / KEYFRAME_W + fy = FY * h / KEYFRAME_H + cx = CX * w / KEYFRAME_W + cy = CY * h / KEYFRAME_H + u = np.arange(w, dtype=np.float64) + 0.5 + v = np.arange(h, dtype=np.float64) + 0.5 + uu, vv = np.meshgrid(u, v) + dx = (uu - cx) / fx + dy = -(vv - cy) / fy + d_cam = np.stack([dx, dy, -np.ones_like(dx)], axis=-1) + d_world = d_cam @ pose[:3, :3].T + d_world /= np.linalg.norm(d_world, axis=-1, keepdims=True) + return pose[:3, 3].copy(), d_world + + +def hit_plane_z(origin, d, z_plane, x0, x1, y0, y1): + """Rectangular axis-aligned patch at z = z_plane. Returns (t, normal) with miss t = inf.""" + dz = d[..., 2] + with np.errstate(divide="ignore", invalid="ignore"): + t = (z_plane - origin[..., 2]) / dz + px = origin[..., 0] + t * d[..., 0] + py = origin[..., 1] + t * d[..., 1] + ok = (t > 1e-6) & (px >= x0) & (px <= x1) & (py >= y0) & (py <= y1) + t = np.where(ok & (dz != 0), t, np.inf) + normal = np.zeros_like(d) + normal[..., 2] = 1.0 + return t, normal + + +def hit_plane_y(origin, d, y_plane, x0, x1, z0, z1): + dy = d[..., 1] + with np.errstate(divide="ignore", invalid="ignore"): + t = (y_plane - origin[..., 1]) / dy + px = origin[..., 0] + t * d[..., 0] + pz = origin[..., 2] + t * d[..., 2] + ok = (t > 1e-6) & (px >= x0) & (px <= x1) & (pz >= z0) & (pz <= z1) + t = np.where(ok & (dy != 0), t, np.inf) + normal = np.zeros_like(d) + normal[..., 1] = 1.0 + return t, normal + + +def hit_box(origin, d, lo, hi): + """Axis-aligned box slab test. Returns (t, normal) with miss t = inf.""" + with np.errstate(divide="ignore", invalid="ignore"): + t1 = (np.asarray(lo) - origin) / d + t2 = (np.asarray(hi) - origin) / d + t_near = np.minimum(t1, t2) # (H, W, 3) + t_far = np.maximum(t1, t2) + t_min = t_near.max(axis=-1) + t_max = t_far.min(axis=-1) + ok = (t_max >= t_min) & (t_max > 0.0) + t = np.where(ok & (t_min > 1e-6), t_min, np.inf) + inside = ok & (t_min <= 1e-6) # camera inside the box: not our scene, call it a miss + t = np.where(inside, np.inf, t) + axis = t_near.argmax(axis=-1) + d_along = np.take_along_axis(d, axis[..., None], axis=-1)[..., 0] + sign = np.sign(-d_along) + normal = np.zeros_like(d) + ii, jj = np.indices(axis.shape) + normal[ii, jj, axis] = sign + return t, normal + + +def cast_scene(scene: Scene, origin, d): + """Nearest hit among ground, wall and the boxes. Returns (t, normal, surface_id).""" + ts = [] + normals = [] + ids = [] + ground_x0, ground_x1 = scene.wall_x0 - 12.0, scene.wall_x1 + 12.0 + t, n = hit_plane_y(origin, d, scene.ground_y, ground_x0, ground_x1, scene.wall_z, scene.wall_z + 30.0) + ts.append(t) + normals.append(n) + ids.append(np.full(t.shape, 1, dtype=np.int8)) # 1 = ground + t, n = hit_plane_z(origin, d, scene.wall_z, scene.wall_x0, scene.wall_x1, scene.ground_y, scene.wall_top) + ts.append(t) + normals.append(n) + ids.append(np.full(t.shape, 2, dtype=np.int8)) # 2 = wall + for idx, box in enumerate([scene.meter, scene.gas], start=3): + if box is None: + continue + aabb = box["aabb"] + t, n = hit_box(origin, d, aabb["lo"], aabb["hi"]) + ts.append(t) + normals.append(n) + ids.append(np.full(t.shape, idx, dtype=np.int8)) # 3 = meter box, 4 = gas box + t_all = np.stack(ts) + nearest = t_all.argmin(axis=0) + t_min = np.take_along_axis(t_all, nearest[None], axis=0)[0] + normal = np.take_along_axis(np.stack(normals), np.broadcast_to(nearest, t_all.shape)[..., None], axis=0)[0] + surface = np.take_along_axis(np.stack(ids), nearest[None], axis=0)[0] + return t_min, normal, surface + + +# -------------------------------------------------------------------------------------- +# Texture +# -------------------------------------------------------------------------------------- + + +def _hash01(*grids: np.ndarray, salt: int = 0) -> np.ndarray: + """Deterministic integer hash of quantised world positions, in [0, 1).""" + h = np.full(grids[0].shape, np.int64(salt) * np.int64(974711) + np.int64(668265263), dtype=np.int64) + for k, grid in enumerate(grids): + h = h ^ np.int64((k + 1) * 374761393) * grid.astype(np.int64) + h = h ^ (h >> np.int64(13)) + h = h * np.int64(1274126177) + h = h ^ (h >> np.int64(16)) + return ((h % np.int64(4096)).astype(np.float64) + 0.5) / 4096.0 + + +def _quant(position: np.ndarray, step: float) -> tuple[np.ndarray, ...]: + return tuple(np.floor(position[..., k] / step).astype(np.int64) for k in range(3)) + + +def shade(scene: Scene, t, normal, surface, origin, d, rng_seed: int) -> np.ndarray: + """RGB per pixel. Textures are functions of the hit point, so they are view-independent.""" + point = origin + t[..., None] * d + # Miss pixels carry t = inf; their coords become 0 here so the texture math + # below never sees inf/nan (the surfaces never select those pixels anyway). + point = np.where(np.isfinite(point), point, 0.0) + image = np.zeros(t.shape + (3,), dtype=np.float64) + ndl = np.clip((normal * SUN).sum(axis=-1), 0.0, 1.0) + + # Base per-surface albedo with multi-scale hash detail, then Lambert + ambient. + for sid, base, coarse, fine in [ + (1, np.array([150.0, 148.0, 143.0]), 0.08, 0.03), # concrete ground + (2, np.array([172.0, 96.0, 72.0]), 0.08, 0.02), # brick facade + (3, np.array([196.0, 198.0, 192.0]), 0.05, 0.02), # meter body + (4, np.array([214.0, 178.0, 42.0]), 0.05, 0.02), # gas meter body + ]: + mask = surface == sid + if not mask.any(): + continue + coarse_n = (_hash01(*_quant(point, coarse), salt=sid) - 0.5) * 22.0 + fine_n = (_hash01(*_quant(point, fine), salt=sid + 17) - 0.5) * 9.0 + albedo = base[None, None, :] + coarse_n[..., None] + fine_n[..., None] + image[mask] = albedo[mask] + + # Brick pattern on the wall: running bond, mortar lines, per-brick variation. + wall = surface == 2 + if wall.any(): + y = point[..., 1] - scene.ground_y + x = point[..., 0] - scene.wall_x0 + row = np.floor(y / 0.075) + xoff = (row % 2.0) * 0.1 + col = np.floor((x + xoff) / 0.20) + fy = y / 0.075 - row + fxx = (x + xoff) / 0.20 - col + mortar = (fy < 0.13) | (fxx < 0.055) + brick_var = (_hash01(row.astype(np.int64), col.astype(np.int64), salt=91) - 0.5) * 30.0 + mortar_color = np.array([186.0, 184.0, 178.0]) + yy, xx = np.indices(t.shape) + sel = yy[wall], xx[wall] + base = image[sel] + base = np.where(mortar[wall][:, None], mortar_color[None, :], base + brick_var[wall][:, None]) + # Sun gradient down the wall: lighter near the top, slight dirt near the ground. + lift = 26.0 * ((point[..., 1][wall] - scene.ground_y) / scene.wall_height - 0.5) + image[sel] = base + lift[:, None] + + # Meter face: dark frame, a dial of rings, small hash texture. + _paint_meter_face(image, surface, point, scene.meter, dial=True) + if scene.gas is not None: + _paint_gas_face(image, surface, point, scene.gas) + + # Windows and vents are painted onto the wall surface (depth still sees the wall). + for win in scene.windows: + inside = ( + wall + & (np.abs(point[..., 0] - win["x"]) <= win["w"] / 2.0) + & (point[..., 1] >= win["sill_y"]) + & (point[..., 1] <= win["sill_y"] + win["h"]) + ) + if not inside.any(): + continue + px = point[..., 0] + py = point[..., 1] + frame = ( + (np.abs(px - win["x"]) > win["w"] / 2.0 - 0.06) + | (py > win["sill_y"] + win["h"] - 0.06) + | (py < win["sill_y"] + 0.06) + ) + glass = np.array([76.0, 94.0, 116.0]) + sheen = 42.0 * np.clip(np.sin((px + py) * 9.0), 0.0, 1.0) ** 2 + color = glass[None, None, :] + sheen[..., None] + color = np.where(frame[..., None], np.array([233.0, 231.0, 226.0])[None, None, :], color) + image[inside] = color[inside] + for vent in scene.vents: + inside = ( + wall + & (np.abs(point[..., 0] - vent["x"]) <= vent["w"] / 2.0) + & (np.abs(point[..., 1] - vent["centre_y"]) <= vent["h"] / 2.0) + ) + if not inside.any(): + continue + py = point[..., 1] + slat = ((py / 0.025) % 1.0) < 0.5 + color = np.where( + slat[..., None], np.array([52.0, 52.0, 54.0])[None, None, :], np.array([72.0, 72.0, 74.0])[None, None, :] + ) + image[inside] = color[inside] + + # Lambert shading everywhere but the painted overlays already include their own contrast. + lit = image * (0.55 + 0.45 * ndl[..., None]) + image = np.where((surface > 0)[..., None], lit, image) + + # Sensor noise, fixed per frame: keeps JPEGs from flattening and mirrors a real camera. + noise = np.random.default_rng(rng_seed).integers(-2, 3, size=image.shape, dtype=np.int16) + image = image + noise + return np.clip(image, 0, 255).astype(np.uint8) + + +def _paint_meter_face(image, surface, point, box, dial: bool) -> None: + front = (surface == 3) & (np.abs(point[..., 2] - (box["aabb"]["hi"][2])) < 1e-6) + if not front.any(): + return + w, h, _ = box["size"] + u = (point[..., 0] - (box["x"] - w / 2.0)) / w + v = (point[..., 1] - (box["aabb"]["lo"][1])) / h + border = ((u < 0.06) | (u > 0.94) | (v < 0.05) | (v > 0.95))[front] + color = image[front] + frame = np.array([58.0, 60.0, 62.0]) + color = np.where(border[:, None], frame[None, :], color) + if dial: + cx, cy = 0.5, 0.42 + r = np.hypot((u[front] - cx) * w, (v[front] - cy) * h) + ring = (r % 0.022) < 0.011 + cap = r < 0.012 + dark = np.array([70.0, 72.0, 74.0]) + light = np.array([168.0, 170.0, 166.0]) + dial_color = np.where((ring | cap)[:, None], dark[None, :], light[None, :]) + body = ~border + color = np.where(body[:, None] & (r < 0.09)[:, None], dial_color, color) + image[front] = color + + +def _paint_gas_face(image, surface, point, box) -> None: + front = (surface == 4) & (np.abs(point[..., 2] - (box["aabb"]["hi"][2])) < 1e-6) + if not front.any(): + return + w, h, _ = box["size"] + u = (point[..., 0] - (box["x"] - w / 2.0)) / w + v = (point[..., 1] - (box["aabb"]["lo"][1])) / h + border = ((u < 0.05) | (u > 0.95) | (v < 0.05) | (v > 0.95))[front] + grille = (((v[front] / 0.06) % 1.0) < 0.4) & ~border + dark = np.array([70.0, 62.0, 24.0]) + color = image[front] + color = np.where(border[:, None], np.array([74.0, 70.0, 52.0])[None, :], color) + color = np.where((~border & grille)[:, None], dark[None, :], color) + image[front] = color + + +# -------------------------------------------------------------------------------------- +# Frame rendering +# -------------------------------------------------------------------------------------- + + +def render_frame( + scene: Scene, pose: np.ndarray, frame_index: int +) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """RGB (640x480x3), depth f32 metres (192x256), confidence u8, feature points.""" + origin, d = ray_grid(pose, KEYFRAME_W, KEYFRAME_H) + t, normal, surface = cast_scene(scene, origin, d) + rgb = shade(scene, t, normal, surface, origin, d, rng_seed=[scene.seed, 1000 + frame_index]) + + d_origin, d_dir = ray_grid(pose, DEPTH_W, DEPTH_H) + dt, _normal, _surface = cast_scene(scene, d_origin, d_dir) + depth = dt.astype(" list[tuple[float, float, float, float, float]]: + """Deterministic 20-byte feature rows: u, v, response, scale, angle (float32 LE).""" + h, w = t.shape + rows = np.arange(FEATURES_STRIDE_PX, h - FEATURES_STRIDE_PX, FEATURES_STRIDE_PX) + cols = np.arange(FEATURES_STRIDE_PX, w - FEATURES_STRIDE_PX, FEATURES_STRIDE_PX) + rr, cc = np.meshgrid(rows, cols) + depth = t[rr, cc] + valid = np.isfinite(depth) + lum = rgb[rr, cc].astype(np.float64).mean(axis=-1) + response = np.abs(lum - 118.0) * valid + order = np.argsort(-response, axis=None)[:FEATURES_PER_FRAME] + rng = np.random.default_rng([seed, 7000 + frame_index]) + points = [] + for flat in order: + r, c = np.unravel_index(flat, response.shape) + if not valid[r, c]: + continue + points.append( + ( + float(cc[r, c] + 0.5), + float(rr[r, c] + 0.5), + float(response[r, c]), + float(rng.uniform(2.0, 5.0)), + 0.0, + ) + ) + return points + + +# -------------------------------------------------------------------------------------- +# Streams +# -------------------------------------------------------------------------------------- + + +def _device_gravity(pose: np.ndarray) -> np.ndarray: + """Gravity in Core Motion device frame, exactly as server.packet.poses.check_gravity_stream expects.""" + return DEVICE_FROM_CAMERA @ (pose[:3, :3].T @ np.array([0.0, -1.0, 0.0])) + + +def _device_from_world(pose: np.ndarray) -> np.ndarray: + return pose[:3, :3] @ DEVICE_FROM_CAMERA.T + + +def _pose_interp(scene: Scene, t: float) -> np.ndarray: + return pose_at(scene, t) + + +def _csv_float(value: float, digits: int = 6) -> str: + return f"{value:.{digits}f}" + + +def _write_gzip_csv(path: Path, lines: list[str]) -> None: + with open(path, "wb") as raw: + with gzip.GzipFile(fileobj=raw, mode="wb", compresslevel=6, mtime=0) as gz: + gz.write(("\n".join(lines) + "\n").encode()) + + +def build_streams(scene: Scene, packet_dir: Path) -> dict: + """motion/heading/location/imuRaw at 100 Hz (location 1 Hz) and arkitPoses at 30 Hz.""" + streams_dir = packet_dir / "streams" + streams_dir.mkdir(parents=True, exist_ok=True) + t0 = SESSION_START_UPTIME + + # --- motion (100 Hz) --- + lines = ["t,qx,qy,qz,qw,rotX,rotY,rotZ,gravX,gravY,gravZ,accX,accY,accZ,magX,magY,magZ,magAccuracy,heading"] + steps = int(round(WALK_SECONDS + 3.5) * 100) + rng = np.random.default_rng([scene.seed, 11]) + acc_noise = rng.uniform(-0.004, 0.004, size=(steps, 3)) + for k in range(steps + 1): + t = k / 100.0 + if t > WALK_SECONDS + 3.5: + break + pose = _pose_interp(scene, t) + quat = rotmat_to_quat_xyzw(_device_from_world(pose)) + grav = _device_gravity(pose) + yaw_prev = _yaw(_pose_interp(scene, max(t - 0.005, 0.0))) + yaw_next = _yaw(_pose_interp(scene, min(t + 0.005, WALK_SECONDS))) + rot = [0.004, (yaw_next - yaw_prev) / 0.01, 0.002] + acc = grav + acc_noise[min(k, steps - 1)] + mag = _device_from_world(pose).T @ np.array([0.0, 0.0, -45.0]) + rows = [f"{t0 + t:.3f}"] + rows += [_csv_float(v, 6) for v in quat] + rows += [_csv_float(v, 4) for v in rot] + rows += [_csv_float(v, 6) for v in grav] + rows += [_csv_float(v, 6) for v in acc] + rows += [_csv_float(v, 4) for v in mag] + rows += ["45.0", "132.4"] + lines.append(",".join(rows)) + _write_gzip_csv(streams_dir / "motion.csv.gz", lines) + + # --- imuRaw (100 Hz) --- + lines = ["t,ax,ay,az,gx,gy,gz"] + for k in range(steps + 1): + t = k / 100.0 + if t > WALK_SECONDS + 3.5: + break + pose = _pose_interp(scene, t) + grav = _device_gravity(pose) + acc = grav + acc_noise[min(k, steps - 1)] + yaw_prev = _yaw(_pose_interp(scene, max(t - 0.005, 0.0))) + yaw_next = _yaw(_pose_interp(scene, min(t + 0.005, WALK_SECONDS))) + rows = [f"{t0 + t:.3f}"] + rows += [_csv_float(v, 6) for v in acc] + rows += [_csv_float(v, 4) for v in [0.004, (yaw_next - yaw_prev) / 0.01, 0.002]] + lines.append(",".join(rows)) + _write_gzip_csv(streams_dir / "imuRaw.csv.gz", lines) + + # --- arkitPoses (30 Hz, keyframe times sit exactly on this grid) --- + lines = ["t,epoch,tracking,tx,ty,tz,qx,qy,qz,qw,fx,fy,cx,cy"] + n_poses = int(round((WALK_SECONDS + 3.5) * 30)) + for k in range(n_poses + 1): + t = k / 30.0 + pose = _pose_interp(scene, t) + quat = rotmat_to_quat_xyzw(_device_from_world(pose)) + rows = [f"{t0 + t:.3f}", "e1", "normal"] + rows += [_csv_float(v, 6) for v in pose[:3, 3]] + rows += [_csv_float(v, 6) for v in quat] + rows += [_csv_float(FX, 3), _csv_float(FY, 3), _csv_float(CX, 3), _csv_float(CY, 3)] + lines.append(",".join(rows)) + _write_gzip_csv(streams_dir / "arkitPoses.csv.gz", lines) + + # --- heading (10 Hz) --- + lines = ["t,timestamp,magneticHeading,trueHeading,headingAccuracy,x,y,z"] + for k in range(int((WALK_SECONDS + 3.5) * 10) + 1): + t = k / 10.0 + stamp = f"2026-10-16T14:00:{int(t):02d}.{int(round((t % 1) * 100)):02d}" + rows = [f"{t0 + t:.3f}", stamp, "132.4", "131.9", "12.0", "24.1", "-3.9", "-41.7"] + lines.append(",".join(rows)) + _write_gzip_csv(streams_dir / "heading.csv.gz", lines) + + # --- location (1 Hz) --- + lines = [ + "t,timestamp,latitude,longitude,altitude,ellipsoidalAltitude," + "horizontalAccuracy,verticalAccuracy,course,courseAccuracy,speed,speedAccuracy" + ] + for k in range(int(WALK_SECONDS + 3.5) + 1): + t = k * 1.0 + speed = WALK_SPEED_MPS if t <= WALK_SECONDS else 0.0 + stamp = f"2026-10-16T14:00:{int(t):02d}.00" + rows = [ + f"{t0 + t:.3f}", + stamp, + "30.267200", + "-97.743100", + "149.0", + "174.2", + "5.0", + "3.0", + "90.0", + "10.0", + _csv_float(speed, 3), + "0.5", + ] + lines.append(",".join(rows)) + _write_gzip_csv(streams_dir / "location.csv.gz", lines) + + return { + "motion": { + "file": "streams/motion.csv.gz", + "rateHz": 100, + "referenceFrame": "xArbitraryZVertical", + "columns": [ + "t", + "qx", + "qy", + "qz", + "qw", + "rotX", + "rotY", + "rotZ", + "gravX", + "gravY", + "gravZ", + "accX", + "accY", + "accZ", + "magX", + "magY", + "magZ", + "magAccuracy", + "heading", + ], + }, + "heading": { + "file": "streams/heading.csv.gz", + "headingOrientation": "portrait", + "headingFilter": 1.0, + "columns": ["t", "timestamp", "magneticHeading", "trueHeading", "headingAccuracy", "x", "y", "z"], + }, + "location": { + "file": "streams/location.csv.gz", + "desiredAccuracy": "best", + "columns": [ + "t", + "timestamp", + "latitude", + "longitude", + "altitude", + "ellipsoidalAltitude", + "horizontalAccuracy", + "verticalAccuracy", + "course", + "courseAccuracy", + "speed", + "speedAccuracy", + ], + }, + "imuRaw": { + "file": "streams/imuRaw.csv.gz", + "rateHz": 100, + "columns": ["t", "ax", "ay", "az", "gx", "gy", "gz"], + }, + "arkitPoses": { + "file": "streams/arkitPoses.csv.gz", + "rateHz": 30, + "columns": ["t", "epoch", "tracking", "tx", "ty", "tz", "qx", "qy", "qz", "qw", "fx", "fy", "cx", "cy"], + }, + } + + +def _yaw(pose: np.ndarray) -> float: + forward = -pose[:3, 2] + return math.atan2(float(forward[0]), -float(forward[2])) + + +# -------------------------------------------------------------------------------------- +# Packet assembly +# -------------------------------------------------------------------------------------- + + +def _round_pose(pose: np.ndarray) -> list[float]: + return [round(float(v), 9) for v in column_major(pose)] + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _file_entry(path: Path, root: Path, role: str) -> dict: + data = path.read_bytes() + return {"path": str(path.relative_to(root)), "bytes": len(data), "sha256": _sha256(data), "role": role} + + +def still_views(scene: Scene) -> list[dict]: + """The meter close-up (head-on) and the oblique still the scale reference needs.""" + meter_c = np.array([scene.meter["x"], scene.meter["centre_y"], scene.wall_z + scene.meter["size"][2] / 2.0]) + close_pos = np.array([scene.meter["x"], scene.meter["centre_y"], scene.wall_z + 0.55]) + oblique_pos = np.array([scene.meter["x"] + 0.38, scene.meter["centre_y"] + 0.12, scene.wall_z + 0.52]) + + def angle_deg(pos: np.ndarray) -> float: + view = meter_c - pos + view = view / np.linalg.norm(view) + # Angle between the camera's viewing axis and the wall plane (0 = head-on): + # the wall normal points out of the wall toward the camera, so use |dot|. + return math.degrees(math.acos(min(1.0, abs(float(view[2]))))) + + return [ + { + "id": "s_meter_close", + "purpose": "meter_close", + "pos": close_pos, + "target": meter_c, + "viewAngleDeg": angle_deg(close_pos), + "timestamp": SESSION_START_UPTIME + WALK_SECONDS + 1.5, + }, + { + "id": "s_meter_oblique", + "purpose": "meter_oblique", + "pos": oblique_pos, + "target": meter_c, + "viewAngleDeg": angle_deg(oblique_pos), + "timestamp": SESSION_START_UPTIME + WALK_SECONDS + 2.5, + }, + ] + + +def generate_packet(spec: dict, out_dir: Path) -> dict: + """Render the packet into out_dir. Returns a summary dict (paths, counts, packet id).""" + scene = Scene(spec) + out_dir = Path(out_dir) + (out_dir / "keyframes").mkdir(parents=True, exist_ok=True) + (out_dir / "stills").mkdir(parents=True, exist_ok=True) + + digest = uuid.uuid5(uuid.NAMESPACE_URL, f"gen_app_capture/{scene.name}/{scene.seed}") + packet_id = digest.hex + session_id = uuid.uuid5(uuid.NAMESPACE_URL, f"gen_app_capture/{scene.name}/{scene.seed}/session").hex.upper() + + frames = [] + files = [] + for index, t in enumerate(keyframe_times()): + pose = pose_at(scene, t) + rgb, depth, confidence, features = render_frame(scene, pose, index) + kid = f"k{index:05d}" + jpg = out_dir / "keyframes" / f"{kid}.jpg" + Image.fromarray(rgb).save(jpg, format="JPEG", quality=JPEG_QUALITY, subsampling=0) + depth_path = out_dir / "keyframes" / f"{kid}.depth.f32" + depth.tofile(depth_path) + conf_path = out_dir / "keyframes" / f"{kid}.confidence.u8" + confidence.tofile(conf_path) + feat_path = out_dir / "keyframes" / f"{kid}.features.bin" + feat_path.write_bytes(b"".join(np.asarray(p, dtype=" tuple[np.ndarray, float]: + lo, hi = np.asarray(aabb["lo"]), np.asarray(aabb["hi"]) + with np.errstate(divide="ignore", invalid="ignore"): + t1 = (lo - origin) / direction + t2 = (hi - origin) / direction + t_near = np.minimum(t1, t2).max() + t_far = np.maximum(t1, t2).min() + if not (t_far >= max(t_near, 0.0)): + raise ValueError("tap ray misses the meter box") + t = max(t_near, 0.0) + position = origin + t * direction + return position, float(np.linalg.norm(position - origin)) + + +def _planes(scene: Scene) -> list[dict]: + cx = (scene.wall_x0 + scene.wall_x1) / 2.0 + cy = scene.ground_y + scene.wall_height / 2.0 + width = scene.wall_x1 - scene.wall_x0 + wall_boundary = [ + [round(scene.wall_x0 - cx, 4), round(scene.ground_y - cy, 4)], + [round(scene.wall_x1 - cx, 4), round(scene.ground_y - cy, 4)], + [round(scene.wall_x1 - cx, 4), round(scene.wall_top - cy, 4)], + [round(scene.wall_x0 - cx, 4), round(scene.wall_top - cy, 4)], + ] + wall = { + "id": "WALL-0001", + "epoch": "e1", + "alignment": "vertical", + "classification": "wall", + "transform": _round_pose( + np.array( + [ + [1.0, 0.0, 0.0, cx], + [0.0, 1.0, 0.0, cy], + [0.0, 0.0, 1.0, scene.wall_z], + [0.0, 0.0, 0.0, 1.0], + ] + ) + ), + "center": [round(cx, 4), round(cy, 4), round(scene.wall_z, 4)], + "planeExtent": {"width": round(width, 4), "height": round(scene.wall_height, 4), "rotationOnYAxis": 0.0}, + "boundary": wall_boundary, + "sourceInliers": 48000, + } + floor = { + "id": "FLOOR-0001", + "epoch": "e1", + "alignment": "horizontal", + "classification": "floor", + "transform": _round_pose( + np.array( + [ + [1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, scene.ground_y], + [0.0, -1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0], + ] + ) + ), + "center": [0.0, round(scene.ground_y, 4), 0.0], + "planeExtent": {"width": 24.0, "height": 30.0, "rotationOnYAxis": 0.0}, + "boundary": [[-12.0, -15.0], [12.0, -15.0], [12.0, 15.0], [-12.0, 15.0]], + "sourceInliers": 51000, + } + return [wall, floor] + + +def build_ground_truth(scene: Scene, packet: dict, tap_pixel: list[float], hit_distance: float) -> dict: + """The facts the pipeline is supposed to recover, in the packet's own world frame.""" + meter_centre = [ + round(scene.meter["x"], 6), + round(scene.meter["centre_y"], 6), + round(scene.wall_z + scene.meter["size"][2] / 2.0, 6), + ] + clearances: dict = {"note": "horizontal distances on the wall plane, metres; no rule values live here"} + hazards = scene.hazards() + if scene.gas is not None: + clearances["meterCentreToGasCentreM"] = round(abs(scene.meter["x"] - scene.gas["x"]), 6) + meter_left_edge = scene.meter["x"] - scene.meter["size"][0] / 2.0 + gas_right_edge = scene.gas["x"] + scene.gas["size"][0] / 2.0 + clearances["meterEdgeToGasEdgeM"] = round(meter_left_edge - gas_right_edge, 6) + tap = packet["taps"][0] + return { + "scene": scene.name, + "seed": scene.seed, + "generator": "backend/scripts/gen_app_capture.py", + "world": "packet.json conventions: ARKit .gravity, y up, origin at the device position at session start", + "units": {"length": "meters", "angle": "degrees"}, + "wall": { + "plane": {"point": [0.0, 0.0, round(scene.wall_z, 6)], "normal": [0.0, 0.0, 1.0]}, + "xRange": [scene.wall_x0, scene.wall_x1], + "bottomY": round(scene.ground_y, 6), + "height": round(scene.wall_height, 6), + }, + "meter": { + "kind": "electric", + "centre": meter_centre, + "size": scene.meter["size"], + "frontFaceZ": round(scene.meter["aabb"]["hi"][2], 6), + "aabb": scene.meter["aabb"], + }, + "hazards": hazards, + "clearances": clearances, + "windows": [ + { + "centre": [round(w["x"], 6), round(w["sill_y"] + w["h"] / 2.0, 6)], + "size": [w["w"], w["h"]], + "sillY": round(w["sill_y"], 6), + } + for w in scene.windows + ], + "vents": [ + { + "centre": [round(v["x"], 6), round(v["centre_y"], 6)], + "size": [v["w"], v["h"]], + } + for v in scene.vents + ], + "capture": { + "keyframes": len(packet["keyframes"]), + "keyframeResolution": [KEYFRAME_W, KEYFRAME_H], + "depthResolution": [DEPTH_W, DEPTH_H], + "walkDistanceToWallM": [1.5, 2.5], + "tap": { + "keyframe": tap["keyframe"], + "pixel": tap_pixel, + "hitPosition": tap["hit"]["position"], + "hitDistanceM": round(hit_distance, 4), + }, + }, + } + + +def load_spec(path: Path) -> dict: + return json.loads(Path(path).read_text()) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--spec", required=True, type=Path, help="scene spec JSON") + parser.add_argument("--out", required=True, type=Path, help="packet output directory") + args = parser.parse_args(argv) + summary = generate_packet(load_spec(args.spec), args.out) + print(json.dumps(summary, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/tests/fixtures/app_capture_synth/clear-wall.json b/backend/tests/fixtures/app_capture_synth/clear-wall.json new file mode 100644 index 000000000..f4a458fe9 --- /dev/null +++ b/backend/tests/fixtures/app_capture_synth/clear-wall.json @@ -0,0 +1,10 @@ +{ + "name": "clear-wall", + "seed": 1401, + "wall": {"xMin": -1.0, "xMax": 5.0, "height": 2.8, "distance": 2.0}, + "camera": {"height": 1.45}, + "meter": {"x": 2.0, "centreHeight": 1.2, "size": [0.45, 0.65, 0.18]}, + "gasMeter": null, + "windows": [{"x": 3.6, "sillHeight": 1.0, "size": [1.2, 1.4]}], + "vents": [{"x": 0.4, "centreHeight": 2.45, "size": [0.3, 0.15]}] +} diff --git a/backend/tests/fixtures/app_capture_synth/gas-left.json b/backend/tests/fixtures/app_capture_synth/gas-left.json new file mode 100644 index 000000000..a29c6094b --- /dev/null +++ b/backend/tests/fixtures/app_capture_synth/gas-left.json @@ -0,0 +1,10 @@ +{ + "name": "gas-left", + "seed": 1402, + "wall": {"xMin": -1.0, "xMax": 5.0, "height": 2.8, "distance": 2.0}, + "camera": {"height": 1.45}, + "meter": {"x": 2.0, "centreHeight": 1.2, "size": [0.45, 0.65, 0.18]}, + "gasMeter": {"x": 1.4, "centreHeight": 1.2, "size": [0.35, 0.5, 0.15]}, + "windows": [{"x": 3.6, "sillHeight": 1.0, "size": [1.2, 1.4]}], + "vents": [{"x": 0.4, "centreHeight": 2.45, "size": [0.3, 0.15]}] +} diff --git a/backend/tests/test_app_capture_synth.py b/backend/tests/test_app_capture_synth.py new file mode 100644 index 000000000..963b188be --- /dev/null +++ b/backend/tests/test_app_capture_synth.py @@ -0,0 +1,212 @@ +"""Synthetic app-capture generator: determinism, ground truth, and real-pipeline validation. + +The generator renders app-shaped packets whose true geometry is exact. These +tests pin the properties that make the packets usable as fixtures: + +- the real ``server.packet`` validator accepts the rendered packets; +- the packed tap replays through the packet's own conventions onto the meter; +- the depth image at the tap pixel agrees with the packed hit distance; +- ground truth matches the scene spec, including hazard clearances; +- re-rendering the same spec and seed reproduces the bytes. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pytest + +from scripts.gen_app_capture import DEPTH_H, DEPTH_W, Scene, generate_packet, load_spec +from server.common.geometry import pixel_ray_arkit +from server.packet.conventions import from_column_major + +SPEC_DIR = Path(__file__).parent / "fixtures" / "app_capture_synth" +SCENES = ("clear-wall", "gas-left") + + +@pytest.fixture(scope="module") +def rendered(tmp_path_factory) -> dict[str, Path]: + out: dict[str, Path] = {} + for scene in SCENES: + spec = load_spec(SPEC_DIR / f"{scene}.json") + target = tmp_path_factory.mktemp(scene) + generate_packet(spec, target) + out[scene] = target + return out + + +def _packet(rendered, scene: str) -> dict: + return json.loads((rendered[scene] / "packet.json").read_text()) + + +def _ground_truth(rendered, scene: str) -> dict: + return json.loads((rendered[scene] / "ground_truth.json").read_text()) + + +def test_packet_layout_matches_fixture_contract(rendered): + """The generator emits the mini-arkit-lidar layout the app and server share.""" + root = rendered["clear-wall"] + packet = _packet(rendered, "clear-wall") + assert packet["format"] == "house-capture-packet" + assert packet["formatVersion"] == "0.4" + assert packet["source"]["kind"] == "synthetic" + assert len(packet["keyframes"]) == 30 + assert len(packet["stills"]) == 2 + assert packet["streams"]["motion"]["file"] == "streams/motion.csv.gz" + assert packet["streams"]["arkitPoses"]["file"] == "streams/arkitPoses.csv.gz" + frame = packet["keyframes"][0] + assert (root / frame["img"]).is_file() + assert (root / frame["depth"]["file"]).stat().st_size == 256 * 192 * 4 + assert (root / frame["depth"]["confidenceFile"]).stat().st_size == 256 * 192 + # iOS fixture manifests reference media relative to the packet directory the same way. + assert all( + entry["path"] in {str(p.relative_to(root)) for p in root.rglob("*") if p.is_file()} for entry in packet["files"] + ) + + +def test_validator_reports_ok(rendered): + """The real server.packet validator and normalizer pass on the generated packets.""" + from server.packet.validate import validate_and_normalize + + for scene in SCENES: + packet, report = validate_and_normalize(rendered[scene] / "packet.json") + assert report.ok, f"{scene}: {report.errors}" + assert [issue.code for issue in report.warnings] == ["synthetic_source"] + assert report.maxTapReplayError < 1e-6 + + +def test_repeated_generation_is_byte_identical(tmp_path, rendered): + """Same spec and seed in, same bytes out — gz timestamps, ids, JPEGs included.""" + spec = load_spec(SPEC_DIR / "clear-wall.json") + second = tmp_path / "again" + generate_packet(spec, second) + first = rendered["clear-wall"] + for path in sorted(first.rglob("*")): + if not path.is_file(): + continue + other = second / path.relative_to(first) + assert other.is_file(), f"missing {path.relative_to(first)}" + assert path.read_bytes() == other.read_bytes(), f"bytes differ: {path.relative_to(first)}" + + +def test_ground_truth_matches_scene_spec(rendered): + gt = _ground_truth(rendered, "gas-left") + assert gt["scene"] == "gas-left" + # Wall plane at z = -2.0, camera on the +z side. + assert gt["wall"]["plane"]["point"][2] == -2.0 + assert gt["wall"]["plane"]["normal"] == [0.0, 0.0, 1.0] + # Meter centre: spec x = 2.0, centreHeight = 1.2, camera height = 1.45. + meter = gt["meter"] + assert meter["centre"][0] == 2.0 + assert meter["centre"][1] == round(-1.45 + 1.2, 6) + assert meter["frontFaceZ"] == round(-2.0 + 0.18, 6) + # The gas meter sits 0.6 m to the left; edges nearly touch at 0.2 m. + assert gt["hazards"][0]["kind"] == "gas_meter" + assert gt["hazards"][0]["centre"][0] == 1.4 + assert gt["clearances"]["meterCentreToGasCentreM"] == pytest.approx(0.6) + assert gt["clearances"]["meterEdgeToGasEdgeM"] == pytest.approx(0.2) + + +def test_clear_wall_has_no_hazards(rendered): + gt = _ground_truth(rendered, "clear-wall") + assert gt["hazards"] == [] + assert "meterCentreToGasCentreM" not in gt["clearances"] + # The two scenes differ in hazards, so they must differ in clearances. + assert _ground_truth(rendered, "gas-left")["clearances"] != gt["clearances"] + + +def test_tap_replays_onto_meter_front_face(rendered): + packet = _packet(rendered, "clear-wall") + gt = _ground_truth(rendered, "clear-wall") + tap = packet["taps"][0] + frame = next(f for f in packet["keyframes"] if f["id"] == tap["keyframe"]) + pose = from_column_major(frame["pose"]) + origin, direction = pixel_ray_arkit(pose, frame["intrinsics"], np.asarray(tap["pixel"])) + replay = np.asarray(origin) + float( + np.dot(direction, np.asarray(tap["hit"]["position"]) - np.asarray(origin)) + ) * np.asarray(direction) + # The replayed ray lands on the meter's front face, at the packed hit point. + assert replay[2] == pytest.approx(gt["meter"]["frontFaceZ"], abs=1e-6) + assert np.linalg.norm(replay - np.asarray(tap["hit"]["position"])) < 1e-4 + assert gt["capture"]["tap"]["hitDistanceM"] == pytest.approx(tap["hit"]["distance"], abs=1e-4) + + +def test_tap_hits_depth_at_the_same_distance(tmp_path, rendered): + """Casting the packed tap ray against the scene reproduces the packed distance.""" + packet = _packet(rendered, "clear-wall") + gt = _ground_truth(rendered, "clear-wall") + spec = load_spec(SPEC_DIR / "clear-wall.json") + scene = Scene(spec) + tap = packet["taps"][0] + frame = next(f for f in packet["keyframes"] if f["id"] == tap["keyframe"]) + pose = from_column_major(frame["pose"]) + origin, direction = pixel_ray_arkit(pose, frame["intrinsics"], np.asarray(tap["pixel"])) + from scripts.gen_app_capture import cast_scene # local import: module-scoped renderer + + origin_grid = np.broadcast_to(np.asarray(origin), (1, 1, 3)) + dir_grid = np.broadcast_to(np.asarray(direction), (1, 1, 3)) + t, _normal, surface = cast_scene(scene, origin_grid, dir_grid) + assert surface[0, 0] == 3, "tap ray should hit the meter box" + assert float(t[0, 0]) == pytest.approx(tap["hit"]["distance"], abs=5e-4) + assert gt["meter"]["aabb"]["lo"][0] <= float(t[0, 0]) * direction[0] + origin[0] <= gt["meter"]["aabb"]["hi"][0] + + +def test_depth_pixels_agree_with_geometry(rendered): + """The depth image around the tap pixel is within one depth pixel of the packed hit.""" + packet = _packet(rendered, "clear-wall") + tap = packet["taps"][0] + frame = next(f for f in packet["keyframes"] if f["id"] == tap["keyframe"]) + depth = np.fromfile(rendered["clear-wall"] / frame["depth"]["file"], dtype=" Date: Fri, 16 Oct 2026 23:26:24 +0000 Subject: [PATCH 2/3] docs(backend): synthetic capture validator reports and ground truth Co-authored-by: Sam Gu --- .../clear-wall-ground-truth.json | 113 ++++++++++++++ .../clear-wall-validator-output.txt | 2 + .../gas-left-ground-truth.json | 141 ++++++++++++++++++ .../gas-left-validator-output.txt | 2 + 4 files changed, 258 insertions(+) create mode 100644 backend/docs/app-capture-synth-evidence/clear-wall-ground-truth.json create mode 100644 backend/docs/app-capture-synth-evidence/clear-wall-validator-output.txt create mode 100644 backend/docs/app-capture-synth-evidence/gas-left-ground-truth.json create mode 100644 backend/docs/app-capture-synth-evidence/gas-left-validator-output.txt diff --git a/backend/docs/app-capture-synth-evidence/clear-wall-ground-truth.json b/backend/docs/app-capture-synth-evidence/clear-wall-ground-truth.json new file mode 100644 index 000000000..b96b1ec2d --- /dev/null +++ b/backend/docs/app-capture-synth-evidence/clear-wall-ground-truth.json @@ -0,0 +1,113 @@ +{ + "scene": "clear-wall", + "seed": 1401, + "generator": "backend/scripts/gen_app_capture.py", + "world": "packet.json conventions: ARKit .gravity, y up, origin at the device position at session start", + "units": { + "length": "meters", + "angle": "degrees" + }, + "wall": { + "plane": { + "point": [ + 0.0, + 0.0, + -2.0 + ], + "normal": [ + 0.0, + 0.0, + 1.0 + ] + }, + "xRange": [ + -1.0, + 5.0 + ], + "bottomY": -1.45, + "height": 2.8 + }, + "meter": { + "kind": "electric", + "centre": [ + 2.0, + -0.25, + -1.91 + ], + "size": [ + 0.45, + 0.65, + 0.18 + ], + "frontFaceZ": -1.82, + "aabb": { + "lo": [ + 1.775, + -0.575, + -2.0 + ], + "hi": [ + 2.225, + 0.07500000000000001, + -1.82 + ] + } + }, + "hazards": [], + "clearances": { + "note": "horizontal distances on the wall plane, metres; no rule values live here" + }, + "windows": [ + { + "centre": [ + 3.6, + 0.25 + ], + "size": [ + 1.2, + 1.4 + ], + "sillY": -0.45 + } + ], + "vents": [ + { + "centre": [ + 0.4, + 1.0 + ], + "size": [ + 0.3, + 0.15 + ] + } + ], + "capture": { + "keyframes": 30, + "keyframeResolution": [ + 640, + 480 + ], + "depthResolution": [ + 256, + 192 + ], + "walkDistanceToWallM": [ + 1.5, + 2.5 + ], + "tap": { + "keyframe": "k00012", + "pixel": [ + 320.0, + 218.728 + ], + "hitPosition": [ + 2.0, + -0.250001, + -1.82 + ], + "hitDistanceM": 1.5266 + } + } +} diff --git a/backend/docs/app-capture-synth-evidence/clear-wall-validator-output.txt b/backend/docs/app-capture-synth-evidence/clear-wall-validator-output.txt new file mode 100644 index 000000000..87423b0b4 --- /dev/null +++ b/backend/docs/app-capture-synth-evidence/clear-wall-validator-output.txt @@ -0,0 +1,2 @@ +{"level":"warning","event":"packet_issue","msg":"synthetic_source","code":"synthetic_source","pointer":"/source/kind","capture_id":"609e3067892652719af2ae64cd3a7c09"} +{"format": "hss-validation-report/v0", "packetId": "609e3067892652719af2ae64cd3a7c09", "formatVersion": "0.4", "tier": "arkit_lidar", "stage": "full", "ok": true, "errors": [], "warnings": [{"code": "synthetic_source", "level": "warning", "pointer": "/source/kind", "message": "synthetic packet", "value": null}], "keyframes": 30, "stills": 2, "taps": 1, "epochs": 1, "files": 127, "filesHashed": 127, "maxTapReplayError": 3.4585401209596967e-10, "imagesUsable": true, "imagesDecodable": 32, "timingsS": {"schema": 0.001902872999835381, "files": 0.19820175800032303, "total": 0.2217083570003524}} diff --git a/backend/docs/app-capture-synth-evidence/gas-left-ground-truth.json b/backend/docs/app-capture-synth-evidence/gas-left-ground-truth.json new file mode 100644 index 000000000..73b86efa2 --- /dev/null +++ b/backend/docs/app-capture-synth-evidence/gas-left-ground-truth.json @@ -0,0 +1,141 @@ +{ + "scene": "gas-left", + "seed": 1402, + "generator": "backend/scripts/gen_app_capture.py", + "world": "packet.json conventions: ARKit .gravity, y up, origin at the device position at session start", + "units": { + "length": "meters", + "angle": "degrees" + }, + "wall": { + "plane": { + "point": [ + 0.0, + 0.0, + -2.0 + ], + "normal": [ + 0.0, + 0.0, + 1.0 + ] + }, + "xRange": [ + -1.0, + 5.0 + ], + "bottomY": -1.45, + "height": 2.8 + }, + "meter": { + "kind": "electric", + "centre": [ + 2.0, + -0.25, + -1.91 + ], + "size": [ + 0.45, + 0.65, + 0.18 + ], + "frontFaceZ": -1.82, + "aabb": { + "lo": [ + 1.775, + -0.575, + -2.0 + ], + "hi": [ + 2.225, + 0.07500000000000001, + -1.82 + ] + } + }, + "hazards": [ + { + "kind": "gas_meter", + "centre": [ + 1.4, + -0.25, + -1.925 + ], + "size": [ + 0.35, + 0.5, + 0.15 + ], + "aabb": { + "lo": [ + 1.2249999999999999, + -0.5, + -2.0 + ], + "hi": [ + 1.575, + 0.0, + -1.85 + ] + } + } + ], + "clearances": { + "note": "horizontal distances on the wall plane, metres; no rule values live here", + "meterCentreToGasCentreM": 0.6, + "meterEdgeToGasEdgeM": 0.2 + }, + "windows": [ + { + "centre": [ + 3.6, + 0.25 + ], + "size": [ + 1.2, + 1.4 + ], + "sillY": -0.45 + } + ], + "vents": [ + { + "centre": [ + 0.4, + 1.0 + ], + "size": [ + 0.3, + 0.15 + ] + } + ], + "capture": { + "keyframes": 30, + "keyframeResolution": [ + 640, + 480 + ], + "depthResolution": [ + 256, + 192 + ], + "walkDistanceToWallM": [ + 1.5, + 2.5 + ], + "tap": { + "keyframe": "k00012", + "pixel": [ + 320.0, + 218.728 + ], + "hitPosition": [ + 2.0, + -0.250001, + -1.82 + ], + "hitDistanceM": 1.5266 + } + } +} diff --git a/backend/docs/app-capture-synth-evidence/gas-left-validator-output.txt b/backend/docs/app-capture-synth-evidence/gas-left-validator-output.txt new file mode 100644 index 000000000..a8df88601 --- /dev/null +++ b/backend/docs/app-capture-synth-evidence/gas-left-validator-output.txt @@ -0,0 +1,2 @@ +{"level":"warning","event":"packet_issue","msg":"synthetic_source","code":"synthetic_source","pointer":"/source/kind","capture_id":"1115853d975a5129a2202ee27d758b1d"} +{"format": "hss-validation-report/v0", "packetId": "1115853d975a5129a2202ee27d758b1d", "formatVersion": "0.4", "tier": "arkit_lidar", "stage": "full", "ok": true, "errors": [], "warnings": [{"code": "synthetic_source", "level": "warning", "pointer": "/source/kind", "message": "synthetic packet", "value": null}], "keyframes": 30, "stills": 2, "taps": 1, "epochs": 1, "files": 127, "filesHashed": 127, "maxTapReplayError": 3.4585401209596967e-10, "imagesUsable": true, "imagesDecodable": 32, "timingsS": {"schema": 0.0017585860000508546, "files": 0.21119022099992435, "total": 0.22321545000022525}} From d6ff45924448a1b089fbca980e4604fc9ccaea97 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Thu, 22 Oct 2026 09:34:58 -0500 Subject: [PATCH 3/3] fix(synth): emit packet-contract depth features and closeups --- backend/scripts/gen_app_capture.py | 71 +++++++++++++++++++------ backend/tests/test_app_capture_synth.py | 45 ++++++++++++++-- 2 files changed, 96 insertions(+), 20 deletions(-) diff --git a/backend/scripts/gen_app_capture.py b/backend/scripts/gen_app_capture.py index 84df2df63..674078428 100644 --- a/backend/scripts/gen_app_capture.py +++ b/backend/scripts/gen_app_capture.py @@ -54,6 +54,7 @@ import hashlib import json import math +import struct import sys import uuid from pathlib import Path @@ -538,14 +539,15 @@ def render_frame( d_origin, d_dir = ray_grid(pose, DEPTH_W, DEPTH_H) dt, _normal, _surface = cast_scene(scene, d_origin, d_dir) - depth = dt.astype("hwi", pose[:3, :3].T, d_dir) + depth = np.where(np.isfinite(dt), dt * -camera_dir[..., 2], 0.0).astype(" list[tuple[float, float, float, float, float]]: - """Deterministic 20-byte feature rows: u, v, response, scale, angle (float32 LE).""" +def _features(rgb, t, pose, frame_index) -> list[tuple[float, float, float, int]]: + """Deterministic app raw-point rows: world x/y/z and a uint64 identifier.""" h, w = t.shape rows = np.arange(FEATURES_STRIDE_PX, h - FEATURES_STRIDE_PX, FEATURES_STRIDE_PX) cols = np.arange(FEATURES_STRIDE_PX, w - FEATURES_STRIDE_PX, FEATURES_STRIDE_PX) @@ -555,19 +557,21 @@ def _features(rgb, t, frame_index, seed) -> list[tuple[float, float, float, floa lum = rgb[rr, cc].astype(np.float64).mean(axis=-1) response = np.abs(lum - 118.0) * valid order = np.argsort(-response, axis=None)[:FEATURES_PER_FRAME] - rng = np.random.default_rng([seed, 7000 + frame_index]) + origin, direction = ray_grid(pose, KEYFRAME_W, KEYFRAME_H) points = [] - for flat in order: + for ordinal, flat in enumerate(order): r, c = np.unravel_index(flat, response.shape) if not valid[r, c]: continue + py, px = int(rr[r, c]), int(cc[r, c]) + world = origin + float(t[py, px]) * direction[py, px] + identifier = (int(frame_index) << 32) | (ordinal + 1) points.append( ( - float(cc[r, c] + 0.5), - float(rr[r, c] + 0.5), - float(response[r, c]), - float(rng.uniform(2.0, 5.0)), - 0.0, + float(world[0]), + float(world[1]), + float(world[2]), + identifier, ) ) return points @@ -844,7 +848,7 @@ def generate_packet(spec: dict, out_dir: Path) -> dict: conf_path = out_dir / "keyframes" / f"{kid}.confidence.u8" confidence.tofile(conf_path) feat_path = out_dir / "keyframes" / f"{kid}.features.bin" - feat_path.write_bytes(b"".join(np.asarray(p, dtype=" dict: img = out_dir / "stills" / f"{view['id']}.jpg" Image.fromarray(rgb).save(img, format="JPEG", quality=JPEG_QUALITY, subsampling=0) files.append(_file_entry(img, out_dir, "still")) - link = frames[min(range(len(frames)), key=lambda i: abs(walk_x(keyframe_times()[i]) - scene.meter["x"]))]["id"] + keyframe_id = f"k_{view['id']}" + depth_path = out_dir / "keyframes" / f"{keyframe_id}.depth.f32" + depth.tofile(depth_path) + conf_path = out_dir / "keyframes" / f"{keyframe_id}.confidence.u8" + confidence.tofile(conf_path) + feat_path = out_dir / "keyframes" / f"{keyframe_id}.features.bin" + feat_path.write_bytes(b"".join(struct.pack(" dict: "h": KEYFRAME_H, "timestamp": round(view["timestamp"], 4), "orientation": 1, - "keyframe": link, + "keyframe": keyframe_id, "intrinsics": [round(v, 3) for v in INTRINSICS], "gravity": [round(float(v), 6) for v in _device_gravity(pose)], } diff --git a/backend/tests/test_app_capture_synth.py b/backend/tests/test_app_capture_synth.py index 963b188be..9e8a5f80b 100644 --- a/backend/tests/test_app_capture_synth.py +++ b/backend/tests/test_app_capture_synth.py @@ -52,7 +52,7 @@ def test_packet_layout_matches_fixture_contract(rendered): assert packet["format"] == "house-capture-packet" assert packet["formatVersion"] == "0.4" assert packet["source"]["kind"] == "synthetic" - assert len(packet["keyframes"]) == 30 + assert len(packet["keyframes"]) == 32 assert len(packet["stills"]) == 2 assert packet["streams"]["motion"]["file"] == "streams/motion.csv.gz" assert packet["streams"]["arkitPoses"]["file"] == "streams/arkitPoses.csv.gz" @@ -73,7 +73,9 @@ def test_validator_reports_ok(rendered): for scene in SCENES: packet, report = validate_and_normalize(rendered[scene] / "packet.json") assert report.ok, f"{scene}: {report.errors}" - assert [issue.code for issue in report.warnings] == ["synthetic_source"] + warning_codes = {issue.code for issue in report.warnings} + assert "synthetic_source" in warning_codes + assert warning_codes <= {"synthetic_source", "pose_jump", "arkit_pose_stream_gap"} assert report.maxTapReplayError < 1e-6 @@ -154,7 +156,7 @@ def test_tap_hits_depth_at_the_same_distance(tmp_path, rendered): def test_depth_pixels_agree_with_geometry(rendered): - """The depth image around the tap pixel is within one depth pixel of the packed hit.""" + """The depth image stores camera-axis depth, with zero for missing rays.""" packet = _packet(rendered, "clear-wall") tap = packet["taps"][0] frame = next(f for f in packet["keyframes"] if f["id"] == tap["keyframe"]) @@ -163,10 +165,43 @@ def test_depth_pixels_agree_with_geometry(rendered): du = min(DEPTH_W - 1, max(0, int(u * DEPTH_W / 640))) dv = min(DEPTH_H - 1, max(0, int(v * DEPTH_H / 480))) patch = depth[max(0, dv - 1) : dv + 2, max(0, du - 1) : du + 2] - finite = patch[np.isfinite(patch)] + assert np.isfinite(depth).all() and np.all(depth >= 0) + confidence = np.fromfile(rendered["clear-wall"] / frame["depth"]["confidenceFile"], dtype=np.uint8).reshape( + DEPTH_H, DEPTH_W + ) + assert int(confidence.max()) <= 2, "ARConfidenceLevel only permits 0, 1, and 2" + finite = patch[patch > 0] assert finite.size, "depth around the tap should be valid" nearest = float(finite[np.argmin(np.abs(finite - tap["hit"]["distance"]))]) - assert nearest == pytest.approx(tap["hit"]["distance"], abs=0.01) + pose = from_column_major(frame["pose"]) + _origin, ray = pixel_ray_arkit(pose, frame["intrinsics"], np.asarray(tap["pixel"])) + camera_ray = pose[:3, :3].T @ ray + expected_axis_depth = tap["hit"]["distance"] * -float(camera_ray[2]) + assert nearest == pytest.approx(expected_axis_depth, abs=0.02) + + +def test_features_use_world_points_and_uint64_ids(rendered): + packet = _packet(rendered, "clear-wall") + root = rendered["clear-wall"] + dtype = np.dtype([("x", "= -1.1) & (records["x"] <= 5.1)) + assert np.all((records["y"] >= -1.5) & (records["y"] <= 1.4)) + assert np.all((records["z"] >= -2.2) & (records["z"] <= -1.7)) + + +def test_closeup_stills_link_to_their_own_keyframes(rendered): + packet = _packet(rendered, "clear-wall") + frames = {frame["id"]: frame for frame in packet["keyframes"]} + assert {frame.get("purpose") for frame in packet["keyframes"]} >= {"meter_close", "meter_oblique"} + for still in packet["stills"]: + linked = frames[still["keyframe"]] + assert linked["purpose"] == still["purpose"] + assert (rendered["clear-wall"] / linked["img"]).is_file() def test_keyframe_times_sit_on_the_30hz_grid(rendered):