From 231d8353f8115e51e8318946e3ea03d4f6153532 Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:41:14 -0500 Subject: [PATCH 01/62] fix(packet): qualify study evidence and proposed validation https://github.com/SamGu-NRX/BaseScanning/pull/22#discussion_r4113999645 https://github.com/SamGu-NRX/BaseScanning/pull/22#discussion_r4114080315 https://github.com/SamGu-NRX/BaseScanning/pull/22#discussion_r4114080318 --- packet/README.md | 14 ++++++++------ packet/validator-checks.md | 3 ++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/packet/README.md b/packet/README.md index 541ce1982..76abdba6e 100644 --- a/packet/README.md +++ b/packet/README.md @@ -85,18 +85,20 @@ performance. 6. **Estimated depth with its uncertainty** (S6). A phone without LiDAR can infer metric depth from the image, but used without its uncertainty it creates surfaces that are not there. Proposal: a depth `source` of `estimated`, which requires a `sigma` file (float32 meters, one - standard deviation per depth pixel, the depth's layout). + standard deviation per depth pixel, the depth's layout). To keep malformed uncertainty from + reaching geometry calculations, the proposed validator must require exactly `w * h` samples + (`4 * w * h` bytes), all finite and non-negative. A missing, truncated or unreadable map is a + validation problem, not an exception. 7. **The frame of a plane's boundary.** 0.4 lists `planes[].boundary` without saying its frame. ARKit's `boundaryVertices` are relative to the anchor transform, not to `planeExtent`'s centre and `rotationOnYAxis`, and packet 1.1 found that mixing the two is an easy mistake. Proposal: carry the anchor-frame boundary in a new field whose frame is stated, rather than giving the existing `boundary` a meaning an existing reader might not share, and have the validator check it lies within `planeExtent` once the centre and rotation are applied. -8. **Photos that frame one span.** With learned depth on ETH3D photos, an along-wall edge came to - 0.9 in p90 when seen within 15° of face-on and 24 in beyond 45° (#59, - `experiments/edge-geometry`), so a distance measured inside one face-on photo is likely far - better than one chained across the walk. Proposal: a still or - keyframe purpose `span` that names the marks the photo frames. +8. **Photos that frame one span.** With learned depth on ETH3D electro photos scaled using the + laser reference, in-plane edges had along-wall error of 0.9 in p90 within 15° of face-on and + 24 in beyond 45° (#59, `experiments/edge-geometry`). Proposal: a still or keyframe purpose + `span` that names the marks the photo frames. 9. **Scale references in view.** One reference near the meter may not be enough: on MARViN's walks ARKit's scale varied 1.8% (robust SD) within a walk (#59, `experiments/drift-anatomy`), so a reference helps where it appears. Proposal: `references[]` with `kind` (door, brick course, and hand-held ones such diff --git a/packet/validator-checks.md b/packet/validator-checks.md index 37a8cf041..f5be618be 100644 --- a/packet/validator-checks.md +++ b/packet/validator-checks.md @@ -52,7 +52,8 @@ case. `geometry/mesh.ply` is a binary little-endian PLY. 0.4 does not fix its index and count types, so this check proposes one layout as well: - `float x, y, z` per vertex, and per face a `uchar` count followed by that many `int` indices, - then an optional `uchar classification`; + then `uchar classification` when `mesh.classified` is true. The property may be absent only + when the flag is false, so a mesh cannot claim face classes without carrying them; - the body length equals exactly what the vertex and face counts imply; - vertex and face counts are non-negative, and every vertex coordinate is finite; - every face is a triangle, and every index satisfies `0 <= index < vertex_count`; From ec357bb56897440ddaa959790feea0ec3642d53c Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:44:30 -0500 Subject: [PATCH 02/62] fix(check): include independent verification in root dispatch https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4114226413 --- Makefile | 17 ++++++++++++----- tests/makefile.sh | 11 ++++++----- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 21d3fd050..138e460e5 100644 --- a/Makefile +++ b/Makefile @@ -1,34 +1,36 @@ # Each suite target runs the commands of its CI job in .github/workflows/. # `make check` runs server, web and ios, then scoring, measure-lab, evals, recon -# and meter-closeup where the branch has them. +# meter-closeup and verification where the branch has them. # tests/makefile.sh checks this dispatch with stub suites. XCODEGEN ?= xcodegen # Must match XCODEGEN_VERSION in .github/workflows/ios.yml; other versions write a different project. XCODEGEN_VERSION := 2.46.0 -# The file each experiment suite's target needs. These suites arrive in separate pull requests, so +# The file each optional suite's target needs. These suites arrive in separate pull requests, so # `check` includes one only when its file exists and the Makefile works before and after each -# merge. Once all five are on main, list them in `check` directly so a deleted folder fails. +# merge. Once all six are on main, list them in `check` directly so a deleted folder fails. SCORING := experiments/scoring/pyproject.toml MEASURE_LAB := experiments/measure-lab/Makefile EVALS := experiments/evals/Makefile RECON := recon/Makefile METER_CLOSEUP := experiments/meter-closeup/Makefile +VERIFICATION := verification/Makefile # $(call present,FILE,TARGET) is TARGET when FILE exists, otherwise nothing. present = $(if $(wildcard $(1)),$(2)) # $(call require,FILE,PR): stop before any command runs when the suite's file is missing. require = @test -f $(1) || { echo "make $@: $(1) is missing; the suite arrives with pull request $(2)." >&2; exit 1; } -.PHONY: check server web ios ios-project smoke scoring measure-lab evals recon meter-closeup +.PHONY: check server web ios ios-project smoke scoring measure-lab evals recon meter-closeup verification check: server web ios \ $(call present,$(SCORING),scoring) \ $(call present,$(MEASURE_LAB),measure-lab) \ $(call present,$(EVALS),evals) \ $(call present,$(RECON),recon) \ - $(call present,$(METER_CLOSEUP),meter-closeup) + $(call present,$(METER_CLOSEUP),meter-closeup) \ + $(call present,$(VERIFICATION),verification) server: cd server && uv sync --locked @@ -85,6 +87,11 @@ meter-closeup: $(call require,$(METER_CLOSEUP),#16) $(MAKE) -C experiments/meter-closeup check +# PR #13 added this suite; leaving it out lets `make check` skip the independent verifier. +verification: + $(call require,$(VERIFICATION),#13) + $(MAKE) -C verification test + # Posts each scene in server/examples to a placement server and prints its decision: # make smoke URL=https://house-scanning-server.vercel.app [KEY_FILE=server/.env.private.local] smoke: diff --git a/tests/makefile.sh b/tests/makefile.sh index 9e7552bd5..361870a41 100755 --- a/tests/makefile.sh +++ b/tests/makefile.sh @@ -27,10 +27,10 @@ EOF done export PATH="$work/bin:$PATH" -suites="scoring measure-lab evals recon meter-closeup" +suites="scoring measure-lab evals recon meter-closeup verification" folder() { case $1 in - recon) echo recon ;; + recon | verification) echo "$1" ;; *) echo "experiments/$1" ;; esac } @@ -66,8 +66,8 @@ fail() { } # 1. `make check` runs server, web and ios, then exactly the experiment suites present, in order, -# for all 32 combinations. "." is the ios build, which runs from the root. -for mask in $(seq 0 31); do +# for all 64 combinations. "." is the ios build, which runs from the root. +for mask in $(seq 0 63); do present=() expected="server web ." i=0 @@ -106,6 +106,7 @@ expect_commands evals "experiments/evals uv sync --locked experiments/evals make test" expect_commands recon "recon make test" expect_commands meter-closeup "experiments/meter-closeup make check" +expect_commands verification "verification make test" # 3. Naming a missing suite fails loudly and runs nothing. for suite in $suites; do @@ -119,7 +120,7 @@ done # 4. A failing command inside a present suite fails `make check`, and nothing runs after it. for FAIL_AT in "experiments/scoring uv" "experiments/measure-lab make" "experiments/evals uv" \ - "experiments/evals make" "recon make" "experiments/meter-closeup make"; do + "experiments/evals make" "recon make" "experiments/meter-closeup make" "verification make"; do make_tree $suites if run_make check; then fail "check passed although $FAIL_AT failed" From a5e976d75c88263a4fb79c6d48d0b9f788367dc5 Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:46:52 -0500 Subject: [PATCH 03/62] fix(verification): keep simulator evidence tied to its own run https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4115325486 https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4115325493 https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4114865933 https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4114352263 https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4114352258 --- verification/hsverify/a11yaudit.py | 22 +++++--- verification/hsverify/simrun.py | 34 +++++++++--- verification/tests/test_a11yaudit.py | 21 +++++++- verification/tests/test_simrun.py | 79 +++++++++++++++++++++++++++- 4 files changed, 142 insertions(+), 14 deletions(-) diff --git a/verification/hsverify/a11yaudit.py b/verification/hsverify/a11yaudit.py index f02b604bb..0873a121e 100644 --- a/verification/hsverify/a11yaudit.py +++ b/verification/hsverify/a11yaudit.py @@ -90,7 +90,7 @@ def audit_problems(screens: list[dict], harness_ran: bool, min_screens: int) -> """Why this run is not evidence of an audit; empty when it is.""" problems = [] if not harness_ran: - problems.append("the audit harness did not run (no TEST SUCCEEDED or TEST FAILED line)") + problems.append("the audit harness did not finish successfully") problems += [f"screen {s.get('index')}: {s['error']}" for s in screens if "error" in s] audited = audited_screens(screens) if not audited: @@ -102,7 +102,7 @@ def audit_problems(screens: list[dict], harness_ran: bool, min_screens: int) -> def run_harness( udid: str, bundle_id: str, args: argparse.Namespace, launch: list[str], out: Path -) -> str: +) -> tuple[str, int]: env = os.environ | { "TEST_RUNNER_AUDIT_BUNDLE": bundle_id, "TEST_RUNNER_AUDIT_ARGS": ARG_SEPARATOR.join(launch), @@ -126,7 +126,7 @@ def run_harness( simrun.wait_for_other_builds(args.build_wait) result = subprocess.run(cmd, env=env, capture_output=True, text=True) (out / "xcodebuild-test.log").write_text(result.stdout + result.stderr) - return result.stdout + return result.stdout, result.returncode def write_report(out: Path, data: dict) -> None: @@ -171,6 +171,7 @@ def write_report(out: Path, data: dict) -> None: (out / "index.html").write_text(page) +@simrun.device_lock() def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--ref", default="origin/t3/ios-mvf") @@ -222,7 +223,11 @@ def main(argv: list[str] | None = None) -> int: if app is None: print(f"Build failed; see {out / 'build.log'}") return 1 - device = simrun.ensure_device() + try: + device = simrun.ensure_device() + except BaseException: + simrun.discard_app(app) + raise udid = device["udid"] try: bundle_id = simrun.bundle_id_of(app) @@ -234,13 +239,18 @@ def main(argv: list[str] | None = None) -> int: ["xcrun", "simctl", "privacy", udid, "grant", "camera", bundle_id], capture_output=True ) print(f"Auditing {bundle_id} at {sha[:12]}...", flush=True) - output = run_harness(udid, bundle_id, args, launch, out) + output, test_returncode = run_harness(udid, bundle_id, args, launch, out) finally: + simrun.discard_app(app) subprocess.run(["xcrun", "simctl", "shutdown", udid], capture_output=True) screens, issues = parse_output(output) symbols = symbol_name_labels(screens) - harness_ran = "** TEST SUCCEEDED **" in output or "** TEST FAILED **" in output + harness_ran = ( + test_returncode == 0 + and "** TEST SUCCEEDED **" in output + and "** TEST FAILED **" not in output + ) problems = audit_problems(screens, harness_ran, args.min_screens) data = { # ok: the run is evidence of an audit. Issues found are counted separately below. diff --git a/verification/hsverify/simrun.py b/verification/hsverify/simrun.py index 235d96d8e..8d42bbeb1 100644 --- a/verification/hsverify/simrun.py +++ b/verification/hsverify/simrun.py @@ -30,6 +30,7 @@ import shutil import subprocess import sys +import tempfile import threading import time from collections.abc import Iterable @@ -52,6 +53,7 @@ DEFAULT_REPORTS = Path.home() / "house-scanning-data" / "reports" / "sim" DERIVED_DATA = Path("/tmp/hs-verify-derived-data") BUILD_LOCK = Path("/tmp/hs-verify-xcodebuild.lock") +DEVICE_LOCK = Path("/tmp/hs-verify-simulator.lock") BUILT_APP_DIR = "built-app" # in the report folder, until the Simulator has installed it @@ -219,11 +221,14 @@ def build_app( built = DERIVED_DATA / "Build" / "Products" / "Debug-iphonesimulator" / f"{scheme}.app" app = None if code == 0 and built.exists(): - app = out / BUILT_APP_DIR / built.name - shutil.rmtree(app.parent, ignore_errors=True) - shutil.copytree(built, app, symlinks=True) + app = Path(tempfile.mkdtemp(prefix=f"{BUILT_APP_DIR}-", dir=out)) / built.name + try: + shutil.copytree(built, app, symlinks=True) + except BaseException: + discard_app(app) + raise text = log_path.read_text(errors="replace") - warnings = sorted(set(re.findall(r"^.*: warning: .*$", text, re.M))) + warnings = sorted(set(re.findall(r"^(?:.*: )?warning: .*$", text, re.M))) errors = sorted(set(re.findall(r"^.*: error: .*$", text, re.M))) result = { "ok": code == 0, @@ -239,7 +244,7 @@ def build_app( def discard_app(app: Path) -> None: """Remove this run's copy of the app once the Simulator has installed its own.""" - if app.parent.name == BUILT_APP_DIR: + if app.parent.name.startswith(f"{BUILT_APP_DIR}-"): shutil.rmtree(app.parent, ignore_errors=True) @@ -529,6 +534,16 @@ def collect_crashes(since: float, out: Path) -> list[str]: return found +@contextlib.contextmanager +def device_lock(): + """Hold the shared Simulator until install, capture and shutdown have finished.""" + DEVICE_LOCK.touch() + with DEVICE_LOCK.open() as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + yield + + +@device_lock() def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--ref", default="origin/main", help="git ref to build") @@ -608,7 +623,12 @@ def main(argv: list[str] | None = None) -> int: flush=True, ) - device = ensure_device(args.udid) + try: + device = ensure_device(args.udid) + except BaseException: + if app is not None: + discard_app(app) + raise rep = RunReport( ref=args.ref, sha=sha, @@ -654,6 +674,8 @@ def main(argv: list[str] | None = None) -> int: if rep.crash_reports: rep.problems.append(f"Crash reports: {', '.join(rep.crash_reports)}") finally: + if app is not None: + discard_app(app) services.close() rep.peak_memory = peak_rss_mb() if not args.keep_booted: diff --git a/verification/tests/test_a11yaudit.py b/verification/tests/test_a11yaudit.py index 6013a20e3..7a2f1fd03 100644 --- a/verification/tests/test_a11yaudit.py +++ b/verification/tests/test_a11yaudit.py @@ -1,3 +1,7 @@ +import argparse +import subprocess + +from hsverify import a11yaudit from hsverify.a11yaudit import audit_problems, audited_screens, parse_output, symbol_name_labels @@ -51,7 +55,7 @@ def test_zero_screens_is_not_evidence(): def test_harness_that_did_not_run_is_not_evidence(): problems = audit_problems(NINE, harness_ran=False, min_screens=9) - assert problems == ["the audit harness did not run (no TEST SUCCEEDED or TEST FAILED line)"] + assert problems == ["the audit harness did not finish successfully"] def test_too_few_distinct_screens(): @@ -79,3 +83,18 @@ def test_error_lines_from_the_harness_are_parsed_as_screens(): "screen 1: app not running", "no screen was audited", ] + + +def test_failed_xctest_cannot_be_clean_evidence(tmp_path, monkeypatch): + output = "\n".join(f'A11Y_SCREEN={{"index":{i},"issues":0}}' for i in range(1, 10)) + output += "\n** TEST SUCCEEDED **" + monkeypatch.setattr(a11yaudit.simrun, "wait_for_other_builds", lambda _: None) + monkeypatch.setattr( + a11yaudit.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess([], 65, output, "failed"), + ) + args = argparse.Namespace(build_wait=0, seconds=1, idle=1) + text, code = a11yaudit.run_harness("mock", "mock", args, [], tmp_path) + screens, _ = parse_output(text) + assert audit_problems(screens, harness_ran=code == 0, min_screens=9) diff --git a/verification/tests/test_simrun.py b/verification/tests/test_simrun.py index 515f29828..b4981cecc 100644 --- a/verification/tests/test_simrun.py +++ b/verification/tests/test_simrun.py @@ -1,11 +1,15 @@ +import contextlib import datetime as dt import io import json import queue import subprocess +import sys from pathlib import Path -from hsverify import simrun +import pytest + +from hsverify import gitref, simrun from hsverify.simrun import ( ShotRecord, app_export, @@ -186,3 +190,76 @@ def fake_build(cmd, **kwargs): assert (app / "Info.plist").read_text() == "run A" simrun.discard_app(app) assert not app.parent.exists() and out.exists() + + +def test_shared_output_keeps_both_app_copies(tmp_path, monkeypatch): + derived = tmp_path / "derived" + built = derived / "Build" / "Products" / "Debug-iphonesimulator" / "HouseScan.app" + monkeypatch.setattr(simrun, "DERIVED_DATA", derived) + monkeypatch.setattr(simrun, "BUILD_LOCK", tmp_path / "lock") + monkeypatch.setattr(simrun, "wait_for_other_builds", lambda _: 0.0) + + def fake_build(cmd, **kwargs): + built.mkdir(parents=True, exist_ok=True) + (built / "Info.plist").write_text("app") + kwargs["stdout"].write("warning: Run script build phase\n/path/File: warning: warning\n") + return subprocess.CompletedProcess(cmd, 0) + + monkeypatch.setattr(simrun.subprocess, "run", fake_build) + tree, out = tmp_path / "tree", tmp_path / "report" + (tree / "ios" / "HouseScan.xcodeproj").mkdir(parents=True) + out.mkdir() + result, first = simrun.build_app(tree, out, 0, "ios/HouseScan.xcodeproj", "HouseScan") + _, second = simrun.build_app(tree, out, 0, "ios/HouseScan.xcodeproj", "HouseScan") + assert first != second and first.exists() and second.exists() + assert len(result["warnings"]) == 2 + simrun.discard_app(first) + assert not first.exists() and second.exists() + + +@pytest.mark.parametrize("failure", ["device", "bundle", "install"]) +def test_failed_preparation_discards_app_copy(tmp_path, monkeypatch, failure): + out = tmp_path / "out" + app = out / "built-app-unique" / "HouseScan.app" + app.mkdir(parents=True) + monkeypatch.setattr(simrun, "DEVICE_LOCK", tmp_path / "device.lock") + monkeypatch.setattr(gitref, "fetch", lambda: None) + monkeypatch.setattr(gitref, "resolve", lambda _: "a" * 40) + monkeypatch.setattr( + gitref, "detached_worktree", lambda *args, **kwargs: contextlib.nullcontext(tmp_path) + ) + monkeypatch.setattr(simrun, "static_c4_check", lambda _: {}) + monkeypatch.setattr(simrun, "build_app", lambda *args: ({"ok": True}, app)) + + def fail(): + raise RuntimeError("preparation failed") + + monkeypatch.setattr( + simrun, "ensure_device", lambda _: fail() if failure == "device" else {"udid": "mock"} + ) + monkeypatch.setattr(simrun, "bundle_id_of", lambda _: fail() if failure == "bundle" else "mock") + monkeypatch.setattr(simrun, "prepare_display", lambda *args: None) + monkeypatch.setattr(simrun, "simctl", lambda *args: fail() if args[0] == "install" else "") + monkeypatch.setattr( + simrun.subprocess, "run", lambda *args, **kwargs: subprocess.CompletedProcess([], 0) + ) + monkeypatch.setattr(simrun, "peak_rss_mb", lambda: {}) + monkeypatch.setattr(simrun.report, "write_sim_report", lambda *args: None) + with pytest.raises(RuntimeError, match="preparation failed"): + simrun.main(["--out", str(out)]) + assert not app.parent.exists() + + +def test_device_lock_excludes_another_runner_and_releases(tmp_path, monkeypatch): + monkeypatch.setattr(simrun, "DEVICE_LOCK", tmp_path / "device.lock") + script = """import fcntl, sys +with open(sys.argv[1]) as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + sys.exit(43) +""" + cmd = [sys.executable, "-c", script, str(simrun.DEVICE_LOCK)] + with simrun.device_lock(): + assert subprocess.run(cmd, timeout=5, check=False).returncode == 43 + assert subprocess.run(cmd, timeout=5, check=False).returncode == 0 From c6442176eb155282ba1ac10b779606d5afc6b574 Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:47:13 -0500 Subject: [PATCH 04/62] fix(verification): validate interior footprints and replay failures https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4114226423 https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4115043212 https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4115325475 https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4114835310 https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4115325477 https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4114352256 https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4114676179 https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4115325497 --- verification/hsverify/replaycheck.py | 8 +++-- verification/hsverify/resultcheck.py | 20 +++++++++-- .../scenes/eth3d-facade/build_scene.py | 7 ++-- verification/tests/test_build_scene.py | 9 +++++ verification/tests/test_replaycheck.py | 18 ++++++++++ verification/tests/test_resultcheck.py | 33 +++++++++++++++++-- 6 files changed, 85 insertions(+), 10 deletions(-) diff --git a/verification/hsverify/replaycheck.py b/verification/hsverify/replaycheck.py index eaf415414..1f6fab57a 100644 --- a/verification/hsverify/replaycheck.py +++ b/verification/hsverify/replaycheck.py @@ -46,7 +46,11 @@ def numbers(value: Any, count: int) -> list[float] | None: return None if not all(isinstance(v, (int, float)) and not isinstance(v, bool) for v in value): return None - return [float(v) for v in value] if all(math.isfinite(v) for v in value) else None + try: + converted = [float(v) for v in value] + except OverflowError: + return None + return converted if all(math.isfinite(v) for v in converted) else None def rigid_problems(m: np.ndarray, tol: float = 1e-3) -> list[str]: @@ -246,7 +250,7 @@ def main(argv: list[str] | None = None) -> int: errors, summary = check_format(folder, session) report: dict = {"session": str(folder), "format_errors": errors, "summary": summary} truth_path = folder / "ground_truth.json" - if truth_path.exists(): + if truth_path.exists() and not errors: report["pose_accuracy"] = accuracy(session, json.loads(truth_path.read_text())) if args.json: print(json.dumps(report, indent=2)) diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index bb276281a..e1ab5a718 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -91,6 +91,7 @@ class RuleSet: battery_height_ft: float | None = None # the rules' headroom.min_ft: how high a wall view without out_ft reached headroom_ft: float | None = None + corner_allowance_ft: float | None = None @classmethod def from_yaml(cls, text: str) -> RuleSet: @@ -150,6 +151,7 @@ def reach(key: str) -> Need: value("sweep", "wall_join_ft"), value("battery", "height_ft"), headroom, + value("route", "corner_allowance_ft"), ) @@ -450,10 +452,11 @@ def off_wall(lo: float, hi: float, outcome: str, wall_id: str) -> str | None: for run in result.get("sweep", []): if run["outcome"] not in ("pass", "unsure"): continue - for start in dict.fromkeys(run["start_ft"]): # a run's two ends bound all its starts + for start in run_starts(run["start_ft"], rules.step_ft): battery = (start, start + rules.width_ft) if (why := off_wall(*battery, run["outcome"], run["wall_id"])) is not None: problems.append(f"sweep {run['outcome']} start {start:.2f} {why}") + break return problems @@ -1109,8 +1112,19 @@ def with_past_ends_asked(scene: dict, requests: list[dict], rules: RuleSet) -> d # --- Case expectations ----------------------------------------------------------------------- -def assumption_mismatches(assumed: dict[str, float], result: dict) -> list[str]: +def assumption_mismatches( + assumed: dict[str, float], result: dict, rules: RuleSet | None = None +) -> list[str]: out = [] + configured = ( + {"battery_width": rules.width_ft, "corner_allowance": rules.corner_allowance_ft} + if rules + else {} + ) + for key, value in assumed.items(): + actual = configured.get(key) + if actual is not None and abs(actual - value) > EPS: + out.append(f"case assumes {key} {value}, rules use {actual}") for key, value in assumed.items(): for check in result.get("checks", []): threshold = check.get("threshold_ft") @@ -1394,7 +1408,7 @@ def with_ground_short_of(scene: dict, rules: RuleSet, margin_ft: float = 0.1) -> m = copy.deepcopy(scene) for entry in m["coverage"]["observed"]: if entry["band"] == "ground": - entry["out_ft"] = min(entry["out_ft"], reach) + entry["out_ft"] = min(entry.get("out_ft", 0.0), reach) return m diff --git a/verification/scenes/eth3d-facade/build_scene.py b/verification/scenes/eth3d-facade/build_scene.py index a5cb9f841..f794a9e4f 100644 --- a/verification/scenes/eth3d-facade/build_scene.py +++ b/verification/scenes/eth3d-facade/build_scene.py @@ -487,12 +487,13 @@ def band_coverage(band, walls, facing, kfs, seen=visible): down so it never claims more. `seen(kf, wall, X, walls)` is `visible`; tests replace it.""" samples = [] # (s0, s1, reach) per sample seen for w in walls: - for s in np.arange(w.s0 + COVERAGE_STEP_FT / 2, w.s0 + w.length, COVERAGE_STEP_FT): + for a in np.arange(w.s0, w.s0 + w.length, COVERAGE_STEP_FT): + b = min(a + COVERAGE_STEP_FT, w.s0 + w.length) + s = (a + b) / 2 fd = next((v for (wid, a, b, v) in facing if wid == w.id and a <= s < b), None) probes = band_probes(band, w, s, fd) if probes is not None and any(all(seen(k, w, X, walls) for X in probes) for k in kfs): - half = COVERAGE_STEP_FT / 2 - samples.append([s - half, s + half, band_reach(band, fd)]) + samples.append([a, b, band_reach(band, fd)]) merged = [] # consecutive samples, also across a corner, keep the smaller reach for a, b, reach in samples: if merged and abs(merged[-1][1] - a) < 1e-6: diff --git a/verification/tests/test_build_scene.py b/verification/tests/test_build_scene.py index d2983bfb7..1e0c6bfd4 100644 --- a/verification/tests/test_build_scene.py +++ b/verification/tests/test_build_scene.py @@ -92,3 +92,12 @@ def only_low(kf, w, X, walls): entries, _ = build_scene.band_coverage("wall", [wall("w", 0.0, 2.0)], [], ["kf"], only_low) assert entries == [] + + +@pytest.mark.parametrize("length", [2.2, 2.3, 0.05]) +def test_non_grid_wall_has_no_excess_or_missing_tail(length): + entries, spans = build_scene.band_coverage( + "wall", [wall("w", 0.0, length)], [], ["kf"], seen_everywhere + ) + assert entries[0]["span_ft"] == [0.0, length] + assert spans == [[0.0, length]] diff --git a/verification/tests/test_replaycheck.py b/verification/tests/test_replaycheck.py index dd830e2e3..ef07b710e 100644 --- a/verification/tests/test_replaycheck.py +++ b/verification/tests/test_replaycheck.py @@ -4,6 +4,7 @@ import numpy as np import pytest +from hsverify import replaycheck from hsverify.replaycheck import ( accuracy, check_format, @@ -136,3 +137,20 @@ def test_accuracy_separates_scale_from_alignment(tmp_path, scale): assert result["max_vertical_axis_mismatch"] == pytest.approx(0, abs=1e-9) expected_final = (1 - scale) * 39 assert result["anchored_error_m_final"] == pytest.approx(expected_final, abs=1e-3) + + +def test_oversized_integer_is_a_format_error(tmp_path): + session = session_with(tmp_path, [[0, 0, 0]]) + session["keyframes"][0]["intrinsics"][0] = 10**400 + errors, _ = check_format(tmp_path, session) + assert any("finite numbers" in e for e in errors) + + +def test_invalid_replay_with_ground_truth_still_reports(tmp_path, capsys): + session = session_with(tmp_path, [[0, 0, 0]]) + del session["keyframes"][0]["pose"] + (tmp_path / "session.json").write_text(json.dumps(session)) + (tmp_path / "ground_truth.json").write_text("{}") + assert replaycheck.main([str(tmp_path), "--json"]) == 1 + data = json.loads(capsys.readouterr().out) + assert data["format_errors"] and "pose_accuracy" not in data diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 5846b7a9a..08935bd3a 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -412,7 +412,7 @@ def test_rules_come_from_rules_yaml_and_a_missing_one_is_loud(): openings: {exempt_bottom_above_ft: null} facing: {min_ft: {value: 3}} headroom: {min_ft: {value: 6.5}} -route: {height_ft: {value: 1}} +route: {height_ft: {value: 1}, corner_allowance_ft: {value: 0.5}} sweep: {step_ft: {value: 0.166666666667}, wall_join_ft: {value: 0.6}} """ rules = RuleSet.from_yaml(text) @@ -1335,7 +1335,7 @@ def test_a_passing_footprint_keeps_the_wall_error_clear_of_its_stretch_end(): ] r = result(spot=False, sweep=[pass_run(6.846, 10.0 - 31 / 12)]) assert footprint_problems(scene, r, RULES) == [ - "sweep pass start 7.42 ends within its error (0.60 ft) of its straight wall's end at 10.00" + "sweep pass start 7.01 ends within its error (0.60 ft) of its straight wall's end at 10.00" ] r["sweep"][0]["outcome"] = "unsure" # unsure may stand that close assert footprint_problems(scene, r, RULES) == [] @@ -1479,3 +1479,32 @@ def test_a_band_gap_under_the_coverage_tolerance_is_rounding(): assert gaps_over([(-6.5, 18.0)], 9.94, 18.011) == [(18.0, 18.011)] assert gaps_over([(0.0, 1.0), (1.009, 3.0)], 0.5, 2.0) == [] assert gaps_over([(0.0, 1.0), (1.011, 3.0)], 0.5, 2.0) == [(1.0, 1.011)] + + +@pytest.mark.parametrize("outcome", ["pass", "unsure"]) +def test_interior_sweep_start_cannot_cross_a_corner(outcome): + scene = copy.deepcopy(SCENE) + scene["walls"][0]["baseline"] = [[-12.0, 0.0], [5.0, 0.0], [5.0, 7.0]] + result = {"sweep": [{"wall_id": "w1", "start_ft": [1.0, 8.0], "outcome": outcome}]} + assert footprint_problems(scene, result, RULES) + + +def test_ground_trimming_keeps_omitted_reach_at_zero(): + scene = copy.deepcopy(SCENE) + scene["coverage"]["observed"] = [ + {"band": "ground", "span_ft": [-12, 0]}, + {"band": "ground", "span_ft": [0, 12], "out_ft": 100}, + ] + trimmed = with_ground_short_of(scene, RULES) + assert trimmed["coverage"]["observed"][0]["out_ft"] == 0 + + +def test_geometric_assumptions_compare_active_rules(): + rules = replace(RULES, corner_allowance_ft=0.5) + assert ( + len(assumption_mismatches({"battery_width": 100, "corner_allowance": 100}, {}, rules)) == 2 + ) + assert ( + assumption_mismatches({"battery_width": rules.width_ft, "corner_allowance": 0.5}, {}, rules) + == [] + ) From 853c2dfa9889cce89f7b8932ccc65a9de9664b79 Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:48:00 -0500 Subject: [PATCH 05/62] fix(verification): reject incomplete evidence and stale input hashes https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4114352255 https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4114865938 https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4114226429 https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4114226433 https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4114331241 https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4114352260 https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4115043204 https://github.com/SamGu-NRX/BaseScanning/pull/13#discussion_r4115043208 --- verification/hsverify/e2e.py | 31 ++++++++----- verification/hsverify/gitref.py | 2 +- verification/hsverify/scoreboard.py | 14 +++++- verification/scoreboard/metrics.yaml | 2 + verification/tests/test_e2e.py | 67 ++++++++++++++++++++++++++- verification/tests/test_gitref.py | 7 +++ verification/tests/test_scoreboard.py | 25 ++++++++++ 7 files changed, 134 insertions(+), 14 deletions(-) diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index 87e8bde35..7d7321584 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -151,9 +151,12 @@ def read_bundle(path: Path) -> tuple[bytes, dict[str, bytes]]: declared = sum(i.file_size for i in infos) if declared > MAX_BUNDLE_BYTES: raise SystemExit(f"{path}: declares {declared} bytes, limit {MAX_BUNDLE_BYTES}") - scene = next((i for i in infos if i.filename.endswith("scene.json")), None) - if scene is None: - raise SystemExit(f"{path}: no scene.json in the bundle") + scenes = [i for i in infos if i.filename == "scene.json"] + if len(scenes) != 1: + raise SystemExit( + f"{path}: expected exactly one canonical scene.json, found {len(scenes)}" + ) + scene = scenes[0] raw = read_bounded(bundle, scene, MAX_SCENE_BYTES) images = { Path(i.filename).name: read_bounded(bundle, i, MAX_IMAGE_BYTES) @@ -467,7 +470,7 @@ def judge( contract += property_problems(url, endpoint, item, result, schemas, rules) if item.real and ms > LATENCY_BUDGET_MS: contract.append(f"real-derived scene took {ms:.0f} ms, budget {LATENCY_BUDGET_MS:.0f}") - mismatches = assumption_mismatches(item.rules_assumed, result) + mismatches = assumption_mismatches(item.rules_assumed, result, rules) expected = expectation_problems(item.expect, result) if item.expect and not mismatches else [] problems = contract + expected @@ -491,7 +494,8 @@ def property_problems( problems: list[str] = [] def resend(name: str, scene: dict) -> dict | None: - other, failure = answer(url, endpoint, variant(item, name, scene)) + transformed = variant(item, name, scene) + other, failure = answer(url, endpoint, transformed) if failure: problems.append(f"{name}: {failure}") return None @@ -499,7 +503,10 @@ def resend(name: str, scene: dict) -> dict | None: if errors: problems.append(f"{name}: result schema: {errors[0]}") return None - problems.extend(f"{name}: {p}" for p in invariant_problems(scene, other, rules=rules)) + problems.extend( + f"{name}: {p}" + for p in invariant_problems(scene, other, sent=transformed.raw, rules=rules) + ) return other # 1. Same input, same answer. @@ -661,8 +668,11 @@ def judge_hostile( and not (endpoint.accepts_zip) ): return { - "status": "skipped", - "skip_reason": "the endpoint takes neither a zip nor an upload", + "status": "fail", + "problems": ["large-input probe was not exercised: endpoint accepts no archive"], + "contract_problems": [ + "large-input probe was not exercised: endpoint accepts no archive" + ], } before = limit.reset() if limit else 0.0 status, payload, ms = post(url, endpoint, item, timeout_s=HOSTILE_BUDGET_MS / 1000 + 5) @@ -712,9 +722,8 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: killed = bool(meta["peak_memory"].get("server_killed")) report = meta | { "counts": counts, - "all_passed": all(r["status"] in ("pass", "skipped") for r in records) - and bool(checked) - and not killed, + # A missing listed scene leaves the run incomplete, even when other scenes passed. + "all_passed": all(r["status"] == "pass" for r in records) and bool(checked) and not killed, "latency_ms": max(real) if real else None, "scenes_answered": sum(1 for r in records if "decision" in r), "scenes_checked": len(checked), diff --git a/verification/hsverify/gitref.py b/verification/hsverify/gitref.py index e711e295d..59f1431d6 100644 --- a/verification/hsverify/gitref.py +++ b/verification/hsverify/gitref.py @@ -24,7 +24,7 @@ def git(*args: str, cwd: Path = REPO) -> str: def fetch() -> None: - git("fetch", "--quiet", "origin") + git("fetch", "--quiet", "--prune", "origin") def resolve(ref: str) -> str: diff --git a/verification/hsverify/scoreboard.py b/verification/hsverify/scoreboard.py index 3cca83fe9..725557158 100644 --- a/verification/hsverify/scoreboard.py +++ b/verification/hsverify/scoreboard.py @@ -750,7 +750,19 @@ def render_markdown(board: Board) -> str: def list_prs() -> list[dict]: out = subprocess.run( - ["gh", "pr", "list", "--repo", GITHUB_REPO, "--state", "open", "--json", PR_FIELDS], + [ + "gh", + "pr", + "list", + "--repo", + GITHUB_REPO, + "--state", + "open", + "--limit", + "10000", + "--json", + PR_FIELDS, + ], check=True, capture_output=True, text=True, diff --git a/verification/scoreboard/metrics.yaml b/verification/scoreboard/metrics.yaml index c73e9cc0c..2013f8c08 100644 --- a/verification/scoreboard/metrics.yaml +++ b/verification/scoreboard/metrics.yaml @@ -162,6 +162,8 @@ metrics: glob: ~/house-scanning-data/reports/e2e/*/report.json # True only when at least one scene was answered and checked and none broke the contract. key: contract_ok + # A failed or incomplete run cannot establish the advertised black-box acceptance. + valid_key: all_passed equals: true require_current_sha: true diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index 13d70f552..a7707e1ec 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -384,7 +384,7 @@ def test_a_prebuilt_zip_goes_as_application_zip_when_the_endpoint_takes_it(refus RefusingServer.content_types.clear() bomb = hostile_inputs()[4] json_only = Endpoint("/p", "application/json") - assert judge_hostile(refusing_url, json_only, bomb, SCHEMAS)["status"] == "skipped" + assert judge_hostile(refusing_url, json_only, bomb, SCHEMAS)["status"] == "fail" zip_too = Endpoint("/p", "application/json", accepts_zip=True) assert judge_hostile(refusing_url, zip_too, bomb, SCHEMAS)["status"] == "pass" assert RefusingServer.content_types == ["application/zip"] @@ -484,3 +484,68 @@ def test_a_200_that_is_not_a_json_object_is_a_failure_not_a_crash(monkeypatch): assert "not a JSON object" in record["contract_problems"][0] hostile = judge_hostile("http://unused", endpoint, hostile_inputs()[3], SCHEMAS) assert hostile["status"] == "fail" and "not a JSON object" in hostile["problems"][0] + + +@pytest.mark.parametrize( + "extra", + [ + {"status": "skipped", "skip_reason": "missing scene"}, + {"status": "fail", "expectation_problems": ["wrong decision"], "contract_problems": []}, + ], +) +def test_run_acceptance_is_separate_from_contract_invariants(tmp_path, extra): + report = e2e.write_report(tmp_path, META, [CHECKED, {"name": "other", **extra}]) + assert not report["all_passed"] + assert report["contract_ok"] + + +def test_unexercised_large_input_probe_fails_contract(): + item = hostile_inputs()[-1] + item.bundle = b"archive" + record = judge_hostile("unused", Endpoint("/place", "application/json"), item, {}) + assert record["status"] == "fail" and record["contract_problems"] + + +def test_bundle_uses_exact_canonical_scene(tmp_path): + archive = tmp_path / "scene.zip" + with zipfile.ZipFile(archive, "w") as z: + z.writestr("old-scene.json", b"old") + z.writestr("scene.json", b"canonical") + assert e2e.read_bundle(archive)[0] == b"canonical" + + +@pytest.mark.parametrize("names", [["old-scene.json"], ["scene.json", "scene.json"]]) +def test_bundle_rejects_missing_or_duplicate_canonical_scene(tmp_path, names): + archive = tmp_path / "scene.zip" + with zipfile.ZipFile(archive, "w") as z: + for name in names: + z.writestr(name, b"{}") + with pytest.raises(SystemExit, match=r"exactly one canonical scene\.json"): + e2e.read_bundle(archive) + + +def test_transformed_answer_checks_its_own_hash(monkeypatch): + calls = [] + original = {"decision": "pass", "stats": {}, "sweep": []} + monkeypatch.setattr( + e2e, "post", lambda *args, **kwargs: (200, json.dumps(original).encode(), 1) + ) + monkeypatch.setattr(e2e, "answer", lambda *args: (original, None)) + monkeypatch.setattr(e2e, "schema_errors", lambda *args: []) + + def check_hash(scene, result, sent=None, rules=None): + calls.append((scene, sent)) + return ["stale input hash"] if sent is not None else [] + + monkeypatch.setattr(e2e, "invariant_problems", check_hash) + monkeypatch.setattr(e2e, "with_ground_short_of", lambda *args: None) + monkeypatch.setattr(e2e, "with_more_error", lambda *args: None) + monkeypatch.setattr(e2e, "tape_to_tap", lambda *args: None) + monkeypatch.setattr(e2e, "with_less_coverage", lambda *args: None) + scene = {"walls": [{"id": "w1", "baseline": [[-1, 0], [1, 0]]}], "meter": {"pos": [0, 1, 0]}} + probe = SceneInput(name="test", scene=scene, raw=json.dumps(scene).encode(), source="test") + failures = e2e.property_problems( + "unused", Endpoint("/p", "application/json"), probe, original, {"result": {}}, RULES + ) + assert calls and all(raw == json.dumps(scene, indent=1).encode() for scene, raw in calls) + assert any("stale input hash" in p for p in failures) diff --git a/verification/tests/test_gitref.py b/verification/tests/test_gitref.py index da2c1511a..3903936d3 100644 --- a/verification/tests/test_gitref.py +++ b/verification/tests/test_gitref.py @@ -12,3 +12,10 @@ def test_two_uses_of_one_sha_get_separate_trees_and_both_clean_up(): def test_show_returns_none_for_a_missing_file(): assert gitref.show("HEAD", "no/such/file") is None assert gitref.show("HEAD", "verification/pyproject.toml") is not None + + +def test_fetch_prunes_deleted_refs(monkeypatch): + calls = [] + monkeypatch.setattr(gitref, "git", lambda *args: calls.append(args)) + gitref.fetch() + assert calls == [("fetch", "--quiet", "--prune", "origin")] diff --git a/verification/tests/test_scoreboard.py b/verification/tests/test_scoreboard.py index 21292cb27..a67c7111b 100644 --- a/verification/tests/test_scoreboard.py +++ b/verification/tests/test_scoreboard.py @@ -2,6 +2,7 @@ import datetime as dt import json import os +import subprocess import textwrap import time @@ -675,3 +676,27 @@ def test_tail_keeps_last_three_lines_with_spaces_collapsed(): sb._tail("a\n\n... [ 82%]\nb\n 87 passed in 1s \n") == "... [ 82%] / b / 87 passed in 1s" ) + + +def test_pr_inventory_exceeds_default_page(monkeypatch): + prs = [{"number": n} for n in range(1, 42)] + + def fake_run(cmd, **kwargs): + assert int(cmd[cmd.index("--limit") + 1]) > 30 + assert kwargs["timeout"] == sb.PR_LIST_TIMEOUT_S + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(prs)) + + monkeypatch.setattr(sb.subprocess, "run", fake_run) + assert sb.list_prs() == prs + + +def test_s2_contract_credit_requires_successful_full_run(tmp_path): + _, metrics = sb.load_metrics() + metric = next(m for m in metrics if m.id == "S2-8") + metric.probe["glob"] = str(tmp_path / "*" / "report.json") + write_report( + tmp_path / "r" / "report.json", + {"sha": SHA, "contract_ok": True, "all_passed": False}, + 100, + ) + assert sb.probe_report(metric.probe, SHA).status == "gap" From b5da197b97f42c0568d4b9424ab3780762837245 Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:49:41 -0500 Subject: [PATCH 06/62] fix(recon): preserve evidence and refuse incomplete cached outputs https://github.com/SamGu-NRX/BaseScanning/pull/20#discussion_r4114128928 https://github.com/SamGu-NRX/BaseScanning/pull/20#discussion_r4114128930 https://github.com/SamGu-NRX/BaseScanning/pull/20#discussion_r4114128938 https://github.com/SamGu-NRX/BaseScanning/pull/20#discussion_r4114194380 https://github.com/SamGu-NRX/BaseScanning/pull/20#discussion_r4114194382 https://github.com/SamGu-NRX/BaseScanning/pull/20#discussion_r4114194384 https://github.com/SamGu-NRX/BaseScanning/pull/20#discussion_r4114621091 https://github.com/SamGu-NRX/BaseScanning/pull/20#discussion_r4114621094 https://github.com/SamGu-NRX/BaseScanning/pull/20#discussion_r4114621095 https://github.com/SamGu-NRX/BaseScanning/pull/20#discussion_r4115044062 https://github.com/SamGu-NRX/BaseScanning/pull/20#discussion_r4115044074 https://github.com/SamGu-NRX/BaseScanning/pull/20#discussion_r4115044084 https://github.com/SamGu-NRX/BaseScanning/pull/20#discussion_r4115044089 https://github.com/SamGu-NRX/BaseScanning/pull/20#discussion_r4115044093 https://github.com/SamGu-NRX/BaseScanning/pull/20#discussion_r4115044096 https://github.com/SamGu-NRX/BaseScanning/pull/20#discussion_r4115044099 https://github.com/SamGu-NRX/BaseScanning/pull/20#discussion_r4115044101 https://github.com/SamGu-NRX/BaseScanning/pull/20#discussion_r4115917428 --- recon/HANDOFF.md | 12 +- recon/models/moge_depth.py | 18 +- recon/recon/capture.py | 98 ++++++++- recon/recon/coverage.py | 45 +++-- recon/recon/depth.py | 49 ++++- recon/recon/pipeline.py | 23 ++- recon/recon/scene.py | 17 +- recon/tests/test_review_debt.py | 348 ++++++++++++++++++++++++++++++++ recon/tests/test_worker.py | 5 +- 9 files changed, 570 insertions(+), 45 deletions(-) create mode 100644 recon/tests/test_review_debt.py diff --git a/recon/HANDOFF.md b/recon/HANDOFF.md index 16b84e388..b952169d2 100644 --- a/recon/HANDOFF.md +++ b/recon/HANDOFF.md @@ -126,17 +126,13 @@ In `recon/` on this branch: ## 6. Known defects -Reviewers confirmed these in the worker at this branch's head, and none is fixed. Each can let the server settle a check on evidence the worker doesn't have. No replacement below has been built or validated. +These defects remain after the review-debt repairs. They can let the server settle a check on evidence the worker does not have. The saved ETH3D results above predate those repairs; dataset acceptance has not been rerun. -- **One view can count as two for free space.** `recon/coverage.py:183-187, 205-243`: `_slab` treats a voxel as observed when its fused weight is above zero, and any single frame gives it weight. Repro: a capture in which one frame alone sees the space in front of a wall cell reports a facing clearance there. Wall and ground cells need two positions 0.25 m apart for the same claim. The server at 903d86f can settle a check on that clearance. A fix needs visibility kept per frame for free space, with the same two-position test as the wall and ground. -- **The overhead strip is narrower than a battery.** `recon/coverage.py:42, 231-242`: overhead space is sampled only 0.10 to 0.56 m out from the wall front. Repro: an eave 0.05 m out from the wall, or 0.7 m out over a 3 ft deep battery, is not found, and the cell reports clearance above it. A fix needs the footprint of the battery the chosen rule places, passed from the server's rules or request. -- **Old facing and overhead entries survive a small meter move.** `recon/scene.py:50-57, 87-89`: when the meter moves 5 cm or less, the bundle's existing `facing` and `overheads` entries are kept and the new ones appended, although the reconstructed wall's line and orientation replaced the phone's. Repro: a bundle with a phone-measured facing gap, and a reconstructed wall rotated a few degrees about the meter, sends both the old gap and the new one. A fix needs the target wall's earlier entries dropped whenever its line is replaced, as they already are when the meter moves more than 5 cm. +- **One view can count as two for free space.** `recon/coverage.py:189-193, 208-246`: `_slab` treats a voxel as observed when its fused weight is above zero, and any single frame gives it weight. Repro: a capture in which one frame alone sees the space in front of a wall cell reports a facing clearance there. Wall and ground cells need two positions 0.25 m apart for the same claim. The server at 903d86f can settle a check on that clearance. A fix needs visibility kept per frame for free space, with the same two-position test as the wall and ground. +- **The overhead strip is narrower than a battery.** `recon/coverage.py:42, 236-245`: overhead space is sampled only 0.10 to 0.56 m out from the wall front. Repro: an eave 0.05 m out from the wall, or 0.7 m out over a 3 ft deep battery, is not found, and the cell reports clearance above it. A fix needs the footprint of the battery the chosen rule places, passed from the server's rules or request. - **The photos-only clearance error is not calibrated.** `recon/pipeline.py:29-35`: facing gaps and overhead clearances from photos carry ±0.56 ft. That comes from the 95% interval of the evals' surface pair-error p90 over 1 to 3 m spans, not from any measurement of where the fused volume puts a boundary. The evals put surface p90 at 5.64 in [4.63, 6.66] and edges at 9.65 in [4.93, 15.01]. Repro: a photos-only overhead clearance measured at 7.10 ft passes the 6.5 ft headroom rule, since 7.10 − 0.56 = 6.54 > 6.5. A fix needs the clearance estimator's own measured uncertainty, or a gate that returns the check unresolved until that exists. -- **Partly occupied space counts as clear.** `recon/coverage.py:223, 235` (threshold at 179): a step out from the wall, or up over the footprint, counts as blocked only when at least 20% of its sampled voxels are occupied, on two steps in a row. That 20% has no source. Otherwise `free_space` reports the space clear through voxels the reconstruction marks occupied. Repro: at 5 cm voxels, a 5 cm rail crossing the facing band (0.3 to 1.8 m high) fills 1 of its 31 height rows, about 3%, and the facing clearance runs straight through it. An obstruction thinner than two steps passes the same way. A fix needs any occupied voxel in the band to end the clear distance or leave the check unresolved. -- **LiDAR without a confidence map is fused unfiltered.** `recon/depth.py:90-92` (accepted at `recon/capture.py:182-189`): a keyframe whose `depth` names a `file` but no `confidenceFile` is accepted, and `lidar()` then keeps every positive sample, including ARKit's low-confidence ones that section 4 says to drop. Repro: a packet keyframe with `"depth": {"file": "k.f32", "w": 256, "h": 192}` has all its samples fused, with no confidence filter. A fix needs a missing confidence map either refused, or treated as that frame having no usable LiDAR. - -Two more are filed as issues for the server team: coverage samples ground on a flat plane at the meter's height rather than the fitted slope (#95), and coverage cells can reach past the fitted wall's ends (#96). +The worker now replaces the target wall's prior measurements, stops clearance at any occupied sampled voxel, and refuses LiDAR without a confidence map. Ground coverage follows the fitted slope, and only whole cells within the fitted wall extent are exported. The regressions are in `tests/test_review_debt.py`; they do not validate the remaining clearance policies above. ## 7. Data diff --git a/recon/models/moge_depth.py b/recon/models/moge_depth.py index 0c049b79b..f26c881dd 100644 --- a/recon/models/moge_depth.py +++ b/recon/models/moge_depth.py @@ -16,6 +16,7 @@ import math import os import sys +import tempfile from pathlib import Path DATA = Path(os.environ.get("HOUSE_SCANNING_DATA", Path.home() / "house-scanning-data")) @@ -28,6 +29,21 @@ MPS_CAP_GB = 3.6 +def write_depth(target: Path, depth) -> None: + import numpy as np + + temporary = None + try: + with tempfile.NamedTemporaryFile(dir=target.parent, suffix=".npz", delete=False) as f: + temporary = Path(f.name) + np.savez(f, depth=depth.astype(np.float32)) + # A killed writer cannot leave a partially written archive at the reusable cache path. + os.replace(temporary, target) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + def main(manifest: Path) -> None: import cv2 import numpy as np @@ -58,7 +74,7 @@ def main(manifest: Path) -> None: out = model.infer(rgb, resolution_level=RESOLUTION_LEVEL, fov_x=fov_x, use_fp16=True) depth = out["depth"].float().cpu().numpy() depth[~out["mask"].cpu().numpy().astype(bool)] = np.nan - np.savez(item["out"], depth=depth.astype(np.float32)) + write_depth(Path(item["out"]), depth) print(f"{Path(item['image']).name}: median {np.nanmedian(depth):.2f} m", file=sys.stderr) del out, rgb if device == "mps": diff --git a/recon/recon/capture.py b/recon/recon/capture.py index 3c573ff87..a585c1282 100644 --- a/recon/recon/capture.py +++ b/recon/recon/capture.py @@ -26,6 +26,9 @@ import hashlib import json import re +import shutil +import struct +import tempfile import zipfile from dataclasses import dataclass, field from pathlib import Path @@ -94,6 +97,10 @@ def __post_init__(self) -> None: ROTATION_TOL = 1e-2 # scene.json rounds poses to 4 decimals; anything further off is not a pose +# Existing bundle admission policy in verification/hsverify/e2e.py, not measured capture maxima. +MAX_BUNDLE_ENTRIES = 2000 +MAX_BUNDLE_BYTES = 512 * 2**20 + def column_major(pose: list[float]) -> np.ndarray: """A 4x4 from 16 column-major numbers, its rotation snapped to the nearest proper rotation. @@ -119,18 +126,101 @@ def unpack(path: Path, work: Path) -> Path: """A folder as is; a zip extracted once under `work`, keyed by its sha256.""" if path.is_dir(): return path - digest = hashlib.sha256(path.read_bytes()).hexdigest()[:16] + if path.stat().st_size > MAX_BUNDLE_BYTES: + raise ValueError(f"{path}: archive exceeds {MAX_BUNDLE_BYTES} bytes") + with path.open("rb") as f: + digest = hashlib.file_digest(f, "sha256").hexdigest()[:16] out = work / "bundles" / digest if not out.exists(): + _zip_directory_limits(path) with zipfile.ZipFile(path) as z: - names = [n for n in z.namelist() if not n.startswith("__MACOSX/")] - for n in names: + infos = z.infolist() + if len(infos) > MAX_BUNDLE_ENTRIES: + raise ValueError(f"{path}: {len(infos)} entries, limit {MAX_BUNDLE_ENTRIES}") + if sum(i.file_size for i in infos) > MAX_BUNDLE_BYTES: + raise ValueError(f"{path}: expanded archive exceeds {MAX_BUNDLE_BYTES} bytes") + members = [i for i in infos if not i.filename.startswith("__MACOSX/")] + for info in members: + n = info.filename if n.startswith("/") or ".." in Path(n).parts: raise ValueError(f"{path}: unsafe entry {n!r}") - z.extractall(out, members=names) + out.parent.mkdir(parents=True, exist_ok=True) + # Extraction stays private until every CRC-checked member finishes. Failed extraction + # cannot leave a partial cache directory that a later run mistakes for a complete one. + stage = Path(tempfile.mkdtemp(dir=out.parent, prefix=f".{digest}-")) + try: + written = 0 + for info in members: + dest = stage / info.filename + if info.is_dir(): + dest.mkdir(parents=True, exist_ok=True) + continue + dest.parent.mkdir(parents=True, exist_ok=True) + with z.open(info) as src, dest.open("wb") as dst: + while chunk := src.read(2**20): + written += len(chunk) + if written > MAX_BUNDLE_BYTES: + raise ValueError(f"{path}: expanded archive exceeds byte limit") + dst.write(chunk) + try: + stage.rename(out) + except OSError: + # Another run of this exact content may have finished first. + if not out.is_dir(): + raise + finally: + if stage.exists(): + shutil.rmtree(stage) return out +def _zip_directory_limits(path: Path) -> None: + """Check central-directory records in bounded reads before ZipFile allocates their metadata. + ZIP64 and split archives exceed this worker's ordinary ZIP admission format and are refused. + The record layouts are PKWARE APPNOTE sections 4.3.12 and 4.3.16.""" + with path.open("rb") as f: + size = path.stat().st_size + f.seek(max(0, size - 22 - 65535)) + tail = f.read(22 + 65535) + end = len(tail) + while True: + pos = tail.rfind(b"PK\x05\x06", 0, end) + if pos < 0: + raise ValueError(f"{path}: missing ZIP end record") + if pos + 22 <= len(tail): + record = struct.unpack("<4s4H2IH", tail[pos : pos + 22]) + if pos + 22 + record[-1] == len(tail): + break + end = pos + _, disk, central_disk, disk_entries, total_entries, length, offset, _ = record + if disk or central_disk or disk_entries != total_entries: + raise ValueError(f"{path}: split ZIP archives are not supported") + if total_entries > MAX_BUNDLE_ENTRIES: + raise ValueError(f"{path}: {total_entries} entries, limit {MAX_BUNDLE_ENTRIES}") + directory_end = offset + length + end_offset = size - len(tail) + pos + if length == 0xFFFFFFFF or offset == 0xFFFFFFFF or directory_end != end_offset: + raise ValueError(f"{path}: ZIP64 or inconsistent ZIP directory") + f.seek(offset) + count, expanded = 0, 0 + while f.tell() < directory_end: + header = f.read(46) + if len(header) != 46 or header[:4] != b"PK\x01\x02": + raise ValueError(f"{path}: invalid ZIP central-directory record") + fields = struct.unpack("<4s6H3I5H2I", header) + count += 1 + expanded += fields[9] + if count > MAX_BUNDLE_ENTRIES: + raise ValueError(f"{path}: {count} entries, limit {MAX_BUNDLE_ENTRIES}") + if expanded > MAX_BUNDLE_BYTES: + raise ValueError(f"{path}: expanded archive exceeds {MAX_BUNDLE_BYTES} bytes") + f.seek(sum(fields[10:13]), 1) + if f.tell() > directory_end: + raise ValueError(f"{path}: ZIP central-directory record exceeds directory") + if count != total_entries: + raise ValueError(f"{path}: inconsistent ZIP entry count") + + def _find(root: Path, name: str) -> Path | None: hits = [p for p in root.rglob(name) if "__MACOSX" not in p.parts] if len(hits) > 1: diff --git a/recon/recon/coverage.py b/recon/recon/coverage.py index a04d50d01..a0a1eaa59 100644 --- a/recon/recon/coverage.py +++ b/recon/recon/coverage.py @@ -31,7 +31,7 @@ from recon.capture import FEET, UP, Frame from recon.depth import Depth from recon.fusion import MAX_RANGE_M, Mesh, Volume -from recon.geometry import WallFrame +from recon.geometry import Ground, WallFrame CELL_M = 0.1524 # half a foot, the app's cell WALL_TOP_M = 1.9812 # 6.5 ft, the headroom the export's wall band claims @@ -110,6 +110,8 @@ def face_offsets(wall: WallFrame, mesh: Mesh, cells: np.ndarray) -> np.ndarray: """Per cell, how far a flat, full-height face stands in front of the fitted plane (0 if none). Flat: in at least 75% of the 10 cm height bins from 0.3 to 1.9 m, the front-most surface within 0.6 m lies within 3 cm of their median. A bush or bin is neither, so it stays an occluder.""" + if not len(cells): + return np.zeros(0) loc = wall.local(mesh.vertices.astype(np.float64)) s, h, out = loc[:, 0], loc[:, 1], loc[:, 2] keep = (out > -0.15) & (out < 0.6) & (h > 0.3) & (h < 1.9) @@ -146,6 +148,7 @@ def wall_and_ground( vol: Volume, mesh: Mesh, cells: np.ndarray, + ground: Ground | None = None, ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: centres = np.array([f.center for f in frames]) faces = face_offsets(wall, mesh, cells) @@ -164,7 +167,10 @@ def wall_and_ground( S = np.broadcast_to(alongs[:, :, None], (len(cells), 2, len(outs))) Oo = np.maximum(np.broadcast_to(outs[None, None, :], S.shape), faces[:, None, None] + 0.05) gpts = wall.world(S, np.zeros_like(S), Oo).reshape(-1, 3) - gseen = two_positions(seen_by(gpts, UP, frames, depths, vol), centres).reshape( + if ground is not None: + gpts[:, 1] = ground.height_at(gpts[:, 0], gpts[:, 2]) + gnormal = UP if ground is None else ground.normal + gseen = two_positions(seen_by(gpts, gnormal, frames, depths, vol), centres).reshape( len(cells), 2, len(outs) ) row_ok = gseen.all(axis=1) # (C, outs) @@ -207,7 +213,8 @@ def free_space(wall: WallFrame, vol: Volume, cells: np.ndarray): The facing gap and clear distance are measured out from the wall's reconstructed front, not from the fitted plane: in front of a pilaster 0.3 m proud, something 1.3 m out of the plane is 1.0 m from the face the battery would stand against. The front is the first free step - past the wall's occupied voxels, so both read up to a voxel short, never long. Overhead + past the wall's occupied voxels. The gap also backs off one voxel from the first occupied + sample, so voxel sampling does not extend it into the obstacle. Overhead clearance is a height above the ground over the footprint in front of that face.""" step = vol.voxel n = len(cells) @@ -217,38 +224,40 @@ def free_space(wall: WallFrame, vol: Volume, cells: np.ndarray): for c in range(n): across = cells[c] + CELL_M * np.array([0.25, 0.5, 0.75]) front = wall_front(wall, vol, across) - hits = 0 for out in np.arange(front + step, front + FACING_MAX_M, step): share, seen = _slab(wall, vol, across, fh, [out]) - hits = hits + 1 if share >= OCCUPIED_SHARE else 0 - if hits == 2: - facing_gap[c] = out - step - front + # Any reconstructed occupied voxel stops clearance, including one-step thin rails. + if share > 0: + facing_gap[c] = max(0.0, out - step - front) break if not seen: break - if hits == 0: - facing_clear[c] = out - front + facing_clear[c] = out - front oo = front + np.arange(OVERHEAD_OUT_M[0], OVERHEAD_OUT_M[1] + 1e-9, step) - hits = 0 for h in np.arange(0.3, OVERHEAD_MAX_M, step): share, seen = _slab(wall, vol, across, [h], oo) - hits = hits + 1 if share >= OCCUPIED_SHARE else 0 - if hits == 2: - over_gap[c] = h - step + if share > 0: + over_gap[c] = max(0.0, h - step) break if not seen: break - if hits == 0: - over_clear[c] = h + over_clear[c] = h return facing_gap, facing_clear, over_gap, over_clear def compute( - wall: WallFrame, frames: list[Frame], depths: dict[str, Depth], vol: Volume, mesh: Mesh + wall: WallFrame, + frames: list[Frame], + depths: dict[str, Depth], + vol: Volume, + mesh: Mesh, + ground: Ground | None = None, ) -> CellCoverage: lo, hi = wall.s_range - cells = np.arange(np.floor(lo / CELL_M) * CELL_M, hi - 1e-9, CELL_M) - wall_ok, ground_out, _ = wall_and_ground(wall, frames, depths, vol, mesh, cells) + # Only complete cells inside the fitted extent can support the full exported half-foot span. + start = np.ceil(lo / CELL_M - 1e-9) * CELL_M + cells = np.arange(start, hi - CELL_M + 1e-9, CELL_M) + wall_ok, ground_out, _ = wall_and_ground(wall, frames, depths, vol, mesh, cells, ground) fg, fc, og, oc = free_space(wall, vol, cells) return CellCoverage(cells, wall_ok, ground_out, fg, fc, og, oc) diff --git a/recon/recon/depth.py b/recon/recon/depth.py index 14889b09e..82a87419a 100644 --- a/recon/recon/depth.py +++ b/recon/recon/depth.py @@ -16,11 +16,14 @@ from __future__ import annotations +import fcntl import hashlib import json import os import subprocess import sys +import zipfile +import zlib from dataclasses import dataclass from pathlib import Path @@ -83,20 +86,22 @@ def rotated_intrinsics(k: np.ndarray, w: float, h: float, turns: int) -> np.ndar def lidar(frame: Frame) -> Depth: d = frame.lidar + if d.confidence is None: + raise ValueError(f"{d.file}: LiDAR depth requires its confidence map") depth = np.fromfile(d.file, dtype=" 0)] = np.nan color = cv2.resize(_image(frame), (d.width, d.height), interpolation=cv2.INTER_AREA) return Depth(depth, scaled(frame.intrinsics, d.width / frame.width), color, "lidar") def _image(frame: Frame) -> np.ndarray: - img = cv2.imread(str(frame.image), cv2.IMREAD_COLOR) + # Poses, intrinsics and LiDAR describe sensor pixels, regardless of JPEG EXIF orientation. + img = cv2.imread(str(frame.image), cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) if img is None: raise FileNotFoundError(frame.image) if img.shape[:2] != (frame.height, frame.width): @@ -169,6 +174,23 @@ def cache_file(work: Path, fid: str, suffix: str) -> Path: def moge(capture: Capture, work: Path) -> dict[str, Depth]: """MoGe-2 depth per frame at up to MAX_SIDE px, in the frame's own (unrotated) orientation, cached under `work` by the capture's content (`moge_cache`).""" + work.mkdir(parents=True, exist_ok=True) + # The lock covers key repair, inputs, inference and reads, so concurrent runs cannot read a + # partially written archive or launch duplicate inference for the same cache root. + with (work / ".moge.lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + return _moge_locked(capture, work) + + +def _cached_depth(path: Path, shape: tuple[int, int]) -> np.ndarray: + with np.load(path, allow_pickle=False) as archive: + d = archive["depth"] + if d.shape != shape or d.dtype.kind != "f": + raise ValueError(f"{path}: expected floating depth with shape {shape}, got {d.shape}") + return d.astype(np.float32) + + +def _moge_locked(capture: Capture, work: Path) -> dict[str, Depth]: work = moge_cache(capture, work) manifest, meta = [], {} for f in capture.frames: @@ -184,7 +206,22 @@ def moge(capture: Capture, work: Path) -> dict[str, Depth]: k_up = rotated_intrinsics(k_small, small.shape[1], small.shape[0], turns) out = cache_file(work, f.id, ".moge2.npz") meta[f.id] = (out, turns, k_small, small) - if not out.exists(): + valid = False + if out.exists(): + try: + _cached_depth(out, np.rot90(small, turns).shape[:2]) + valid = True + except ( + ValueError, + OSError, + KeyError, + EOFError, + TypeError, + zipfile.BadZipFile, + zlib.error, + ): + out.unlink() + if not valid: manifest.append({"image": str(up_path), "fx": float(k_up[0]), "out": str(out)}) if manifest: (work / "moge2.json").write_text(json.dumps(manifest, indent=1)) @@ -203,7 +240,7 @@ def moge(capture: Capture, work: Path) -> dict[str, Depth]: ) depths = {} for fid, (out, turns, k_small, small) in meta.items(): - d = np.rot90(np.load(out)["depth"], -turns).astype(np.float32) + d = np.rot90(_cached_depth(out, np.rot90(small, turns).shape[:2]), -turns) if d.shape != small.shape[:2]: raise ValueError(f"{out}: depth {d.shape} does not match the image {small.shape[:2]}") depths[fid] = Depth(np.ascontiguousarray(d), k_small, small, "moge2") diff --git a/recon/recon/pipeline.py b/recon/recon/pipeline.py index 15794a1cc..2c0b5ded1 100644 --- a/recon/recon/pipeline.py +++ b/recon/recon/pipeline.py @@ -57,7 +57,7 @@ def reconstruct( # Only frames with a depth map can see anything (depth.depth_maps: LiDAR frames saved # without depth get none). seeing = [f for f in capture.frames if f.id in depths] - cov = coverage.compute(wall, seeing, depths, vol, surface) + cov = coverage.compute(wall, seeing, depths, vol, surface, ground) _log(f"coverage over {len(cov.cells)} cells ({time.perf_counter() - t0:.0f} s)") return { "depths": depths, @@ -124,7 +124,28 @@ def write_model(r: dict, path: Path) -> None: def run( bundle: Path, out: Path, work: Path, depth_mode: str, server_url: str | None, move_meter: bool ) -> dict: + owned = ( + "model.glb", + "geometry.json", + "scene.json", + "coverage.json", + "report.md", + "result.json", + "site-plan.svg", + ) + bundle_path, out_path = bundle.resolve(), out.resolve() + if bundle.is_dir(): + overlap = bundle_path.is_relative_to(out_path) or out_path.is_relative_to(bundle_path) + else: + overlap = bundle_path in {(out_path / name).resolve() for name in owned} + if overlap: + raise ValueError( + f"output {out} overlaps capture input {bundle}; use a separate output folder" + ) out.mkdir(parents=True, exist_ok=True) + # A failed or --no-server rerun must not leave an earlier run's answer beside new outputs. + for name in owned: + (out / name).unlink(missing_ok=True) capture = cap.load(bundle, work) _log(f"{capture.source}: {len(capture.frames)} frames from {bundle}") t0 = time.perf_counter() diff --git a/recon/recon/scene.py b/recon/recon/scene.py index 6f0ce4bcc..c0ee6db48 100644 --- a/recon/recon/scene.py +++ b/recon/recon/scene.py @@ -4,8 +4,8 @@ A scan bundle's scene is updated: the meter's wall takes the fitted line (over the union of its fitted extent and the phone's baseline), `coverage.observed` is replaced by the occlusion-aware -coverage, and the measured facing gaps and overhead clearances are added to whatever the phone -marked. A Measure Lab session gets a new scene with the same parts. +coverage, and the target wall's measured facing gaps and overhead clearances are replaced. +A Measure Lab session gets a new scene with the same parts. The fitted wall takes the source of the depth that built it (`pipeline.WALL_SOURCE`: "mesh" from LiDAR, "plane" from photos alone) and no `plus_minus_ft`; a bundle's tapped bound is removed. The @@ -83,10 +83,17 @@ def build( target["baseline"] = [_plan_ft(wall.world(s, 0.0)) for s in (lo, hi)] target["source"] = wall_source target.pop("plus_minus_ft", None) - doc.setdefault("coverage", {})["observed"] = observed(cov)[:500] + entries = observed(cov) + # scene.schema.json caps observed entries at 500; a partial export hides valid observations. + if len(entries) > 500: + raise ValueError( + f"coverage.observed has {len(entries)} entries; scene contract limit is 500" + ) + doc.setdefault("coverage", {})["observed"] = entries facing, overheads = measurements(cov, wid, plus_minus_ft) - doc["facing"] = [*doc.get("facing", []), *facing] - doc["overheads"] = [*doc.get("overheads", []), *overheads] + # Replacing the wall invalidates its old spans even when the meter stays in place. + doc["facing"] = [*[m for m in doc.get("facing", []) if m["wall_id"] != wid], *facing] + doc["overheads"] = [*[m for m in doc.get("overheads", []) if m["wall_id"] != wid], *overheads] return doc diff --git a/recon/tests/test_review_debt.py b/recon/tests/test_review_debt.py new file mode 100644 index 000000000..65b6ac0a9 --- /dev/null +++ b/recon/tests/test_review_debt.py @@ -0,0 +1,348 @@ +"""Regressions for review findings that survived the reconstruction handoff.""" + +import fcntl +import importlib.util +import io +import json +import struct +import zipfile +from dataclasses import replace +from pathlib import Path + +import cv2 +import numpy as np +import pytest +from test_worker import ( + NO_COVERAGE, + WALL, + _all_visible_setup, + _capture_named_scan, + _fake_moge, + _lidar_frames, + _slab_volume, +) + +from recon import capture as cap +from recon import coverage, depth, geometry, pipeline, scene +from recon.coverage import CELL_M + + +def test_replacing_wall_drops_its_old_measurements_and_keeps_other_walls(): + old_facing = {"wall_id": "w", "span_ft": [-1, 1], "depth_ft": 20} + old_overhead = {"wall_id": "w", "span_ft": [-1, 1], "clearance_ft": 10} + other = {"wall_id": "v", "span_ft": [1, 2], "depth_ft": 3} + prior = { + "meter": {"wall_id": "w", "pos": [0, 5, 0]}, + "walls": [{"id": "w", "baseline": [[-1, 0], [1, 0]]}], + "facing": [old_facing, other], + "overheads": [old_overhead], + } + capture = cap.Capture("scan-bundle", Path("b"), [], 0.0, WALL.meter, None, prior) + rotated = replace(WALL, along=np.array([np.cos(0.1), 0, np.sin(0.1)])) + doc = scene.build(capture, rotated, NO_COVERAGE, "mesh", 0.5) + assert doc["facing"] == [other] and doc["overheads"] == [] + assert prior["facing"] == [old_facing, other] + + +@pytest.mark.parametrize("n", [500, 501]) +def test_export_observations_refuses_contract_overflow_without_slicing(monkeypatch, n): + entries = [{"band": "wall", "span_ft": [i, i + 0.5]} for i in range(n)] + monkeypatch.setattr(scene, "observed", lambda _: entries) + capture = cap.Capture("measure-lab", Path("b"), [], 0.0, None, None) + if n == 501: + with pytest.raises(ValueError, match=r"501 entries.*limit is 500"): + scene.build(capture, WALL, NO_COVERAGE, "mesh", 0.5) + else: + assert ( + scene.build(capture, WALL, NO_COVERAGE, "mesh", 0.5)["coverage"]["observed"] == entries + ) + + +def test_coverage_exports_only_cells_inside_fitted_extent(monkeypatch): + wall, frames, depths, vol, surface, _ = _all_visible_setup() + monkeypatch.setattr(coverage, "seen_by", lambda points, *_: np.ones((len(points), 2), bool)) + monkeypatch.setattr( + coverage, "free_space", lambda wall, vol, cells: (np.zeros(len(cells)),) * 4 + ) + wall = replace(wall, s_range=(0.9 * CELL_M, 2.1 * CELL_M)) + cov = coverage.compute(wall, frames, depths, vol, surface) + np.testing.assert_allclose(cov.cells, [CELL_M]) + entries = coverage.observed(cov) + assert all(0.5 <= e["span_ft"][0] < e["span_ft"][1] <= 1.0 for e in entries) + cov = coverage.compute(replace(wall, s_range=(0.01, 0.02)), frames, depths, vol, surface) + assert cov.cells.size == 0 and coverage.observed(cov) == [] + + +def test_ground_coverage_uses_local_slope_height_and_normal(monkeypatch): + setup = _all_visible_setup() + ground = geometry.Ground(np.zeros(3), np.array([0.0, 1, -0.2]), 0.0, 10) + sampled = [] + + def see(points, normal, *_): + if normal is ground.normal: + sampled.append(points.copy()) + seen = np.abs(points[:, 1] - 0.2 * points[:, 2]) < 1e-9 + else: + seen = np.ones(len(points), bool) + return np.repeat(seen[:, None], 2, axis=1) + + monkeypatch.setattr(coverage, "seen_by", see) + _, reach, _ = coverage.wall_and_ground(*setup, ground) + assert sampled and np.all(sampled[0][:, 1] > 0) + np.testing.assert_allclose(reach, coverage.GROUND_MAX_M) + + +def test_depth_test_sees_sloped_ground_and_rejects_horizontal_datum(): + # Two cameras look vertically down at y = 0.2*z. Pixel-ray intersections supply metric depth. + from test_worker import DOWN, GROUND_FRAMES, H, K, W + + frames = GROUND_FRAMES + v, _u = np.mgrid[:H, :W] + dz_per_depth = (v + 0.5 - K[3]) / K[1] + d = 1.5 / (1 + 0.2 * dz_per_depth) + depths = { + f.id: depth.Depth(d.astype(np.float32), K, np.zeros((H, W, 3), np.uint8), "lidar") + for f in frames + } + assert np.array_equal(frames[0].cam_to_world[:3, :3], DOWN) + + class EmptyRays: + voxel = 0.05 + + def blocked(self, start, ends, stop_short): + return np.zeros(len(ends), bool) + + points = np.array([[0.0, 0.12, 0.6], [0.0, -0.12, -0.6]]) + normal = np.array([0.0, 1, -0.2]) / np.sqrt(1.04) + assert coverage.seen_by(points, normal, frames, depths, EmptyRays()).all() + horizontal = points.copy() + horizontal[:, 1] = 0.0 + assert not coverage.seen_by(horizontal, normal, frames, depths, EmptyRays()).any() + + +def test_reconstruction_passes_fitted_ground_to_coverage(monkeypatch): + wall, frames, depths, _, surface, _ = _all_visible_setup() + capture = cap.Capture("measure-lab", Path("b"), frames, 0.0, wall.meter, None) + ground = geometry.Ground(np.zeros(3), np.array([0.0, 1, -0.2]), 0.0, 10) + volume = _slab_volume([], 7) + monkeypatch.setattr(pipeline.fusion, "integrate", lambda *_: volume) + monkeypatch.setattr(pipeline.fusion, "mesh", lambda _: surface) + monkeypatch.setattr(pipeline.geometry, "fit_ground", lambda *_: ground) + monkeypatch.setattr(pipeline.geometry, "wall_lines", lambda *_: ["line"]) + monkeypatch.setattr(pipeline.geometry, "choose_wall", lambda *_: "line") + monkeypatch.setattr(pipeline.geometry, "wall_frame", lambda *_: wall) + + def compute(*args): + assert args[-1] is ground + return NO_COVERAGE + + monkeypatch.setattr(pipeline.coverage, "compute", compute) + assert pipeline.reconstruct(capture, depths, {"source": "lidar"})["coverage"] is NO_COVERAGE + + +@pytest.mark.parametrize("band", ["facing", "overhead"]) +def test_one_voxel_obstruction_ends_clearance(band): + wall, *_ = _all_visible_setup() + vol = _slab_volume([(-0.3, 0.0)], 7.0) + if band == "facing": + # One 5 cm row at y=1 m, one step at z=1 m: under 20% of the facing slab. + vol.tsdf[:, 20, 30] = -1 + else: + # One outward voxel at z=0.3 m, one height step at y=1.5 m. + vol.tsdf[:, 30, 16] = -1 + fg, fc, og, oc = coverage.free_space(wall, vol, np.array([0.0])) + if band == "facing": + assert np.isfinite(fg[0]) and 0.8 < fg[0] < 1.0 and fc[0] < 1.0 + else: + assert np.isfinite(og[0]) and 1.3 < og[0] < 1.5 and oc[0] < 1.5 + + +def test_lidar_without_confidence_is_refused_before_depth_read(tmp_path): + frames = _lidar_frames(tmp_path, {"a": True}) + f = replace(frames[0], lidar=replace(frames[0].lidar, confidence=None)) + f.lidar.file.unlink() + with pytest.raises(ValueError, match="requires its confidence map"): + depth.lidar(f) + + +@pytest.mark.parametrize("orientation", [3, 6]) +def test_keyframe_jpeg_exif_does_not_rotate_sensor_pixels(tmp_path, orientation): + image = np.zeros((16, 24, 3), np.uint8) + image[:8, :12] = 255 + ok, jpeg = cv2.imencode(".jpg", image) + assert ok + tiff = struct.pack("<2sHIH", b"II", 42, 8, 1) + tiff += struct.pack("H", len(exif) + 2) + exif + raw[2:] + path = tmp_path / "k.jpg" + path.write_bytes(tagged) + frame = cap.Frame("k", path, 24, 16, np.array([24.0, 24, 12, 8]), np.eye(4)) + sensor = cv2.imdecode(jpeg, cv2.IMREAD_COLOR) + np.testing.assert_array_equal(depth._image(frame), sensor) + default = cv2.imread(str(path), cv2.IMREAD_COLOR) + assert default.shape != sensor.shape or not np.array_equal(default, sensor) + + +def corrupt_compressed_npz() -> bytes: + archive = io.BytesIO() + np.savez_compressed(archive, depth=np.ones((2, 2), np.float32)) + raw = bytearray(archive.getvalue()) + name_bytes, extra_bytes = struct.unpack_from(" list[cap.Frame]: lid = None if has: np.full((2, 2), 2.0, " Date: Tue, 13 Oct 2026 18:06:09 -0500 Subject: [PATCH 07/62] fix(recon): rebuild caches with unsupported ZIP compression https://github.com/SamGu-NRX/BaseScanning/pull/240#discussion_r4264960688 --- recon/recon/depth.py | 2 ++ recon/tests/test_review_debt.py | 24 +++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/recon/recon/depth.py b/recon/recon/depth.py index 82a87419a..70c49b2c2 100644 --- a/recon/recon/depth.py +++ b/recon/recon/depth.py @@ -217,6 +217,8 @@ def _moge_locked(capture: Capture, work: Path) -> dict[str, Depth]: KeyError, EOFError, TypeError, + # ZIP corruption can name an unsupported compression method. + NotImplementedError, zipfile.BadZipFile, zlib.error, ): diff --git a/recon/tests/test_review_debt.py b/recon/tests/test_review_debt.py index 65b6ac0a9..e1c951e3a 100644 --- a/recon/tests/test_review_debt.py +++ b/recon/tests/test_review_debt.py @@ -195,7 +195,27 @@ def corrupt_compressed_npz() -> bytes: return bytes(raw) -@pytest.mark.parametrize("bad", [b"PK\x03\x04truncated", b"not an npz", corrupt_compressed_npz()]) +def unsupported_compression_npz() -> bytes: + archive = io.BytesIO() + np.savez(archive, depth=np.ones((2, 2), np.float32)) + raw = bytearray(archive.getvalue()) + central = raw.index(b"PK\x01\x02") + # Both ZIP headers agree on an unsupported method, so the failure occurs on member read. + struct.pack_into(" Date: Tue, 13 Oct 2026 18:15:08 -0500 Subject: [PATCH 08/62] experiments(drift-anatomy): test A3 on wall-parallel windows; say A4's meter is each window's start A3 split error along and across each window's travel, but a wall plane removes error along the wall's normal, which is the across axis only while the walk runs straight along the wall. Windows whose reference path stays within 0.25 m or 0.5 m of its start-to-end line now get their own table. On MARViN at 20 ft along is 0.98 and 0.95 of the p90 there, so the conclusion holds on the windows that test it. Graded rows are unchanged. A4 already ranges from a fixed meter: each window is one session that starts at the meter, and the range runs from that start to the window's end. METHODS, README and the generated report now say so. https://github.com/SamGu-NRX/BaseScanning/pull/59#discussion_r4113839737 https://github.com/SamGu-NRX/BaseScanning/pull/59#discussion_r4115917036 https://github.com/SamGu-NRX/BaseScanning/pull/59#discussion_r4113839728 https://github.com/SamGu-NRX/BaseScanning/pull/59#discussion_r4115917030 Co-Authored-By: Claude Opus 5.5 --- experiments/drift-anatomy/METHODS.md | 4 +- experiments/drift-anatomy/README.md | 4 +- experiments/drift-anatomy/metrics.py | 19 +++ .../drift-anatomy/results/drift_anatomy.json | 108 +++++++++++++++++- .../drift-anatomy/results/drift_anatomy.md | 19 ++- experiments/drift-anatomy/run.py | 72 ++++++++++-- .../drift-anatomy/tests/test_metrics.py | 16 +++ 7 files changed, 219 insertions(+), 23 deletions(-) diff --git a/experiments/drift-anatomy/METHODS.md b/experiments/drift-anatomy/METHODS.md index 7fba32f7e..e5d4a588f 100644 --- a/experiments/drift-anatomy/METHODS.md +++ b/experiments/drift-anatomy/METHODS.md @@ -9,7 +9,7 @@ The server allows 0.16 ft of error per foot along the wall. That allowance domin - **A1, return to the meter.** When a walk comes back within 0.3 m of an earlier point more than 30 s later, ARKit's closure residual r is the error in the displacement between the two visits. Does r bound the largest error between the visits? - **A2, scale.** Is ARKit's scale one number per walk? This measures the spread of scale over sliding 5 m windows within a walk. - **A3, the wall plane.** What share of the error runs across the direction of travel, which a wall plane would remove, compared with along it? -- **A4, a range to the meter.** A second phone left at the meter ranges the walking phone by UWB (σ 10 cm, simulated) at every window end. What is the p90 left over? +- **A4, a range to the meter.** A second phone left at the meter ranges the walking phone by UWB (σ 10 cm, simulated) at every window end. What is the p90 left over? Each window is one session that starts at the meter: the walking phone leaves the meter at the window's start, the second phone stays there, and the range runs from that start to the window's end. Sliding the start along a walk samples many meter placements; the error depends on the path walked from the meter, not on where in the dataset the meter sits. ## Pass criteria, fixed before the run @@ -24,7 +24,7 @@ All four fail. Tables: [results/drift_anatomy.md](results/drift_anatomy.md). - **A1.** 30 of 56 MARViN loops (54%). ADVIO has no revisit under 0.3 m while ARKit tracks. Scale and heading error cancel on return, so r misses them. - **A2.** 5.7% SD on MARViN, 19% on ADVIO: drop. A few outlier windows inflate MARViN's SD; its robust SD is 1.8%. -- **A3.** At 20 ft, along is 0.92 of the p90 on MARViN and 0.76 on ADVIO. +- **A3.** At 20 ft, along is 0.92 of the p90 on MARViN and 0.76 on ADVIO. Added after review: across-travel is a wall's normal only while the walk runs straight along the wall, and the walks have no wall annotations. On windows whose path stays within 0.25 m of its start-to-end line, along is 0.98 of the p90 on MARViN and 0.75 on ADVIO at 20 ft; within 0.5 m, 0.95 and 0.75. The conclusion holds on the windows that test it. - **A4.** MARViN goes from 10.4 and 13.6 in to 7.7 and 8.5 in. At σ 10 cm the range's own p90 is 6.5 in, so the bar was nearly out of reach. At 5 cm: 4.9 and 6.1 in. **Run:** `uv run python run.py`. Needs the evals checkout at 190ed33 (`EVALS_HARNESS`) and MARViN and ADVIO in `~/house-scanning-data`. diff --git a/experiments/drift-anatomy/README.md b/experiments/drift-anatomy/README.md index e745d756f..1ae0e79f6 100644 --- a/experiments/drift-anatomy/README.md +++ b/experiments/drift-anatomy/README.md @@ -2,7 +2,7 @@ ## Question -The server allows 0.16 ft of error per foot along the wall, which dominates a check's error bar beyond a few feet. Which part of ARKit's error is worth attacking? A1 asks whether returning to the meter bounds it, A2 whether scale is one number per walk, A3 how much runs across travel, where a wall plane would help, and A4 what a UWB range to the meter leaves. The data is MARViN (iPhone 14 Pro Max) and ADVIO (iPhone 6s); [METHODS.md](METHODS.md) has the details. +The server allows 0.16 ft of error per foot along the wall, which dominates a check's error bar beyond a few feet. Which part of ARKit's error is worth attacking? A1 asks whether returning to the meter bounds it, A2 whether scale is one number per walk, A3 how much runs across travel, where a wall plane would help, and A4 what a UWB range to the meter leaves. Each window stands for a session that starts at the meter, so in A4 the second phone sits at the window's start. The data is MARViN (iPhone 14 Pro Max) and ADVIO (iPhone 6s); [METHODS.md](METHODS.md) has the details. ## Pass criteria, fixed before the run (397507f) @@ -17,7 +17,7 @@ All four fail ([tables](results/drift_anatomy.md)). - **A1.** The closure residual bounds the peak error in 30 of 56 loops. Scale and heading error cancel on return. - **A2.** Scale spread within a walk is 1.8% robust SD (5.7% SD) on MARViN. -- **A3.** At 20 ft, the along-travel p90 is 9.6 of 10.4 in. +- **A3.** At 20 ft, the along-travel p90 is 9.6 of 10.4 in. On the windows that walk straight, which is where across-travel is a wall's normal, along is 0.95 to 0.98 of the total on MARViN (added after review). - **A4.** UWB cuts 20 ft from 10.4 to 7.7 in at σ 10 cm, and to 4.9 in at σ 5 cm. **Run:** `uv run python run.py`: 20 s. Needs the evals checkout at 190ed33 and MARViN and ADVIO in `~/house-scanning-data`. diff --git a/experiments/drift-anatomy/metrics.py b/experiments/drift-anatomy/metrics.py index 1107729a4..f329e769a 100644 --- a/experiments/drift-anatomy/metrics.py +++ b/experiments/drift-anatomy/metrics.py @@ -167,6 +167,25 @@ def along_across(err: np.ndarray, ref_disp: np.ndarray) -> tuple[np.ndarray, np. return along, across +def chord_deviation(ref: np.ndarray, i: np.ndarray, j: np.ndarray) -> np.ndarray: + """Per window, the largest horizontal distance of the reference path from the straight line + through its start ref[i] and its (fractional) end at(ref, j). A wall plane removes the error + along the wall's normal, which is the across-travel axis only while the walk runs straight + along the wall; this measures how far a window strays from that.""" + out = np.empty(len(i)) + for k, (a, b) in enumerate(zip(i, j, strict=True)): + start = ref[a, [0, 2]] + end = at(ref, np.array([b]))[0, [0, 2]] + pts = np.vstack([ref[a : int(np.floor(b)) + 1, [0, 2]], end]) - start + chord = end - start + n = np.linalg.norm(chord) + if n == 0: + raise ValueError(f"window {a}..{b} ends where it starts and has no travel line") + u = chord / n + out[k] = np.abs(u[1] * pts[:, 0] - u[0] * pts[:, 1]).max() + return out + + # A4: a range to the meter diff --git a/experiments/drift-anatomy/results/drift_anatomy.json b/experiments/drift-anatomy/results/drift_anatomy.json index 73d0a8375..edd2313e4 100644 --- a/experiments/drift-anatomy/results/drift_anatomy.json +++ b/experiments/drift-anatomy/results/drift_anatomy.json @@ -1357,30 +1357,78 @@ "p90_along_in": 5.93244053666383, "p90_across_in": 2.383144858716218, "p90_horizontal_in": 6.356644425002893, + "along_share_of_total_p90": 0.9332660661857188, "median_horizontal_in": 2.6181875496018105, "p90_vertical_in": 1.9654002913094888, "p90_3d_in": 6.796782660764225, - "along_share_of_total_p90": 0.9332660661857188 + "wall_parallel": { + "0.25": { + "windows": 3601, + "p90_along_in": 5.826966988094975, + "p90_across_in": 2.111570110459862, + "p90_horizontal_in": 6.152786774170161, + "along_share_of_total_p90": 0.9470451686310665 + }, + "0.50": { + "windows": 4489, + "p90_along_in": 5.877316441113764, + "p90_across_in": 2.256740731176376, + "p90_horizontal_in": 6.281388835683555, + "along_share_of_total_p90": 0.9356714884016857 + } + } }, "20": { "windows": 4956, "p90_along_in": 9.614029066697613, "p90_across_in": 3.8201651149007376, "p90_horizontal_in": 10.416259256830854, + "along_share_of_total_p90": 0.9229828895044881, "median_horizontal_in": 4.395873486077264, "p90_vertical_in": 2.9551007554751285, "p90_3d_in": 10.87021034634674, - "along_share_of_total_p90": 0.9229828895044881 + "wall_parallel": { + "0.25": { + "windows": 1416, + "p90_along_in": 9.604249774006092, + "p90_across_in": 2.6576477424629545, + "p90_horizontal_in": 9.791322826235511, + "along_share_of_total_p90": 0.9808939960872128 + }, + "0.50": { + "windows": 2735, + "p90_along_in": 9.735246181368389, + "p90_across_in": 2.9849409636746014, + "p90_horizontal_in": 10.254954545799945, + "along_share_of_total_p90": 0.9493212415412987 + } + } }, "30": { "windows": 4866, "p90_along_in": 12.50959372814818, "p90_across_in": 5.464582640836947, "p90_horizontal_in": 13.628585737497145, + "along_share_of_total_p90": 0.917893754282206, "median_horizontal_in": 6.016112751925093, "p90_vertical_in": 3.6388989773814653, "p90_3d_in": 14.139117730080716, - "along_share_of_total_p90": 0.917893754282206 + "wall_parallel": { + "0.25": { + "windows": 556, + "p90_along_in": 11.52265233012735, + "p90_across_in": 3.0105234637262224, + "p90_horizontal_in": 11.707672299015773, + "along_share_of_total_p90": 0.9841966904980782 + }, + "0.50": { + "windows": 1506, + "p90_along_in": 12.273519598588699, + "p90_across_in": 3.6974588667790194, + "p90_horizontal_in": 12.509892612405189, + "along_share_of_total_p90": 0.981105112478576 + } + } } }, "advio": { @@ -1389,30 +1437,78 @@ "p90_along_in": 40.16740160527824, "p90_across_in": 41.217205108912495, "p90_horizontal_in": 55.603313400377814, + "along_share_of_total_p90": 0.7223922307659694, "median_horizontal_in": 26.202438486533016, "p90_vertical_in": 3.697033252704125, "p90_3d_in": 55.7048750882303, - "along_share_of_total_p90": 0.7223922307659694 + "wall_parallel": { + "0.25": { + "windows": 1831, + "p90_along_in": 38.817104640899515, + "p90_across_in": 40.40824203776524, + "p90_horizontal_in": 54.051646176045516, + "along_share_of_total_p90": 0.7181484263119888 + }, + "0.50": { + "windows": 1905, + "p90_along_in": 39.71706496159873, + "p90_across_in": 40.95345003105173, + "p90_horizontal_in": 55.31558133196048, + "along_share_of_total_p90": 0.7180086334670921 + } + } }, "20": { "windows": 1903, "p90_along_in": 72.05520337403567, "p90_across_in": 77.9870905225656, "p90_horizontal_in": 94.28551696290192, + "along_share_of_total_p90": 0.76422345334742, "median_horizontal_in": 50.231545101074694, "p90_vertical_in": 6.375715770353499, "p90_3d_in": 94.60404389848367, - "along_share_of_total_p90": 0.76422345334742 + "wall_parallel": { + "0.25": { + "windows": 1572, + "p90_along_in": 66.36786396952603, + "p90_across_in": 73.94467380934859, + "p90_horizontal_in": 88.94005061866382, + "along_share_of_total_p90": 0.7462089745606566 + }, + "0.50": { + "windows": 1720, + "p90_along_in": 70.01436124445905, + "p90_across_in": 77.12953122417956, + "p90_horizontal_in": 93.0226402125133, + "along_share_of_total_p90": 0.7526593642634624 + } + } }, "30": { "windows": 1889, "p90_along_in": 97.37446063335635, "p90_across_in": 110.39001216955775, "p90_horizontal_in": 132.53687138322834, + "along_share_of_total_p90": 0.7346971421394095, "median_horizontal_in": 73.64854529941651, "p90_vertical_in": 9.174807119748339, "p90_3d_in": 132.57989402774612, - "along_share_of_total_p90": 0.7346971421394095 + "wall_parallel": { + "0.25": { + "windows": 1421, + "p90_along_in": 93.12387571665461, + "p90_across_in": 102.7530290645741, + "p90_horizontal_in": 124.45301489257115, + "along_share_of_total_p90": 0.7482653256495223 + }, + "0.50": { + "windows": 1549, + "p90_along_in": 99.08655440233662, + "p90_across_in": 107.2678362551467, + "p90_horizontal_in": 129.17438173809424, + "along_share_of_total_p90": 0.7670758943769378 + } + } } } }, diff --git a/experiments/drift-anatomy/results/drift_anatomy.md b/experiments/drift-anatomy/results/drift_anatomy.md index bece27d8d..1f7bebdde 100644 --- a/experiments/drift-anatomy/results/drift_anatomy.md +++ b/experiments/drift-anatomy/results/drift_anatomy.md @@ -132,9 +132,26 @@ The evals' windows: from every image (MARViN) or every 0.5 s (ADVIO) until the r Criterion at 20 ft: along p90 at most 0.60 of the horizontal total p90. MARViN 0.92, FAIL; ADVIO 0.76, FAIL. +Added after review: a wall plane removes the error along the wall's normal, and that is the across-travel axis only while the walk runs straight along the wall. In windows that turn, across holds some along-wall error and along holds some error a plane would remove, so the graded split above cannot by itself say whether a plane helps. The walks have no wall annotations, so the windows below stand in for a walk along a wall: those whose reference path stays within the given distance of its own start-to-end line. The widths are untuned and not pre-registered. + +| Data | Walked | Within | Windows | Along | Across | Horizontal total | Along / total | +| --- | --- | --- | --- | --- | --- | --- | --- | +| MARViN | 10 ft | 0.25 m | 3601 | 5.8 | 2.1 | 6.2 | 0.95 | +| MARViN | 10 ft | 0.50 m | 4489 | 5.9 | 2.3 | 6.3 | 0.94 | +| MARViN | 20 ft | 0.25 m | 1416 | 9.6 | 2.7 | 9.8 | 0.98 | +| MARViN | 20 ft | 0.50 m | 2735 | 9.7 | 3.0 | 10.3 | 0.95 | +| MARViN | 30 ft | 0.25 m | 556 | 11.5 | 3.0 | 11.7 | 0.98 | +| MARViN | 30 ft | 0.50 m | 1506 | 12.3 | 3.7 | 12.5 | 0.98 | +| ADVIO | 10 ft | 0.25 m | 1831 | 38.8 | 40.4 | 54.1 | 0.72 | +| ADVIO | 10 ft | 0.50 m | 1905 | 39.7 | 41.0 | 55.3 | 0.72 | +| ADVIO | 20 ft | 0.25 m | 1572 | 66.4 | 73.9 | 88.9 | 0.75 | +| ADVIO | 20 ft | 0.50 m | 1720 | 70.0 | 77.1 | 93.0 | 0.75 | +| ADVIO | 30 ft | 0.25 m | 1421 | 93.1 | 102.8 | 124.5 | 0.75 | +| ADVIO | 30 ft | 0.50 m | 1549 | 99.1 | 107.3 | 129.2 | 0.77 | + ## A4, a range to the meter -A simulated range from each window's start to its end: the reference's 3-D distance plus N(0, sigma), 20 draws per window (seed 0). ARKit's end moves radially in the horizontal plane about the start, keeping its height, onto the range. p90 horizontal error, inches. sigma 0.10 m is graded; 0.05 and 0.20 m are not. +Each window is a session that starts at the meter, where the second phone stays. A simulated range from each window's start to its end: the reference's 3-D distance plus N(0, sigma), 20 draws per window (seed 0). ARKit's end moves radially in the horizontal plane about the start, keeping its height, onto the range. p90 horizontal error, inches. sigma 0.10 m is graded; 0.05 and 0.20 m are not. | Data | Walked | Before | After, 0.05 m | After, 0.10 m | After, 0.20 m | | --- | --- | --- | --- | --- | --- | diff --git a/experiments/drift-anatomy/run.py b/experiments/drift-anatomy/run.py index 51556068c..396e4a777 100644 --- a/experiments/drift-anatomy/run.py +++ b/experiments/drift-anatomy/run.py @@ -17,6 +17,7 @@ from metrics import ( along_across, at, + chord_deviation, find_loops, local_scale, loop_errors, @@ -56,6 +57,11 @@ RANGE_SIGMAS_M = (0.05, 0.10, 0.20) GRADED_SIGMA_M = 0.10 RANGE_DRAWS = 20 +# A3, added after review: the across-travel axis is a wall's normal only while the walk runs +# straight along the wall, and the walks carry no wall annotations. Windows whose reference path +# stays within this distance of its own start-to-end line stand in for a walk along a wall. Both +# widths are untuned guesses at how straight a homeowner keeps to a wall, reported side by side. +WALL_PARALLEL_M = (0.25, 0.5) SEED = 0 @@ -236,15 +242,31 @@ def a2(walks: list[Walk]) -> dict: # A3 and A4 -def window_errors(ws: list[Walk], ft: int, window_pairs) -> tuple[np.ndarray, np.ndarray]: - """ARKit's displacement turned by the heading offset at each window's start, and the - reference's, pooled over walks; the evals' windows (every start sample, exactly ft of path).""" - est, ref = [], [] +def window_errors( + ws: list[Walk], ft: int, window_pairs +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """ARKit's displacement turned by the heading offset at each window's start, the reference's, + and how far the reference path strays from its start-to-end line, pooled over walks; the + evals' windows (every start sample, exactly ft of path).""" + est, ref, dev = [], [], [] for w in ws: i, j = window_pairs(w.ref, ft * FEET, w.start_step) est.append(turn(at(w.ark, j) - w.ark[i], w.delta[i])) ref.append(at(w.ref, j) - w.ref[i]) - return np.concatenate(est), np.concatenate(ref) + dev.append(chord_deviation(w.ref, i, j)) + return np.concatenate(est), np.concatenate(ref), np.concatenate(dev) + + +def split_p90(err: np.ndarray, ref: np.ndarray) -> dict: + along, across = along_across(err, ref) + horiz = np.hypot(along, across) + return { + "windows": len(err), + "p90_along_in": p90(np.abs(along)) / INCH, + "p90_across_in": p90(np.abs(across)) / INCH, + "p90_horizontal_in": p90(horiz) / INCH, + "along_share_of_total_p90": p90(np.abs(along)) / p90(horiz), + } def a3_a4(walks: list[Walk]) -> tuple[dict, dict]: @@ -254,19 +276,21 @@ def a3_a4(walks: list[Walk]) -> tuple[dict, dict]: for dataset, ws in by_dataset(walks).items(): a3[dataset], a4[dataset] = {}, {} for ft in WINDOW_FT: - est, ref = window_errors(ws, ft, drift.window_pairs) + est, ref, dev = window_errors(ws, ft, drift.window_pairs) err = est - ref along, across = along_across(err, ref) horiz = np.hypot(along, across) a3[dataset][ft] = { - "windows": len(err), - "p90_along_in": p90(np.abs(along)) / INCH, - "p90_across_in": p90(np.abs(across)) / INCH, - "p90_horizontal_in": p90(horiz) / INCH, + **split_p90(err, ref), "median_horizontal_in": float(np.median(horiz)) / INCH, "p90_vertical_in": p90(np.abs(err[:, 1])) / INCH, "p90_3d_in": p90(np.linalg.norm(err, axis=1)) / INCH, - "along_share_of_total_p90": p90(np.abs(along)) / p90(horiz), + "wall_parallel": { + f"{m:.2f}": split_p90(err[dev <= m], ref[dev <= m]) + if (dev <= m).any() + else None + for m in WALL_PARALLEL_M + }, } true_range = np.linalg.norm(ref, axis=1) a4[dataset][ft] = {"before_p90_in": p90(horiz) / INCH} @@ -457,10 +481,34 @@ def markdown(meta: dict, r1: dict, r2: dict, r3: dict, r4: dict, r5: dict) -> st for d, x in r3.items() ) + ".", + "", + "Added after review: a wall plane removes the error along the wall's normal, and that is " + "the across-travel axis only while the walk runs straight along the wall. In windows that " + "turn, across holds some along-wall error and along holds some error a plane would remove, " + "so the graded split above cannot by itself say whether a plane helps. The walks have no " + "wall annotations, so the windows below stand in for a walk along a wall: those whose " + "reference path stays within the given distance of its own start-to-end line. The widths " + "are untuned and not pre-registered.", + "", + "| Data | Walked | Within | Windows | Along | Across | Horizontal total | Along / total |", + "| --- | --- | --- | --- | --- | --- | --- | --- |", + ] + for d, x in r3.items(): + for ft, y in x.items(): + for m, z in y["wall_parallel"].items(): + if z is None: + L.append(f"| {names[d]} | {ft} ft | {m} m | 0 | | | | |") + continue + L.append( + f"| {names[d]} | {ft} ft | {m} m | {z['windows']} | {z['p90_along_in']:.1f} | " + f"{z['p90_across_in']:.1f} | {z['p90_horizontal_in']:.1f} | " + f"{z['along_share_of_total_p90']:.2f} |" + ) + L += [ "", "## A4, a range to the meter", "", - f"A simulated range from each window's start to its end: the reference's 3-D distance plus " + f"Each window is a session that starts at the meter, where the second phone stays. A simulated range from each window's start to its end: the reference's 3-D distance plus " f"N(0, sigma), {RANGE_DRAWS} draws per window (seed {SEED}). ARKit's end moves radially in " "the horizontal plane about the start, keeping its height, onto the range. p90 horizontal " "error, inches. sigma 0.10 m is graded; 0.05 and 0.20 m are not.", diff --git a/experiments/drift-anatomy/tests/test_metrics.py b/experiments/drift-anatomy/tests/test_metrics.py index 3b49b968e..6a607e373 100644 --- a/experiments/drift-anatomy/tests/test_metrics.py +++ b/experiments/drift-anatomy/tests/test_metrics.py @@ -6,6 +6,7 @@ from metrics import ( along_across, at, + chord_deviation, find_loops, local_scale, loop_errors, @@ -189,6 +190,21 @@ def test_along_across_is_pythagorean_and_turn_invariant(): np.testing.assert_allclose(c2, across, atol=1e-12) +def test_chord_deviation_is_zero_on_a_straight_walk_and_the_corner_offset_on_a_turn(): + lap = square_lap(side_m=15.0) # 1 m samples: corners at 15, 30, 45 + i, j = np.array([0, 10]), np.array([12.5, 20.0]) + dev = chord_deviation(lap, i, j) + assert dev[0] == pytest.approx(0.0, abs=1e-12) # 12.5 m along the first side + # From (10, 0) round the corner to (15, 5): the corner is 5 / sqrt(2) off the chord. + assert dev[1] == pytest.approx(5 / np.sqrt(2), abs=1e-12) + + +def test_chord_deviation_refuses_a_window_with_no_travel(): + p = np.c_[np.zeros(5), np.ones(5), np.zeros(5)] + with pytest.raises(ValueError, match="no travel line"): + chord_deviation(p, np.array([0]), np.array([3.0])) + + # Range to the meter From bfb6c19150fb1515fcea1f4e7d3c54805d2c4bda Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Tue, 13 Oct 2026 18:16:20 -0500 Subject: [PATCH 09/62] experiments(meter-closeup): drop a saved review verdict once its number changes The review page restored browser answers from a fixed localStorage key by photo id, so after a label correction or dataset update a stale Keep showed as done and was exported for a number nobody had checked. Each saved answer now records the number it was given for, and startingAnswers keeps it only while the page still shows that number; otherwise the ingested answer, if any, applies and the photo is open again. Answers saved before this change carry no number and are dropped the same way. https://github.com/SamGu-NRX/BaseScanning/pull/16#discussion_r4114120267 Co-Authored-By: Claude Opus 5.5 --- experiments/meter-closeup/METHODS.md | 2 +- .../meter-closeup/src/meter_eval/review.py | 17 +++++++---- .../meter-closeup/tests/test_review_keep.py | 30 +++++++++++++++++++ 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/experiments/meter-closeup/METHODS.md b/experiments/meter-closeup/METHODS.md index 666b52599..22c203395 100644 --- a/experiments/meter-closeup/METHODS.md +++ b/experiments/meter-closeup/METHODS.md @@ -17,7 +17,7 @@ Two AI models, not people, read every photo, so every number is only as good as - **Loose** (73 photos, used for the headline): reader 1's number equals any number reader 2 listed, or one contains the other and the shorter has at least 6 characters, and neither reader doubted a character. - **Strict** (62 photos): both readers' main numbers are identical and both were sure. Reading and ranking results are reported under both rules. -To confirm the labels by hand, run `make review` and open `~/house-scanning-data/meter-closeup/review.html`. Each photo shows whole, beside the agreed number. Press K to keep the number, or F to type a correction. Then run `uv run python -m meter_eval.review ingest ` and rebuild with `uv run python -m meter_eval.labels`. A kept number counts as agreed under both rules; a corrected one does not until someone keeps it. Ingest merges new answers into earlier ones, and a rebuilt page shows each earlier correction in place of the readers' number, so Keep confirms the correction even in a new browser. +To confirm the labels by hand, run `make review` and open `~/house-scanning-data/meter-closeup/review.html`. Each photo shows whole, beside the agreed number. Press K to keep the number, or F to type a correction. Then run `uv run python -m meter_eval.review ingest ` and rebuild with `uv run python -m meter_eval.labels`. A kept number counts as agreed under both rules; a corrected one does not until someone keeps it. A verdict saved in the browser stays only while the page shows the number it was given for, so after a label correction that photo opens again. Ingest merges new answers into earlier ones, and a rebuilt page shows each earlier correction in place of the readers' number, so Keep confirms the correction even in a new browser. ## Keeping meter numbers out of the repository diff --git a/experiments/meter-closeup/src/meter_eval/review.py b/experiments/meter-closeup/src/meter_eval/review.py index 6eb71c770..7c318f55a 100644 --- a/experiments/meter-closeup/src/meter_eval/review.py +++ b/experiments/meter-closeup/src/meter_eval/review.py @@ -222,12 +222,19 @@ def ingest(path: str) -> None: ', page, re.DOTALL).group(1)) + return row, items + + _, items = reload() + stored = {"m01": {**keep_answer(None, items[0]), "shown": items[0]["number"]}} + rows = on_page("answerRows(items, stored)", items=items, stored=stored) + assert rows == [ + ["id", "verdict", "number", "shown", "shown_known"], + ["m01", "keep", "", "1111111", "yes"], + ] + if legacy_csv: + rows = [row[:3] for row in rows] + download = tmp_path / "answers.csv" + with download.open("w", newline="") as handle: + csv.writer(handle).writerows(rows) + review.ingest(str(download)) + imported = review.read_answers(human)["m01"] + assert imported.get("shown", "") == ("" if legacy_csv else "1111111") + if not legacy_csv: + before, items = reload() + assert before["human_check"] == "keep" + assert on_page("startingAnswers(items, {})", items=items) == {"m01": imported} + + write(tmp_path / "labels_reader1.csv", [{**reader, "meter_number": new_number}]) + after, items = reload() + assert items[0]["number"] == ("" if new_number == "NONE" else new_number) + assert after.get("human_check", "") == "" + assert after.get("number_agreed", "no") == after.get("number_agreed_strict", "no") == "no" + # Both a fresh browser and the stale original browser must reopen the changed label. + assert on_page("startingAnswers(items, {})", items=items) == {} + assert on_page("startingAnswers(items, stored)", items=items, stored=stored) == {} + assert on_page("answerRows(items, startingAnswers(items, stored))", items=items, stored=stored)[ + 1 + ] == [ + "m01", + "", + "", + "", + "", + ] + + +def test_a_keep_for_a_different_shown_number_does_not_inherit_an_earlier_correction(): + old = {"m01": {"verdict": "fix", "number": "1111111"}} + new = {"m01": {"verdict": "keep", "number": "", "shown": "2222222"}} + assert review.merge(old, new) == new + + +@pytest.mark.parametrize("legacy_csv", [False, True]) +def test_csv_distinguishes_a_reviewed_empty_label_from_missing_binding( + tmp_path, monkeypatch, legacy_csv +): + items = [{"id": "m01", "number": "", "corrected": False, "answer": None}] + kept = {"m01": {"verdict": "keep", "number": "", "shown": ""}} + rows = on_page("answerRows(items, kept)", items=items, kept=kept) + if legacy_csv: + rows = [row[:3] for row in rows] + download = tmp_path / "download.csv" + with download.open("w", newline="") as handle: + csv.writer(handle).writerows(rows) + human = tmp_path / "human.csv" + monkeypatch.setattr(review, "HUMAN_LABELS", human) + review.ingest(str(download)) + imported = review.read_answers(human) + items[0]["answer"] = imported["m01"] + if legacy_csv: + assert "shown" not in imported["m01"] + assert on_page("startingAnswers(items, {})", items=items) == {} + else: + assert imported == kept + assert on_page("startingAnswers(items, {})", items=items) == kept + # Writing and re-ingesting the human file must not invent a binding on a second pass. + review.ingest(str(human)) + assert review.read_answers(human) == imported + + +def test_a_bound_keep_of_the_prior_correction_retains_its_label_binding(): + old = {"m01": {"verdict": "fix", "number": "1111111"}} + new = {"m01": {"verdict": "keep", "number": "", "shown": "1111111"}} + assert review.merge(old, new) == { + "m01": {"verdict": "keep", "number": "1111111", "shown": "1111111"} + } From 10f0fb14634a1c9aca57b2d52673c7fdc694cf84 Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:10:51 -0500 Subject: [PATCH 45/62] fix: reject ambiguous archives and incomplete acceptance evidence Bind ZIP admission to the footer stdlib will parse and refuse ZIP64 overrides before metadata allocation. S2-8 requires execution of all five hostile probes, and geometric fixtures declare the public rule values their expectations depend on. The root verifier target now uses the locked dependency sync from CI. Tiny archive controls, shortened CLI reports, real-case overrides and dispatch failures cover the new review findings. --- Makefile | 3 +- recon/recon/capture.py | 30 ++++-- recon/tests/test_review_debt.py | 43 +++++++++ tests/makefile.sh | 5 +- verification/e2e/cases/c5-no-coverage.json | 4 +- verification/e2e/cases/d-gas-drift.json | 1 + .../e2e/cases/d-reach-drift-left.json | 1 + .../e2e/cases/d-reach-drift-right.json | 1 + .../e2e/cases/g01-unseen-beyond-garage.json | 4 +- .../e2e/cases/g03-negative-s-near-edge.json | 5 +- .../e2e/cases/g05-inside-corner-gas.json | 4 +- .../e2e/cases/g06-regulator-footprint.json | 1 + verification/e2e/cases/g07-headroom-fail.json | 1 + .../e2e/cases/g07-headroom-margin.json | 1 + .../e2e/cases/g07-headroom-partial-span.json | 1 + verification/e2e/cases/g07-headroom-pass.json | 1 + .../e2e/cases/g07-headroom-unobserved.json | 1 + .../e2e/cases/g08a-garage-blocks-route.json | 4 +- .../cases/g08b-missing-wall-blocks-route.json | 4 +- verification/e2e/cases/g09-reach-14.json | 1 + verification/e2e/cases/g09-reach-15.json | 1 + verification/e2e/cases/g09-reach-16.json | 1 + verification/e2e/cases/g09-reach-20.json | 1 + .../e2e/cases/g09-reach-20p2-pm03.json | 1 + .../e2e/cases/g09-reach-20p4-pm03.json | 1 + verification/e2e/cases/g09-reach-21.json | 1 + verification/e2e/cases/g09-vertical-run.json | 1 + .../e2e/cases/g10-derived-compound-error.json | 1 + verification/e2e/cases/g10-e03-fail.json | 1 + .../e2e/cases/g10-e03-lower-edge.json | 1 + verification/e2e/cases/g10-e03-pass.json | 1 + .../e2e/cases/g10-e03-upper-edge.json | 1 + verification/e2e/cases/g10-e05-fail.json | 1 + .../e2e/cases/g10-e05-lower-edge.json | 1 + verification/e2e/cases/g10-e05-pass.json | 1 + .../e2e/cases/g10-e05-upper-edge.json | 1 + verification/e2e/cases/g10-e15-fail.json | 1 + .../e2e/cases/g10-e15-lower-edge.json | 1 + verification/e2e/cases/g10-e15-pass.json | 1 + .../e2e/cases/g10-e15-upper-edge.json | 1 + .../e2e/cases/g11a-unexplored-corner.json | 1 + .../cases/g11b-both-ends-real-all-fail.json | 4 +- .../e2e/cases/g11c-unobserved-ground.json | 1 + .../e2e/cases/g12a-subgrid-start.json | 1 + .../e2e/cases/g12b-exact-fit-last-start.json | 5 +- .../g13a-unsure-never-outranks-pass.json | 1 + verification/e2e/cases/g13b-unsure-alone.json | 1 + verification/e2e/cases/generate.py | 11 ++- verification/hsverify/e2e.py | 38 ++++++-- verification/scoreboard/metrics.yaml | 3 +- verification/tests/test_e2e.py | 94 +++++++++++++++++++ verification/tests/test_generate_cases.py | 78 +++++++++++++++ verification/tests/test_scoreboard.py | 12 +++ 53 files changed, 351 insertions(+), 35 deletions(-) diff --git a/Makefile b/Makefile index 138e460e5..88a036e4b 100644 --- a/Makefile +++ b/Makefile @@ -87,9 +87,10 @@ meter-closeup: $(call require,$(METER_CLOSEUP),#16) $(MAKE) -C experiments/meter-closeup check -# PR #13 added this suite; leaving it out lets `make check` skip the independent verifier. +# Match verification CI's locked sync so a stale lockfile cannot pass the local check. verification: $(call require,$(VERIFICATION),#13) + cd verification && uv sync --locked $(MAKE) -C verification test # Posts each scene in server/examples to a placement server and prints its decision: diff --git a/recon/recon/capture.py b/recon/recon/capture.py index a585c1282..378a32320 100644 --- a/recon/recon/capture.py +++ b/recon/recon/capture.py @@ -182,16 +182,20 @@ def _zip_directory_limits(path: Path) -> None: size = path.stat().st_size f.seek(max(0, size - 22 - 65535)) tail = f.read(22 + 65535) - end = len(tail) - while True: - pos = tail.rfind(b"PK\x05\x06", 0, end) - if pos < 0: - raise ValueError(f"{path}: missing ZIP end record") - if pos + 22 <= len(tail): - record = struct.unpack("<4s4H2IH", tail[pos : pos + 22]) - if pos + 22 + record[-1] == len(tail): - break - end = pos + # Match ZipFile's footer selection: a no-comment footer at EOF, otherwise the last + # signature. Backtracking past malformed signatures can validate a different directory + # from the one ZipFile later allocates. Refuse that ambiguity before constructing it. + if len(tail) >= 22 and tail[-22:-18] == b"PK\x05\x06" and tail[-2:] == b"\0\0": + pos = len(tail) - 22 + else: + pos = tail.rfind(b"PK\x05\x06") + if pos < 0: + raise ValueError(f"{path}: missing ZIP end record") + if pos + 22 > len(tail): + raise ValueError(f"{path}: truncated ZIP end record") + record = struct.unpack("<4s4H2IH", tail[pos : pos + 22]) + if pos + 22 + record[-1] != len(tail): + raise ValueError(f"{path}: ambiguous ZIP end record or invalid comment length") _, disk, central_disk, disk_entries, total_entries, length, offset, _ = record if disk or central_disk or disk_entries != total_entries: raise ValueError(f"{path}: split ZIP archives are not supported") @@ -199,6 +203,12 @@ def _zip_directory_limits(path: Path) -> None: raise ValueError(f"{path}: {total_entries} entries, limit {MAX_BUNDLE_ENTRIES}") directory_end = offset + length end_offset = size - len(tail) + pos + # ZipFile checks this locator even when the ordinary footer has no ZIP64 sentinels. + # It must not replace the directory fields after the bounded scan below. + if end_offset >= 20: + f.seek(end_offset - 20) + if f.read(4) == b"PK\x06\x07": + raise ValueError(f"{path}: ZIP64 archive metadata is not supported") if length == 0xFFFFFFFF or offset == 0xFFFFFFFF or directory_end != end_offset: raise ValueError(f"{path}: ZIP64 or inconsistent ZIP directory") f.seek(offset) diff --git a/recon/tests/test_review_debt.py b/recon/tests/test_review_debt.py index e1c951e3a..372d6edf0 100644 --- a/recon/tests/test_review_debt.py +++ b/recon/tests/test_review_debt.py @@ -351,6 +351,49 @@ def test_zip_entry_count_is_checked_before_metadata_allocation(tmp_path, monkeyp cap.unpack(path, tmp_path / "work") +@pytest.mark.parametrize("truncated", [False, True]) +def test_zip_comment_cannot_select_a_different_footer(tmp_path, monkeypatch, truncated): + path = tmp_path / "bundle.zip" + fake = struct.pack("<4s4H2IH", b"PK\x05\x06", 0, 0, 4, 4, 184, 0, 1) + comment = b"PK\x05\x06x" if truncated else fake + b"tail" + with zipfile.ZipFile(path, "w") as z: + z.writestr("one", b"one") + z.comment = comment + # The private stdlib parser reads only the bounded footer, without constructing ZipInfo + # objects. It chooses the later signature despite its claimed one-byte comment ending early. + if not truncated: + with path.open("rb") as f: + parsed = zipfile._EndRecData(f) + assert parsed[zipfile._ECD_ENTRIES_TOTAL] == 4 + assert parsed[zipfile._ECD_LOCATION] == path.stat().st_size - len(comment) + monkeypatch.setattr(zipfile, "ZipFile", lambda *_: pytest.fail("allocated ZIP metadata")) + with pytest.raises(ValueError, match=r"ambiguous ZIP end record|truncated ZIP end record"): + cap.unpack(path, tmp_path / "work") + assert not (tmp_path / "work").exists() + + +def test_zip_with_an_ordinary_comment_extracts(tmp_path): + path = tmp_path / "bundle.zip" + with zipfile.ZipFile(path, "w") as z: + z.writestr("scan/scene.json", '{"schema_version": "1.0"}') + z.comment = b"ordinary scan bundle comment" + out = cap.unpack(path, tmp_path / "work") + assert json.loads((out / "scan/scene.json").read_text()) == {"schema_version": "1.0"} + + +def test_zip64_locator_cannot_override_a_checked_ordinary_footer(tmp_path, monkeypatch): + path = tmp_path / "bundle.zip" + info = zipfile.ZipInfo("one") + # A locator in the final central record's file comment is immediately before the footer, + # where stdlib probes for ZIP64 even without sentinel values in the ordinary footer. + info.comment = struct.pack("<4sIQI", b"PK\x06\x07", 0, 0, 1) + with zipfile.ZipFile(path, "w") as z: + z.writestr(info, b"one") + monkeypatch.setattr(zipfile, "ZipFile", lambda *_: pytest.fail("allocated ZIP metadata")) + with pytest.raises(ValueError, match="ZIP64 archive metadata is not supported"): + cap.unpack(path, tmp_path / "work") + + def test_failed_zip_extraction_does_not_publish_partial_bundle(tmp_path, monkeypatch): path = tmp_path / "bundle.zip" with zipfile.ZipFile(path, "w") as z: diff --git a/tests/makefile.sh b/tests/makefile.sh index 361870a41..d44bd7c51 100755 --- a/tests/makefile.sh +++ b/tests/makefile.sh @@ -106,7 +106,8 @@ expect_commands evals "experiments/evals uv sync --locked experiments/evals make test" expect_commands recon "recon make test" expect_commands meter-closeup "experiments/meter-closeup make check" -expect_commands verification "verification make test" +expect_commands verification "verification uv sync --locked +verification make test" # 3. Naming a missing suite fails loudly and runs nothing. for suite in $suites; do @@ -120,7 +121,7 @@ done # 4. A failing command inside a present suite fails `make check`, and nothing runs after it. for FAIL_AT in "experiments/scoring uv" "experiments/measure-lab make" "experiments/evals uv" \ - "experiments/evals make" "recon make" "experiments/meter-closeup make" "verification make"; do + "experiments/evals make" "recon make" "experiments/meter-closeup make" "verification uv" "verification make"; do make_tree $suites if run_make check; then fail "check passed although $FAIL_AT failed" diff --git a/verification/e2e/cases/c5-no-coverage.json b/verification/e2e/cases/c5-no-coverage.json index 29e9225b6..4777661c2 100644 --- a/verification/e2e/cases/c5-no-coverage.json +++ b/verification/e2e/cases/c5-no-coverage.json @@ -67,7 +67,9 @@ } ] }, - "rules_assumed": {}, + "rules_assumed": { + "battery_width": 2.5833333333333335 + }, "expect": { "decision_in": [ "manual_review" diff --git a/verification/e2e/cases/d-gas-drift.json b/verification/e2e/cases/d-gas-drift.json index e94419104..3beac4277 100644 --- a/verification/e2e/cases/d-gas-drift.json +++ b/verification/e2e/cases/d-gas-drift.json @@ -119,6 +119,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0, "route_length": 20.0 }, diff --git a/verification/e2e/cases/d-reach-drift-left.json b/verification/e2e/cases/d-reach-drift-left.json index 1f015c85b..2b03a1f4e 100644 --- a/verification/e2e/cases/d-reach-drift-left.json +++ b/verification/e2e/cases/d-reach-drift-left.json @@ -107,6 +107,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/d-reach-drift-right.json b/verification/e2e/cases/d-reach-drift-right.json index 826109a2e..e8dd80374 100644 --- a/verification/e2e/cases/d-reach-drift-right.json +++ b/verification/e2e/cases/d-reach-drift-right.json @@ -107,6 +107,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/g01-unseen-beyond-garage.json b/verification/e2e/cases/g01-unseen-beyond-garage.json index d382da71b..b11340fe9 100644 --- a/verification/e2e/cases/g01-unseen-beyond-garage.json +++ b/verification/e2e/cases/g01-unseen-beyond-garage.json @@ -124,7 +124,9 @@ ] } }, - "rules_assumed": {}, + "rules_assumed": { + "battery_width": 2.5833333333333335 + }, "expect": { "missing_evidence_empty": true, "sweep_runs": [ diff --git a/verification/e2e/cases/g03-negative-s-near-edge.json b/verification/e2e/cases/g03-negative-s-near-edge.json index c7d60170b..9b66fdccf 100644 --- a/verification/e2e/cases/g03-negative-s-near-edge.json +++ b/verification/e2e/cases/g03-negative-s-near-edge.json @@ -204,7 +204,10 @@ ] } }, - "rules_assumed": {}, + "rules_assumed": { + "battery_width": 2.5833333333333335, + "corner_allowance": 0.5 + }, "expect": { "spot": { "wall_id": "w1", diff --git a/verification/e2e/cases/g05-inside-corner-gas.json b/verification/e2e/cases/g05-inside-corner-gas.json index 78e61e185..d9542b1ac 100644 --- a/verification/e2e/cases/g05-inside-corner-gas.json +++ b/verification/e2e/cases/g05-inside-corner-gas.json @@ -149,7 +149,9 @@ } }, "rules_assumed": { - "gas": 3.0 + "battery_width": 2.5833333333333335, + "gas": 3.0, + "corner_allowance": 0.5 }, "expect": { "decision_not": [ diff --git a/verification/e2e/cases/g06-regulator-footprint.json b/verification/e2e/cases/g06-regulator-footprint.json index 2d12e8c84..6d941c54e 100644 --- a/verification/e2e/cases/g06-regulator-footprint.json +++ b/verification/e2e/cases/g06-regulator-footprint.json @@ -150,6 +150,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g07-headroom-fail.json b/verification/e2e/cases/g07-headroom-fail.json index 4cbbcc947..406c90015 100644 --- a/verification/e2e/cases/g07-headroom-fail.json +++ b/verification/e2e/cases/g07-headroom-fail.json @@ -120,6 +120,7 @@ ] }, "rules_assumed": { + "battery_width": 2.5833333333333335, "headroom": 6.5 }, "expect": { diff --git a/verification/e2e/cases/g07-headroom-margin.json b/verification/e2e/cases/g07-headroom-margin.json index 385c3d930..54126f030 100644 --- a/verification/e2e/cases/g07-headroom-margin.json +++ b/verification/e2e/cases/g07-headroom-margin.json @@ -120,6 +120,7 @@ ] }, "rules_assumed": { + "battery_width": 2.5833333333333335, "headroom": 6.5 }, "expect": { diff --git a/verification/e2e/cases/g07-headroom-partial-span.json b/verification/e2e/cases/g07-headroom-partial-span.json index 7da273f41..40166dffa 100644 --- a/verification/e2e/cases/g07-headroom-partial-span.json +++ b/verification/e2e/cases/g07-headroom-partial-span.json @@ -120,6 +120,7 @@ ] }, "rules_assumed": { + "battery_width": 2.5833333333333335, "headroom": 6.5 }, "expect": { diff --git a/verification/e2e/cases/g07-headroom-pass.json b/verification/e2e/cases/g07-headroom-pass.json index 5184547de..b817e2d94 100644 --- a/verification/e2e/cases/g07-headroom-pass.json +++ b/verification/e2e/cases/g07-headroom-pass.json @@ -120,6 +120,7 @@ ] }, "rules_assumed": { + "battery_width": 2.5833333333333335, "headroom": 6.5 }, "expect": { diff --git a/verification/e2e/cases/g07-headroom-unobserved.json b/verification/e2e/cases/g07-headroom-unobserved.json index 4f224cdd9..302eaaf85 100644 --- a/verification/e2e/cases/g07-headroom-unobserved.json +++ b/verification/e2e/cases/g07-headroom-unobserved.json @@ -102,6 +102,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "headroom": 6.5 }, "expect": { diff --git a/verification/e2e/cases/g08a-garage-blocks-route.json b/verification/e2e/cases/g08a-garage-blocks-route.json index 8d44257c0..f405b0dbf 100644 --- a/verification/e2e/cases/g08a-garage-blocks-route.json +++ b/verification/e2e/cases/g08a-garage-blocks-route.json @@ -124,7 +124,9 @@ ] } }, - "rules_assumed": {}, + "rules_assumed": { + "battery_width": 2.5833333333333335 + }, "expect": { "decision_not": [ "pass" diff --git a/verification/e2e/cases/g08b-missing-wall-blocks-route.json b/verification/e2e/cases/g08b-missing-wall-blocks-route.json index 6b20e34a5..d72f7fff0 100644 --- a/verification/e2e/cases/g08b-missing-wall-blocks-route.json +++ b/verification/e2e/cases/g08b-missing-wall-blocks-route.json @@ -131,7 +131,9 @@ ] } }, - "rules_assumed": {}, + "rules_assumed": { + "battery_width": 2.5833333333333335 + }, "expect": { "decision_not": [ "pass" diff --git a/verification/e2e/cases/g09-reach-14.json b/verification/e2e/cases/g09-reach-14.json index c38cb8b27..3b02b7295 100644 --- a/verification/e2e/cases/g09-reach-14.json +++ b/verification/e2e/cases/g09-reach-14.json @@ -131,6 +131,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/g09-reach-15.json b/verification/e2e/cases/g09-reach-15.json index 2a3beb2ff..515b99fd9 100644 --- a/verification/e2e/cases/g09-reach-15.json +++ b/verification/e2e/cases/g09-reach-15.json @@ -131,6 +131,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/g09-reach-16.json b/verification/e2e/cases/g09-reach-16.json index 3d44710e6..b044445e2 100644 --- a/verification/e2e/cases/g09-reach-16.json +++ b/verification/e2e/cases/g09-reach-16.json @@ -131,6 +131,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/g09-reach-20.json b/verification/e2e/cases/g09-reach-20.json index 74e3350cf..861f163ce 100644 --- a/verification/e2e/cases/g09-reach-20.json +++ b/verification/e2e/cases/g09-reach-20.json @@ -131,6 +131,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/g09-reach-20p2-pm03.json b/verification/e2e/cases/g09-reach-20p2-pm03.json index b47ac4c51..22cefea9b 100644 --- a/verification/e2e/cases/g09-reach-20p2-pm03.json +++ b/verification/e2e/cases/g09-reach-20p2-pm03.json @@ -131,6 +131,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/g09-reach-20p4-pm03.json b/verification/e2e/cases/g09-reach-20p4-pm03.json index 7ff4ecd72..2b8cfe85e 100644 --- a/verification/e2e/cases/g09-reach-20p4-pm03.json +++ b/verification/e2e/cases/g09-reach-20p4-pm03.json @@ -131,6 +131,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/g09-reach-21.json b/verification/e2e/cases/g09-reach-21.json index 2ea2706f7..6cbbe2d8a 100644 --- a/verification/e2e/cases/g09-reach-21.json +++ b/verification/e2e/cases/g09-reach-21.json @@ -131,6 +131,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/g09-vertical-run.json b/verification/e2e/cases/g09-vertical-run.json index ea0be5f32..b13ec63a0 100644 --- a/verification/e2e/cases/g09-vertical-run.json +++ b/verification/e2e/cases/g09-vertical-run.json @@ -144,6 +144,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/g10-derived-compound-error.json b/verification/e2e/cases/g10-derived-compound-error.json index d17aeb721..164a591d7 100644 --- a/verification/e2e/cases/g10-derived-compound-error.json +++ b/verification/e2e/cases/g10-derived-compound-error.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e03-fail.json b/verification/e2e/cases/g10-e03-fail.json index 9fdef2f3d..99acb6e5d 100644 --- a/verification/e2e/cases/g10-e03-fail.json +++ b/verification/e2e/cases/g10-e03-fail.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e03-lower-edge.json b/verification/e2e/cases/g10-e03-lower-edge.json index 2596a533b..48452bea5 100644 --- a/verification/e2e/cases/g10-e03-lower-edge.json +++ b/verification/e2e/cases/g10-e03-lower-edge.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e03-pass.json b/verification/e2e/cases/g10-e03-pass.json index ecc3a6cd2..ad77a1311 100644 --- a/verification/e2e/cases/g10-e03-pass.json +++ b/verification/e2e/cases/g10-e03-pass.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e03-upper-edge.json b/verification/e2e/cases/g10-e03-upper-edge.json index b2014bcc3..5568ed123 100644 --- a/verification/e2e/cases/g10-e03-upper-edge.json +++ b/verification/e2e/cases/g10-e03-upper-edge.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e05-fail.json b/verification/e2e/cases/g10-e05-fail.json index b9e3050c5..39732668e 100644 --- a/verification/e2e/cases/g10-e05-fail.json +++ b/verification/e2e/cases/g10-e05-fail.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e05-lower-edge.json b/verification/e2e/cases/g10-e05-lower-edge.json index 95736f521..bfa44a8ef 100644 --- a/verification/e2e/cases/g10-e05-lower-edge.json +++ b/verification/e2e/cases/g10-e05-lower-edge.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e05-pass.json b/verification/e2e/cases/g10-e05-pass.json index 69f63c6be..9270ff2d4 100644 --- a/verification/e2e/cases/g10-e05-pass.json +++ b/verification/e2e/cases/g10-e05-pass.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e05-upper-edge.json b/verification/e2e/cases/g10-e05-upper-edge.json index f59d16188..b110ad62c 100644 --- a/verification/e2e/cases/g10-e05-upper-edge.json +++ b/verification/e2e/cases/g10-e05-upper-edge.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e15-fail.json b/verification/e2e/cases/g10-e15-fail.json index 1f3d62be4..c09577b41 100644 --- a/verification/e2e/cases/g10-e15-fail.json +++ b/verification/e2e/cases/g10-e15-fail.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e15-lower-edge.json b/verification/e2e/cases/g10-e15-lower-edge.json index a11fae0c5..920148f7c 100644 --- a/verification/e2e/cases/g10-e15-lower-edge.json +++ b/verification/e2e/cases/g10-e15-lower-edge.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e15-pass.json b/verification/e2e/cases/g10-e15-pass.json index b5ea1c602..6afe24d62 100644 --- a/verification/e2e/cases/g10-e15-pass.json +++ b/verification/e2e/cases/g10-e15-pass.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e15-upper-edge.json b/verification/e2e/cases/g10-e15-upper-edge.json index 5b4204ba3..36c014efa 100644 --- a/verification/e2e/cases/g10-e15-upper-edge.json +++ b/verification/e2e/cases/g10-e15-upper-edge.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g11a-unexplored-corner.json b/verification/e2e/cases/g11a-unexplored-corner.json index d837ed903..d878a851e 100644 --- a/verification/e2e/cases/g11a-unexplored-corner.json +++ b/verification/e2e/cases/g11a-unexplored-corner.json @@ -138,6 +138,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g11b-both-ends-real-all-fail.json b/verification/e2e/cases/g11b-both-ends-real-all-fail.json index 60fc39539..17a8379c2 100644 --- a/verification/e2e/cases/g11b-both-ends-real-all-fail.json +++ b/verification/e2e/cases/g11b-both-ends-real-all-fail.json @@ -197,7 +197,9 @@ } }, "rules_assumed": { - "gas": 3.0 + "battery_width": 2.5833333333333335, + "gas": 3.0, + "corner_allowance": 0.5 }, "expect": { "decision_in": [ diff --git a/verification/e2e/cases/g11c-unobserved-ground.json b/verification/e2e/cases/g11c-unobserved-ground.json index 6ddd3c15f..8018fa307 100644 --- a/verification/e2e/cases/g11c-unobserved-ground.json +++ b/verification/e2e/cases/g11c-unobserved-ground.json @@ -138,6 +138,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g12a-subgrid-start.json b/verification/e2e/cases/g12a-subgrid-start.json index c05e3a148..bd75a484f 100644 --- a/verification/e2e/cases/g12a-subgrid-start.json +++ b/verification/e2e/cases/g12a-subgrid-start.json @@ -140,6 +140,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "opening": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g12b-exact-fit-last-start.json b/verification/e2e/cases/g12b-exact-fit-last-start.json index 6ad22a994..33e40ed14 100644 --- a/verification/e2e/cases/g12b-exact-fit-last-start.json +++ b/verification/e2e/cases/g12b-exact-fit-last-start.json @@ -168,7 +168,10 @@ ] } }, - "rules_assumed": {}, + "rules_assumed": { + "battery_width": 2.5833333333333335, + "corner_allowance": 0.5 + }, "expect": { "spot": { "wall_id": "w1", diff --git a/verification/e2e/cases/g13a-unsure-never-outranks-pass.json b/verification/e2e/cases/g13a-unsure-never-outranks-pass.json index 90a65f0a7..9ed6d39e7 100644 --- a/verification/e2e/cases/g13a-unsure-never-outranks-pass.json +++ b/verification/e2e/cases/g13a-unsure-never-outranks-pass.json @@ -138,6 +138,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g13b-unsure-alone.json b/verification/e2e/cases/g13b-unsure-alone.json index a92e83c01..b262a3323 100644 --- a/verification/e2e/cases/g13b-unsure-alone.json +++ b/verification/e2e/cases/g13b-unsure-alone.json @@ -138,6 +138,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/generate.py b/verification/e2e/cases/generate.py index e30079fec..1377f0924 100644 --- a/verification/e2e/cases/generate.py +++ b/verification/e2e/cases/generate.py @@ -17,6 +17,8 @@ # Battery footprint from the lane C review's parameter table: 31 in wide, 22 in deep. W = 31 / 12 D = 11 / 6 +# Public server/rules.yaml route.corner_allowance_ft, a day-1 placeholder, not measured. +CORNER_ALLOWANCE = 0.5 METER_Y = 4.0 # Walls placed at z = -D put a flush battery's front edge at z = 0 exactly in floating point, # so a gas strip at z = d is measured as exactly d. The margin cases need that: 3.3 - 0.3 and @@ -159,7 +161,8 @@ def case(cid, source, description, scene, expect, rules_assumed=None): "source": source, "description": description, "scene": scene, - "rules_assumed": rules_assumed or {}, + # The fixed fixture spans and sweep ranges were derived with this battery width. + "rules_assumed": {"battery_width": W} | (rules_assumed or {}), "expect": expect, } ) @@ -230,6 +233,7 @@ def straight_scene(x0, x1, z=0.0, meter_x=0.0, objects=(), ground=None, cov=None "sweep_runs": [{"wall_id": "w1", "start_ft": [-8.4, 6.9], "outcome": "fail"}], "missing_evidence_empty": True, }, + {"corner_allowance": CORNER_ALLOWANCE}, ) # g05: inside corner, gas meter on the return wall at [4, 4]. @@ -258,7 +262,7 @@ def straight_scene(x0, x1, z=0.0, meter_x=0.0, objects=(), ground=None, cov=None {"wall_id": "w1", "start_ft": [6, 7.4], "outcome": "fail", "failing_match": "gas"} ], }, - {"gas": 3.0}, + {"gas": 3.0, "corner_allowance": CORNER_ALLOWANCE}, ) # g06: gas meter body 10 ft out, regulator reaching [7, 23/6]. @@ -448,7 +452,7 @@ def g10_scene(d, e, wall_pm=0.0): {"wall_id": "w2", "start_ft": [8, 11.4], "outcome": "fail", "failing_match": "gas"}, ], }, - {"gas": 3.0}, + {"gas": 3.0, "corner_allowance": CORNER_ALLOWANCE}, ) # g11c: observed starts fail gas; the ground right of s = 1 was never seen (closed gate). @@ -531,6 +535,7 @@ def g10_scene(d, e, wall_pm=0.0): "coverage": full_coverage((-3, 6 + W)), }, {"spot": {"wall_id": "w1", "span_within": [6, 6 + W]}, "missing_evidence_empty": True}, + {"corner_allowance": CORNER_ALLOWANCE}, ) diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index 7d7321584..6ce9cf93b 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -577,6 +577,15 @@ def resend(name: str, scene: dict) -> dict | None: HOSTILE_BUDGET_MS = 10_000.0 +HOSTILE_PROBE_NAMES = ( + "hostile: 150 objects, each its own error", + "hostile: coordinates of 1e300 ft", + "hostile: a 5000 ft wall", + "hostile: nested 5000 deep", + "hostile: 400 MB scene.json in a small zip", +) + + def hostile_inputs() -> list[SceneInput]: """Small requests that cost a careless server a lot. Built here, not stored, so the repository carries no blobs; none takes the harness more than a few MB to build.""" @@ -620,13 +629,11 @@ def hostile_inputs() -> list[SceneInput]: } nested = ('{"meter": ' + "[" * 5000 + "]" * 5000 + "}").encode() items = [ - SceneInput( - "hostile: 150 objects, each its own error", crowded, json.dumps(crowded).encode() - ), - SceneInput("hostile: coordinates of 1e300 ft", huge, json.dumps(huge).encode()), - SceneInput("hostile: a 5000 ft wall", long_wall, json.dumps(long_wall).encode()), - SceneInput("hostile: nested 5000 deep", {}, nested), - SceneInput("hostile: 400 MB scene.json in a small zip", {}, b"", bundle=zip_bomb(400)), + SceneInput(HOSTILE_PROBE_NAMES[0], crowded, json.dumps(crowded).encode()), + SceneInput(HOSTILE_PROBE_NAMES[1], huge, json.dumps(huge).encode()), + SceneInput(HOSTILE_PROBE_NAMES[2], long_wall, json.dumps(long_wall).encode()), + SceneInput(HOSTILE_PROBE_NAMES[3], {}, nested), + SceneInput(HOSTILE_PROBE_NAMES[4], {}, b"", bundle=zip_bomb(400)), ] for item in items: item.hostile = True @@ -676,7 +683,7 @@ def judge_hostile( } before = limit.reset() if limit else 0.0 status, payload, ms = post(url, endpoint, item, timeout_s=HOSTILE_BUDGET_MS / 1000 + 5) - record = {"http_status": status, "latency_ms": round(ms, 1)} + record = {"http_status": status, "latency_ms": round(ms, 1), "hostile_probe_executed": True} grew = limit.peak_mb - before if limit else 0.0 if limit: record["server_growth_mb"] = round(grew, 1) @@ -720,10 +727,16 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: exported = [r for r in records if r.get("app_export")] contract_problems = sum(len(r.get("contract_problems", [])) for r in records) killed = bool(meta["peak_memory"].get("server_killed")) + all_passed = all(r["status"] == "pass" for r in records) and bool(checked) and not killed + contract_ok = bool(checked) and contract_problems == 0 and not killed + executed = sorted( + {r["name"] for r in records if r.get("hostile") and r.get("hostile_probe_executed") is True} + ) + hostile_complete = set(HOSTILE_PROBE_NAMES).issubset(executed) report = meta | { "counts": counts, # A missing listed scene leaves the run incomplete, even when other scenes passed. - "all_passed": all(r["status"] == "pass" for r in records) and bool(checked) and not killed, + "all_passed": all_passed, "latency_ms": max(real) if real else None, "scenes_answered": sum(1 for r in records if "decision" in r), "scenes_checked": len(checked), @@ -733,7 +746,12 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: # expectations are counted separately because a case can be wrong itself. "contract_problem_count": contract_problems, # The contract holds only on evidence: at least one scene checked, none broken. - "contract_ok": bool(checked) and contract_problems == 0 and not killed, + "contract_ok": contract_ok, + "hostile_probes_required": list(HOSTILE_PROBE_NAMES), + "hostile_probes_executed": executed, + "hostile_probes_complete": hostile_complete, + # Local --no-hostile success proves only that shortened run, not S2-8's full guarantee. + "full_contract_ok": all_passed and contract_ok and hostile_complete, "expectation_problem_count": sum(len(r.get("expectation_problems", [])) for r in records), "real_scene_passed": any(r.get("real") and r["status"] == "pass" for r in records), # Null when the run had no app export, so the scoreboard looks for an older run. diff --git a/verification/scoreboard/metrics.yaml b/verification/scoreboard/metrics.yaml index 2013f8c08..9d93cd12f 100644 --- a/verification/scoreboard/metrics.yaml +++ b/verification/scoreboard/metrics.yaml @@ -161,7 +161,8 @@ metrics: report: glob: ~/house-scanning-data/reports/e2e/*/report.json # True only when at least one scene was answered and checked and none broke the contract. - key: contract_ok + # Requires every hostile probe; shortened --no-hostile runs cannot satisfy this field. + key: full_contract_ok # A failed or incomplete run cannot establish the advertised black-box acceptance. valid_key: all_passed equals: true diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index a7707e1ec..e4900ffcb 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -549,3 +549,97 @@ def check_hash(scene, result, sent=None, rules=None): ) assert calls and all(raw == json.dumps(scene, indent=1).encode() for scene, raw in calls) assert any("stale input hash" in p for p in failures) + + +def hostile_records(): + return [ + { + "name": name, + "hostile": True, + "hostile_probe_executed": True, + "http_status": 422, + "latency_ms": 1.0, + "status": "pass", + "contract_problems": [], + } + for name in e2e.HOSTILE_PROBE_NAMES + ] + + +@pytest.mark.parametrize("missing", range(5)) +def test_every_required_hostile_probe_must_be_exercised(tmp_path, missing): + probes = hostile_records() + omitted = probes.pop(missing) + probes.append(probes[0].copy()) # duplicate evidence cannot replace the missing probe + report = e2e.write_report(tmp_path, META, [CHECKED, *probes]) + assert report["all_passed"] and report["contract_ok"] + assert not report["hostile_probes_complete"] and not report["full_contract_ok"] + assert omitted["name"] not in report["hostile_probes_executed"] + + +def test_probe_name_without_execution_is_not_full_evidence(tmp_path): + probes = hostile_records() + del probes[0]["hostile_probe_executed"] + assert not e2e.write_report(tmp_path, META, [CHECKED, *probes])["full_contract_ok"] + assert e2e.write_report(tmp_path, META, [CHECKED, *hostile_records()])["full_contract_ok"] + + +def test_hostile_execution_is_recorded_only_after_sending(monkeypatch): + monkeypatch.setattr(e2e, "post", lambda *args, **kwargs: (422, b"{}", 1.0)) + probe = SceneInput(e2e.HOSTILE_PROBE_NAMES[0], {}, b"{}", hostile=True) + record = judge("unused", Endpoint("/place", "application/json"), probe, SCHEMAS, RULES) + assert record["hostile_probe_executed"] is True + probe.bundle = b"archive" + record = judge("unused", Endpoint("/place", "application/json"), probe, SCHEMAS, RULES) + assert not record.get("hostile_probe_executed", False) + + +@pytest.mark.parametrize("no_hostile", [False, True]) +def test_cli_shortened_run_cannot_claim_full_contract(tmp_path, monkeypatch, no_hostile): + from pathlib import Path + + # Only public rules are read. All HTTP, git and hostile payload generation are mocked. + public_rules = (Path(__file__).resolve().parents[2] / "server" / "rules.yaml").read_text() + cases = tmp_path / "cases" + cases.mkdir() + (cases / "scene.json").write_text(json.dumps(SCENE)) + monkeypatch.setattr(e2e.gitref, "fetch", lambda: None) + monkeypatch.setattr(e2e.gitref, "resolve", lambda _: "a" * 40) + monkeypatch.setattr( + e2e.gitref, "show", lambda sha, path: public_rules if path.endswith("rules.yaml") else "{}" + ) + monkeypatch.setattr( + e2e.urllib.request, + "urlopen", + lambda *args, **kwargs: io.BytesIO(json.dumps(openapi({"/place": json_op()})).encode()), + ) + monkeypatch.setattr( + e2e, + "hostile_inputs", + lambda: [SceneInput(name, {}, b"{}", hostile=True) for name in e2e.HOSTILE_PROBE_NAMES], + ) + + def fake_judge(url, endpoint, probe, *args): + if probe.hostile: + return next(r for r in hostile_records() if r["name"] == probe.name) + return CHECKED | {"name": probe.name} + + monkeypatch.setattr(e2e, "judge", fake_judge) + monkeypatch.setattr(e2e, "peak_rss_mb", lambda: {}) + out = tmp_path / "out" + args = ["--server-url", "http://unused", "--cases", str(cases), "--out", str(out)] + if no_hostile: + args.append("--no-hostile") + assert e2e.main(args) == 0 + data = json.loads((out / "report.json").read_text()) + assert data["all_passed"] and data["contract_ok"] + assert data["full_contract_ok"] is (not no_hostile) + assert data["hostile_probes_executed"] == ( + [] if no_hostile else sorted(e2e.HOSTILE_PROBE_NAMES) + ) + from hsverify import scoreboard + + metric = next(m for m in scoreboard.load_metrics()[1] if m.id == "S2-8") + metric.probe["glob"] = str(out / "report.json") + status = scoreboard.probe_report(metric.probe, data["sha"]).status + assert status == ("gap" if no_hostile else "met") diff --git a/verification/tests/test_generate_cases.py b/verification/tests/test_generate_cases.py index 01c7cdfac..a37800816 100644 --- a/verification/tests/test_generate_cases.py +++ b/verification/tests/test_generate_cases.py @@ -39,3 +39,81 @@ def test_the_committed_cases_are_what_the_generator_writes(tmp_path): committed = SCRIPT.parent for f in tmp_path.glob("*.json"): assert f.read_text() == (committed / f.name).read_text(), f.name + + +def test_fixture_geometric_baselines_match_public_rules(): + import yaml + + rules = yaml.safe_load((SCRIPT.parents[3] / "server" / "rules.yaml").read_text()) + assert pytest.approx(rules["battery"]["width_ft"]["value"]) == generate.W + assert rules["route"]["corner_allowance_ft"]["value"] == generate.CORNER_ALLOWANCE + for case in generate.CASES: + assert case["rules_assumed"]["battery_width"] == generate.W + for cid in [ + "g03-negative-s-near-edge", + "g05-inside-corner-gas", + "g11b-both-ends-real-all-fail", + "g12b-exact-fit-last-start", + ]: + case = next(c for c in generate.CASES if c["id"] == cid) + assert case["rules_assumed"]["corner_allowance"] == generate.CORNER_ALLOWANCE + + +@pytest.mark.parametrize("origin", ["committed", "generated"]) +@pytest.mark.parametrize("override", ["width", "corner"]) +def test_real_case_rule_override_skips_stale_expectation(tmp_path, monkeypatch, origin, override): + from dataclasses import replace + + from hsverify import e2e + from hsverify.resultcheck import RuleSet, expectation_problems + + if origin == "generated": + generate.main(tmp_path) + folder = tmp_path + else: + folder = SCRIPT.parent + fixture = e2e.load_input(folder / "g12b-exact-fit-last-start.json") + public = (SCRIPT.parents[3] / "server" / "rules.yaml").read_text() + baseline = RuleSet.from_yaml(public) + rules = ( + replace(baseline, width_ft=4.0) + if override == "width" + else replace(baseline, corner_allowance_ft=10.0) + ) + # A wider battery cannot fit the exact-width segment; a high corner allowance loses reach. + answer = {"decision": "reject", "spot": None, "checks": [], "sweep": []} + assert expectation_problems(fixture.expect, answer) # stale fit assertion would fail + monkeypatch.setattr( + e2e, "post", lambda *args, **kwargs: (200, json.dumps(answer).encode(), 1.0) + ) + monkeypatch.setattr(e2e, "schema_errors", lambda *args: []) + monkeypatch.setattr(e2e, "invariant_problems", lambda *args, **kwargs: []) + monkeypatch.setattr(e2e, "property_problems", lambda *args: []) + + def forbidden(*args): + pytest.fail("a mismatched case must not apply its stale expectations") + + monkeypatch.setattr(e2e, "expectation_problems", forbidden) + record = e2e.judge( + "unused", + e2e.Endpoint("/place", "application/json"), + fixture, + {"scene": {}, "result": {}}, + rules, + ) + assert record["status"] == "assumption mismatch" and record["assumption_mismatches"] + assert record["expectation_problems"] == [] + meta = { + "peak_memory": {}, + "server_ref": "public", + "server_sha": "a" * 40, + "started_at": "test", + "endpoint": "mock", + "load_average_1_5_15": [], + } + probes = [ + {"name": name, "hostile": True, "hostile_probe_executed": True, "status": "pass"} + for name in e2e.HOSTILE_PROBE_NAMES + ] + report = e2e.write_report(tmp_path, meta, [record, *probes]) + assert not report["all_passed"] and not report["full_contract_ok"] diff --git a/verification/tests/test_scoreboard.py b/verification/tests/test_scoreboard.py index a67c7111b..f8b2f5e44 100644 --- a/verification/tests/test_scoreboard.py +++ b/verification/tests/test_scoreboard.py @@ -700,3 +700,15 @@ def test_s2_contract_credit_requires_successful_full_run(tmp_path): 100, ) assert sb.probe_report(metric.probe, SHA).status == "gap" + + +@pytest.mark.parametrize("full", [None, False, True]) +def test_s2_requires_hostile_execution_evidence_from_report(tmp_path, full): + metric = real_metric("S2-8") + metric.probe["glob"] = str(tmp_path / "*" / "report.json") + data = {"sha": SHA, "contract_ok": True, "all_passed": True} + if full is not None: + data["full_contract_ok"] = full + write_report(tmp_path / "r" / "report.json", data, 100) + status = sb.probe_report(metric.probe, SHA).status + assert (status == "met") is (full is True) From b6bc4adbe8da850816f518cc696fbc10a96fc790 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Wed, 14 Oct 2026 11:26:53 -0500 Subject: [PATCH 46/62] test(live-dots): evaluate fog mutations before assertions --- .../ReplayPlaybackTests.swift | 99 ++++++++++++------- 1 file changed, 66 insertions(+), 33 deletions(-) diff --git a/experiments/live-dots/Tests/LiveDotsCoreTests/ReplayPlaybackTests.swift b/experiments/live-dots/Tests/LiveDotsCoreTests/ReplayPlaybackTests.swift index 9cd737469..c002be63e 100644 --- a/experiments/live-dots/Tests/LiveDotsCoreTests/ReplayPlaybackTests.swift +++ b/experiments/live-dots/Tests/LiveDotsCoreTests/ReplayPlaybackTests.swift @@ -11,18 +11,22 @@ struct ReplayPlaybackTests { var continuity = FogContinuity() playback.togglePlayback() playback.tick(now: 0) - #expect(continuity.consume(request(playback))) + var resetsFog = continuity.consume(request(playback)) + #expect(resetsFog) playback.tick(now: 0.2) let before = request(playback) - #expect(!continuity.consume(before)) + resetsFog = continuity.consume(before) + #expect(!resetsFog) playback.scrub(to: 0) let after = request(playback) #expect(after.keyframe == before.keyframe) #expect(after.time == 0 && !after.isSettled) #expect(after.seekGeneration == before.seekGeneration + 1) - #expect(continuity.consume(after)) + resetsFog = continuity.consume(after) + #expect(resetsFog) playback.tick(now: 0.3) - #expect(!continuity.consume(request(playback))) + resetsFog = continuity.consume(request(playback)) + #expect(!resetsFog) } @Test func `natural playback eases across same and adjacent frames but explicit seeks reset`() { @@ -30,16 +34,21 @@ struct ReplayPlaybackTests { var continuity = FogContinuity() playback.togglePlayback() playback.tick(now: 0) - #expect(continuity.consume(request(playback))) + var resetsFog = continuity.consume(request(playback)) + #expect(resetsFog) playback.tick(now: 0.1) - #expect(!continuity.consume(request(playback))) + resetsFog = continuity.consume(request(playback)) + #expect(!resetsFog) playback.tick(now: 0.3) #expect(playback.keyframe == 1) - #expect(!continuity.consume(request(playback))) + resetsFog = continuity.consume(request(playback)) + #expect(!resetsFog) playback.scrub(to: 2) - #expect(continuity.consume(request(playback))) + resetsFog = continuity.consume(request(playback)) + #expect(resetsFog) playback.scrub(to: 1) - #expect(continuity.consume(request(playback))) + resetsFog = continuity.consume(request(playback)) + #expect(resetsFog) } @Test(arguments: [false, true]) func `resuming a settled scrub resets at the birth time`(reduceMotion: Bool) { @@ -48,14 +57,17 @@ struct ReplayPlaybackTests { playback.scrub(to: 2) let preview = request(playback, reduceMotion: reduceMotion) #expect(preview.isSettled) - #expect(continuity.consume(preview)) + var resetsFog = continuity.consume(preview) + #expect(resetsFog) playback.togglePlayback() let resumed = request(playback, reduceMotion: reduceMotion) #expect(resumed.time == Schedule.start(of: 2) && !resumed.isSettled) #expect(resumed.seekGeneration == preview.seekGeneration + 1) #expect(resumed.reduceMotion == reduceMotion) - #expect(continuity.consume(resumed)) - #expect(!continuity.consume(resumed)) + resetsFog = continuity.consume(resumed) + #expect(resetsFog) + resetsFog = continuity.consume(resumed) + #expect(!resetsFog) } @Test func `scrub and resume between rendered frames still invalidate the old mask`() { @@ -63,13 +75,16 @@ struct ReplayPlaybackTests { var continuity = FogContinuity() playback.togglePlayback() playback.tick(now: 0) - #expect(continuity.consume(request(playback))) + var resetsFog = continuity.consume(request(playback)) + #expect(resetsFog) playback.tick(now: 0.2) - #expect(!continuity.consume(request(playback))) + resetsFog = continuity.consume(request(playback)) + #expect(!resetsFog) playback.togglePlayback() playback.scrub(to: 0) playback.togglePlayback() - #expect(continuity.consume(request(playback))) + resetsFog = continuity.consume(request(playback)) + #expect(resetsFog) } @Test func `pause and resume without a scrub preserve the playhead and fog easing`() { @@ -77,10 +92,12 @@ struct ReplayPlaybackTests { var continuity = FogContinuity() playback.togglePlayback() playback.tick(now: 0) - #expect(continuity.consume(request(playback))) + var resetsFog = continuity.consume(request(playback)) + #expect(resetsFog) playback.tick(now: 0.2) let before = request(playback) - #expect(!continuity.consume(before)) + resetsFog = continuity.consume(before) + #expect(!resetsFog) playback.togglePlayback() playback.tick(now: 5) playback.togglePlayback() @@ -88,7 +105,8 @@ struct ReplayPlaybackTests { let resumed = request(playback) #expect(resumed.time == before.time) #expect(resumed.seekGeneration == before.seekGeneration) - #expect(!continuity.consume(resumed)) + resetsFog = continuity.consume(resumed) + #expect(!resetsFog) playback.tick(now: 10.02) #expect(abs(playback.playhead - before.time - 0.02) < 1e-5) } @@ -101,12 +119,14 @@ struct ReplayPlaybackTests { playback.tick(now: 10) #expect(!playback.isPlaying) let final = request(playback) - #expect(continuity.consume(final)) + var resetsFog = continuity.consume(final) + #expect(resetsFog) playback.togglePlayback() let restarted = request(playback) #expect(restarted.keyframe == 0 && restarted.time == 0) #expect(restarted.seekGeneration == final.seekGeneration + 1) - #expect(continuity.consume(restarted)) + resetsFog = continuity.consume(restarted) + #expect(resetsFog) } @Test func `mode changes fog reentry and settled previews invalidate continuity`() { @@ -114,31 +134,44 @@ struct ReplayPlaybackTests { var continuity = FogContinuity() playback.togglePlayback() var frame = request(playback) - #expect(continuity.consume(frame)) - #expect(!continuity.consume(frame)) + var resetsFog = continuity.consume(frame) + #expect(resetsFog) + resetsFog = continuity.consume(frame) + #expect(!resetsFog) frame.mode = .noLidar - #expect(continuity.consume(frame)) - #expect(!continuity.consume(frame)) + resetsFog = continuity.consume(frame) + #expect(resetsFog) + resetsFog = continuity.consume(frame) + #expect(!resetsFog) frame.fog = .off - #expect(!continuity.consume(frame)) + resetsFog = continuity.consume(frame) + #expect(!resetsFog) frame.fog = .on - #expect(continuity.consume(frame)) + resetsFog = continuity.consume(frame) + #expect(resetsFog) frame.time = 1e5 - #expect(continuity.consume(frame)) + resetsFog = continuity.consume(frame) + #expect(resetsFog) frame.time = 0 - #expect(continuity.consume(frame)) - #expect(!continuity.consume(frame)) + resetsFog = continuity.consume(frame) + #expect(resetsFog) + resetsFog = continuity.consume(frame) + #expect(!resetsFog) } @Test func `offscreen clock rewinds and nonadjacent jumps reset without a player`() { var continuity = FogContinuity() var frame = FrameRequest(mode: .lidar, keyframe: 0, time: 0.2, reduceMotion: false, fog: .on) - #expect(continuity.consume(frame)) + var resetsFog = continuity.consume(frame) + #expect(resetsFog) frame.time = 0.1 - #expect(continuity.consume(frame)) + resetsFog = continuity.consume(frame) + #expect(resetsFog) frame.keyframe = 3 frame.time = Schedule.start(of: 3) - #expect(continuity.consume(frame)) - #expect(!continuity.consume(frame)) + resetsFog = continuity.consume(frame) + #expect(resetsFog) + resetsFog = continuity.consume(frame) + #expect(!resetsFog) } } From df7b75d7e142f1b94eda469b4320b66e1fcdfbb2 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Wed, 14 Oct 2026 11:33:54 -0500 Subject: [PATCH 47/62] fix(verification): bound sweep sampling before geometry checks --- verification/hsverify/resultcheck.py | 53 ++++++++++++++++++---- verification/tests/test_resultcheck.py | 63 ++++++++++++++++++++++++-- 2 files changed, 105 insertions(+), 11 deletions(-) diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index e1ab5a718..2ea978f2a 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -29,7 +29,7 @@ import hashlib import itertools import math -from collections.abc import Callable +from collections.abc import Callable, Iterator from dataclasses import dataclass from typing import Any @@ -38,6 +38,8 @@ EPS = 1e-6 NUMBER_TOL_FT = 1e-5 +# Bound verifier work even when a server span or rules step requests billions of samples. +MAX_SWEEP_SAMPLES = 10_000 # Spot centres and offsets are rounded by the server; a hundredth of a foot is 1/8 inch. OFFSET_TOL_FT = 0.01 @@ -306,6 +308,8 @@ def margin_problem(check: dict) -> str | None: def invariant_problems( scene: dict, result: dict, sent: bytes | None = None, rules: RuleSet | None = None ) -> list[str]: + if rules is not None and (sampling := sweep_sampling_problems(scene, result, rules)): + return sampling problems: list[str] = [] decision = result["decision"] spot = result.get("spot") @@ -419,6 +423,8 @@ def footprint_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: """A battery that passes or is left to a person backs onto one straight stretch of scanned wall along its whole width; a footprint past the stretch's end (a corner, the end of the wall, a gap) can't sit flush, so the server fails it (server/solver.py check_backing).""" + if sampling := sweep_sampling_problems(scene, result, rules): + return sampling pieces = straight_pieces(scene, rules) if not pieces: return [] @@ -673,17 +679,44 @@ def measured_band_gap( return None -def run_starts(span: list[float], step_ft: float | None) -> list[float]: - """Starts sampled from a sweep run: its first, every step after it, and its last. +def run_starts(span: list[float], step_ft: float | None) -> Iterator[float]: + """Validate eagerly, then sample a run lazily: first, every step after it, and last. A sample, not the server's own list: the server also evaluates boundaries and midpoints - that a run reports only as its two ends.""" - if step_ft is None or step_ft <= 0: - raise ValueError("checking a sweep run needs the rules' sweep.step_ft") + that a run reports only as its two ends. Oversized runs fail rather than lose coverage + through truncation.""" + if step_ft is None or not math.isfinite(step_ft) or step_ft <= 0: + raise ValueError("checking a sweep run needs finite positive rules' sweep.step_ft") a, b = span + if not math.isfinite(a) or not math.isfinite(b) or a > b: + raise ValueError("sweep start_ft must be finite and ordered") + steps = (b - a) / step_ft + if not math.isfinite(steps) or steps > MAX_SWEEP_SAMPLES: + raise ValueError(f"sweep sampling exceeds the {MAX_SWEEP_SAMPLES} sample limit") # Every step strictly before the last start, the first included however short the run. - count = max(0, math.ceil((b - a) / step_ft - EPS)) - return [a + k * step_ft for k in range(count)] + [b] + count = max(int(a < b), math.ceil(steps - EPS)) + if count + 1 > MAX_SWEEP_SAMPLES: + raise ValueError(f"sweep sampling exceeds the {MAX_SWEEP_SAMPLES} sample limit") + return itertools.chain((a + k * step_ft for k in range(count)), (b,)) + + +def sweep_sampling_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: + """Reject invalid runs before footprint, coverage or height checks sample them.""" + ends = chain_ends_s(scene, rules) + problems = [] + for run in result.get("sweep", []): + if run["outcome"] not in ("pass", "unsure"): + continue + a, b = run["start_ft"] + prefix = f"sweep {run['outcome']} starts {run['start_ft']}: " + if ends is not None and (a < ends[0] - COLLINEAR_FT or b > ends[1] + COLLINEAR_FT): + problems.append(prefix + f"start_ft span exceeds scanned range [{ends[0]}, {ends[1]}]") + continue + try: + run_starts(run["start_ft"], rules.step_ft) + except ValueError as exc: + problems.append(prefix + str(exc)) + return problems # Checks every result lists besides those with coverage needs (server README, "What settles @@ -717,6 +750,8 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: check's area around that start's battery with that battery's own position error, and the wall and cable route back to the meter over the whole run. Checking the whole run as one battery at its farthest start's error would ask for more than any single start needs.""" + if sampling := sweep_sampling_problems(scene, result, rules): + return sampling problems: list[str] = [] route_wall = observed( scene, "wall", rules.route_height_ft, beyond=True, wall_default_ft=rules.headroom_ft @@ -1048,6 +1083,8 @@ def declared_height_problems(scene: dict, result: dict, rules: RuleSet) -> list[ it unsure for one within the battery's error of it (server/solver.py at 7133283, check_backing), so wall_backing passes only if every declared wall overlapping [s0 - e, s1 + e] is taller than battery.height_ft.""" + if sampling := sweep_sampling_problems(scene, result, rules): + return sampling declared = {w["id"]: w["height_ft"] for w in scene.get("walls", []) if "height_ft" in w} if not declared or rules.battery_height_ft is None: return [] diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 08935bd3a..526a11bba 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -6,6 +6,7 @@ import pytest from hsverify.resultcheck import ( + MAX_SWEEP_SAMPLES, GroundPlan, Need, RuleSet, @@ -875,8 +876,8 @@ def test_each_start_in_a_sweep_run_needs_only_its_own_reach(): def test_sweep_run_starts_are_sampled_at_the_rules_step_plus_the_ends(): - assert run_starts([1.0, 1.5], 1 / 6) == pytest.approx([1.0, 7 / 6, 8 / 6, 1.5]) - assert run_starts([2.0, 2.0], 1 / 6) == [2.0] + assert list(run_starts([1.0, 1.5], 1 / 6)) == pytest.approx([1.0, 7 / 6, 8 / 6, 1.5]) + assert list(run_starts([2.0, 2.0], 1 / 6)) == [2.0] with pytest.raises(ValueError, match=r"sweep\.step_ft"): run_starts([1.0, 2.0], None) @@ -1083,7 +1084,7 @@ def test_a_captured_request_claims_only_the_reach_it_asked_for(): def test_a_sweep_run_shorter_than_a_step_keeps_its_first_start(): # Review 4114352251: run_starts([0, 0.4], 1) returned only 0.4. - assert run_starts([0.0, 0.4], 1.0) == [0.0, 0.4] + assert list(run_starts([0.0, 0.4], 1.0)) == [0.0, 0.4] def test_a_passing_spot_must_pass_in_the_sweep_too(): @@ -1508,3 +1509,59 @@ def test_geometric_assumptions_compare_active_rules(): assumption_mismatches({"battery_width": rules.width_ft, "corner_allowance": 0.5}, {}, rules) == [] ) + + +@pytest.mark.parametrize("outcome", ["pass", "unsure"]) +@pytest.mark.parametrize("span", [[0, 1e9], [-1e9, 0], [1e9, 1e9]]) +def test_out_of_scan_sweep_fails_before_sampling(monkeypatch, outcome, span): + def no_sampling(*args): + pytest.fail("out-of-scan sweep reached sampling") + + monkeypatch.setattr("hsverify.resultcheck.run_starts", no_sampling) + r = result(spot=False, sweep=[pass_run(*span) | {"outcome": outcome}]) + message = f"sweep {outcome} starts {span}: start_ft span exceeds scanned range [-12.0, 12.0]" + for judge in ( + invariant_problems, + footprint_problems, + coverage_problems, + declared_height_problems, + ): + assert judge(SCENE, r, rules=RULES) == [message] + + +def test_sweep_sampling_is_lazy_and_keeps_the_limit_endpoint(): + samples = run_starts([0, MAX_SWEEP_SAMPLES - 1], 1) + assert iter(samples) is samples + assert next(samples) == 0 + remaining = list(samples) + assert len(remaining) == MAX_SWEEP_SAMPLES - 1 + assert remaining[-1] == MAX_SWEEP_SAMPLES - 1 + assert list(run_starts([0, 1e-9], 1)) == [0, 1e-9] + + +@pytest.mark.parametrize( + "span,step", + [([0, 1e9], 1 / 6), ([0, MAX_SWEEP_SAMPLES], 1), ([0, 1], 1e-300), ([-1e308, 1e308], 1)], +) +def test_sweep_sample_limit_is_checked_before_iteration(span, step): + with pytest.raises(ValueError, match="10000 sample limit"): + run_starts(span, step) + + +@pytest.mark.parametrize("span", [[2, 1], [0, float("inf")], [float("nan"), 1]]) +def test_sweep_sampler_refuses_invalid_spans(span): + with pytest.raises(ValueError, match="finite and ordered"): + run_starts(span, 1) + + +@pytest.mark.parametrize("step", [None, 0, -1, float("inf"), float("nan")]) +def test_sweep_sampler_refuses_invalid_steps(step): + with pytest.raises(ValueError, match="finite positive"): + run_starts([0, 1], step) + + +def test_in_scan_sweep_with_tiny_rule_step_fails_loudly(): + r = result(spot=False, sweep=[pass_run(0, 1) | {"outcome": "unsure"}]) + assert invariant_problems(SCENE, r, rules=replace(RULES, step_ft=1e-300)) == [ + "sweep unsure starts [0, 1]: sweep sampling exceeds the 10000 sample limit" + ] From aaacc0fc0c5e50b94c7a461aa394dd06b8f8d42d Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Wed, 14 Oct 2026 11:44:51 -0500 Subject: [PATCH 48/62] fix(verification): distinguish reported rules from public assumptions --- recon/HANDOFF.md | 5 +- verification/README.md | 8 ++ verification/hsverify/e2e.py | 65 +++++++++++++--- verification/hsverify/resultcheck.py | 73 ++++++++++++------ verification/tests/test_e2e.py | 94 +++++++++++++++++++++-- verification/tests/test_generate_cases.py | 5 +- verification/tests/test_resultcheck.py | 94 +++++++++++++++++++++-- 7 files changed, 298 insertions(+), 46 deletions(-) diff --git a/recon/HANDOFF.md b/recon/HANDOFF.md index b952169d2..0f26dd199 100644 --- a/recon/HANDOFF.md +++ b/recon/HANDOFF.md @@ -128,10 +128,13 @@ In `recon/` on this branch: These defects remain after the review-debt repairs. They can let the server settle a check on evidence the worker does not have. The saved ETH3D results above predate those repairs; dataset acceptance has not been rerun. -- **One view can count as two for free space.** `recon/coverage.py:189-193, 208-246`: `_slab` treats a voxel as observed when its fused weight is above zero, and any single frame gives it weight. Repro: a capture in which one frame alone sees the space in front of a wall cell reports a facing clearance there. Wall and ground cells need two positions 0.25 m apart for the same claim. The server at 903d86f can settle a check on that clearance. A fix needs visibility kept per frame for free space, with the same two-position test as the wall and ground. - **The overhead strip is narrower than a battery.** `recon/coverage.py:42, 236-245`: overhead space is sampled only 0.10 to 0.56 m out from the wall front. Repro: an eave 0.05 m out from the wall, or 0.7 m out over a 3 ft deep battery, is not found, and the cell reports clearance above it. A fix needs the footprint of the battery the chosen rule places, passed from the server's rules or request. - **The photos-only clearance error is not calibrated.** `recon/pipeline.py:29-35`: facing gaps and overhead clearances from photos carry ±0.56 ft. That comes from the 95% interval of the evals' surface pair-error p90 over 1 to 3 m spans, not from any measurement of where the fused volume puts a boundary. The evals put surface p90 at 5.64 in [4.63, 6.66] and edges at 9.65 in [4.93, 15.01]. Repro: a photos-only overhead clearance measured at 7.10 ft passes the 6.5 ft headroom rule, since 7.10 − 0.56 = 6.54 > 6.5. A fix needs the clearance estimator's own measured uncertainty, or a gate that returns the check unresolved until that exists. +### Fixed in the worker + +Free-space coverage now uses `SeenFromTwo` in `compute()`: facing and overhead samples need visibility from two camera positions at least 0.25 m apart. One-view and nearby-camera regressions are in `tests/test_coverage_views.py`. + The worker now replaces the target wall's prior measurements, stops clearance at any occupied sampled voxel, and refuses LiDAR without a confidence map. Ground coverage follows the fitted slope, and only whole cells within the fitted wall extent are exported. The regressions are in `tests/test_review_debt.py`; they do not validate the remaining clearance policies above. ## 7. Data diff --git a/verification/README.md b/verification/README.md index b20d4031c..3862d5a14 100644 --- a/verification/README.md +++ b/verification/README.md @@ -19,6 +19,14 @@ Reports go to `~/house-scanning-data/reports/`, outside git, because replay fram non-commercial datasets. What each check asserts is in the docstrings of `hsverify/e2e.py`, `resultcheck.py` and `simrun.py`. +When a case's battery width or corner allowance differs from public rules, e2e uses the +value reported in the result's spot or rule-keyed check threshold. If the result omits it, +the report says "assumed public rules; server may override" and marks the case +`assumptions unverified`. It skips the rule-dependent checks and case expectations, while +retaining rule-independent invariants. These records cannot establish acceptance. Exit +code 2 means the run is inconclusive for this reason; exit code 1 still means another +failure or incomplete check. The verifier does not read private backend files. + **Result.** At `t3/server` `3baa338`, with the app's upload from `c820cf1` (#10's freeze, synthetic replay, hosted server), 49 of 50 scenes pass (45 answered and checked; slowest real scene 0.48 s). The ETH3D scene fails: after its one requested capture the server leaves pool unsure with no diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index 939a0d9d3..0c57c3540 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -72,6 +72,8 @@ mirror_scene, more_error_problems, outcome_lengths, + public_assumption_warnings, + reported_rules, schema_errors, tape_to_tap, unobserved_checks, @@ -575,23 +577,44 @@ def judge( if problems: return record | {"status": "fail", "problems": problems[:20], "contract_problems": problems} - # Contract problems hold for any scene; expectation problems depend on one case's geometry. - contract = invariant_problems(item.scene, result, sent=item.raw, rules=rules) - contract += property_problems(url, endpoint, item, result, schemas, rules) + # Unreported overrides cannot establish rule-dependent geometry or case expectations. + mismatches = assumption_mismatches(item.rules_assumed, result) + warnings = public_assumption_warnings(item.rules_assumed, result, rules) + active_rules = reported_rules(result, rules) + contract = invariant_problems( + item.scene, result, sent=item.raw, rules=None if warnings else active_rules + ) + if not warnings: + contract += property_problems(url, endpoint, item, result, schemas, active_rules) + else: + record["rule_checks_unverified"] = True if item.real and ms > LATENCY_BUDGET_MS: contract.append(f"real-derived scene took {ms:.0f} ms, budget {LATENCY_BUDGET_MS:.0f}") - mismatches = assumption_mismatches(item.rules_assumed, result, rules) - expected = expectation_problems(item.expect, result) if item.expect and not mismatches else [] + expected = ( + expectation_problems(item.expect, result) + if item.expect and not mismatches and not warnings + else [] + ) problems = contract + expected record["decision"] = result["decision"] record["outcome_lengths_ft"] = outcome_lengths(result) if mismatches: record["assumption_mismatches"] = mismatches + if warnings: + record["assumption_warnings"] = warnings record["contract_problems"] = contract record["expectation_problems"] = expected record["problems"] = problems - record["status"] = "fail" if problems else "assumption mismatch" if mismatches else "pass" + record["status"] = ( + "fail" + if problems + else "assumption mismatch" + if mismatches + else "assumptions unverified" + if warnings + else "pass" + ) return record @@ -615,7 +638,9 @@ def resend(name: str, scene: dict) -> dict | None: return None problems.extend( f"{name}: {p}" - for p in invariant_problems(scene, other, sent=transformed.raw, rules=rules) + for p in invariant_problems( + scene, other, sent=transformed.raw, rules=reported_rules(other, rules) + ) ) return other @@ -838,7 +863,12 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: contract_problems = sum(len(r.get("contract_problems", [])) for r in records) killed = bool(meta["peak_memory"].get("server_killed")) all_passed = all(r["status"] == "pass" for r in records) and bool(checked) and not killed - contract_ok = bool(checked) and contract_problems == 0 and not killed + contract_ok = ( + bool(checked) + and contract_problems == 0 + and not killed + and not any(r.get("rule_checks_unverified") for r in checked) + ) executed = sorted( {r["name"] for r in records if r.get("hostile") and r.get("hostile_probe_executed") is True} ) @@ -897,7 +927,11 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: "| --- | --- | --- | --- | --- |", ] for r in records: - issues = "; ".join(r.get("problems", []) + r.get("assumption_mismatches", [])) + issues = "; ".join( + r.get("problems", []) + + r.get("assumption_mismatches", []) + + r.get("assumption_warnings", []) + ) issues = issues or r.get("skip_reason", "") lines.append( f"| {r['name']}{' (real)' if r.get('real') else ''} | {r['status']} | " @@ -996,7 +1030,7 @@ def main(argv: list[str] | None = None) -> int: else "" ) ) - for problem in record.get("problems", [])[:6]: + for problem in (record.get("problems", []) + record.get("assumption_warnings", []))[:6]: print(f" - {problem}") meta = { @@ -1016,7 +1050,16 @@ def main(argv: list[str] | None = None) -> int: } report = write_report(out, meta, records) print(f"Report: {out / 'report.md'}") - return 0 if report["all_passed"] else 1 + if report["all_passed"]: + return 0 + # An unreported override is incomplete evidence, not a proven server failure. + if ( + not meta["peak_memory"].get("server_killed") + and all(r["status"] in ("pass", "assumptions unverified") for r in records) + and any(r["status"] == "assumptions unverified" for r in records) + ): + return 2 + return 1 if __name__ == "__main__": diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index acc57cc74..d24120c73 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -30,7 +30,7 @@ import itertools import math from collections.abc import Callable, Iterator -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Any import yaml @@ -1149,28 +1149,57 @@ def with_past_ends_asked(scene: dict, requests: list[dict], rules: RuleSet) -> d # --- Case expectations ----------------------------------------------------------------------- -def assumption_mismatches( - assumed: dict[str, float], result: dict, rules: RuleSet | None = None +def reported_assumption_values(key: str, result: dict) -> list[tuple[str, float]]: + """Rule values carried by C2 spots or check thresholds, without reading server files.""" + values = [] + if key == "battery_width": + for name in ("spot", "nearest_considered"): + spot = result.get(name) + if spot is not None and "width_ft" in spot: + values.append((f"{name}.width_ft", spot["width_ft"])) + rule_key = { + "battery_width": "battery.width_ft", + "corner_allowance": "route.corner_allowance_ft", + }.get(key) + for check in result.get("checks", []): + threshold = check.get("threshold_ft") + matches = check.get("rule", {}).get("key") == rule_key if rule_key else key in check["id"] + if matches and threshold is not None: + values.append((check["id"], threshold)) + return values + + +def reported_rules(result: dict, rules: RuleSet) -> RuleSet: + """Use reported geometry for invariants too, so public defaults cannot reject an override.""" + changes = {} + for key, field in (("battery_width", "width_ft"), ("corner_allowance", "corner_allowance_ft")): + if values := reported_assumption_values(key, result): + changes[field] = values[0][1] + return replace(rules, **changes) + + +def assumption_mismatches(assumed: dict[str, float], result: dict) -> list[str]: + return [ + f"case assumes {key} threshold {value}, server's {source} uses {actual}" + for key, value in assumed.items() + for source, actual in reported_assumption_values(key, result) + if abs(actual - value) > EPS + ] + + +def public_assumption_warnings( + assumed: dict[str, float], result: dict, rules: RuleSet ) -> list[str]: - out = [] - configured = ( - {"battery_width": rules.width_ft, "corner_allowance": rules.corner_allowance_ft} - if rules - else {} - ) - for key, value in assumed.items(): - actual = configured.get(key) - if actual is not None and abs(actual - value) > EPS: - out.append(f"case assumes {key} {value}, rules use {actual}") - for key, value in assumed.items(): - for check in result.get("checks", []): - threshold = check.get("threshold_ft") - if key in check["id"] and threshold is not None and abs(threshold - value) > EPS: - out.append( - f"case assumes {key} threshold {value}, server's {check['id']} uses " - f"{check['threshold_ft']}" - ) - return out + configured = {"battery_width": rules.width_ft, "corner_allowance": rules.corner_allowance_ft} + return [ + f"case assumes {key} {value}, public rules use {configured[key]}; " + "assumed public rules; server may override; result does not report this value, " + "so case expectations are unverified" + for key, value in assumed.items() + if configured.get(key) is not None + and not reported_assumption_values(key, result) + and abs(configured[key] - value) > EPS + ] def expectation_problems(expect: dict, result: dict) -> list[str]: diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index b7dddf81c..b7b9102e5 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -6,6 +6,7 @@ import threading import time import zipfile +from dataclasses import replace from http.server import BaseHTTPRequestHandler, HTTPServer, ThreadingHTTPServer from typing import ClassVar @@ -867,7 +868,10 @@ def test_hostile_execution_is_recorded_only_after_sending(monkeypatch): @pytest.mark.parametrize("no_hostile", [False, True]) -def test_cli_shortened_run_cannot_claim_full_contract(tmp_path, monkeypatch, no_hostile): +@pytest.mark.parametrize("unverified", [False, True]) +def test_cli_shortened_run_cannot_claim_full_contract( + tmp_path, monkeypatch, no_hostile, unverified +): from pathlib import Path # Only public rules are read. All HTTP, git and hostile payload generation are mocked. @@ -894,7 +898,19 @@ def test_cli_shortened_run_cannot_claim_full_contract(tmp_path, monkeypatch, no_ def fake_judge(url, endpoint, probe, *args): if probe.hostile: return next(r for r in hostile_records() if r["name"] == probe.name) - return CHECKED | {"name": probe.name} + return ( + CHECKED + | {"name": probe.name} + | ( + { + "status": "assumptions unverified", + "rule_checks_unverified": True, + "assumption_warnings": ["assumed public rules; server may override"], + } + if unverified + else {} + ) + ) monkeypatch.setattr(e2e, "judge", fake_judge) monkeypatch.setattr(e2e, "peak_rss_mb", lambda: {}) @@ -902,10 +918,11 @@ def fake_judge(url, endpoint, probe, *args): args = ["--server-url", "http://unused", "--cases", str(cases), "--out", str(out)] if no_hostile: args.append("--no-hostile") - assert e2e.main(args) == 0 + assert e2e.main(args) == (2 if unverified else 0) data = json.loads((out / "report.json").read_text()) - assert data["all_passed"] and data["contract_ok"] - assert data["full_contract_ok"] is (not no_hostile) + assert data["all_passed"] is (not unverified) + assert data["contract_ok"] is (not unverified) + assert data["full_contract_ok"] is (not no_hostile and not unverified) assert data["hostile_probes_executed"] == ( [] if no_hostile else sorted(e2e.HOSTILE_PROBE_NAMES) ) @@ -914,4 +931,69 @@ def fake_judge(url, endpoint, probe, *args): metric = next(m for m in scoreboard.load_metrics()[1] if m.id == "S2-8") metric.probe["glob"] = str(out / "report.json") status = scoreboard.probe_report(metric.probe, data["sha"]).status - assert status == ("gap" if no_hostile else "met") + assert status == ("gap" if no_hostile or unverified else "met") + + +@pytest.mark.parametrize("reported_width", [None, 31 / 12, 4]) +@pytest.mark.parametrize("contract_failure", [False, True]) +def test_judge_distinguishes_reported_override_mismatch_and_unverified_defaults( + tmp_path, monkeypatch, reported_width, contract_failure +): + answer = {"decision": "manual_review", "spot": None, "checks": [], "sweep": []} + if reported_width is not None: + answer["checks"] = [ + { + "id": "wall_backing", + "threshold_ft": reported_width, + "rule": {"key": "battery.width_ft"}, + } + ] + monkeypatch.setattr(e2e, "post", lambda *args: (200, json.dumps(answer).encode(), 1)) + monkeypatch.setattr(e2e, "schema_errors", lambda *args: []) + seen = [] + + def invariants(scene, result, sent=None, rules=None): + seen.append(None if rules is None else rules.width_ft) + return ( + ["stats.input_sha256 does not match the scene bytes sent"] if contract_failure else [] + ) + + monkeypatch.setattr(e2e, "invariant_problems", invariants) + properties = [] + monkeypatch.setattr( + e2e, "property_problems", lambda *args: properties.append(args[-1].width_ft) or [] + ) + expectations = [] + monkeypatch.setattr(e2e, "expectation_problems", lambda *args: expectations.append(True) or []) + item = SceneInput( + "override", + SCENE, + b"{}", + expect={"decision_in": ["manual_review"]}, + rules_assumed={"battery_width": 31 / 12}, + ) + record = judge( + "unused", Endpoint("/place", "application/json"), item, SCHEMAS, replace(RULES, width_ft=4) + ) + assert seen == [reported_width] + if contract_failure: + assert record["status"] == "fail" and record["contract_problems"] + assert record["problems"] == ["stats.input_sha256 does not match the scene bytes sent"] + else: + assert record["problems"] == [] + if reported_width is None: + assert record["status"] == ("fail" if contract_failure else "assumptions unverified") + assert "assumed public rules; server may override" in record["assumption_warnings"][0] + assert not properties and not expectations + elif reported_width == 31 / 12: + assert record["status"] == ("fail" if contract_failure else "pass") + assert properties == [31 / 12] and expectations == [True] + else: + assert record["status"] == ("fail" if contract_failure else "assumption mismatch") + assert record["assumption_mismatches"] and not expectations + report = e2e.write_report(tmp_path, META, [record, *hostile_records()]) + assert report["all_passed"] is (reported_width == 31 / 12 and not contract_failure) + assert report["full_contract_ok"] is (reported_width == 31 / 12 and not contract_failure) + if reported_width is None: + assert not report["contract_ok"] + assert "assumed public rules; server may override" in (tmp_path / "report.md").read_text() diff --git a/verification/tests/test_generate_cases.py b/verification/tests/test_generate_cases.py index a37800816..4e7a4b4fb 100644 --- a/verification/tests/test_generate_cases.py +++ b/verification/tests/test_generate_cases.py @@ -101,7 +101,10 @@ def forbidden(*args): {"scene": {}, "result": {}}, rules, ) - assert record["status"] == "assumption mismatch" and record["assumption_mismatches"] + assert record["status"] == "assumptions unverified" and record["assumption_warnings"] + assert "assumed public rules; server may override" in record["assumption_warnings"][0] + assert record["rule_checks_unverified"] + assert record["contract_problems"] == [] assert record["expectation_problems"] == [] meta = { "peak_memory": {}, diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 11826df86..2a16bc044 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -29,7 +29,9 @@ observed, outcome_at, outcome_lengths, + public_assumption_warnings, reach_gaps, + reported_rules, run_starts, straight_pieces, tape_to_tap, @@ -1558,17 +1560,89 @@ def test_ground_trimming_keeps_omitted_reach_at_zero(): assert trimmed["coverage"]["observed"][0]["out_ft"] == 0 -def test_geometric_assumptions_compare_active_rules(): - rules = replace(RULES, corner_allowance_ft=0.5) +def test_geometric_assumptions_use_reported_override_instead_of_public_defaults(): + public = replace(RULES, width_ft=4, corner_allowance_ft=10) + answer = result() + answer["checks"].append( + { + "id": "corner_allowance", + "threshold_ft": 0.5, + "rule": {"key": "route.corner_allowance_ft"}, + } + ) + assumed = {"battery_width": RULES.width_ft, "corner_allowance": 0.5} + assert assumption_mismatches(assumed, answer) == [] + assert public_assumption_warnings(assumed, answer, public) == [] + active = reported_rules(answer, public) + assert (active.width_ft, active.corner_allowance_ft) == (RULES.width_ft, 0.5) + assert (public.width_ft, public.corner_allowance_ft) == (4, 10) + + +def test_geometric_assumptions_detect_a_real_reported_mismatch(): + answer = { + "spot": {"width_ft": 4}, + "checks": [ + { + "id": "route_corner", + "threshold_ft": 10, + "rule": {"key": "route.corner_allowance_ft"}, + } + ], + } assert ( - len(assumption_mismatches({"battery_width": 100, "corner_allowance": 100}, {}, rules)) == 2 + len( + assumption_mismatches( + {"battery_width": RULES.width_ft, "corner_allowance": 0.5}, answer + ) + ) + == 2 ) + + +def test_unreported_geometric_override_is_a_warning_not_a_mismatch(): + public = replace(RULES, width_ft=4, corner_allowance_ft=10) + assumed = {"battery_width": RULES.width_ft, "corner_allowance": 0.5} + assert assumption_mismatches(assumed, {}) == [] + warnings = public_assumption_warnings(assumed, {}, public) + assert len(warnings) == 2 + assert all("assumed public rules; server may override" in text for text in warnings) assert ( - assumption_mismatches({"battery_width": rules.width_ft, "corner_allowance": 0.5}, {}, rules) - == [] + public_assumption_warnings({"battery_width": 4, "corner_allowance": 10}, {}, public) == [] ) +@pytest.mark.parametrize("source", ["spot", "nearest_considered", "check"]) +def test_reported_width_sources_override_public_width(source): + answer = ( + { + "checks": [ + { + "id": "wall_backing", + "threshold_ft": RULES.width_ft, + "rule": {"key": "battery.width_ft"}, + } + ] + } + if source == "check" + else {source: {"width_ft": RULES.width_ft}} + ) + assert reported_rules(answer, replace(RULES, width_ft=4)).width_ft == RULES.width_ft + assert assumption_mismatches({"battery_width": RULES.width_ft}, answer) == [] + + +def test_other_width_thresholds_do_not_report_battery_width(): + answer = { + "checks": [ + { + "id": "meter_working_space", + "threshold_ft": 4, + "rule": {"key": "meter_working_space.width_ft"}, + } + ] + } + assert reported_rules(answer, RULES).width_ft == RULES.width_ft + + @pytest.mark.parametrize("outcome", ["pass", "unsure"]) @pytest.mark.parametrize("span", [[0, 1e9], [-1e9, 0], [1e9, 1e9]]) def test_out_of_scan_sweep_fails_before_sampling(monkeypatch, outcome, span): @@ -1623,3 +1697,13 @@ def test_in_scan_sweep_with_tiny_rule_step_fails_loudly(): assert invariant_problems(SCENE, r, rules=replace(RULES, step_ft=1e-300)) == [ "sweep unsure starts [0, 1]: sweep sampling exceeds the 10000 sample limit" ] + + +def test_reported_override_prevents_a_false_footprint_failure(): + public = replace(RULES, width_ft=4) + answer = result(spot=False, sweep=[pass_run(9, 9) | {"outcome": "unsure"}]) + backing = next(c for c in answer["checks"] if c["id"] == "wall_backing") + backing["threshold_ft"] = RULES.width_ft + backing["rule"]["key"] = "battery.width_ft" + assert footprint_problems(SCENE, answer, public) + assert footprint_problems(SCENE, answer, reported_rules(answer, public)) == [] From 851814da763584f9c6b0309735dcf669ded024e2 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Wed, 14 Oct 2026 12:39:05 -0500 Subject: [PATCH 49/62] fix(pipeline): bound response geometry and preserve capture caches --- recon/recon/capture.py | 95 ++++++-- recon/recon/coverage.py | 8 +- recon/recon/pipeline.py | 3 + recon/tests/test_vet_pipeline.py | 108 +++++++++ verification/hsverify/bundle_limits.py | 61 +++++ verification/hsverify/e2e.py | 50 +++- verification/hsverify/resultcheck.py | 293 +++++++++++++++++++++-- verification/hsverify/simrun.py | 16 +- verification/tests/test_bundle_limits.py | 94 ++++++++ verification/tests/test_e2e.py | 82 +++++++ verification/tests/test_resultcheck.py | 121 +++++++++- verification/tests/test_simrun.py | 19 ++ 12 files changed, 886 insertions(+), 64 deletions(-) create mode 100644 recon/tests/test_vet_pipeline.py create mode 100644 verification/hsverify/bundle_limits.py create mode 100644 verification/tests/test_bundle_limits.py diff --git a/recon/recon/capture.py b/recon/recon/capture.py index 378a32320..e0733cd7c 100644 --- a/recon/recon/capture.py +++ b/recon/recon/capture.py @@ -23,6 +23,7 @@ from __future__ import annotations +import fcntl import hashlib import json import re @@ -30,6 +31,7 @@ import struct import tempfile import zipfile +import zlib from dataclasses import dataclass, field from pathlib import Path @@ -122,32 +124,78 @@ def outward_of(along: np.ndarray) -> np.ndarray: return np.array([-along[2], 0.0, along[0]]) -def unpack(path: Path, work: Path) -> Path: - """A folder as is; a zip extracted once under `work`, keyed by its sha256.""" +def bundle_root(path: Path, work: Path) -> Path: + """The capture folder, including a ZIP's cache path before extraction or cleanup.""" if path.is_dir(): return path if path.stat().st_size > MAX_BUNDLE_BYTES: raise ValueError(f"{path}: archive exceeds {MAX_BUNDLE_BYTES} bytes") with path.open("rb") as f: digest = hashlib.file_digest(f, "sha256").hexdigest()[:16] - out = work / "bundles" / digest - if not out.exists(): - _zip_directory_limits(path) - with zipfile.ZipFile(path) as z: - infos = z.infolist() - if len(infos) > MAX_BUNDLE_ENTRIES: - raise ValueError(f"{path}: {len(infos)} entries, limit {MAX_BUNDLE_ENTRIES}") - if sum(i.file_size for i in infos) > MAX_BUNDLE_BYTES: - raise ValueError(f"{path}: expanded archive exceeds {MAX_BUNDLE_BYTES} bytes") - members = [i for i in infos if not i.filename.startswith("__MACOSX/")] - for info in members: - n = info.filename - if n.startswith("/") or ".." in Path(n).parts: - raise ValueError(f"{path}: unsafe entry {n!r}") - out.parent.mkdir(parents=True, exist_ok=True) + return work / "bundles" / digest + + +def _bundle_intact(root: Path, members: list[zipfile.ZipInfo]) -> bool: + """Check the extracted file set and each file's size and CRC against the archive.""" + if root.is_symlink() or not root.is_dir(): + return False + files = {Path(i.filename): i for i in members if not i.is_dir()} + directories = {Path(i.filename) for i in members if i.is_dir()} + for name in files.keys() | directories: + directories.update(p for p in name.parents if p != Path(".")) + expected = files.keys() | directories + try: + for entry in root.rglob("*"): + name = entry.relative_to(root) + if entry.is_symlink() or name not in expected: + return False + if any(not (root / name).is_dir() for name in directories): + return False + for name, info in files.items(): + entry = root / name + if not entry.is_file() or entry.stat().st_size != info.file_size: + return False + crc, size = 0, 0 + with entry.open("rb") as source: + while chunk := source.read(2**20): + size += len(chunk) + if size > info.file_size: + return False + crc = zlib.crc32(chunk, crc) + if size != info.file_size or crc != info.CRC: + return False + except OSError: + return False + return True + + +def unpack(path: Path, work: Path) -> Path: + """A folder as is; a ZIP cache reused only while its extracted members remain intact.""" + out = bundle_root(path, work) + if path.is_dir(): + return out + _zip_directory_limits(path) + with zipfile.ZipFile(path) as z: + infos = z.infolist() + if len(infos) > MAX_BUNDLE_ENTRIES: + raise ValueError(f"{path}: {len(infos)} entries, limit {MAX_BUNDLE_ENTRIES}") + if sum(i.file_size for i in infos) > MAX_BUNDLE_BYTES: + raise ValueError(f"{path}: expanded archive exceeds {MAX_BUNDLE_BYTES} bytes") + members = [i for i in infos if not i.filename.startswith("__MACOSX/")] + for info in members: + n = info.filename + if n.startswith("/") or ".." in Path(n).parts: + raise ValueError(f"{path}: unsafe entry {n!r}") + out.parent.mkdir(parents=True, exist_ok=True) + # Cache verification and replacement share a lock so one repair cannot delete another + # run's newly published extraction. The lock stays outside the replaceable directory. + with (work / f".bundle-{out.name}.lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + if _bundle_intact(out, members): + return out # Extraction stays private until every CRC-checked member finishes. Failed extraction # cannot leave a partial cache directory that a later run mistakes for a complete one. - stage = Path(tempfile.mkdtemp(dir=out.parent, prefix=f".{digest}-")) + stage = Path(tempfile.mkdtemp(dir=out.parent, prefix=f".{out.name}-")) try: written = 0 for info in members: @@ -162,12 +210,11 @@ def unpack(path: Path, work: Path) -> Path: if written > MAX_BUNDLE_BYTES: raise ValueError(f"{path}: expanded archive exceeds byte limit") dst.write(chunk) - try: - stage.rename(out) - except OSError: - # Another run of this exact content may have finished first. - if not out.is_dir(): - raise + if out.is_symlink() or out.is_file(): + out.unlink() + elif out.exists(): + shutil.rmtree(out) + stage.rename(out) finally: if stage.exists(): shutil.rmtree(stage) diff --git a/recon/recon/coverage.py b/recon/recon/coverage.py index a0a1eaa59..d40f9d1c8 100644 --- a/recon/recon/coverage.py +++ b/recon/recon/coverage.py @@ -39,7 +39,7 @@ GROUND_MAX_M = 3.048 # 10 ft out FACING_MAX_M = 6.0 FACING_BAND_M = (0.3, 1.8) -OVERHEAD_OUT_M = (0.1, 0.56) # above the battery's footprint (1.83 ft deep) +OVERHEAD_OUT_M = (0.0, 0.56) # the whole existing 1.83 ft footprint, from the first free wall step OVERHEAD_MAX_M = 3.0 MAX_ANGLE = np.radians(65) # the app's limit on how oblique a view may be IMAGE_MARGIN = 0.03 @@ -233,7 +233,11 @@ def free_space(wall: WallFrame, vol: Volume, cells: np.ndarray): if not seen: break facing_clear[c] = out - front - oo = front + np.arange(OVERHEAD_OUT_M[0], OVERHEAD_OUT_M[1] + 1e-9, step) + # Include both footprint edges. Skipping the first 10 cm missed a 1.90 m eave in the + # full-depth regression, while a voxel step alone can end short of the outer edge. + oo = front + np.append( + np.arange(OVERHEAD_OUT_M[0], OVERHEAD_OUT_M[1], step), OVERHEAD_OUT_M[1] + ) for h in np.arange(0.3, OVERHEAD_MAX_M, step): share, seen = _slab(wall, vol, across, [h], oo) if share > 0: diff --git a/recon/recon/pipeline.py b/recon/recon/pipeline.py index 2c0b5ded1..c27829686 100644 --- a/recon/recon/pipeline.py +++ b/recon/recon/pipeline.py @@ -138,6 +138,9 @@ def run( overlap = bundle_path.is_relative_to(out_path) or out_path.is_relative_to(bundle_path) else: overlap = bundle_path in {(out_path / name).resolve() for name in owned} + if not overlap and bundle.is_file(): + capture_root = cap.bundle_root(bundle, work).resolve() + overlap = capture_root.is_relative_to(out_path) or out_path.is_relative_to(capture_root) if overlap: raise ValueError( f"output {out} overlaps capture input {bundle}; use a separate output folder" diff --git a/recon/tests/test_vet_pipeline.py b/recon/tests/test_vet_pipeline.py new file mode 100644 index 000000000..ab6fa05f0 --- /dev/null +++ b/recon/tests/test_vet_pipeline.py @@ -0,0 +1,108 @@ +"""Input preservation and full-depth footprint checks for the reconstruction worker.""" + +import zipfile + +import numpy as np +import pytest +from test_worker import WALL + +from recon import capture as cap +from recon import coverage, pipeline + + +def _bundle(tmp_path): + path = tmp_path / "bundle.zip" + with zipfile.ZipFile(path, "w") as archive: + archive.writestr("scene.json", b"capture input") + archive.writestr("photos/frame.jpg", b"image bytes") + work = tmp_path / "work" + return path, work, cap.unpack(path, work) + + +@pytest.mark.parametrize("relation", ["same", "ancestor", "descendant"]) +def test_zip_output_overlap_is_refused_before_cleanup(tmp_path, monkeypatch, relation): + bundle, work, root = _bundle(tmp_path) + out = {"same": root, "ancestor": root.parent, "descendant": root / "outputs"}[relation] + out.mkdir(exist_ok=True) + sentinel = out / "model.glb" + sentinel.write_bytes(b"previous output") + + def load(*_): + pytest.fail("capture load reached before overlap rejection") + + monkeypatch.setattr(pipeline.cap, "load", load) + with pytest.raises(ValueError, match="overlaps capture input"): + pipeline.run(bundle, out, work, "auto", None, False) + assert (root / "scene.json").read_bytes() == b"capture input" + assert sentinel.read_bytes() == b"previous output" + + +@pytest.mark.parametrize("damage", ["missing", "truncated", "changed", "symlink", "extra"]) +def test_damaged_extracted_bundle_is_rebuilt(tmp_path, damage): + bundle, work, root = _bundle(tmp_path) + member = root / "photos/frame.jpg" + if damage == "missing": + member.unlink() + elif damage == "truncated": + member.write_bytes(b"image") + elif damage == "changed": + member.write_bytes(b"other bytes") + elif damage == "symlink": + outside = tmp_path / "outside.jpg" + outside.write_bytes(b"image bytes") + member.unlink() + member.symlink_to(outside) + else: + (root / "unexpected.json").write_bytes(b"foreign metadata") + assert cap.unpack(bundle, work) == root + assert member.is_file() and not member.is_symlink() + assert member.read_bytes() == b"image bytes" + assert not (root / "unexpected.json").exists() + if damage == "symlink": + assert outside.read_bytes() == b"image bytes" + + +def test_intact_extracted_bundle_is_verified_without_reextracting(tmp_path, monkeypatch): + bundle, work, root = _bundle(tmp_path) + original = (root / "scene.json").stat().st_mtime_ns + monkeypatch.setattr( + zipfile.ZipFile, "open", lambda *_: pytest.fail("re-extracted intact cache") + ) + assert cap.unpack(bundle, work) == root + assert (root / "scene.json").stat().st_mtime_ns == original + + +class _FootprintVolume: + voxel = 0.05 + + def __init__(self, band, obstruction): + self.band = band + self.obstruction = obstruction + + def occupied(self, points): + low, high = self.band + return ( + self.obstruction + & (points[:, 2] >= low) + & (points[:, 2] <= high) + & (points[:, 1] >= 1.9 - 1e-9) + ) + + def observed(self, points): + low, high = self.band + missing = (points[:, 2] >= low) & (points[:, 2] <= high) & (points[:, 1] >= 1.9 - 1e-9) + return np.ones(len(points), bool) if self.obstruction else ~missing + + +@pytest.mark.parametrize("band", [(0.05, 0.10), (0.609, 0.611)], ids=["near_wall", "far_edge"]) +@pytest.mark.parametrize("obstruction", [True, False], ids=["eave", "unobserved"]) +def test_overhead_checks_full_footprint_depth(band, obstruction): + # The first free step is 0.05 m out, so the 0.56 m footprint ends at z=0.61 m. + _, _, gap, clear = coverage.free_space( + WALL, _FootprintVolume(band, obstruction), np.array([0.0]) + ) + assert clear[0] < 1.9 + if obstruction: + assert gap[0] == pytest.approx(1.85) + else: + assert np.isnan(gap[0]) diff --git a/verification/hsverify/bundle_limits.py b/verification/hsverify/bundle_limits.py new file mode 100644 index 000000000..6f4eb3885 --- /dev/null +++ b/verification/hsverify/bundle_limits.py @@ -0,0 +1,61 @@ +"""Bound ZIP metadata before the verification harness constructs a ZipFile.""" + +import struct +from pathlib import Path + + +def zip_directory_limits(path: Path, max_entries: int, max_bytes: int) -> None: + """Check central-directory records in bounded reads before ZipFile allocates their metadata. + ZIP64 and split archives exceed this worker's ordinary ZIP admission format and are refused. + The record layouts are PKWARE APPNOTE sections 4.3.12 and 4.3.16.""" + with path.open("rb") as f: + size = path.stat().st_size + f.seek(max(0, size - 22 - 65535)) + tail = f.read(22 + 65535) + # Match ZipFile's footer selection: a no-comment footer at EOF, otherwise the last + # signature. Backtracking past malformed signatures can validate a different directory + # from the one ZipFile later allocates. Refuse that ambiguity before constructing it. + if len(tail) >= 22 and tail[-22:-18] == b"PK\x05\x06" and tail[-2:] == b"\0\0": + pos = len(tail) - 22 + else: + pos = tail.rfind(b"PK\x05\x06") + if pos < 0: + raise ValueError(f"{path}: missing ZIP end record") + if pos + 22 > len(tail): + raise ValueError(f"{path}: truncated ZIP end record") + record = struct.unpack("<4s4H2IH", tail[pos : pos + 22]) + if pos + 22 + record[-1] != len(tail): + raise ValueError(f"{path}: ambiguous ZIP end record or invalid comment length") + _, disk, central_disk, disk_entries, total_entries, length, offset, _ = record + if disk or central_disk or disk_entries != total_entries: + raise ValueError(f"{path}: split ZIP archives are not supported") + if total_entries > max_entries: + raise ValueError(f"{path}: {total_entries} entries, limit {max_entries}") + directory_end = offset + length + end_offset = size - len(tail) + pos + # ZipFile checks this locator even when the ordinary footer has no ZIP64 sentinels. + # It must not replace the directory fields after the bounded scan below. + if end_offset >= 20: + f.seek(end_offset - 20) + if f.read(4) == b"PK\x06\x07": + raise ValueError(f"{path}: ZIP64 archive metadata is not supported") + if length == 0xFFFFFFFF or offset == 0xFFFFFFFF or directory_end != end_offset: + raise ValueError(f"{path}: ZIP64 or inconsistent ZIP directory") + f.seek(offset) + count, expanded = 0, 0 + while f.tell() < directory_end: + header = f.read(46) + if len(header) != 46 or header[:4] != b"PK\x01\x02": + raise ValueError(f"{path}: invalid ZIP central-directory record") + fields = struct.unpack("<4s6H3I5H2I", header) + count += 1 + expanded += fields[9] + if count > max_entries: + raise ValueError(f"{path}: {count} entries, limit {max_entries}") + if expanded > max_bytes: + raise ValueError(f"{path}: expanded archive exceeds {max_bytes} bytes") + f.seek(sum(fields[10:13]), 1) + if f.tell() > directory_end: + raise ValueError(f"{path}: ZIP central-directory record exceeds directory") + if count != total_entries: + raise ValueError(f"{path}: inconsistent ZIP entry count") diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index 6ce9cf93b..aa4194756 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -36,6 +36,7 @@ import contextlib import datetime as dt import hashlib +import http.client import io import json import os @@ -55,6 +56,7 @@ from pathlib import Path from hsverify import gitref, memory +from hsverify.bundle_limits import zip_directory_limits from hsverify.memory import peak_rss_mb from hsverify.resultcheck import ( RuleSet, @@ -144,13 +146,20 @@ def read_bounded(bundle: zipfile.ZipFile, info: zipfile.ZipInfo, limit: int) -> def read_bundle(path: Path) -> tuple[bytes, dict[str, bytes]]: """scene.json and the JPEGs of a C1 zip bundle, within the limits above.""" + if path.stat().st_size > MAX_BUNDLE_BYTES: + raise SystemExit(f"{path}: archive exceeds {MAX_BUNDLE_BYTES} bytes") + try: + zip_directory_limits(path, MAX_BUNDLE_ENTRIES, MAX_BUNDLE_BYTES) + except ValueError as exc: + raise SystemExit(str(exc)) from exc with zipfile.ZipFile(path) as bundle: - infos = [i for i in bundle.infolist() if not i.is_dir()] + infos = bundle.infolist() if len(infos) > MAX_BUNDLE_ENTRIES: raise SystemExit(f"{path}: {len(infos)} entries, limit {MAX_BUNDLE_ENTRIES}") declared = sum(i.file_size for i in infos) if declared > MAX_BUNDLE_BYTES: raise SystemExit(f"{path}: declares {declared} bytes, limit {MAX_BUNDLE_BYTES}") + infos = [i for i in infos if not i.is_dir()] scenes = [i for i in infos if i.filename == "scene.json"] if len(scenes) != 1: raise SystemExit( @@ -357,6 +366,27 @@ def bundle_zip(item: SceneInput) -> bytes: return buf.getvalue() +# Fixed harness admission limit to bound answers from a server under test. This is not a +# measured maximum result size; it matches the 4 MiB transport policy in PR #242. +MAX_RESPONSE_BYTES = 4 * 2**20 + + +def read_response(response) -> bytes: + declared = response.headers.get("Content-Length") + if declared is not None: + declared = int(declared) + if declared < 0: + raise ValueError("negative response Content-Length") + if declared > MAX_RESPONSE_BYTES: + raise ValueError(f"response exceeds {MAX_RESPONSE_BYTES} bytes") + payload = response.read(MAX_RESPONSE_BYTES + 1) + if len(payload) > MAX_RESPONSE_BYTES: + raise ValueError(f"response exceeds {MAX_RESPONSE_BYTES} bytes") + if declared is not None and len(payload) < declared: + raise ValueError(f"truncated response: {len(payload)} bytes, expected {declared}") + return payload + + def post( url: str, endpoint: Endpoint, item: SceneInput, timeout_s: float = 60 ) -> tuple[int | None, bytes, float]: @@ -381,11 +411,19 @@ def post( ) start = time.perf_counter() try: - with urllib.request.urlopen(request, timeout=timeout_s) as response: - status, payload = response.status, response.read() - except urllib.error.HTTPError as exc: - status, payload = exc.code, exc.read() - except (urllib.error.URLError, TimeoutError, ConnectionError) as exc: + try: + response = urllib.request.urlopen(request, timeout=timeout_s) + except urllib.error.HTTPError as exc: + response = exc + with response: + status, payload = response.status, read_response(response) + except ( + urllib.error.URLError, + TimeoutError, + ConnectionError, + http.client.HTTPException, + ValueError, + ) as exc: status, payload = None, str(exc).encode() return status, payload, (time.perf_counter() - start) * 1000 diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 2ea978f2a..a7a1eca77 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -28,9 +28,10 @@ import copy import hashlib import itertools +import json import math from collections.abc import Callable, Iterator -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Any import yaml @@ -40,6 +41,11 @@ NUMBER_TOL_FT = 1e-5 # Bound verifier work even when a server span or rules step requests billions of samples. MAX_SWEEP_SAMPLES = 10_000 +# The existing public-pool regression has 55 starts and a worst-case ground +# reach of 14.15 ft: its conservative grid bound is below 50 million visits. +# This cap retains that case while bounding aggregate work; it is a resource +# policy, not a measured runtime or a physical scan limit. +MAX_GEOMETRY_SAMPLES = 50_000_000 # Spot centres and offsets are rounded by the server; a hundredth of a foot is 1/8 inch. OFFSET_TOL_FT = 0.01 @@ -308,8 +314,9 @@ def margin_problem(check: dict) -> str | None: def invariant_problems( scene: dict, result: dict, sent: bytes | None = None, rules: RuleSet | None = None ) -> list[str]: - if rules is not None and (sampling := sweep_sampling_problems(scene, result, rules)): + if sampling := geometry_admission_problems(scene, result, rules): return sampling + result = deduplicate_result_runs(result) problems: list[str] = [] decision = result["decision"] spot = result.get("spot") @@ -423,8 +430,9 @@ def footprint_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: """A battery that passes or is left to a person backs onto one straight stretch of scanned wall along its whole width; a footprint past the stretch's end (a corner, the end of the wall, a gap) can't sit flush, so the server fails it (server/solver.py check_backing).""" - if sampling := sweep_sampling_problems(scene, result, rules): + if sampling := geometry_admission_problems(scene, result, rules): return sampling + result = deduplicate_result_runs(result) pieces = straight_pieces(scene, rules) if not pieces: return [] @@ -679,7 +687,7 @@ def measured_band_gap( return None -def run_starts(span: list[float], step_ft: float | None) -> Iterator[float]: +def run_start_count(span: list[float], step_ft: float | None) -> int: """Validate eagerly, then sample a run lazily: first, every step after it, and last. A sample, not the server's own list: the server also evaluates boundaries and midpoints @@ -697,25 +705,240 @@ def run_starts(span: list[float], step_ft: float | None) -> Iterator[float]: count = max(int(a < b), math.ceil(steps - EPS)) if count + 1 > MAX_SWEEP_SAMPLES: raise ValueError(f"sweep sampling exceeds the {MAX_SWEEP_SAMPLES} sample limit") - return itertools.chain((a + k * step_ft for k in range(count)), (b,)) + return count + 1 -def sweep_sampling_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: - """Reject invalid runs before footprint, coverage or height checks sample them.""" - ends = chain_ends_s(scene, rules) - problems = [] +def run_starts(span: list[float], step_ft: float | None) -> Iterator[float]: + count = run_start_count(span, step_ft) + a, b = span + return itertools.chain((a + k * step_ft for k in range(count - 1)), (b,)) + + +def deduplicate_result_runs(result: dict) -> dict: + """Identical records cannot add geometry evidence or multiply sampling work.""" + unique = {} for run in result.get("sweep", []): - if run["outcome"] not in ("pass", "unsure"): + unique.setdefault(json.dumps(run, sort_keys=True), run) + return result | {"sweep": list(unique.values())} + + +def reported_dimension_problems(result: dict, rules: RuleSet | None) -> list[str]: + """Reject inconsistent C2 geometry before its dimensions can override public rules.""" + problems = [] + + def finite(value: Any) -> bool: + try: + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + ) + except OverflowError: + return False + + spots = [(name, result.get(name)) for name in ("spot", "nearest_considered")] + for name, spot in spots: + if spot is None: + continue + for field in ("width_ft", "depth_ft", "height_ft"): + if field in spot and (not finite(spot[field]) or spot[field] <= 0): + problems.append(f"{name}.{field} must be finite and positive") + span = spot.get("span_ft") + if span is not None: + if len(span) != 2 or not all(finite(v) for v in span) or span[0] >= span[1]: + problems.append(f"{name}.span_ft must be finite and increasing") + elif ( + finite(spot.get("width_ft")) + and abs(span[1] - span[0] - spot["width_ft"]) > OFFSET_TOL_FT + ): + problems.append(f"{name}.width_ft does not match span_ft") + corners = spot.get("footprint") + if corners is None: continue + if len(corners) != 4 or any( + len(point) != 2 or not all(finite(v) for v in point) for point in corners + ): + problems.append(f"{name}.footprint must have four finite plan corners") + continue + # C2 orders back-left, back-right, front-right, front-left. Use the verifier's existing + # hundredth-foot position tolerance when comparing its rounded plan edges and spans. + for field, edges in (("width_ft", ((0, 1), (2, 3))), ("depth_ft", ((1, 2), (3, 0)))): + if finite(spot.get(field)) and any( + abs(math.dist(corners[a], corners[b]) - spot[field]) > OFFSET_TOL_FT + for a, b in edges + ): + problems.append(f"{name}.{field} does not match footprint") + + widths = [ + (f"{name}.width_ft", spot["width_ft"]) + for name, spot in spots + if spot is not None and "width_ft" in spot + ] + widths += [ + (c["id"], c["threshold_ft"]) + for c in result.get("checks", []) + if c.get("rule", {}).get("key") == "battery.width_ft" and c.get("threshold_ft") is not None + ] + for source, width in widths: + if not finite(width) or width <= 0: + message = f"{source} must be finite and positive" + if message not in problems: + problems.append(message) + valid_widths = [(source, width) for source, width in widths if finite(width) and width > 0] + if valid_widths and any( + abs(width - valid_widths[0][1]) > NUMBER_TOL_FT for _, width in valid_widths + ): + problems.append("reported battery widths disagree") + first, second = (spot for _, spot in spots) + if first is not None and second is not None: + for field in ("depth_ft", "height_ft"): + if ( + finite(first.get(field)) + and finite(second.get(field)) + and abs(first[field] - second[field]) > NUMBER_TOL_FT + ): + problems.append(f"spot and nearest_considered {field} disagree") + return problems + + +def geometry_admission_problems(scene: dict, result: dict, rules: RuleSet | None) -> list[str]: + """Bound all sampling before any footprint, coverage or height check runs. + + Count sweep starts and a conservative upper bound for ground grid point visits, + including split-band edges. Bounds use whole runs, never enumerate their starts. + """ + if problems := reported_dimension_problems(result, rules): + return problems + # Use validated reported dimensions only to estimate work, before the caller adopts them. + widths = [ + c["threshold_ft"] + for c in result.get("checks", []) + if c.get("rule", {}).get("key") == "battery.width_ft" and c.get("threshold_ft") is not None + ] + widths += [ + s["width_ft"] + for key in ("spot", "nearest_considered") + if (s := result.get(key)) is not None and "width_ft" in s + ] + depths = [ + s["depth_ft"] + for key in ("spot", "nearest_considered") + if (s := result.get(key)) is not None and "depth_ft" in s + ] + if rules is not None: + rules = replace( + rules, + width_ft=max(widths, default=rules.width_ft), + depth_ft=max([rules.depth_ft, *depths]), + ) + problems = [] + bodies = [] + ends = chain_ends_s(scene, rules) if rules is not None else None + for name in ("spot", "nearest_considered"): + if (spot := result.get(name)) is not None: + bodies.append((name, spot["span_ft"], spot["wall_id"], 1, True)) + for run in deduplicate_result_runs(result)["sweep"]: a, b = run["start_ft"] prefix = f"sweep {run['outcome']} starts {run['start_ft']}: " - if ends is not None and (a < ends[0] - COLLINEAR_FT or b > ends[1] + COLLINEAR_FT): - problems.append(prefix + f"start_ft span exceeds scanned range [{ends[0]}, {ends[1]}]") + if not math.isfinite(a) or not math.isfinite(b) or a > b: + problems.append(prefix + "start_ft must be finite and ordered") continue - try: - run_starts(run["start_ft"], rules.step_ft) - except ValueError as exc: - problems.append(prefix + str(exc)) + if run["outcome"] in ("pass", "unsure") and rules is not None: + if ends is not None and (a < ends[0] - COLLINEAR_FT or b > ends[1] + COLLINEAR_FT): + problems.append( + prefix + f"start_ft span exceeds scanned range [{ends[0]}, {ends[1]}]" + ) + continue + try: + count = run_start_count(run["start_ft"], rules.step_ft) + except ValueError as exc: + problems.append(prefix + str(exc)) + continue + bodies.append( + (prefix, [a, b + rules.width_ft], run["wall_id"], count, run["outcome"] == "pass") + ) + for name, span, _, _, _ in bodies: + if any(not math.isfinite(v) for v in span) or span[0] > span[1]: + problems.append(f"{name}: span_ft must be finite and ordered") + if problems or rules is None: + return problems + total = 0 + + def grid_count(length: float) -> int: + cells = length / GroundPlan.STEP_FT + if not math.isfinite(cells) or cells > MAX_GEOMETRY_SAMPLES: + return MAX_GEOMETRY_SAMPLES + 1 + return 2 * max(1, math.ceil(max(0, cells))) + 1 + + for name, (lo, hi), wid, count, ground in bodies: + # Sweep ends name starts; footprints may extend beyond the chain and are checked below. + start_hi = hi - rules.width_ft if name.startswith("sweep") else hi + if ends is not None and (lo < ends[0] - COLLINEAR_FT or start_hi > ends[1] + COLLINEAR_FT): + label = "start_ft span" if name.startswith("sweep") else "span_ft" + problems.append( + f"{name}{'' if name.endswith(': ') else ': '}" + f"{label} exceeds scanned range [{ends[0]}, {ends[1]}]" + ) + continue + total += count * (2 + (len(rules.needs) if ground else 0)) + if ground and rules.wall_join_ft is not None: + segments = segments_s(scene, rules) or [] + # Error changes at wall boundaries. Evaluate both sides and endpoints, not the grid. + width = rules.width_ft if name.startswith("sweep") else hi - lo + starts = {lo, max(lo, hi - width)} + boundaries = {edge for _, a, b, _, _ in segments for edge in (a, b)} + boundaries.update( + (left[2] + right[1]) / 2 for left, right in itertools.pairwise(segments) + ) + for edge_at in boundaries: + for edge in (edge_at, edge_at - width, edge_at - width / 2): + for offset in (-EPS, EPS): + if lo <= edge + offset <= hi - width: + starts.add(edge + offset) + for check, needs in rules.needs.items(): + for need in needs: + if need.band != "ground": + continue + error = ( + 0.0 + if not need.widen + else max( + (along_error if need.along else battery_error)( + scene, rules, wid, x, x + width + ) + for x in starts + ) + ) + radius = need.radius + error + splits = 4 + 2 * sum( + e["band"] == "ground" for e in scene.get("coverage", {}).get("observed", []) + ) + along = grid_count(width + 2 * radius) + 3 * splits + out = grid_count(rules.depth_ft + radius) + # Spot coverage and missing-evidence checks can both visit the grid. + repeats = ( + 1 + if name.startswith("sweep") + else 1 + + sum( + c["id"] == check and c["outcome"] == "pass" + for c in result.get("checks", []) + ) + ) + total += count * along * out * repeats + if total > MAX_GEOMETRY_SAMPLES: + problems.append( + f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples" + ) + break + # A check-only reported width is still untrusted geometry, even without a chosen spot/run. + for width in [*widths, *depths]: + if not math.isfinite(width) or width <= 0: + problems.append("reported battery width must be finite and positive") + break + total += grid_count(width) + if total > MAX_GEOMETRY_SAMPLES and not any("sample budget" in p for p in problems): + problems.append(f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples") return problems @@ -750,8 +973,9 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: check's area around that start's battery with that battery's own position error, and the wall and cable route back to the meter over the whole run. Checking the whole run as one battery at its farthest start's error would ask for more than any single start needs.""" - if sampling := sweep_sampling_problems(scene, result, rules): + if sampling := geometry_admission_problems(scene, result, rules): return sampling + result = deduplicate_result_runs(result) problems: list[str] = [] route_wall = observed( scene, "wall", rules.route_height_ft, beyond=True, wall_default_ft=rules.headroom_ft @@ -870,6 +1094,9 @@ def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = the reason and its requests; leaving both out would never ask the homeowner for the view that settles the check. Without a spot and without the reason, every position fails anyway, no photo would change the answer, and C2 lets missing_evidence stay empty.""" + if sampling := geometry_admission_problems(scene, result, rules): + return sampling + result = deduplicate_result_runs(result) codes = {r["code"] for r in result.get("reasons", [])} spot = result.get("spot") unseen = unobserved_checks(result) @@ -1054,19 +1281,34 @@ def covers( return False for s in self._samples(p, q): # Up to out_lo the stretch's own band already saw the ground. - deeper = [o for o in self._samples(out_lo, need(s)) if o > out_lo + SERVER_EPS] - for out in deeper: + for out in self._samples(out_lo, need(s)): + if out <= out_lo + SERVER_EPS: + continue x = self.point(s_mid, s, out) if x is None or (self.required(x) and not self.seen(x)): return False return True @classmethod - def _samples(cls, a: float, b: float) -> list[float]: - n = max(1, math.ceil((b - a) / cls.STEP_FT)) - edges = [a + (b - a) * i / n for i in range(n + 1)] - centres = [a + (b - a) * (i + 0.5) / n for i in range(n)] - return sorted(edges + centres) + def _samples(cls, a: float, b: float) -> Iterator[float]: + cells = (b - a) / cls.STEP_FT + if not math.isfinite(cells) or cells > MAX_GEOMETRY_SAMPLES: + raise ValueError( + f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples" + ) + n = max(1, math.ceil(max(0, cells))) + if 2 * n + 1 > MAX_GEOMETRY_SAMPLES: + raise ValueError( + f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples" + ) + + def points(): + for i in range(n): + yield a + (b - a) * i / n + yield a + (b - a) * (i + 0.5) / n + yield b + + return points() def chain_ends_s(scene: dict, rules: RuleSet) -> tuple[float, float] | None: @@ -1083,8 +1325,9 @@ def declared_height_problems(scene: dict, result: dict, rules: RuleSet) -> list[ it unsure for one within the battery's error of it (server/solver.py at 7133283, check_backing), so wall_backing passes only if every declared wall overlapping [s0 - e, s1 + e] is taller than battery.height_ft.""" - if sampling := sweep_sampling_problems(scene, result, rules): + if sampling := geometry_admission_problems(scene, result, rules): return sampling + result = deduplicate_result_runs(result) declared = {w["id"]: w["height_ft"] for w in scene.get("walls", []) if "height_ft" in w} if not declared or rules.battery_height_ft is None: return [] diff --git a/verification/hsverify/simrun.py b/verification/hsverify/simrun.py index 8d42bbeb1..bb5346cee 100644 --- a/verification/hsverify/simrun.py +++ b/verification/hsverify/simrun.py @@ -8,7 +8,7 @@ a screenshot once it has been on screen for `--settle` seconds; a state replaced sooner is captured immediately and flagged `transient`, because its image may already show the next state. The run ends at `--until`, after `--idle` seconds without a new state, or at -`--timeout`. A state named with `--require` that never appeared makes the run fail. If the app +`--timeout`. States named with `--require` must appear in the order given. If the app logged a scan bundle, it is copied into the report as scan.zip (required with `--require-export`) and named in report.json's `app_export`, so the report folder can be replayed as an app export. Reports go outside git @@ -331,7 +331,17 @@ def wait_for_log_stream(events: queue.Queue, timeout_s: float) -> str | None: def missing_required(required: list[str], states: list[ShotRecord]) -> list[str]: seen = {s.state for s in states} - return [f"required state {name} never appeared" for name in required if name not in seen] + missing = [f"required state {name} never appeared" for name in required if name not in seen] + if missing: + return missing + remaining = iter(required) + next_required = next(remaining, None) + for state in states: + if state.state == next_required: + next_required = next(remaining, None) + if next_required is not None: + return [f"required states did not appear in order: {' -> '.join(required)}"] + return [] def logged_message(line: str) -> str: @@ -563,7 +573,7 @@ def main(argv: list[str] | None = None) -> int: action="append", default=[], metavar="STATE", - help="a state the run must reach; one that never appears is a problem (exit 1)", + help="a state the run must reach; repeat in the required journey order (exit 1 on failure)", ) parser.add_argument( "--require-export", diff --git a/verification/tests/test_bundle_limits.py b/verification/tests/test_bundle_limits.py new file mode 100644 index 000000000..cc3a8f1aa --- /dev/null +++ b/verification/tests/test_bundle_limits.py @@ -0,0 +1,94 @@ +import ast +import io +import struct +import zipfile +from pathlib import Path + +import pytest + +from hsverify.bundle_limits import zip_directory_limits + + +def recon_guard(max_entries, max_bytes): + # Execute only the stdlib guard so this parity check needs no recon model dependencies. + source = Path(__file__).resolve().parents[2] / "recon" / "recon" / "capture.py" + tree = ast.parse(source.read_text()) + guard = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "_zip_directory_limits" + ) + namespace = { + "Path": Path, + "struct": struct, + "MAX_BUNDLE_ENTRIES": max_entries, + "MAX_BUNDLE_BYTES": max_bytes, + } + exec(compile(ast.Module(body=[guard], type_ignores=[]), str(source), "exec"), namespace) + return namespace["_zip_directory_limits"] + + +def ordinary_zip(): + data = io.BytesIO() + with zipfile.ZipFile(data, "w") as archive: + archive.writestr("scene.json", b"{}") + archive.writestr("frames/", b"") + archive.writestr("frames/a.jpg", b"image") + return bytearray(data.getvalue()) + + +@pytest.mark.parametrize( + "mutation", + [ + "ordinary", + "entries", + "bytes", + "false_count", + "comment", + "footer_in_comment", + "split", + "directory_offset", + "directory_record", + "zip64_locator", + "truncated", + ], +) +def test_admission_matches_recon_guard(tmp_path, mutation): + data = ordinary_zip() + entries, expanded = 3, 7 + footer = len(data) - 22 + directory = struct.unpack_from("oops", b"[1, 2]"): monkeypatch.setattr(e2e, "post", lambda *args, body=body, **kwargs: (200, body, 5.0)) diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 526a11bba..11b8a5322 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -6,6 +6,7 @@ import pytest from hsverify.resultcheck import ( + MAX_GEOMETRY_SAMPLES, MAX_SWEEP_SAMPLES, GroundPlan, Need, @@ -753,9 +754,9 @@ def test_an_overhead_seen_clear_all_the_way_up_settles_headroom(): def test_battery_clearance_is_needed_when_its_rule_reaches_past_gas_or_a_battery_is_marked(): - # BAND_RULES set battery_clearance's radius (100) above gas's (1): the server evaluates it, + # BAND_RULES set battery_clearance's radius (2) above gas's (1): the server evaluates it, # so a result that leaves it out is flagged and its coverage is needed anyway. - wide = replace(RULES, needs=RULES.needs | {"battery_clearance": (Need("ground", 100.0),)}) + wide = replace(RULES, needs=RULES.needs | {"battery_clearance": (Need("ground", 2.0),)}) r = result(checks=[check()], sweep=[PASS_RUN]) msgs = invariant_problems(SCENE, r, rules=wide) assert "the result leaves out check battery_clearance, which the rules define" in msgs @@ -1223,9 +1224,9 @@ def test_a_passing_spot_must_stand_on_the_scanned_wall(): ], ) r["spot"]["span_ft"] = [11.0, 11.0 + 31 / 12] + r["spot"]["footprint"] = [[11, 0], [11 + 31 / 12, 0], [11 + 31 / 12, 22 / 12], [11, 22 / 12]] msgs = invariant_problems(broadly_seen(), r, rules=RULES) - assert any("spot [11.00, 13.58] runs past its straight wall [-12.00, 12.00]" in m for m in msgs) - assert any("sweep pass start 11.00 runs past its straight wall" in m for m in msgs) + assert msgs == ["spot: span_ft exceeds scanned range [-12.0, 12.0]"] def test_walls_meeting_in_a_straight_line_are_one_stretch_and_a_corner_is_not(): @@ -1565,3 +1566,115 @@ def test_in_scan_sweep_with_tiny_rule_step_fails_loudly(): assert invariant_problems(SCENE, r, rules=replace(RULES, step_ft=1e-300)) == [ "sweep unsure starts [0, 1]: sweep sampling exceeds the 10000 sample limit" ] + + +@pytest.mark.parametrize("outcome", ["pass", "unsure", "fail"]) +@pytest.mark.parametrize("span", [[float("nan"), 1], [0, float("inf")], [2, 1]]) +def test_vet_all_sweep_spans_are_validated_before_downstream(monkeypatch, outcome, span): + def no_downstream(*args): + pytest.fail("invalid geometry reached downstream sampling") + + monkeypatch.setattr("hsverify.resultcheck.run_starts", no_downstream) + answer = result(spot=False, sweep=[pass_run(*span) | {"outcome": outcome}]) + assert any("finite and ordered" in p for p in invariant_problems(SCENE, answer, rules=RULES)) + + +def test_vet_identical_unsure_runs_are_sampled_only_once(monkeypatch): + calls = [] + + def starts(*args): + calls.append(True) + return iter([1.0]) + + monkeypatch.setattr("hsverify.resultcheck.run_starts", starts) + run = pass_run(1, 9) | {"outcome": "unsure"} + answer = result(spot=False, sweep=[copy.deepcopy(run) for _ in range(10_000)]) + assert footprint_problems(SCENE, answer, RULES) == [] + assert len(calls) == 1 + + +def test_vet_many_distinct_admissible_runs_exceed_one_response_budget(monkeypatch): + def no_sampling(*args): + pytest.fail("over-budget response reached sampling") + + monkeypatch.setattr("hsverify.resultcheck.run_starts", no_sampling) + scene = copy.deepcopy(SCENE) + scene["walls"][0]["baseline"] = [[-2000, 0], [2000, 0]] + runs = [pass_run(i / 1000, 1000 + i / 1000) | {"outcome": "unsure"} for i in range(5000)] + answer = result(spot=False, sweep=runs) + assert invariant_problems(scene, answer, rules=RULES) == [ + f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples" + ] + + +def test_vet_huge_spot_with_empty_sweep_stops_before_geometry(monkeypatch): + def no_geometry(*args): + pytest.fail("invalid spot reached the ground grid") + + monkeypatch.setattr(GroundPlan, "_samples", no_geometry) + answer = result(sweep=[]) + answer["spot"]["span_ft"] = [1, 1e9] + problems = invariant_problems(SCENE, answer, rules=RULES) + assert problems and any("span_ft" in p for p in problems) + + +def test_vet_geometry_budget_counts_spot_ground_grid_before_iteration(monkeypatch): + def no_geometry(*args): + pytest.fail("over-budget spot reached the ground grid") + + monkeypatch.setattr(GroundPlan, "_samples", no_geometry) + answer = result(sweep=[]) + huge_reach = replace(RULES, needs={"gas_clearance": (Need("ground", 1e9),)}) + assert invariant_problems(SCENE, answer, rules=huge_reach) == [ + f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples" + ] + + +def test_vet_geometry_budget_counts_check_only_reported_width(monkeypatch): + answer = result(spot=False, sweep=[]) + answer["checks"].append( + {"id": "width", "threshold_ft": 1e9, "rule": {"key": "battery.width_ft"}} + ) + assert invariant_problems(SCENE, answer, rules=RULES) == [ + f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples" + ] + + +def test_vet_ground_samples_are_lazy_and_preserve_grid_edges_and_centres(): + points = GroundPlan._samples(0, 0.1) + assert iter(points) is points + assert list(points) == pytest.approx([0, 0.025, 0.05, 0.075, 0.1]) + + +def test_vet_repeated_spot_checks_count_toward_the_shared_budget(monkeypatch): + def no_geometry(*args): + pytest.fail("repeated checks reached geometry sampling") + + monkeypatch.setattr(GroundPlan, "_samples", no_geometry) + answer = result(sweep=[], checks=[passing("ground_surface") for _ in range(1000)]) + broad = replace(RULES, needs={"ground_surface": (Need("ground", 10),)}) + assert invariant_problems(SCENE, answer, rules=broad) == [ + f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples" + ] + + +def test_vet_budget_includes_midpoint_direction_switches(monkeypatch): + scene = copy.deepcopy(SCENE) + scene["meter"]["plus_minus_ft"] = 8.8 + scene["walls"] = [ + {"id": "w1", "baseline": [[0, 0], [10, 0]], "plus_minus_ft": 0}, + {"id": "w2", "baseline": [[10, 0], [9, 0]], "plus_minus_ft": 0}, + {"id": "w3", "baseline": [[9, 0], [109, 0]], "plus_minus_ft": 0}, + ] + broad = replace( + RULES, width_ft=10, depth_ft=2, step_ft=0.1, needs={"ground_surface": (Need("ground", 0),)} + ) + answer = result(spot=False, sweep=[pass_run(0, 10)]) + + def no_sampling(*args): + pytest.fail("midpoint error exceeded budget after admission") + + monkeypatch.setattr("hsverify.resultcheck.run_starts", no_sampling) + assert invariant_problems(scene, answer, rules=broad) == [ + f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples" + ] diff --git a/verification/tests/test_simrun.py b/verification/tests/test_simrun.py index b4981cecc..3f5fef6d2 100644 --- a/verification/tests/test_simrun.py +++ b/verification/tests/test_simrun.py @@ -109,6 +109,25 @@ def test_missing_required_state_is_a_problem(): assert missing_required([], []) == [] +def test_required_states_reject_reversed_complete_journey(): + required = ["onboarding", "capture", "uploading", "result"] + assert missing_required(required, [shot(name) for name in reversed(required)]) == [ + "required states did not appear in order: onboarding -> capture -> uploading -> result" + ] + + +def test_required_states_allow_other_states_and_a_repeated_journey(): + required = ["capture", "uploading", "result"] + states = ["result", "capture", "capture", "review", "uploading", "result"] + assert missing_required(required, [shot(name) for name in states]) == [] + + +def test_repeated_required_state_needs_a_separate_occurrence(): + assert missing_required(["capture", "capture"], [shot("capture")]) == [ + "required states did not appear in order: capture -> capture" + ] + + def test_sim_app_requires_the_result_state(): makefile = (Path(__file__).resolve().parents[1] / "Makefile").read_text() recipe = makefile.split("\nsim-app:\n", 1)[1].split("\n\n", 1)[0] From 17425e7ac46ea6b304b0613a58099899682991f7 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Wed, 14 Oct 2026 12:39:13 -0500 Subject: [PATCH 50/62] fix(verification): reject invalid reported dimensions and allow new failures --- verification/hsverify/e2e.py | 8 +++ verification/hsverify/resultcheck.py | 75 +++++++++++++++++++++++++- verification/tests/test_e2e.py | 24 +++++++++ verification/tests/test_resultcheck.py | 70 ++++++++++++++++++++++++ 4 files changed, 175 insertions(+), 2 deletions(-) diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index 0c57c3540..c9755475c 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -73,6 +73,7 @@ more_error_problems, outcome_lengths, public_assumption_warnings, + reported_dimension_problems, reported_rules, schema_errors, tape_to_tap, @@ -577,6 +578,10 @@ def judge( if problems: return record | {"status": "fail", "problems": problems[:20], "contract_problems": problems} + dimensions = reported_dimension_problems(result, rules) + if dimensions: + return record | {"status": "fail", "problems": dimensions, "contract_problems": dimensions} + # Unreported overrides cannot establish rule-dependent geometry or case expectations. mismatches = assumption_mismatches(item.rules_assumed, result) warnings = public_assumption_warnings(item.rules_assumed, result, rules) @@ -636,6 +641,9 @@ def resend(name: str, scene: dict) -> dict | None: if errors: problems.append(f"{name}: result schema: {errors[0]}") return None + if dimensions := reported_dimension_problems(other, rules): + problems.extend(f"{name}: {p}" for p in dimensions) + return None problems.extend( f"{name}: {p}" for p in invariant_problems( diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index d24120c73..1f3c729a7 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -308,6 +308,8 @@ def margin_problem(check: dict) -> str | None: def invariant_problems( scene: dict, result: dict, sent: bytes | None = None, rules: RuleSet | None = None ) -> list[str]: + if dimensions := reported_dimension_problems(result, rules): + return dimensions if rules is not None and (sampling := sweep_sampling_problems(scene, result, rules)): return sampling problems: list[str] = [] @@ -1171,6 +1173,8 @@ def reported_assumption_values(key: str, result: dict) -> list[tuple[str, float] def reported_rules(result: dict, rules: RuleSet) -> RuleSet: """Use reported geometry for invariants too, so public defaults cannot reject an override.""" + if problems := reported_dimension_problems(result, rules): + raise ValueError("; ".join(problems)) changes = {} for key, field in (("battery_width", "width_ft"), ("corner_allowance", "corner_allowance_ft")): if values := reported_assumption_values(key, result): @@ -1178,6 +1182,71 @@ def reported_rules(result: dict, rules: RuleSet) -> RuleSet: return replace(rules, **changes) +def reported_dimension_problems(result: dict, rules: RuleSet | None) -> list[str]: + """Reject inconsistent C2 geometry before its dimensions can override public rules.""" + problems = [] + + def finite(value: Any) -> bool: + return ( + isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + ) + + spots = [(name, result.get(name)) for name in ("spot", "nearest_considered")] + for name, spot in spots: + if spot is None: + continue + for field in ("width_ft", "depth_ft", "height_ft"): + if field in spot and (not finite(spot[field]) or spot[field] <= 0): + problems.append(f"{name}.{field} must be finite and positive") + span = spot.get("span_ft") + if span is not None: + if len(span) != 2 or not all(finite(v) for v in span) or span[0] >= span[1]: + problems.append(f"{name}.span_ft must be finite and increasing") + elif ( + finite(spot.get("width_ft")) + and abs(span[1] - span[0] - spot["width_ft"]) > OFFSET_TOL_FT + ): + problems.append(f"{name}.width_ft does not match span_ft") + corners = spot.get("footprint") + if corners is None: + continue + if len(corners) != 4 or any( + len(point) != 2 or not all(finite(v) for v in point) for point in corners + ): + problems.append(f"{name}.footprint must have four finite plan corners") + continue + # C2 orders back-left, back-right, front-right, front-left. Use the verifier's existing + # hundredth-foot position tolerance when comparing its rounded plan edges and spans. + for field, edges in (("width_ft", ((0, 1), (2, 3))), ("depth_ft", ((1, 2), (3, 0)))): + if finite(spot.get(field)) and any( + abs(math.dist(corners[a], corners[b]) - spot[field]) > OFFSET_TOL_FT + for a, b in edges + ): + problems.append(f"{name}.{field} does not match footprint") + + widths = reported_assumption_values("battery_width", result) + for source, width in widths: + if not finite(width) or width <= 0: + message = f"{source} must be finite and positive" + if message not in problems: + problems.append(message) + valid_widths = [(source, width) for source, width in widths if finite(width) and width > 0] + if valid_widths and any( + abs(width - valid_widths[0][1]) > NUMBER_TOL_FT for _, width in valid_widths + ): + problems.append("reported battery widths disagree") + first, second = (spot for _, spot in spots) + if first is not None and second is not None: + for field in ("depth_ft", "height_ft"): + if ( + finite(first.get(field)) + and finite(second.get(field)) + and abs(first[field] - second[field]) > NUMBER_TOL_FT + ): + problems.append(f"spot and nearest_considered {field} disagree") + return problems + + def assumption_mismatches(assumed: dict[str, float], result: dict) -> list[str]: return [ f"case assumes {key} threshold {value}, server's {source} uses {actual}" @@ -1414,12 +1483,14 @@ def outcomes_near(result: dict, wall_id: str, start_ft: float) -> tuple[set[str] def more_error_problems(before: dict, after: dict, change: str) -> list[str]: """With more error, a start may only move toward UNSURE: UNSURE stays, PASS and FAIL stay - or become UNSURE. A start the original sweep never evaluated is read as `outcomes_near`.""" + or become UNSURE. A start the original sweep never evaluated is read as `outcomes_near`. + A wall with no original run has no outcome to contradict a new FAIL, but cannot newly PASS. + """ problems = [] for wall_id, s in probe_starts(before, after): b = outcome_at(after, wall_id, s) allowed, was = outcomes_near(before, wall_id, s) - if b is None or b == "unsure" or b in allowed: + if b is None or b == "unsure" or b in allowed or (not allowed and b == "fail"): continue problems.append(f"{change}: start {wall_id} {s:.3f} went from {was} to {b}") if before["decision"] != "pass" and after["decision"] == "pass": diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index b7b9102e5..63bbbc74e 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -997,3 +997,27 @@ def invariants(scene, result, sent=None, rules=None): if reported_width is None: assert not report["contract_ok"] assert "assumed public rules; server may override" in (tmp_path / "report.md").read_text() + + +@pytest.mark.parametrize("width", [-1, 0, float("nan"), float("inf")]) +def test_vet_judge_rejects_invalid_width_before_rule_consumers(monkeypatch, width): + answer = { + "decision": "reject", + "spot": None, + "sweep": [], + "checks": [{"id": "width", "threshold_ft": width, "rule": {"key": "battery.width_ft"}}], + } + monkeypatch.setattr(e2e, "post", lambda *args: (200, json.dumps(answer).encode(), 1)) + monkeypatch.setattr(e2e, "schema_errors", lambda *args: []) + consumers = [] + monkeypatch.setattr( + e2e, "invariant_problems", lambda *args, **kwargs: consumers.append("invariants") or [] + ) + monkeypatch.setattr( + e2e, "property_problems", lambda *args: consumers.append("properties") or [] + ) + item = SceneInput("invalid-width", SCENE, b"{}") + record = judge("unused", Endpoint("/place", "application/json"), item, SCHEMAS, RULES) + assert record["status"] == "fail" + assert record["contract_problems"] == ["width must be finite and positive"] + assert consumers == [] diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 2a16bc044..1d1c20ec0 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -1707,3 +1707,73 @@ def test_reported_override_prevents_a_false_footprint_failure(): backing["rule"]["key"] = "battery.width_ft" assert footprint_problems(SCENE, answer, public) assert footprint_problems(SCENE, answer, reported_rules(answer, public)) == [] + + +@pytest.mark.parametrize("field", ["width_ft", "depth_ft", "height_ft"]) +@pytest.mark.parametrize("value", [-1, 0, float("nan"), float("inf")]) +def test_vet_reported_dimensions_refuse_invalid_numbers(field, value): + answer = result() + answer["spot"][field] = value + with pytest.raises(ValueError, match=f"spot.{field} must be finite and positive"): + reported_rules(answer, RULES) + + +@pytest.mark.parametrize("source", ["spot", "nearest_considered", "check"]) +def test_vet_reported_width_sources_must_agree(source): + answer = result() + answer["nearest_considered"] = copy.deepcopy(answer["spot"]) + answer["checks"].append( + {"id": "width", "threshold_ft": RULES.width_ft, "rule": {"key": "battery.width_ft"}} + ) + if source == "check": + answer["checks"][-1]["threshold_ft"] = 4 + else: + answer[source]["width_ft"] = 4 + with pytest.raises(ValueError, match="reported battery widths disagree"): + reported_rules(answer, RULES) + + +@pytest.mark.parametrize("source", ["spot", "nearest_considered"]) +@pytest.mark.parametrize("value", [4, 1e9]) +def test_vet_reported_width_must_match_span_and_footprint(source, value): + answer = result() + answer[source] = copy.deepcopy(answer["spot"]) + answer[source]["width_ft"] = value + if source == "nearest_considered": + answer["spot"] = None + with pytest.raises(ValueError, match="does not match"): + reported_rules(answer, RULES) + + +def test_vet_reported_depth_must_match_footprint(): + answer = result() + answer["spot"]["depth_ft"] = 4 + with pytest.raises(ValueError, match="depth_ft does not match footprint"): + reported_rules(answer, RULES) + + +@pytest.mark.parametrize("value", [-1, 0, float("nan"), float("inf")]) +def test_vet_reported_width_threshold_must_be_finite_positive(value): + answer = { + "checks": [{"id": "width", "threshold_ft": value, "rule": {"key": "battery.width_ft"}}] + } + with pytest.raises(ValueError, match="width must be finite and positive"): + reported_rules(answer, RULES) + + +@pytest.mark.parametrize("outcome", ["fail", "unsure"]) +@pytest.mark.parametrize("old_wall_has_run", [False, True]) +def test_vet_more_error_may_add_nonpassing_run_on_previously_unswept_wall( + outcome, old_wall_has_run +): + before = swept(run(0, 4, "fail")) if old_wall_has_run else swept() + after = swept(*before["sweep"], run(10, 12, outcome, wall="w2")) + assert more_error_problems(before, after, "more error") == [] + + +def test_vet_more_error_cannot_add_pass_on_previously_unswept_wall(): + before = swept(run(0, 4, "pass"), decision="pass") + after = swept(*before["sweep"], run(10, 12, "pass", wall="w2"), decision="pass") + assert more_error_problems(before, after, "more error") == [ + f"more error: start w2 {s:.3f} went from no run on its wall to pass" for s in (10, 11, 12) + ] From 03b0f0e35e477bed74c6912fb5377f1f18f6576a Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Wed, 14 Oct 2026 12:47:07 -0500 Subject: [PATCH 51/62] docs(recon): distinguish repaired footprint edges from depth policy --- recon/HANDOFF.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recon/HANDOFF.md b/recon/HANDOFF.md index b952169d2..68dc5d8a8 100644 --- a/recon/HANDOFF.md +++ b/recon/HANDOFF.md @@ -129,7 +129,7 @@ In `recon/` on this branch: These defects remain after the review-debt repairs. They can let the server settle a check on evidence the worker does not have. The saved ETH3D results above predate those repairs; dataset acceptance has not been rerun. - **One view can count as two for free space.** `recon/coverage.py:189-193, 208-246`: `_slab` treats a voxel as observed when its fused weight is above zero, and any single frame gives it weight. Repro: a capture in which one frame alone sees the space in front of a wall cell reports a facing clearance there. Wall and ground cells need two positions 0.25 m apart for the same claim. The server at 903d86f can settle a check on that clearance. A fix needs visibility kept per frame for free space, with the same two-position test as the wall and ground. -- **The overhead strip is narrower than a battery.** `recon/coverage.py:42, 236-245`: overhead space is sampled only 0.10 to 0.56 m out from the wall front. Repro: an eave 0.05 m out from the wall, or 0.7 m out over a 3 ft deep battery, is not found, and the cell reports clearance above it. A fix needs the footprint of the battery the chosen rule places, passed from the server's rules or request. +- **The overhead strip is narrower than a battery.** `recon/coverage.py:42, 236-245`: overhead space is sampled only 0.10 to 0.56 m out from the wall front. The worker now samples the rear edge and the full existing 0.56 m depth, including its outer edge. A 0.7 m eave over a 3 ft deep battery still lies beyond that assumed depth. A fix needs the footprint of the battery the chosen rule places, passed from the server's rules or request. - **The photos-only clearance error is not calibrated.** `recon/pipeline.py:29-35`: facing gaps and overhead clearances from photos carry ±0.56 ft. That comes from the 95% interval of the evals' surface pair-error p90 over 1 to 3 m spans, not from any measurement of where the fused volume puts a boundary. The evals put surface p90 at 5.64 in [4.63, 6.66] and edges at 9.65 in [4.93, 15.01]. Repro: a photos-only overhead clearance measured at 7.10 ft passes the 6.5 ft headroom rule, since 7.10 − 0.56 = 6.54 > 6.5. A fix needs the clearance estimator's own measured uncertainty, or a gate that returns the check unresolved until that exists. The worker now replaces the target wall's prior measurements, stops clearance at any occupied sampled voxel, and refuses LiDAR without a confidence map. Ground coverage follows the fitted slope, and only whole cells within the fitted wall extent are exported. The regressions are in `tests/test_review_debt.py`; they do not validate the remaining clearance policies above. From 2d959c7dd800e71d94f92d2b82421b8c48b46bef Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Wed, 14 Oct 2026 12:47:26 -0500 Subject: [PATCH 52/62] docs(recon): describe the remaining battery depth assumption --- recon/HANDOFF.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recon/HANDOFF.md b/recon/HANDOFF.md index 68dc5d8a8..2113681fd 100644 --- a/recon/HANDOFF.md +++ b/recon/HANDOFF.md @@ -129,7 +129,7 @@ In `recon/` on this branch: These defects remain after the review-debt repairs. They can let the server settle a check on evidence the worker does not have. The saved ETH3D results above predate those repairs; dataset acceptance has not been rerun. - **One view can count as two for free space.** `recon/coverage.py:189-193, 208-246`: `_slab` treats a voxel as observed when its fused weight is above zero, and any single frame gives it weight. Repro: a capture in which one frame alone sees the space in front of a wall cell reports a facing clearance there. Wall and ground cells need two positions 0.25 m apart for the same claim. The server at 903d86f can settle a check on that clearance. A fix needs visibility kept per frame for free space, with the same two-position test as the wall and ground. -- **The overhead strip is narrower than a battery.** `recon/coverage.py:42, 236-245`: overhead space is sampled only 0.10 to 0.56 m out from the wall front. The worker now samples the rear edge and the full existing 0.56 m depth, including its outer edge. A 0.7 m eave over a 3 ft deep battery still lies beyond that assumed depth. A fix needs the footprint of the battery the chosen rule places, passed from the server's rules or request. +- **Overhead sampling assumes one battery depth.** `recon/coverage.py`: the worker samples both edges of the existing 0.56 m footprint. A 0.7 m eave over a 3 ft deep battery still lies beyond that assumed depth. Other battery depths need the footprint passed from the server's rules or request. - **The photos-only clearance error is not calibrated.** `recon/pipeline.py:29-35`: facing gaps and overhead clearances from photos carry ±0.56 ft. That comes from the 95% interval of the evals' surface pair-error p90 over 1 to 3 m spans, not from any measurement of where the fused volume puts a boundary. The evals put surface p90 at 5.64 in [4.63, 6.66] and edges at 9.65 in [4.93, 15.01]. Repro: a photos-only overhead clearance measured at 7.10 ft passes the 6.5 ft headroom rule, since 7.10 − 0.56 = 6.54 > 6.5. A fix needs the clearance estimator's own measured uncertainty, or a gate that returns the check unresolved until that exists. The worker now replaces the target wall's prior measurements, stops clearance at any occupied sampled voxel, and refuses LiDAR without a confidence map. Ground coverage follows the fitted slope, and only whole cells within the fitted wall extent are exported. The regressions are in `tests/test_review_debt.py`; they do not validate the remaining clearance policies above. From 6c0639e890dca08aed43131e18363abe859137ee Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Wed, 14 Oct 2026 13:06:06 -0500 Subject: [PATCH 53/62] fix(pipeline): require two views for occupied clearance measurements --- experiments/drift-anatomy/METHODS.md | 2 +- experiments/drift-anatomy/README.md | 2 +- recon/recon/coverage.py | 11 ++-- recon/tests/test_coverage_batches.py | 11 ++-- recon/tests/test_coverage_views.py | 69 ++++++++++++++++++++++++++ verification/hsverify/resultcheck.py | 5 ++ verification/tests/test_e2e.py | 34 +++++++++++++ verification/tests/test_resultcheck.py | 32 ++++++++++++ 8 files changed, 156 insertions(+), 10 deletions(-) diff --git a/experiments/drift-anatomy/METHODS.md b/experiments/drift-anatomy/METHODS.md index bfbbeb58d..346076b6b 100644 --- a/experiments/drift-anatomy/METHODS.md +++ b/experiments/drift-anatomy/METHODS.md @@ -20,7 +20,7 @@ The server allows 0.16 ft of error per foot along the wall. That allowance domin ## Result -All four fail. Tables: [results/drift_anatomy.md](results/drift_anatomy.md). +A1, A2 and A4 fail their numeric criteria. A3 fails the 60% criterion for a travel-axis proxy; this is not evidence that a wall-plane experiment failed. Wall-plane benefit remains untested without wall annotations. Tables: [results/drift_anatomy.md](results/drift_anatomy.md). - **A1.** 30 of 56 MARViN loops (54%). ADVIO has no revisit under 0.3 m while ARKit tracks. Scale and heading error cancel on return, so r misses them. - **A2.** 5.7% SD on MARViN, 19% on ADVIO: drop. A few outlier windows inflate MARViN's SD; its robust SD is 1.8%. diff --git a/experiments/drift-anatomy/README.md b/experiments/drift-anatomy/README.md index 612572a78..c5c68944f 100644 --- a/experiments/drift-anatomy/README.md +++ b/experiments/drift-anatomy/README.md @@ -13,7 +13,7 @@ The server allows 0.16 ft of error per foot along the wall, which dominates a ch ## Result -All four fail ([tables](results/drift_anatomy.md)). +A1, A2 and A4 fail their numeric criteria. A3 fails the 60% criterion for a travel-axis proxy; the unannotated walks cannot test wall-plane benefit ([tables](results/drift_anatomy.md)). - **A1.** The closure residual bounds the peak error in 30 of 56 loops. Scale and heading error cancel on return. - **A2.** Scale spread within a walk is 1.8% robust SD (5.7% SD) on MARViN. diff --git a/recon/recon/coverage.py b/recon/recon/coverage.py index 4f4687016..1b2a0f314 100644 --- a/recon/recon/coverage.py +++ b/recon/recon/coverage.py @@ -19,8 +19,8 @@ Facing and overhead space needs no surface: it is observed where the fused volume shows it seen empty (every voxel observed, none occupied) and two camera positions at least 0.25 m apart each -saw through it (`SeenFromTwo`), up to the first occupied voxel (a measured gap or clearance) or -the first sample short of that (seen clear only that far). +saw through it (`SeenFromTwo`). An occupied slab always stops the march, but supports a measured +gap or clearance only when that slab also passes the observation test. """ from __future__ import annotations @@ -353,7 +353,9 @@ def free_space(wall: WallFrame, vol: Volume | SeenFromTwo, cells: np.ndarray): for out, hit, observed in zip(chunk, occupied, seen, strict=True): # Any occupied sampled voxel stops clearance, including one-step thin rails. if hit: - facing_gap[c] = max(0.0, out - step - front) + # A fused hit alone does not establish a two-view measurement. + if observed: + facing_gap[c] = max(0.0, out - step - front) stopped = True break if not observed: @@ -374,7 +376,8 @@ def free_space(wall: WallFrame, vol: Volume | SeenFromTwo, cells: np.ndarray): stopped = False for h, hit, observed in zip(chunk, occupied, seen, strict=True): if hit: - over_gap[c] = max(0.0, h - step) + if observed: + over_gap[c] = max(0.0, h - step) stopped = True break if not observed: diff --git a/recon/tests/test_coverage_batches.py b/recon/tests/test_coverage_batches.py index c776f0533..204d6b807 100644 --- a/recon/tests/test_coverage_batches.py +++ b/recon/tests/test_coverage_batches.py @@ -32,18 +32,21 @@ def scalar_march(wall, volume, cells): for out in np.arange(front + step, front + coverage.FACING_MAX_M, step): share, seen = coverage._slab(wall, volume, across, heights, [out]) if share > 0: - fg[c] = max(0, out - step - front) + if seen: + fg[c] = max(0, out - step - front) break if not seen: break fc[c] = out - front - outs = front + np.arange( - coverage.OVERHEAD_OUT_M[0], coverage.OVERHEAD_OUT_M[1] + 1e-9, step + outs = front + np.append( + np.arange(coverage.OVERHEAD_OUT_M[0], coverage.OVERHEAD_OUT_M[1], step), + coverage.OVERHEAD_OUT_M[1], ) for height in np.arange(0.3, coverage.OVERHEAD_MAX_M, step): share, seen = coverage._slab(wall, volume, across, [height], outs) if share > 0: - og[c] = max(0, height - step) + if seen: + og[c] = max(0, height - step) break if not seen: break diff --git a/recon/tests/test_coverage_views.py b/recon/tests/test_coverage_views.py index cace017df..306ece85e 100644 --- a/recon/tests/test_coverage_views.py +++ b/recon/tests/test_coverage_views.py @@ -8,6 +8,7 @@ from recon import coverage from recon.capture import Frame from recon.depth import Depth +from recon.geometry import WallFrame class ClearVolume: @@ -117,3 +118,71 @@ def test_one_view_or_one_position_never_claims_free_space(): frame.cam_to_world[:3, 3] = [0, 1, 4] view = coverage.SeenFromTwo(ClearVolume(), frames, depths) assert not view.observed(np.array([[0, 1, 1]])).any() + + +class FirstSlabObstacle(ClearVolume): + def __init__(self, band): + self.band = band + + def occupied(self, points): + if self.band == "facing": + # A thin hit in the first outward slab, too small to become the wall front. + return np.isclose(points[:, 2], 0.1) & np.isclose(points[:, 1], 1.0) + return np.isclose(points[:, 1], 0.3) & np.isclose(points[:, 2], 0.3) + + +def free_space_from_positions(raw, xs): + frames, depths = views(len(xs)) + for frame, x in zip(frames, xs, strict=True): + frame.cam_to_world[:3, :3] = np.eye(3) + frame.cam_to_world[:3, 3] = [x, 1.0, 4.0] + depths[frame.id].depth[:] = 4.0 + wall = WallFrame( + np.array([0.0, 1.0, 0.0]), np.array([1.0, 0, 0]), np.array([0, 0, 1.0]), 0.0, (0, 0.3) + ) + cells = np.array([0.0]) + arrays = coverage.free_space(wall, coverage.SeenFromTwo(raw, frames, depths), cells) + return coverage.CellCoverage(cells, np.zeros(1, bool), np.zeros(1), *arrays) + + +@pytest.mark.parametrize("band", ["facing", "overhead"]) +@pytest.mark.parametrize( + ("xs", "supported"), + [([0.075], False), ([0.025, 0.125], False), ([-0.125, 0.275], True)], + ids=["one_camera", "nearby_cameras", "separated_cameras"], +) +def test_first_occupied_slab_needs_two_views_before_exporting_measurement(band, xs, supported): + cov = free_space_from_positions(FirstSlabObstacle(band), xs) + gap = cov.facing_gap if band == "facing" else cov.overhead_clearance + clear = cov.facing_clear if band == "facing" else cov.overhead_clear + entries = [entry for entry in coverage.observed(cov) if entry["band"] == band] + measurements = coverage.measurements(cov, "wall", 0.5)[0 if band == "facing" else 1] + # An occupied first slab never counts as empty, even when its measurement is unsupported. + assert clear[0] == 0.0 + if supported: + assert gap[0] == pytest.approx(0.0 if band == "facing" else 0.25) + assert entries == [{"band": band, "span_ft": [0.0, 0.5]}] + assert len(measurements) == 1 + else: + assert np.isnan(gap[0]) + assert entries == [] + assert measurements == [] + + +@pytest.mark.parametrize( + ("xs", "supported"), + [([0.075], False), ([0.025, 0.125], False), ([-0.125, 0.275], True)], + ids=["one_camera", "nearby_cameras", "separated_cameras"], +) +def test_empty_volume_needs_two_views_before_exporting_clear_space(xs, supported): + cov = free_space_from_positions(ClearVolume(), xs) + assert np.isnan(cov.facing_gap[0]) and np.isnan(cov.overhead_clearance[0]) + assert coverage.measurements(cov, "wall", 0.5) == ([], []) + entries = coverage.observed(cov) + if supported: + assert cov.facing_clear[0] > 0 and cov.overhead_clear[0] > 0 + assert {entry["band"] for entry in entries} == {"facing", "overhead"} + assert all(entry["out_ft"] > 0 for entry in entries) + else: + assert cov.facing_clear[0] == cov.overhead_clear[0] == 0.0 + assert entries == [] diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 1fe22cc41..5609bf3a2 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -784,6 +784,11 @@ def finite(value: Any) -> bool: message = f"{source} must be finite and positive" if message not in problems: problems.append(message) + for check in result.get("checks", []): + if check.get("rule", {}).get("key") == "route.corner_allowance_ft": + allowance = check.get("threshold_ft") + if allowance is not None and (not finite(allowance) or allowance < 0): + problems.append(f"{check['id']} corner allowance must be finite and nonnegative") valid_widths = [(source, width) for source, width in widths if finite(width) and width > 0] if valid_widths and any( abs(width - valid_widths[0][1]) > NUMBER_TOL_FT for _, width in valid_widths diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index f0eb65990..276a5074b 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -1151,3 +1151,37 @@ def no_adoption(*args): else ["width must be finite and positive"] ) assert consumers == [] + + +@pytest.mark.parametrize("allowance", [-1, float("nan"), float("inf")]) +def test_breadth_invalid_corner_allowance_stops_before_rule_adoption(monkeypatch, allowance): + answer = { + "decision": "reject", + "spot": None, + "sweep": [], + "checks": [ + { + "id": "route_corner", + "threshold_ft": allowance, + "rule": {"key": "route.corner_allowance_ft"}, + } + ], + } + monkeypatch.setattr(e2e, "post", lambda *args: (200, json.dumps(answer).encode(), 1)) + monkeypatch.setattr(e2e, "schema_errors", lambda *args: []) + + def no_adoption(*args): + pytest.fail("invalid corner allowance reached rule adoption") + + monkeypatch.setattr(e2e, "reported_rules", no_adoption) + record = judge( + "unused", + Endpoint("/place", "application/json"), + SceneInput("corner", SCENE, b"{}"), + SCHEMAS, + RULES, + ) + assert record["status"] == "fail" + assert record["contract_problems"] == [ + "route_corner corner allowance must be finite and nonnegative" + ] diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 52197525e..c3a21d21c 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -1890,3 +1890,35 @@ def no_sampling(*args): assert invariant_problems(scene, answer, rules=broad) == [ f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples" ] + + +@pytest.mark.parametrize("allowance", [-1, float("nan"), float("inf"), -float("inf")]) +def test_breadth_corner_allowance_must_be_finite_nonnegative(allowance): + answer = { + "checks": [ + { + "id": "route_corner", + "threshold_ft": allowance, + "rule": {"key": "route.corner_allowance_ft"}, + } + ] + } + with pytest.raises(ValueError, match="corner allowance must be finite and nonnegative"): + reported_rules(answer, RULES) + assert invariant_problems(SCENE, answer, rules=RULES) == [ + "route_corner corner allowance must be finite and nonnegative" + ] + + +@pytest.mark.parametrize("allowance", [0, 0.5]) +def test_breadth_zero_or_positive_corner_allowance_is_valid(allowance): + answer = { + "checks": [ + { + "id": "route_corner", + "threshold_ft": allowance, + "rule": {"key": "route.corner_allowance_ft"}, + } + ] + } + assert reported_rules(answer, RULES).corner_allowance_ft == allowance From 68cb6859ea6ee9ed09f8209ec7786c5cb0956f69 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Wed, 14 Oct 2026 13:55:25 -0500 Subject: [PATCH 54/62] fix(verification): meter actual geometry and comparison work --- verification/hsverify/e2e.py | 144 +++-- verification/hsverify/intervalindex.py | 175 ++++++ verification/hsverify/resultcheck.py | 667 +++++++++++++--------- verification/tests/test_e2e.py | 10 +- verification/tests/test_generate_cases.py | 2 +- verification/tests/test_intervalindex.py | 113 ++++ verification/tests/test_judge_meter.py | 294 ++++++++++ verification/tests/test_resultcheck.py | 70 +-- verification/tests/test_runtime_meter.py | 181 ++++++ 9 files changed, 1311 insertions(+), 345 deletions(-) create mode 100644 verification/hsverify/intervalindex.py create mode 100644 verification/tests/test_intervalindex.py create mode 100644 verification/tests/test_judge_meter.py create mode 100644 verification/tests/test_runtime_meter.py diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index aa4194756..6ef459931 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -51,7 +51,7 @@ import urllib.request import uuid import zipfile -from collections.abc import Iterator +from collections.abc import Callable, Iterator from dataclasses import dataclass, field from pathlib import Path @@ -60,10 +60,13 @@ from hsverify.memory import peak_rss_mb from hsverify.resultcheck import ( RuleSet, + WorkLimitExceeded, + WorkMeter, assumption_mismatches, comparable, coverage_blocks, expectation_problems, + geometry_admission_problems, invariant_problems, less_coverage_problems, mirror_scene, @@ -469,7 +472,10 @@ def judge( schemas: dict, rules: RuleSet, limit: memory.TreeLimit | None = None, + *, + meter: WorkMeter | None = None, ) -> dict: + meter = meter if meter is not None else WorkMeter() record: dict = { "name": item.name, "source": item.source, @@ -499,36 +505,92 @@ def judge( failure = [f"HTTP 200 with a body that is not a JSON object: {payload[:200]!r}"] return record | {"status": "fail", "problems": failure, "contract_problems": failure} record["result"] = result - problems = [f"result schema: {e}" for e in schema_errors(result, schemas["result"])] + + def check(name: str, function: Callable[[], list[str]]) -> list[str]: + try: + return function() + except WorkLimitExceeded: + return [f"verification work limit exceeded ({meter.samples} samples)"] + except Exception as exc: + return [f"{name}: {type(exc).__name__}: {exc}"] + + problems = check( + "result schema", + lambda: [f"result schema: {e}" for e in schema_errors(result, schemas["result"])], + ) if problems: return record | {"status": "fail", "problems": problems[:20], "contract_problems": problems} - # Contract problems hold for any scene; expectation problems depend on one case's geometry. - contract = invariant_problems(item.scene, result, sent=item.raw, rules=rules) - contract += property_problems(url, endpoint, item, result, schemas, rules) + # Admission errors stop geometry, including report summaries that subtract spans. + contract = check( + "geometry admission", + lambda: geometry_admission_problems(item.scene, result, rules, meter=meter), + ) + admitted = not contract + if admitted: + contract += check( + "invariants", + lambda: invariant_problems(item.scene, result, sent=item.raw, rules=rules, meter=meter), + ) + if not contract: + contract += check( + "properties", + lambda: property_problems(url, endpoint, item, result, schemas, rules, meter=meter), + ) if item.real and ms > LATENCY_BUDGET_MS: contract.append(f"real-derived scene took {ms:.0f} ms, budget {LATENCY_BUDGET_MS:.0f}") - mismatches = assumption_mismatches(item.rules_assumed, result, rules) - expected = expectation_problems(item.expect, result) if item.expect and not mismatches else [] - problems = contract + expected + mismatches: list[str] = [] + expected: list[str] = [] + exhausted = any(p.startswith("verification work limit exceeded (") for p in contract) + if admitted and not exhausted: + + def check_assumptions() -> list[str]: + mismatches.extend(assumption_mismatches(item.rules_assumed, result, rules)) + return [] + + contract += check("assumptions", check_assumptions) + if item.expect and not mismatches: + expected = check( + "expectations", lambda: expectation_problems(item.expect, result, meter=meter) + ) + + def summarize() -> list[str]: + record["outcome_lengths_ft"] = outcome_lengths(result) + return [] - record["decision"] = result["decision"] - record["outcome_lengths_ft"] = outcome_lengths(result) + contract += check("outcome lengths", summarize) + if "decision" in result: + record["decision"] = result["decision"] if mismatches: record["assumption_mismatches"] = mismatches + record["verification_samples"] = meter.samples record["contract_problems"] = contract record["expectation_problems"] = expected - record["problems"] = problems - record["status"] = "fail" if problems else "assumption mismatch" if mismatches else "pass" + record["problems"] = contract + expected + record["status"] = ( + "fail" if record["problems"] else "assumption mismatch" if mismatches else "pass" + ) return record def property_problems( - url: str, endpoint: Endpoint, item: SceneInput, result: dict, schemas: dict, rules: RuleSet + url: str, + endpoint: Endpoint, + item: SceneInput, + result: dict, + schemas: dict, + rules: RuleSet, + *, + meter: WorkMeter | None = None, + rule_checks: bool = True, ) -> list[str]: """The same scene resent unchanged and transformed. Every answer to a transformed scene must also satisfy the invariants, and the transforms that only lose information must not make the server surer.""" + meter = meter if meter is not None else WorkMeter() + checked_rules = rules if rule_checks else None + if admission := geometry_admission_problems(item.scene, result, checked_rules, meter=meter): + return admission problems: list[str] = [] def resend(name: str, scene: dict) -> dict | None: @@ -537,14 +599,29 @@ def resend(name: str, scene: dict) -> dict | None: if failure: problems.append(f"{name}: {failure}") return None - errors = schema_errors(other, schemas["result"]) - if errors: - problems.append(f"{name}: result schema: {errors[0]}") + check_name = "result schema" + try: + errors = schema_errors(other, schemas["result"]) + if errors: + problems.append(f"{name}: result schema: {errors[0]}") + return None + check_name = "geometry admission" + admission = geometry_admission_problems(scene, other, checked_rules, meter=meter) + if admission: + problems.extend(f"{name}: {p}" for p in admission) + return None + check_name = "invariants" + invariants = invariant_problems( + scene, other, sent=transformed.raw, rules=checked_rules, meter=meter + ) + if invariants: + problems.extend(f"{name}: {p}" for p in invariants) + return None + except WorkLimitExceeded: + raise + except Exception as exc: + problems.append(f"{name} {check_name}: {type(exc).__name__}: {exc}") return None - problems.extend( - f"{name}: {p}" - for p in invariant_problems(scene, other, sent=transformed.raw, rules=rules) - ) return other # 1. Same input, same answer. @@ -572,19 +649,22 @@ def resend(name: str, scene: dict) -> dict | None: if "coverage" in item.scene: bare = {k: v for k, v in item.scene.items() if k != "coverage"} if (other := resend("no coverage", bare)) is not None: - problems += less_coverage_problems(result, other, "no coverage") - for name, scene in ( - ("less coverage", with_less_coverage(item.scene)), - ("ground short of the largest clearance", with_ground_short_of(item.scene, rules)), - ): - if scene is not None and (other := resend(name, scene)) is not None: - problems += less_coverage_problems(result, other, name) - for name, scene in ( - ("more error", with_more_error(item.scene, rules)), - ("tape re-measured by tap", tape_to_tap(item.scene)), - ): + problems += less_coverage_problems(result, other, "no coverage", meter=meter) + coverage_variants = [("less coverage", with_less_coverage(item.scene))] + if rule_checks: + coverage_variants.append( + ("ground short of the largest clearance", with_ground_short_of(item.scene, rules)) + ) + for name, scene in coverage_variants: if scene is not None and (other := resend(name, scene)) is not None: - problems += more_error_problems(result, other, name) + problems += less_coverage_problems(result, other, name, meter=meter) + if rule_checks: + for name, scene in ( + ("more error", with_more_error(item.scene, rules)), + ("tape re-measured by tap", tape_to_tap(item.scene)), + ): + if scene is not None and (other := resend(name, scene)) is not None: + problems += more_error_problems(result, other, name, meter=meter) # 4. Showing everything the result asked for settles every check that was unsure only # because an area was unobserved (the chosen spot may move; three rounds at most). diff --git a/verification/hsverify/intervalindex.py b/verification/hsverify/intervalindex.py new file mode 100644 index 000000000..0a2e596c9 --- /dev/null +++ b/verification/hsverify/intervalindex.py @@ -0,0 +1,175 @@ +"""Compare sweep outcomes without scanning every run for each sampled start.""" + +from __future__ import annotations + +import heapq +from dataclasses import dataclass +from typing import Protocol + + +class Meter(Protocol): + def charge(self, n: int = 1) -> None: ... + + +def _charge(meter: Meter | None, n: int = 1) -> None: + if meter is not None: + meter.charge(n) + + +def _bisect(values: list[float], value: float, meter: Meter | None, *, right=False) -> int: + lo, hi = 0, len(values) + while lo < hi: + _charge(meter) + mid = (lo + hi) // 2 + if values[mid] < value or (right and values[mid] == value): + lo = mid + 1 + else: + hi = mid + return lo + + +@dataclass +class _WallIndex: + endpoints: list[float] + at_endpoint: list[str | None] + after_endpoint: list[str | None] + starts: list[float] + start_outcomes: list[str] + ends: list[float] + end_outcomes: list[str] + + +class SweepIndex: + """Index closed intervals, choosing the first original run when intervals overlap. + + Endpoint outcomes and the outcomes between endpoints are stored separately. This keeps + an interval's inclusive endpoint, including EPS expansion, without changing overlap order. + """ + + def __init__(self, result: dict, meter: Meter | None = None, eps: float = 1e-6): + self.meter = meter + groups: dict[str, list[dict]] = {} + for run in result.get("sweep", []): + _charge(meter) + groups.setdefault(run["wall_id"], []).append(run) + self.walls = {wall: self._build(runs, eps) for wall, runs in groups.items()} + + def _build(self, runs: list[dict], eps: float) -> _WallIndex: + events = [] + starts, ends = {}, {} + for rank, run in enumerate(runs): + _charge(self.meter) + a, b = run["start_ft"] + events.extend(((a - eps, 0, rank), (b + eps, 1, rank))) + starts.setdefault(a, run["outcome"]) + ends.setdefault(b, run["outcome"]) + events.sort() + active = [False] * len(runs) + heap: list[int] = [] + endpoints, at_endpoint, after_endpoint = [], [], [] + + def first() -> str | None: + while heap and not active[heap[0]]: + _charge(self.meter) + heapq.heappop(heap) + return runs[heap[0]]["outcome"] if heap else None + + pos = 0 + while pos < len(events): + value = events[pos][0] + ending = [] + while pos < len(events) and events[pos][0] == value: + _charge(self.meter) + _, kind, rank = events[pos] + if kind == 0: + active[rank] = True + heapq.heappush(heap, rank) + else: + ending.append(rank) + pos += 1 + endpoints.append(value) + at_endpoint.append(first()) + for rank in ending: + _charge(self.meter) + active[rank] = False + after_endpoint.append(first()) + start_values, end_values = sorted(starts), sorted(ends) + return _WallIndex( + endpoints, + at_endpoint, + after_endpoint, + start_values, + [starts[a] for a in start_values], + end_values, + [ends[b] for b in end_values], + ) + + def outcome_at(self, wall_id: str, start_ft: float) -> str | None: + _charge(self.meter) + wall = self.walls.get(wall_id) + if wall is None: + return None + i = _bisect(wall.endpoints, start_ft, self.meter) + if i < len(wall.endpoints) and wall.endpoints[i] == start_ft: + return wall.at_endpoint[i] + return wall.after_endpoint[i - 1] if i else None + + def outcomes_near(self, wall_id: str, start_ft: float) -> tuple[set[str], str]: + if (inside := self.outcome_at(wall_id, start_ft)) is not None: + return {inside}, inside + wall = self.walls.get(wall_id) + if wall is None: + return set(), "no run on its wall" + left = _bisect(wall.ends, start_ft, self.meter) - 1 + right = _bisect(wall.starts, start_ft, self.meter, right=True) + if left < 0 or right == len(wall.starts): + return {"fail"}, "past the original sweep" + a, b = wall.end_outcomes[left], wall.start_outcomes[right] + return {a, b}, f"unswept beside {a} and {b}" + + +def probe_starts(*results: dict, meter: Meter | None = None) -> list[tuple[str, float]]: + points = set() + for result in results: + for run in result.get("sweep", []): + _charge(meter, 3) + a, b = run["start_ft"] + # Halving first avoids overflowing the sum of two admitted finite endpoints. + points.update( + ((run["wall_id"], a), (run["wall_id"], b), (run["wall_id"], a / 2 + b / 2)) + ) + return sorted(points) + + +def more_error_problems( + before: dict, after: dict, change: str, meter: Meter | None = None +) -> list[str]: + original, changed = SweepIndex(before, meter), SweepIndex(after, meter) + problems = [] + for wall_id, s in probe_starts(before, after, meter=meter): + _charge(meter) + b = changed.outcome_at(wall_id, s) + allowed, was = original.outcomes_near(wall_id, s) + if b is None or b == "unsure" or b in allowed or (not allowed and b == "fail"): + continue + problems.append(f"{change}: start {wall_id} {s:.3f} went from {was} to {b}") + if before["decision"] != "pass" and after["decision"] == "pass": + problems.append(f"{change}: the decision became pass") + return problems + + +def less_coverage_problems( + before: dict, after: dict, change: str, meter: Meter | None = None +) -> list[str]: + original, changed = SweepIndex(before, meter), SweepIndex(after, meter) + problems = [] + for wall_id, s in probe_starts(before, after, meter=meter): + _charge(meter) + if changed.outcome_at(wall_id, s) != "pass": + continue + allowed, was = original.outcomes_near(wall_id, s) + if "pass" not in allowed: + problems.append(f"{change}: start {wall_id} {s:.3f} went from {was} to pass") + if before["decision"] != "pass" and after["decision"] == "pass": + problems.append(f"{change}: the decision became pass") + return problems diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index a7a1eca77..8212ad087 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -31,21 +31,96 @@ import json import math from collections.abc import Callable, Iterator -from dataclasses import dataclass, replace +from dataclasses import dataclass from typing import Any import yaml from jsonschema import Draft202012Validator +from .intervalindex import ( + SweepIndex, +) +from .intervalindex import ( + less_coverage_problems as indexed_less_coverage_problems, +) +from .intervalindex import ( + more_error_problems as indexed_more_error_problems, +) +from .intervalindex import ( + probe_starts as indexed_probe_starts, +) + EPS = 1e-6 NUMBER_TOL_FT = 1e-5 # Bound verifier work even when a server span or rules step requests billions of samples. MAX_SWEEP_SAMPLES = 10_000 -# The existing public-pool regression has 55 starts and a worst-case ground -# reach of 14.15 ft: its conservative grid bound is below 50 million visits. -# This cap retains that case while bounding aggregate work; it is a resource -# policy, not a measured runtime or a physical scan limit. -MAX_GEOMETRY_SAMPLES = 50_000_000 +# On 2026-10-14, 35 public fixture judgments without contract problems, including +# property variants, peaked at 80,788 units for pass-clear-side-wall. Another 468 +# distinct saved responses matched to current fixtures peaked at 392,115 invariant +# units for ETH3D facade. Allow 20 times that larger recorded peak for other scans +# and their property variants. This is an operational bound, not a calibration of +# measurement accuracy or evidence that the samples prove physical clearance. +MAX_WORK_SAMPLES = 20 * 392_115 + + +class WorkLimitExceeded(Exception): + def __init__(self, samples: int) -> None: + self.samples = samples + super().__init__(f"verification work limit exceeded ({samples} samples)") + + +class WorkMeter: + """Count actual geometry work across one response and its property comparisons.""" + + def __init__(self, limit: int | None = None) -> None: + self.limit = MAX_WORK_SAMPLES if limit is None else limit + self.samples = 0 + self.cache: dict[tuple, Any] = {} + self.inputs: dict[int, tuple[Any, int]] = {} + self.input_values: dict[Any, int] = {} + + def charge(self, n: int = 1) -> None: + if n < 0: + raise ValueError("verification work charge must be nonnegative") + self.samples += n + if self.samples > self.limit: + raise WorkLimitExceeded(self.samples) + + def input_key(self, value: Any) -> Any: + # judge's scenes and rule sets are immutable during a response's checks. Keep the + # source alive so a later transformed scene cannot reuse its object id. + identity = id(value) + if identity not in self.inputs: + frozen = _input_key(value, self) + token = self.input_values.setdefault(frozen, len(self.input_values)) + self.inputs[identity] = (value, token) + return self.inputs[identity][1] + + +def finite(value: Any) -> bool: + """JSON integers may exceed the float range accepted by math.isfinite.""" + try: + return ( + isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + ) + except (OverflowError, TypeError, ValueError): + return False + + +def _input_key(value: Any, meter: WorkMeter | None = None) -> Any: + if meter is not None: + meter.charge() + if isinstance(value, dict): + return tuple((key, _input_key(item, meter)) for key, item in sorted(value.items())) + if isinstance(value, (tuple, list)): + return tuple(_input_key(item, meter) for item in value) + if isinstance(value, RuleSet): + return _input_key(vars(value), meter) + if isinstance(value, Need): + return _input_key(vars(value), meter) + return value + + # Spot centres and offsets are rounded by the server; a hundredth of a foot is 1/8 inch. OFFSET_TOL_FT = 0.01 @@ -191,6 +266,7 @@ def observed( min_out_ft: float = 0.0, beyond: bool = False, wall_default_ft: float | None = None, + meter: WorkMeter | None = None, ) -> list[tuple[float, float]]: """Merged observed s-intervals for a band, counting only entries whose view reached `min_out_ft` (out from the wall for ground and facing, up for wall and overhead). With @@ -201,8 +277,10 @@ def observed( headroom.min_ft) and settles any height up to it inclusively, `beyond` or not; S2 chose this category default at 6200098 (server/scene.py `observed_intervals`). None keeps the old reading, all the way up.""" + meter = meter if meter is not None else WorkMeter() spans = [] for entry in scene.get("coverage", {}).get("observed", []): + meter.charge() if entry["band"] != band: continue if band == "wall" and "out_ft" not in entry and wall_default_ft is not None: @@ -220,6 +298,7 @@ def observed( spans.sort() merged: list[list[float]] = [] for a, b in spans: + meter.charge() if merged and a <= merged[-1][1] + EPS: merged[-1][1] = max(merged[-1][1], b) else: @@ -227,8 +306,19 @@ def observed( return [(a, b) for a, b in merged] -def covers(intervals: list[tuple[float, float]], a: float, b: float, slack: float = 0.0) -> bool: - return any(lo - slack <= a + EPS and b - EPS <= hi + slack for lo, hi in intervals) +def covers( + intervals: list[tuple[float, float]], + a: float, + b: float, + slack: float = 0.0, + meter: WorkMeter | None = None, +) -> bool: + meter = meter if meter is not None else WorkMeter() + for lo, hi in intervals: + meter.charge() + if lo - slack <= a + EPS and b - EPS <= hi + slack: + return True + return False # A gap in a 1D band narrower than this is rounding between the capture's spans and the unrolled @@ -237,12 +327,14 @@ def covers(intervals: list[tuple[float, float]], a: float, b: float, slack: floa def gaps_over( - intervals: list[tuple[float, float]], a: float, b: float + intervals: list[tuple[float, float]], a: float, b: float, meter: WorkMeter | None = None ) -> list[tuple[float, float]]: """The parts of [a, b] no interval covers, as the server's missing() finds them: gaps narrower than COVERAGE_TOLERANCE_FT don't count.""" + meter = meter if meter is not None else WorkMeter() gaps, at = [], a for lo, hi in sorted(intervals): + meter.charge() if hi <= at: continue if lo > at: @@ -312,9 +404,14 @@ def margin_problem(check: dict) -> str | None: def invariant_problems( - scene: dict, result: dict, sent: bytes | None = None, rules: RuleSet | None = None + scene: dict, + result: dict, + sent: bytes | None = None, + rules: RuleSet | None = None, + meter: WorkMeter | None = None, ) -> list[str]: - if sampling := geometry_admission_problems(scene, result, rules): + meter = meter if meter is not None else WorkMeter() + if sampling := geometry_admission_problems(scene, result, rules, meter=meter): return sampling result = deduplicate_result_runs(result) problems: list[str] = [] @@ -384,15 +481,27 @@ def invariant_problems( if max(abs(want[0] - got[0]), abs(want[1] - got[1])) > OFFSET_TOL_FT: problems.append(f"spot.meter_offset_ft {got} != centre - meter {want}") - problems += spot_sweep_problems(result) + def checked(name: str, run: Callable[[], list[str]]) -> None: + try: + problems.extend(run()) + except WorkLimitExceeded: + raise + except Exception as exc: + problems.append(f"{name}: {type(exc).__name__}: {exc}") + + checked("spot_sweep", lambda: spot_sweep_problems(result, meter=meter)) if rules is not None: - problems += footprint_problems(scene, result, rules) - problems += spot_position_problems(scene, result, rules) - problems += missing_evidence_problems(scene, result, rules) + checked("footprint", lambda: footprint_problems(scene, result, rules, meter=meter)) + checked("spot_position", lambda: spot_position_problems(scene, result, rules, meter=meter)) + checked( + "missing_evidence", lambda: missing_evidence_problems(scene, result, rules, meter=meter) + ) if rules is not None: - problems += coverage_problems(scene, result, rules) - problems += declared_height_problems(scene, result, rules) - problems += dropped_check_problems(scene, result, rules) + checked("coverage", lambda: coverage_problems(scene, result, rules, meter=meter)) + checked( + "declared_height", lambda: declared_height_problems(scene, result, rules, meter=meter) + ) + checked("dropped_checks", lambda: dropped_check_problems(scene, result, rules)) return problems @@ -406,12 +515,16 @@ def invariant_problems( COLLINEAR_FT = 0.05 -def straight_pieces(scene: dict, rules: RuleSet) -> list[tuple[float, float]]: +def straight_pieces( + scene: dict, rules: RuleSet, meter: WorkMeter | None = None +) -> list[tuple[float, float]]: """The s-extent of each straight stretch of scanned wall a battery can back onto: baseline segments joined where the next one starts at the previous one's end and continues its line, within a wall or across two walls that meet.""" + meter = meter if meter is not None else WorkMeter() pieces: list[list] = [] # [s0, s1, start point, end point] - for _, s0, s1, a, b in segments_s(scene, rules) or []: + for _, s0, s1, a, b in segments_s(scene, rules, meter=meter) or []: + meter.charge() if pieces: prev = pieces[-1] length = math.dist(prev[2], prev[3]) @@ -426,23 +539,36 @@ def straight_pieces(scene: dict, rules: RuleSet) -> list[tuple[float, float]]: return [(p[0], p[1]) for p in pieces] -def footprint_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: +def footprint_problems( + scene: dict, result: dict, rules: RuleSet, meter: WorkMeter | None = None +) -> list[str]: """A battery that passes or is left to a person backs onto one straight stretch of scanned wall along its whole width; a footprint past the stretch's end (a corner, the end of the wall, a gap) can't sit flush, so the server fails it (server/solver.py check_backing).""" - if sampling := geometry_admission_problems(scene, result, rules): + meter = meter if meter is not None else WorkMeter() + if sampling := geometry_admission_problems(scene, result, rules, meter=meter): return sampling result = deduplicate_result_runs(result) - pieces = straight_pieces(scene, rules) + pieces = straight_pieces(scene, rules, meter=meter) if not pieces: return [] def off_wall(lo: float, hi: float, outcome: str, wall_id: str) -> str | None: + meter.charge() # The stretch it starts on, or else ends on: the one it would back onto. A start on a # boundary belongs to the stretch it runs onto, an end to the one it runs off. - piece = next((p for p in pieces if p[0] - EPS <= lo < p[1] - EPS), None) or next( - (p for p in pieces if p[0] + EPS < hi <= p[1] + EPS), None - ) + piece = None + for candidate in pieces: + meter.charge() + if candidate[0] - EPS <= lo < candidate[1] - EPS: + piece = candidate + break + if piece is None: + for candidate in pieces: + meter.charge() + if candidate[0] + EPS < hi <= candidate[1] + EPS: + piece = candidate + break if piece is None: return "is not on any scanned wall" if lo < piece[0] - COLLINEAR_FT or hi > piece[1] + COLLINEAR_FT: @@ -451,8 +577,9 @@ def off_wall(lo: float, hi: float, outcome: str, wall_id: str) -> str | None: return None # A pass also keeps its position error clear of the stretch's ends: within it the # battery may not sit flush, and the server leaves it unsure (check_backing). - e = battery_error(scene, rules, wall_id, lo, hi) + e = battery_error(scene, rules, wall_id, lo, hi, meter=meter) for end, clear in ((piece[0], lo - e - piece[0]), (piece[1], piece[1] - hi - e)): + meter.charge() if clear < -COLLINEAR_FT: return f"ends within its error ({e:.2f} ft) of its straight wall's end at {end:.2f}" return None @@ -467,6 +594,7 @@ def off_wall(lo: float, hi: float, outcome: str, wall_id: str) -> str | None: if run["outcome"] not in ("pass", "unsure"): continue for start in run_starts(run["start_ft"], rules.step_ft): + meter.charge() battery = (start, start + rules.width_ft) if (why := off_wall(*battery, run["outcome"], run["wall_id"])) is not None: problems.append(f"sweep {run['outcome']} start {start:.2f} {why}") @@ -474,11 +602,14 @@ def off_wall(lo: float, hi: float, outcome: str, wall_id: str) -> str | None: return problems -def spot_position_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: +def spot_position_problems( + scene: dict, result: dict, rules: RuleSet, meter: WorkMeter | None = None +) -> list[str]: """Where the app places the battery must be where the result says it stands: spot.center is its footprint's centre, and that is the middle of span_ft on the wall, half the battery's depth out. meter_offset_ft is checked against center elsewhere, so a centre moved together with its offset is caught here.""" + meter = meter if meter is not None else WorkMeter() spot = result.get("spot") if spot is None: return [] @@ -494,7 +625,7 @@ def spot_position_problems(scene: dict, result: dict, rules: RuleSet) -> list[st f"[{fx:.2f}, {fz:.2f}]" ) lo, hi = spot["span_ft"] - plan = GroundPlan(scene, rules) + plan = GroundPlan(scene, rules, meter=meter) want = plan.point((lo + hi) / 2, (lo + hi) / 2, rules.depth_ft / 2) if want is not None and math.dist((cx, cz), want) > COLLINEAR_FT: problems.append( @@ -504,7 +635,7 @@ def spot_position_problems(scene: dict, result: dict, rules: RuleSet) -> list[st return problems -def spot_sweep_problems(result: dict) -> list[str]: +def spot_sweep_problems(result: dict, meter: WorkMeter | None = None) -> list[str]: """The chosen spot is one of the evaluated starts, so the sweep must give its start the spot's own outcome, and a passing spot must lie in a passing run. A result could otherwise report a pass at a start its own sweep says fails, with stats that agree with the sweep.""" @@ -512,7 +643,7 @@ def spot_sweep_problems(result: dict) -> list[str]: if spot is None: return [] start = spot["span_ft"][0] - got = outcome_at(result, spot["wall_id"], start) + got = outcome_at(result, spot["wall_id"], start, meter=meter) if got == spot["outcome"]: return [] verb = {"pass": "passes", "fail": "fails"}.get(spot["outcome"], "is unsure") @@ -532,23 +663,31 @@ def required_span(lo: float, hi: float, radius: float) -> tuple[float, float]: return lo - radius, hi + radius -def battery_error(scene: dict, rules: RuleSet, wall_id: str, lo: float, hi: float) -> float: +def battery_error( + scene: dict, rules: RuleSet, wall_id: str, lo: float, hi: float, meter: WorkMeter | None = None +) -> float: """Position error of a battery over [lo, hi] on `wall_id`, as server 3baa338 takes it for its checks (solver.py _errors): its wall's error at its far edge from the meter, plus the larger of the meter's error in plan and the battery's slide along the walls (slide).""" + meter = meter if meter is not None else WorkMeter() wall = next((w for w in scene["walls"] if w["id"] == wall_id), None) if wall is None: raise ValueError(f"the result names wall {wall_id!r}, which the scene does not have") return wall_error_at(wall, rules, max(abs(lo), abs(hi))) + max( - meter_error(scene, rules), slide(scene, rules, lo, hi) + meter_error(scene, rules), slide(scene, rules, lo, hi, meter=meter) ) -def along_error(scene: dict, rules: RuleSet, wall_id: str, lo: float, hi: float) -> float: +def along_error( + scene: dict, rules: RuleSet, wall_id: str, lo: float, hi: float, meter: WorkMeter | None = None +) -> float: """The battery's error against spans measured along the walls: its wall's at its far edge plus its slide, without the meter's plan error (server 3baa338 _errors, along_err).""" + meter = meter if meter is not None else WorkMeter() wall = next(w for w in scene["walls"] if w["id"] == wall_id) - return wall_error_at(wall, rules, max(abs(lo), abs(hi))) + slide(scene, rules, lo, hi) + return wall_error_at(wall, rules, max(abs(lo), abs(hi))) + slide( + scene, rules, lo, hi, meter=meter + ) def wall_error_at(wall: dict, rules: RuleSet, s: float) -> float: @@ -564,13 +703,16 @@ def meter_error(scene: dict, rules: RuleSet) -> float: return scene.get("meter", {}).get("plus_minus_ft", rules.errors["meter"]) -def slide(scene: dict, rules: RuleSet, lo: float, hi: float) -> float: +def slide( + scene: dict, rules: RuleSet, lo: float, hi: float, meter: WorkMeter | None = None +) -> float: """How far a battery over [lo, hi] can move along the walls against spans measured in s when the meter or a corner between them moves within its error (server 3baa338 _slide): the meter's error times the difference between its wall's direction and the battery's, plus each other wall between the meter and the battery's near edge, at its far end, times the same difference for that wall. 0 on the meter's own wall.""" - segments = segments_s(scene, rules) + meter = meter if meter is not None else WorkMeter() + segments = segments_s(scene, rules, meter=meter) if not segments: return 0.0 walls = {w["id"]: w for w in scene["walls"]} @@ -581,7 +723,11 @@ def along(seg) -> Point: return ((q[0] - p[0]) / n, (q[1] - p[1]) / n) def under(s: float): - return min(segments, key=lambda g: max(g[1] - s, s - g[2], 0.0)) + def distance(g): + meter.charge() + return max(g[1] - s, s - g[2], 0.0) + + return min(segments, key=distance) battery = along(under((lo + hi) / 2)) @@ -590,6 +736,7 @@ def turn(seg) -> float: return math.hypot(battery[0] - a[0], battery[1] - a[1]) meter_wall = scene["meter"]["wall_id"] + meter.charge(len(segments)) meter_seg = min( (g for g in segments if g[0] == meter_wall), key=lambda g: max(g[1], -g[2], 0.0) ) @@ -597,6 +744,7 @@ def turn(seg) -> float: a0, b0 = min(0.0, near), max(0.0, near) total = meter_error(scene, rules) * turn(meter_seg) for seg in segments: + meter.charge() a, b = max(seg[1], a0), min(seg[2], b0) if b - a <= EPS: continue @@ -610,7 +758,13 @@ def wall_error(wall: dict, rules: RuleSet) -> float: def reach_gaps( - scene: dict, rules: RuleSet, check: str, wall_id: str, lo: float, hi: float + scene: dict, + rules: RuleSet, + check: str, + wall_id: str, + lo: float, + hi: float, + meter: WorkMeter | None = None, ) -> list[tuple[str, str]]: """(band, what is missing) for each Need of `check` a battery over [lo, hi] does not have. @@ -627,38 +781,57 @@ def reach_gaps( - facing and overhead bands: over [lo - e, hi + e], every place the battery may sit, beyond the check's distance unless measured. """ + meter = meter if meter is not None else WorkMeter() + key = ("reach_gaps", meter.input_key(scene), meter.input_key(rules), check, wall_id, lo, hi) + if key in meter.cache: + return meter.cache[key] gaps = [] for need in rules.needs[check]: + meter.charge() if not need.widen: error = 0.0 elif need.along: - error = along_error(scene, rules, wall_id, lo, hi) + error = along_error(scene, rules, wall_id, lo, hi, meter=meter) else: - error = battery_error(scene, rules, wall_id, lo, hi) + error = battery_error(scene, rules, wall_id, lo, hi, meter=meter) reach = need.radius + error if need.band == "ground": - plan = GroundPlan(scene, rules) if rules.wall_join_ft is not None else None - gap = ground_gap(scene, lo, hi, rules.depth_ft, reach, plan) + plan = GroundPlan(scene, rules, meter=meter) if rules.wall_join_ft is not None else None + gap = ground_gap(scene, lo, hi, rules.depth_ft, reach, plan, meter=meter) elif need.band == "wall": a, b = required_span(lo, hi, reach) wall = observed( - scene, "wall", need.height, beyond=True, wall_default_ft=rules.headroom_ft + scene, + "wall", + need.height, + beyond=True, + wall_default_ft=rules.headroom_ft, + meter=meter, ) - seen = not gaps_over(wall, a, b) + seen = not gaps_over(wall, a, b, meter=meter) gap = None if seen else f"wall [{a:.2f}, {b:.2f}] observed higher than {need.height} ft" else: a, b = required_span(lo, hi, reach) - gap = measured_band_gap(scene, need.band, wall_id, a, b, need.height) + gap = measured_band_gap(scene, need.band, wall_id, a, b, need.height, meter=meter) if gap: gaps.append((need.band, gap)) + meter.cache[key] = gaps return gaps def measured_band_gap( - scene: dict, band: str, wall_id: str, lo: float, hi: float, need: float + scene: dict, + band: str, + wall_id: str, + lo: float, + hi: float, + need: float, + meter: WorkMeter | None = None, ) -> str | None: """The first stretch of [lo, hi] the facing or overhead band leaves unsettled: not observed, or observed short of `need` where no measurement covers it.""" + meter = meter if meter is not None else WorkMeter() + meter.charge(len(scene.get("coverage", {}).get("observed", []))) entries = [ (*sorted(e["span_ft"]), reached(e)) for e in scene.get("coverage", {}).get("observed", []) @@ -672,15 +845,22 @@ def measured_band_gap( edges = {x for e in entries for x in e[:2]} | {x for m in measured for x in m} cuts = sorted({lo, hi} | {x for x in edges if lo < x < hi}) for p, q in itertools.pairwise(cuts): + meter.charge() if q - p <= EPS: continue + meter.charge(len(entries)) seen = [out for x, y, out in entries if x <= p + EPS and q - EPS <= y] where = f"{band} [{p:.2f}, {q:.2f}] observed" if not seen: if q - p < COVERAGE_TOLERANCE_FT: continue return f"{where}, none seen" - settled = any(x <= p + EPS and q - EPS <= y for x, y in measured) + settled = False + for x, y in measured: + meter.charge() + if x <= p + EPS and q - EPS <= y: + settled = True + break # Strictly beyond, as the server requires (server/solver.py at 6c7ca23). if not settled and max(seen) <= need: return f"{where} out to {need:.2f} ft or measured, seen {max(seen):.2f} ft" @@ -693,13 +873,13 @@ def run_start_count(span: list[float], step_ft: float | None) -> int: A sample, not the server's own list: the server also evaluates boundaries and midpoints that a run reports only as its two ends. Oversized runs fail rather than lose coverage through truncation.""" - if step_ft is None or not math.isfinite(step_ft) or step_ft <= 0: + if step_ft is None or not finite(step_ft) or step_ft <= 0: raise ValueError("checking a sweep run needs finite positive rules' sweep.step_ft") a, b = span - if not math.isfinite(a) or not math.isfinite(b) or a > b: + if not finite(a) or not finite(b) or a > b: raise ValueError("sweep start_ft must be finite and ordered") steps = (b - a) / step_ft - if not math.isfinite(steps) or steps > MAX_SWEEP_SAMPLES: + if not finite(steps) or steps > MAX_SWEEP_SAMPLES: raise ValueError(f"sweep sampling exceeds the {MAX_SWEEP_SAMPLES} sample limit") # Every step strictly before the last start, the first included however short the run. count = max(int(a < b), math.ceil(steps - EPS)) @@ -726,16 +906,6 @@ def reported_dimension_problems(result: dict, rules: RuleSet | None) -> list[str """Reject inconsistent C2 geometry before its dimensions can override public rules.""" problems = [] - def finite(value: Any) -> bool: - try: - return ( - isinstance(value, (int, float)) - and not isinstance(value, bool) - and math.isfinite(value) - ) - except OverflowError: - return False - spots = [(name, result.get(name)) for name in ("spot", "nearest_considered")] for name, spot in spots: if spot is None: @@ -784,6 +954,11 @@ def finite(value: Any) -> bool: message = f"{source} must be finite and positive" if message not in problems: problems.append(message) + for check in result.get("checks", []): + if check.get("rule", {}).get("key") == "route.corner_allowance_ft": + allowance = check.get("threshold_ft") + if allowance is not None and (not finite(allowance) or allowance < 0): + problems.append("reported corner allowance must be finite and nonnegative") valid_widths = [(source, width) for source, width in widths if finite(width) and width > 0] if valid_widths and any( abs(width - valid_widths[0][1]) > NUMBER_TOL_FT for _, width in valid_widths @@ -801,46 +976,56 @@ def finite(value: Any) -> bool: return problems -def geometry_admission_problems(scene: dict, result: dict, rules: RuleSet | None) -> list[str]: - """Bound all sampling before any footprint, coverage or height check runs. - - Count sweep starts and a conservative upper bound for ground grid point visits, - including split-band edges. Bounds use whole runs, never enumerate their starts. - """ - if problems := reported_dimension_problems(result, rules): - return problems - # Use validated reported dimensions only to estimate work, before the caller adopts them. - widths = [ - c["threshold_ft"] - for c in result.get("checks", []) - if c.get("rule", {}).get("key") == "battery.width_ft" and c.get("threshold_ft") is not None - ] - widths += [ - s["width_ft"] - for key in ("spot", "nearest_considered") - if (s := result.get(key)) is not None and "width_ft" in s - ] - depths = [ - s["depth_ft"] - for key in ("spot", "nearest_considered") - if (s := result.get(key)) is not None and "depth_ft" in s - ] - if rules is not None: - rules = replace( - rules, - width_ft=max(widths, default=rules.width_ft), - depth_ft=max([rules.depth_ft, *depths]), - ) - problems = [] - bodies = [] - ends = chain_ends_s(scene, rules) if rules is not None else None +def geometry_admission_problems( + scene: dict, result: dict, rules: RuleSet | None, meter: WorkMeter | None = None +) -> list[str]: + """Validate response numbers and references before geometry. Work is counted as it runs.""" + meter = meter if meter is not None else WorkMeter() + problems = reported_dimension_problems(result, rules) + + def numbers(value: Any, path: str) -> None: + if isinstance(value, dict): + for key, item in value.items(): + numbers(item, f"{path}.{key}" if path else key) + elif isinstance(value, (tuple, list)): + for i, item in enumerate(value): + numbers(item, f"{path}[{i}]") + elif isinstance(value, (int, float)) and not isinstance(value, bool) and not finite(value): + problems.append(f"{path} must be finite") + + numbers(result, "result") + numbers(scene, "scene") + wall_ids = {wall["id"] for wall in scene.get("walls", [])} + bodies = [(name, result.get(name)) for name in ("spot", "nearest_considered")] + bodies += [(f"sweep[{i}]", run) for i, run in enumerate(result.get("sweep", []))] + for name, body in bodies: + if body is not None and body.get("wall_id") not in wall_ids: + problems.append(f"{name} names unknown wall_id {body.get('wall_id')!r}") + for run in result.get("sweep", []): + a, b = run["start_ft"] + if not finite(a) or not finite(b) or a > b: + problems.append( + f"sweep {run['outcome']} starts {run['start_ft']}: " + "start_ft must be finite and ordered" + ) + elif not finite(b - a): + problems.append(f"sweep {run['outcome']} start_ft span length must be finite") + if problems: + return list(dict.fromkeys(problems)) + ends = ( + chain_ends_s(scene, rules, meter=meter) + if rules is not None and rules.wall_join_ft is not None + else None + ) for name in ("spot", "nearest_considered"): if (spot := result.get(name)) is not None: - bodies.append((name, spot["span_ft"], spot["wall_id"], 1, True)) + lo, hi = spot["span_ft"] + if ends is not None and (lo < ends[0] - COLLINEAR_FT or hi > ends[1] + COLLINEAR_FT): + problems.append(f"{name}: span_ft exceeds scanned range [{ends[0]}, {ends[1]}]") for run in deduplicate_result_runs(result)["sweep"]: a, b = run["start_ft"] prefix = f"sweep {run['outcome']} starts {run['start_ft']}: " - if not math.isfinite(a) or not math.isfinite(b) or a > b: + if a > b: problems.append(prefix + "start_ft must be finite and ordered") continue if run["outcome"] in ("pass", "unsure") and rules is not None: @@ -850,95 +1035,9 @@ def geometry_admission_problems(scene: dict, result: dict, rules: RuleSet | None ) continue try: - count = run_start_count(run["start_ft"], rules.step_ft) + run_start_count(run["start_ft"], rules.step_ft) except ValueError as exc: problems.append(prefix + str(exc)) - continue - bodies.append( - (prefix, [a, b + rules.width_ft], run["wall_id"], count, run["outcome"] == "pass") - ) - for name, span, _, _, _ in bodies: - if any(not math.isfinite(v) for v in span) or span[0] > span[1]: - problems.append(f"{name}: span_ft must be finite and ordered") - if problems or rules is None: - return problems - total = 0 - - def grid_count(length: float) -> int: - cells = length / GroundPlan.STEP_FT - if not math.isfinite(cells) or cells > MAX_GEOMETRY_SAMPLES: - return MAX_GEOMETRY_SAMPLES + 1 - return 2 * max(1, math.ceil(max(0, cells))) + 1 - - for name, (lo, hi), wid, count, ground in bodies: - # Sweep ends name starts; footprints may extend beyond the chain and are checked below. - start_hi = hi - rules.width_ft if name.startswith("sweep") else hi - if ends is not None and (lo < ends[0] - COLLINEAR_FT or start_hi > ends[1] + COLLINEAR_FT): - label = "start_ft span" if name.startswith("sweep") else "span_ft" - problems.append( - f"{name}{'' if name.endswith(': ') else ': '}" - f"{label} exceeds scanned range [{ends[0]}, {ends[1]}]" - ) - continue - total += count * (2 + (len(rules.needs) if ground else 0)) - if ground and rules.wall_join_ft is not None: - segments = segments_s(scene, rules) or [] - # Error changes at wall boundaries. Evaluate both sides and endpoints, not the grid. - width = rules.width_ft if name.startswith("sweep") else hi - lo - starts = {lo, max(lo, hi - width)} - boundaries = {edge for _, a, b, _, _ in segments for edge in (a, b)} - boundaries.update( - (left[2] + right[1]) / 2 for left, right in itertools.pairwise(segments) - ) - for edge_at in boundaries: - for edge in (edge_at, edge_at - width, edge_at - width / 2): - for offset in (-EPS, EPS): - if lo <= edge + offset <= hi - width: - starts.add(edge + offset) - for check, needs in rules.needs.items(): - for need in needs: - if need.band != "ground": - continue - error = ( - 0.0 - if not need.widen - else max( - (along_error if need.along else battery_error)( - scene, rules, wid, x, x + width - ) - for x in starts - ) - ) - radius = need.radius + error - splits = 4 + 2 * sum( - e["band"] == "ground" for e in scene.get("coverage", {}).get("observed", []) - ) - along = grid_count(width + 2 * radius) + 3 * splits - out = grid_count(rules.depth_ft + radius) - # Spot coverage and missing-evidence checks can both visit the grid. - repeats = ( - 1 - if name.startswith("sweep") - else 1 - + sum( - c["id"] == check and c["outcome"] == "pass" - for c in result.get("checks", []) - ) - ) - total += count * along * out * repeats - if total > MAX_GEOMETRY_SAMPLES: - problems.append( - f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples" - ) - break - # A check-only reported width is still untrusted geometry, even without a chosen spot/run. - for width in [*widths, *depths]: - if not math.isfinite(width) or width <= 0: - problems.append("reported battery width must be finite and positive") - break - total += grid_count(width) - if total > MAX_GEOMETRY_SAMPLES and not any("sample budget" in p for p in problems): - problems.append(f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples") return problems @@ -967,18 +1066,26 @@ def dropped_check_problems(scene: dict, result: dict, rules: RuleSet) -> list[st ] -def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: +def coverage_problems( + scene: dict, result: dict, rules: RuleSet, meter: WorkMeter | None = None +) -> list[str]: """Missing coverage is never a pass (C5), to the reach each check's rule looks out to (see reach_gaps). A sweep run that passes needs, at each start sampled from it (run_starts), every check's area around that start's battery with that battery's own position error, and the wall and cable route back to the meter over the whole run. Checking the whole run as one battery at its farthest start's error would ask for more than any single start needs.""" - if sampling := geometry_admission_problems(scene, result, rules): + meter = meter if meter is not None else WorkMeter() + if sampling := geometry_admission_problems(scene, result, rules, meter=meter): return sampling result = deduplicate_result_runs(result) problems: list[str] = [] route_wall = observed( - scene, "wall", rules.route_height_ft, beyond=True, wall_default_ft=rules.headroom_ft + scene, + "wall", + rules.route_height_ft, + beyond=True, + wall_default_ft=rules.headroom_ft, + meter=meter, ) # Coverage is needed for every check the rules define for this scene, whether or not the # result lists it: a result that drops a check must not skip its coverage. @@ -993,10 +1100,10 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: # The route starts at the meter, which may stand its error either side of s = 0 # (server README, route_path), within the scanned chain. m = meter_error(scene, rules) - ends = chain_ends_s(scene, rules) or (-math.inf, math.inf) + ends = chain_ends_s(scene, rules, meter=meter) or (-math.inf, math.inf) route_lo = max(min(lo, -m), ends[0]) route_hi = min(max(hi, m), ends[1]) - if gaps_over(route_wall, route_lo, route_hi): + if gaps_over(route_wall, route_lo, route_hi, meter=meter): problems.append( f"sweep pass for starts {run['start_ft']} but the wall and cable route " f"[{route_lo:.2f}, {route_hi:.2f}] were not all observed higher than " @@ -1005,8 +1112,11 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: for check in sorted(set(rules.needs) & evaluated): gaps = {} # each gap once, at the first start that needs it for start in run_starts(run["start_ft"], rules.step_ft): + meter.charge() battery = (start, start + rules.width_ft) - for _, gap in reach_gaps(scene, rules, check, run["wall_id"], *battery): + for _, gap in reach_gaps( + scene, rules, check, run["wall_id"], *battery, meter=meter + ): gaps.setdefault(gap, start) problems += [ f"sweep pass for starts {run['start_ft']} but at start {start:.2f} {check} " @@ -1017,11 +1127,10 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: spot = result.get("spot") if spot is not None: lo, hi = spot["span_ft"] - for check in result.get("checks", []): - if check["outcome"] != "pass" or check["id"] not in rules.needs: - continue - for _, gap in reach_gaps(scene, rules, check["id"], spot["wall_id"], lo, hi): - problems.append(f"check {check['id']} passes but needs {gap}") + passing = {check["id"] for check in result.get("checks", []) if check["outcome"] == "pass"} + for check_id in sorted(passing & set(rules.needs)): + for _, gap in reach_gaps(scene, rules, check_id, spot["wall_id"], lo, hi, meter=meter): + problems.append(f"check {check_id} passes but needs {gap}") for request in result.get("missing_evidence", []): if request["kind"] != "band" or "span_ft" not in request or "band" not in request: @@ -1033,8 +1142,8 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: # only where the band was seen all the way (no `out_ft`). default = GROUND_FAR_FT if request["band"] == "ground" else math.inf far = request.get("out_ft", default) - seen = observed(scene, request["band"], far, wall_default_ft=rules.headroom_ft) - if b - a > EPS and covers(seen, a, b): + seen = observed(scene, request["band"], far, wall_default_ft=rules.headroom_ft, meter=meter) + if b - a > EPS and covers(seen, a, b, meter=meter): problems.append( f"missing_evidence asks for {request['band']} {request['span_ft']}, " "which the scene lists as observed" @@ -1049,10 +1158,13 @@ def ground_gap( depth: float, radius: float, plan: GroundPlan | None = None, + meter: WorkMeter | None = None, ) -> str | None: """The first stretch of ground seen less far out than a battery over [lo, hi] needs (see coverage_problems), or None. With `plan`, ground not seen in the battery's own wall's band still counts where it lies in another wall's observed band in plan (GroundPlan).""" + meter = meter if meter is not None else WorkMeter() + meter.charge(len(scene.get("coverage", {}).get("observed", []))) entries = [ (*sorted(e["span_ft"]), e.get("out_ft", 0.0)) for e in scene.get("coverage", {}).get("observed", []) @@ -1061,6 +1173,7 @@ def ground_gap( a, b = lo - radius, hi + radius cuts = sorted({a, lo, hi, b} | {x for e in entries for x in e[:2] if a < x < b}) for p, q in itertools.pairwise(cuts): + meter.charge() if q - p <= EPS: continue @@ -1071,6 +1184,7 @@ def need_at(s: float) -> float: return depth + math.sqrt(max(0.0, radius**2 - d**2)) need = need_at(q if q <= lo else p if p >= hi else (lo + hi) / 2) + meter.charge(len(entries)) seen = max((out for x, y, out in entries if x <= p + EPS and q - EPS <= y), default=None) if seen is None or seen + SEEN_GROWTH_FT < need: if plan is not None and plan.covers((lo + hi) / 2, p, q, seen or 0.0, need_at): @@ -1082,7 +1196,9 @@ def need_at(s: float) -> float: return None -def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = None) -> list[str]: +def missing_evidence_problems( + scene: dict, result: dict, rules: RuleSet | None = None, meter: WorkMeter | None = None +) -> list[str]: """When coverage is what stands in the way (reason unobserved_area), every check left unsure for an unobserved area is named by a request, and a clearance check by a request of each band it lacks around the chosen spot: a check that forgets one band leaves the next capture @@ -1094,12 +1210,13 @@ def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = the reason and its requests; leaving both out would never ask the homeowner for the view that settles the check. Without a spot and without the reason, every position fails anyway, no photo would change the answer, and C2 lets missing_evidence stay empty.""" - if sampling := geometry_admission_problems(scene, result, rules): + meter = meter if meter is not None else WorkMeter() + if sampling := geometry_admission_problems(scene, result, rules, meter=meter): return sampling result = deduplicate_result_runs(result) codes = {r["code"] for r in result.get("reasons", [])} spot = result.get("spot") - unseen = unobserved_checks(result) + unseen = list(dict.fromkeys(unobserved_checks(result))) requests = result.get("missing_evidence", []) if "unobserved_area" not in codes: if spot is None or not unseen: @@ -1112,13 +1229,13 @@ def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = "but no reason unobserved_area and no missing_evidence" ] lo, hi = spot["span_ft"] - asked = with_past_ends_asked(scene, requests, rules) + asked = with_past_ends_asked(scene, requests, rules, meter=meter) return [ f"check {c} is unsure and needs {gap}, not past an end the result asks to walk " f"past, but there is no reason unobserved_area and no {band} request" for c in unseen if c in rules.needs - for band, gap in reach_gaps(asked, rules, c, spot["wall_id"], lo, hi) + for band, gap in reach_gaps(asked, rules, c, spot["wall_id"], lo, hi, meter=meter) ] if not requests: return ["reason unobserved_area but missing_evidence is empty"] @@ -1132,11 +1249,11 @@ def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = if rules is None or spot is None: return problems lo, hi = spot["span_ft"] - asked = with_past_ends_asked(scene, requests, rules) - for c in unobserved_checks(result): + asked = with_past_ends_asked(scene, requests, rules, meter=meter) + for c in unseen: if c not in rules.needs: continue - for band, gap in reach_gaps(asked, rules, c, spot["wall_id"], lo, hi): + for band, gap in reach_gaps(asked, rules, c, spot["wall_id"], lo, hi, meter=meter): if (band, c) not in named: problems.append( f"check {c} is unsure and needs {gap}, but no {band} request names it" @@ -1151,7 +1268,9 @@ def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = Point = tuple[float, float] -def segments_s(scene: dict, rules: RuleSet) -> list[tuple[str, float, float, Point, Point]] | None: +def segments_s( + scene: dict, rules: RuleSet, meter: WorkMeter | None = None +) -> list[tuple[str, float, float, Point, Point]] | None: """Each straight baseline segment as (wall id, s at its start, s at its end, start point, end point), with s measured from the meter's projection, or None without a meter wall. @@ -1159,6 +1278,7 @@ def segments_s(scene: dict, rules: RuleSet) -> list[tuple[str, float, float, Poi when it is a gap: wider than both walls' own errors together (no drift), capped at sweep.wall_join_ft. Narrower, the walls meet and s skips the space (server/scene.py at 7133283, `meets`).""" + meter = meter if meter is not None else WorkMeter() if rules.wall_join_ft is None: raise ValueError("laying out the walls needs the rules' sweep.wall_join_ft") walls = scene.get("walls", []) @@ -1169,12 +1289,14 @@ def segments_s(scene: dict, rules: RuleSet) -> list[tuple[str, float, float, Poi mx, _, mz = scene["meter"]["pos"] s, prev = 0.0, None for wall in walls: + meter.charge(len(wall["baseline"])) pts = [(float(pt[0]), float(pt[1])) for pt in wall["baseline"]] if prev is not None: space = math.dist(prev[0], pts[0]) meets = min(rules.wall_join_ft, max(prev[1] + wall_error(wall, rules), SERVER_EPS)) s += space if space > meets else 0.0 for p, q in itertools.pairwise(pts): + meter.charge() seg = math.dist(p, q) if wall["id"] == scene["meter"]["wall_id"] and seg > 0: t = ((mx - p[0]) * (q[0] - p[0]) + (mz - p[1]) * (q[1] - p[1])) / seg**2 @@ -1190,13 +1312,17 @@ def segments_s(scene: dict, rules: RuleSet) -> list[tuple[str, float, float, Poi return [(wid, a - meter_s[1], b - meter_s[1], p, q) for wid, a, b, p, q in raw] -def wall_spans_s(scene: dict, rules: RuleSet) -> dict[str, tuple[float, float]] | None: +def wall_spans_s( + scene: dict, rules: RuleSet, meter: WorkMeter | None = None +) -> dict[str, tuple[float, float]] | None: """Each wall's stretch of s (see segments_s), or None without a meter wall.""" - segments = segments_s(scene, rules) + meter = meter if meter is not None else WorkMeter() + segments = segments_s(scene, rules, meter=meter) if segments is None: return None spans: dict[str, tuple[float, float]] = {} for wid, a, b, _, _ in segments: + meter.charge() lo, hi = spans.get(wid, (a, b)) spans[wid] = (min(lo, a), max(hi, b)) return spans @@ -1212,14 +1338,17 @@ class GroundPlan: # point of a region was seen: a thin gap shaped to avoid them can still be missed. STEP_FT = 0.05 - def __init__(self, scene: dict, rules: RuleSet) -> None: - self.segments = segments_s(scene, rules) or [] + def __init__(self, scene: dict, rules: RuleSet, meter: WorkMeter | None = None) -> None: + self.meter = meter if meter is not None else WorkMeter() + self.segments = segments_s(scene, rules, meter=self.meter) or [] self.strips = [] # (origin, along, outward, from s, to s, out) for e in scene.get("coverage", {}).get("observed", []): + self.meter.charge() if e["band"] != "ground": continue a, b = sorted(e["span_ft"]) for _, s0, s1, p, q in self.segments: + self.meter.charge() if min(b, s1) - max(a, s0) > EPS and s1 - s0 > EPS: along, outward = self._frame(p, q) strip = (p, along, outward, max(a, s0) - s0, min(b, s1) - s0) @@ -1233,7 +1362,12 @@ def _frame(p: Point, q: Point) -> tuple[Point, Point]: def point(self, s_mid: float, s: float, out: float) -> Point | None: """The plan point at s along the line of the segment under s_mid, `out` in front.""" - seg = next((g for g in self.segments if g[1] - EPS <= s_mid <= g[2] + EPS), None) + seg = None + for candidate in self.segments: + self.meter.charge() + if candidate[1] - EPS <= s_mid <= candidate[2] + EPS: + seg = candidate + break if seg is None: return None _, s0, _, p, q = seg @@ -1243,6 +1377,7 @@ def point(self, s_mid: float, s: float, out: float) -> Point | None: def seen(self, x: Point) -> bool: for origin, along, outward, t0, t1, out in self.strips: + self.meter.charge() dx, dz = x[0] - origin[0], x[1] - origin[1] t = dx * along[0] + dz * along[1] n = dx * outward[0] + dz * outward[1] @@ -1256,6 +1391,7 @@ def required(self, x: Point) -> bool: front of some wall segment, or past either end of the chain. Past a corner, behind the next wall, is neither: the house.""" for i, (_, _, _, p, q) in enumerate(self.segments): + self.meter.charge() along, outward = self._frame(p, q) dx, dz = x[0] - p[0], x[1] - p[1] t = dx * along[0] + dz * along[1] @@ -1280,8 +1416,10 @@ def covers( if not self.strips: return False for s in self._samples(p, q): + self.meter.charge() # Up to out_lo the stretch's own band already saw the ground. for out in self._samples(out_lo, need(s)): + self.meter.charge() if out <= out_lo + SERVER_EPS: continue x = self.point(s_mid, s, out) @@ -1291,52 +1429,56 @@ def covers( @classmethod def _samples(cls, a: float, b: float) -> Iterator[float]: + if not finite(a) or not finite(b): + raise ValueError("ground sampling bounds must be finite") cells = (b - a) / cls.STEP_FT - if not math.isfinite(cells) or cells > MAX_GEOMETRY_SAMPLES: - raise ValueError( - f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples" - ) + if not finite(cells): + raise ValueError("ground sampling span must be finite") n = max(1, math.ceil(max(0, cells))) - if 2 * n + 1 > MAX_GEOMETRY_SAMPLES: - raise ValueError( - f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples" - ) def points(): for i in range(n): - yield a + (b - a) * i / n - yield a + (b - a) * (i + 0.5) / n + yield a + (b - a) * (i / n) + yield a + (b - a) * ((i + 0.5) / n) yield b return points() -def chain_ends_s(scene: dict, rules: RuleSet) -> tuple[float, float] | None: +def chain_ends_s( + scene: dict, rules: RuleSet, meter: WorkMeter | None = None +) -> tuple[float, float] | None: """s of the wall chain's left and right ends, or None without a meter wall.""" - spans = wall_spans_s(scene, rules) + meter = meter if meter is not None else WorkMeter() + spans = wall_spans_s(scene, rules, meter=meter) if spans is None: return None return min(a for a, _ in spans.values()), max(b for _, b in spans.values()) -def declared_height_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: +def declared_height_problems( + scene: dict, result: dict, rules: RuleSet, meter: WorkMeter | None = None +) -> list[str]: """A battery backs onto a wall that declares `height_ft` only if the wall is taller than the battery. The server fails wall_backing for a lower wall surely behind the battery and leaves it unsure for one within the battery's error of it (server/solver.py at 7133283, check_backing), so wall_backing passes only if every declared wall overlapping [s0 - e, s1 + e] is taller than battery.height_ft.""" - if sampling := geometry_admission_problems(scene, result, rules): + meter = meter if meter is not None else WorkMeter() + if sampling := geometry_admission_problems(scene, result, rules, meter=meter): return sampling result = deduplicate_result_runs(result) declared = {w["id"]: w["height_ft"] for w in scene.get("walls", []) if "height_ft" in w} if not declared or rules.battery_height_ft is None: return [] - spans = wall_spans_s(scene, rules) or {} + spans = wall_spans_s(scene, rules, meter=meter) or {} height = rules.battery_height_ft def short_near(wall_id: str, lo: float, hi: float) -> str | None: - e = battery_error(scene, rules, wall_id, lo, hi) + meter.charge() + e = battery_error(scene, rules, wall_id, lo, hi, meter=meter) for wid, h in declared.items(): + meter.charge() a, b = spans.get(wid, (math.inf, -math.inf)) near = a < hi + e - SERVER_EPS and b > lo - e + SERVER_EPS if near and h - height <= SERVER_EPS: @@ -1354,6 +1496,7 @@ def short_near(wall_id: str, lo: float, hi: float) -> str | None: if run["outcome"] != "pass": continue for start in run_starts(run["start_ft"], rules.step_ft): + meter.charge() short = short_near(run["wall_id"], start, start + rules.width_ft) if short: problems.append( @@ -1363,21 +1506,25 @@ def short_near(wall_id: str, lo: float, hi: float) -> str | None: return problems -def with_past_ends_asked(scene: dict, requests: list[dict], rules: RuleSet) -> dict: +def with_past_ends_asked( + scene: dict, requests: list[dict], rules: RuleSet, meter: WorkMeter | None = None +) -> dict: """The scene as if everything past each end the result asks to walk past were observed. Past an unexplored end the wall may turn, so a view along the same line settles nothing; the server asks to keep walking (`past_end`) instead of for a band there, and that request covers every band beyond the end. Only a request at the chain's actual end, on a side not marked `limit`, earns that credit.""" + meter = meter if meter is not None else WorkMeter() kinds = scene.get("coverage", {}).get("ends", {}) past = [r for r in requests if r["kind"] == "past_end" and "span_ft" in r and "side" in r] - ends = chain_ends_s(scene, rules) if past else None + ends = chain_ends_s(scene, rules, meter=meter) if past else None if not past or ends is None: return scene m = copy.deepcopy(scene) observed_ = m.setdefault("coverage", {}).setdefault("observed", []) for r in past: + meter.charge() side = r["side"] end = ends[0] if side == "left" else ends[1] if kinds.get(side, {}).get("kind") == "limit" or abs(r["span_ft"][0] - end) > END_TOL_FT: @@ -1416,7 +1563,7 @@ def assumption_mismatches( return out -def expectation_problems(expect: dict, result: dict) -> list[str]: +def expectation_problems(expect: dict, result: dict, meter: WorkMeter | None = None) -> list[str]: problems: list[str] = [] decision = result["decision"] if "decision_in" in expect and decision not in expect["decision_in"]: @@ -1486,8 +1633,10 @@ def expectation_problems(expect: dict, result: dict) -> list[str]: if field in rule and (got is None or abs(got - rule[field]) > NUMBER_TOL_FT): problems.append(f"check {check['id']}: {field} {got}, expected {rule[field]}") - for rule in expect.get("start_outcomes", []): - got = outcome_at(result, rule["wall_id"], rule["start_ft"]) + starts = expect.get("start_outcomes", []) + index = SweepIndex(result, meter=meter) if starts else None + for rule in starts: + got = index.outcome_at(rule["wall_id"], rule["start_ft"]) if got != rule["outcome"]: why = f" ({rule['why']})" if "why" in rule else "" problems.append( @@ -1573,50 +1722,30 @@ def comparable(result: dict) -> dict: # numbers: more measurement error or less coverage can only make the server less sure. -def outcome_at(result: dict, wall_id: str, start_ft: float) -> str | None: - """Outcome of the sweep run containing this battery start, or None if none contains it.""" - for run in result.get("sweep", []): - a, b = run["start_ft"] - if run["wall_id"] == wall_id and a - EPS <= start_ft <= b + EPS: - return run["outcome"] - return None +def outcome_at( + result: dict, wall_id: str, start_ft: float, meter: WorkMeter | None = None +) -> str | None: + meter = meter if meter is not None else WorkMeter() + return SweepIndex(result, meter=meter).outcome_at(wall_id, start_ft) -def probe_starts(*results: dict) -> list[tuple[str, float]]: - """Every run's ends and middle, from all results: where the outcomes can be compared.""" - points = set() - for result in results: - for run in result.get("sweep", []): - a, b = run["start_ft"] - points |= {(run["wall_id"], a), (run["wall_id"], b), (run["wall_id"], (a + b) / 2)} - return sorted(points) +def probe_starts(*results: dict, meter: WorkMeter | None = None) -> list[tuple[str, float]]: + meter = meter if meter is not None else WorkMeter() + return indexed_probe_starts(*results, meter=meter) -def more_error_problems(before: dict, after: dict, change: str) -> list[str]: - """With more error, a start may only move toward UNSURE: UNSURE stays, PASS and FAIL stay - or become UNSURE.""" - problems = [] - for wall_id, s in probe_starts(before, after): - a, b = outcome_at(before, wall_id, s), outcome_at(after, wall_id, s) - if a is None or b is None or a == b or b == "unsure": - continue - problems.append(f"{change}: start {wall_id} {s:.3f} went from {a} to {b}") - if before["decision"] != "pass" and after["decision"] == "pass": - problems.append(f"{change}: the decision became pass") - return problems +def more_error_problems( + before: dict, after: dict, change: str, meter: WorkMeter | None = None +) -> list[str]: + meter = meter if meter is not None else WorkMeter() + return indexed_more_error_problems(before, after, change, meter=meter) -def less_coverage_problems(before: dict, after: dict, change: str) -> list[str]: - """With less observed, no start may newly PASS.""" - problems = [ - f"{change}: start {wall_id} {s:.3f} went from {outcome_at(before, wall_id, s)} to pass" - for wall_id, s in probe_starts(before, after) - if outcome_at(after, wall_id, s) == "pass" - and outcome_at(before, wall_id, s) not in (None, "pass") - ] - if before["decision"] != "pass" and after["decision"] == "pass": - problems.append(f"{change}: the decision became pass") - return problems +def less_coverage_problems( + before: dict, after: dict, change: str, meter: WorkMeter | None = None +) -> list[str]: + meter = meter if meter is not None else WorkMeter() + return indexed_less_coverage_problems(before, after, change, meter=meter) def chain_length_ft(scene: dict) -> float: diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index 1bf3ca398..060f9156d 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -297,7 +297,8 @@ def test_server_breaking_coverage_rule_fails(fake_url): assert record["status"] == "fail" joined = "\n".join(record["problems"]) assert "ruled out for this case" in joined - assert "no coverage: decision pass for a scene with no coverage at all" in joined + assert "decision pass without a passing spot" in joined + assert "no coverage:" not in joined # Initial invariant failure stops transformed requests. assert re.search(r"needs ground \[.*\] observed, none seen", joined) @@ -615,7 +616,7 @@ def test_transformed_answer_checks_its_own_hash(monkeypatch): monkeypatch.setattr(e2e, "answer", lambda *args: (original, None)) monkeypatch.setattr(e2e, "schema_errors", lambda *args: []) - def check_hash(scene, result, sent=None, rules=None): + def check_hash(scene, result, sent=None, rules=None, *, meter=None): calls.append((scene, sent)) return ["stale input hash"] if sent is not None else [] @@ -624,7 +625,10 @@ def check_hash(scene, result, sent=None, rules=None): monkeypatch.setattr(e2e, "with_more_error", lambda *args: None) monkeypatch.setattr(e2e, "tape_to_tap", lambda *args: None) monkeypatch.setattr(e2e, "with_less_coverage", lambda *args: None) - scene = {"walls": [{"id": "w1", "baseline": [[-1, 0], [1, 0]]}], "meter": {"pos": [0, 1, 0]}} + scene = { + "walls": [{"id": "w1", "baseline": [[-1, 0], [1, 0]]}], + "meter": {"pos": [0, 1, 0], "wall_id": "w1"}, + } probe = SceneInput(name="test", scene=scene, raw=json.dumps(scene).encode(), source="test") failures = e2e.property_problems( "unused", Endpoint("/p", "application/json"), probe, original, {"result": {}}, RULES diff --git a/verification/tests/test_generate_cases.py b/verification/tests/test_generate_cases.py index a37800816..89228e128 100644 --- a/verification/tests/test_generate_cases.py +++ b/verification/tests/test_generate_cases.py @@ -88,7 +88,7 @@ def test_real_case_rule_override_skips_stale_expectation(tmp_path, monkeypatch, ) monkeypatch.setattr(e2e, "schema_errors", lambda *args: []) monkeypatch.setattr(e2e, "invariant_problems", lambda *args, **kwargs: []) - monkeypatch.setattr(e2e, "property_problems", lambda *args: []) + monkeypatch.setattr(e2e, "property_problems", lambda *args, **kwargs: []) def forbidden(*args): pytest.fail("a mismatched case must not apply its stale expectations") diff --git a/verification/tests/test_intervalindex.py b/verification/tests/test_intervalindex.py new file mode 100644 index 000000000..36b6f327b --- /dev/null +++ b/verification/tests/test_intervalindex.py @@ -0,0 +1,113 @@ +import random + +import pytest + +from hsverify.intervalindex import ( + SweepIndex, + less_coverage_problems, + more_error_problems, + probe_starts, +) + + +class CountingMeter: + def __init__(self, limit=None): + self.samples = 0 + self.limit = limit + + def charge(self, n=1): + self.samples += n + if self.limit is not None and self.samples > self.limit: + raise WorkLimit + + +class WorkLimit(Exception): + pass + + +def result(*runs, decision="fail"): + return {"decision": decision, "sweep": list(runs)} + + +def run(a, b, outcome="fail", wall_id="w1"): + return {"wall_id": wall_id, "start_ft": [a, b], "outcome": outcome} + + +def test_thirty_thousand_runs_compare_with_linear_logarithmic_work(): + answer = result(*(run(i / 1e4, i / 1e4) for i in range(30_000))) + meter = CountingMeter() + assert more_error_problems(answer, answer, "more error", meter) == [] + assert 30_000 < meter.samples < 100 * 30_000 + + +def test_first_original_run_wins_overlaps_including_eps_endpoints(): + answer = result(run(2, 4, "fail"), run(0, 6, "pass"), run(4, 4, "unsure")) + index = SweepIndex(answer) + for start in (2 - 1e-6, 2, 3, 4, 4 + 1e-6): + assert index.outcome_at("w1", start) == "fail" + assert index.outcome_at("w1", 2 - 2e-6) == "pass" + assert index.outcome_at("w1", 4 + 2e-6) == "pass" + + +def test_index_agrees_with_original_linear_lookup_on_overlapping_runs(): + rng = random.Random(731) + runs = [ + run( + a := rng.randrange(-20, 20), + a + rng.randrange(0, 10), + rng.choice(["pass", "fail", "unsure"]), + ) + for _ in range(200) + ] + index = SweepIndex(result(*runs)) + points = {x for r in runs for edge in r["start_ft"] for x in (edge - 1e-6, edge, edge + 1e-6)} + points.update(rng.uniform(-30, 30) for _ in range(500)) + for start in points: + expected = next( + ( + r["outcome"] + for r in runs + if r["start_ft"][0] - 1e-6 <= start <= r["start_ft"][1] + 1e-6 + ), + None, + ) + assert index.outcome_at("w1", start) == expected + + +def test_gap_neighbours_and_equal_endpoint_ties_preserve_original_order(): + index = SweepIndex(result(run(0, 1, "fail"), run(0, 1, "pass"), run(3, 4, "unsure"))) + assert index.outcomes_near("w1", 2) == ({"fail", "unsure"}, "unswept beside fail and unsure") + assert index.outcomes_near("w1", -1) == ({"fail"}, "past the original sweep") + assert index.outcomes_near("w1", 5) == ({"fail"}, "past the original sweep") + assert index.outcomes_near("missing", 0) == (set(), "no run on its wall") + + +def test_newly_evaluated_wall_can_fail_but_cannot_pass(): + before = result(run(0, 1)) + assert more_error_problems(before, result(run(0, 1, "fail", "w2")), "error") == [] + assert more_error_problems(before, result(run(0, 1, "pass", "w2")), "error") + assert less_coverage_problems(before, result(run(0, 1, "pass", "w2")), "coverage") + + +def test_unswept_gap_can_keep_neighbour_pass_but_not_create_one(): + after = result(run(2, 2, "pass")) + assert more_error_problems(result(run(0, 1, "pass"), run(3, 4)), after, "error") == [] + assert less_coverage_problems(result(run(0, 1, "pass"), run(3, 4)), after, "coverage") == [] + assert less_coverage_problems(result(run(0, 1), run(3, 4)), after, "coverage") + + +@pytest.mark.parametrize("operation", [SweepIndex, probe_starts]) +def test_work_limit_propagates_from_index_or_probe_creation(operation): + with pytest.raises(WorkLimit): + operation(result(run(0, 1)), meter=CountingMeter(limit=0)) + + +@pytest.mark.parametrize("comparison", [more_error_problems, less_coverage_problems]) +def test_work_limit_propagates_from_comparison(comparison): + answer = result(run(0, 1, "pass")) + with pytest.raises(WorkLimit): + comparison(answer, answer, "change", meter=CountingMeter(limit=1)) + + +def test_probe_midpoint_does_not_overflow_finite_endpoints(): + assert probe_starts(result(run(1e308, 1e308))) == [("w1", 1e308)] diff --git a/verification/tests/test_judge_meter.py b/verification/tests/test_judge_meter.py new file mode 100644 index 000000000..77fea4f18 --- /dev/null +++ b/verification/tests/test_judge_meter.py @@ -0,0 +1,294 @@ +"""Hostile answers must still produce a report, with bounded verification work.""" + +import copy +import hashlib +import json +from pathlib import Path + +import pytest + +from hsverify import e2e +from hsverify.resultcheck import RuleSet, WorkMeter + +ROOT = Path(__file__).resolve().parents[2] +RULES = RuleSet.from_yaml((ROOT / "server/rules.yaml").read_text()) +SCHEMAS = { + "scene": {}, + "result": json.loads((ROOT / "server/schemas/result.schema.json").read_text()), +} +SCENE = { + "meter": {"pos": [0, 4, 0], "wall_id": "w1"}, + "walls": [{"id": "w1", "baseline": [[-30, 0], [30, 0]]}], +} +ENDPOINT = e2e.Endpoint("/place", "application/json") +META = { + "started_at": "2026-10-14", + "server_ref": "public test fixture", + "server_sha": "a" * 40, + "endpoint": "/place", + "load_average_1_5_15": [0, 0, 0], + "peak_memory": {}, +} + + +def probe(): + return e2e.SceneInput("hostile answer", SCENE, json.dumps(SCENE).encode()) + + +def response(): + return { + "schema_version": "1.0", + "decision": "manual_review", + "summary": "No spot chosen", + "reasons": [], + "policy": { + "id": "demo", + "version": "1", + "auto_approve": False, + "sources": ["public"], + "rules_sha256": "a" * 64, + }, + "spot": None, + "route": None, + "checks": [], + "missing_evidence": [], + "ends": { + side: {"kind": "limit", "s_ft": s, "point": [s, 0]} + for side, s in (("left", -30), ("right", 30)) + }, + "sweep": [], + "stats": { + "candidates": 0, + "pass": 0, + "unsure": 0, + "fail": 0, + "elapsed_ms": 1, + "input_sha256": hashlib.sha256(probe().raw).hexdigest(), + }, + } + + +def serve(monkeypatch, result): + assert not e2e.schema_errors(result, SCHEMAS["result"]) + monkeypatch.setattr(e2e, "post", lambda *args: (200, json.dumps(result).encode(), 1)) + + +def forbid_properties(*args, **kwargs): + raise AssertionError("a rejected answer reached property comparisons") + + +@pytest.mark.parametrize("exception", [OverflowError, ValueError, RuntimeError]) +def test_check_exception_is_written_to_report(monkeypatch, tmp_path, exception): + serve(monkeypatch, response()) + + def crash(*args, **kwargs): + raise exception("hostile response") + + monkeypatch.setattr(e2e, "invariant_problems", crash) + monkeypatch.setattr(e2e, "property_problems", forbid_properties) + record = e2e.judge("unused", ENDPOINT, probe(), SCHEMAS, RULES) + assert record["status"] == "fail" + assert record["problems"] == [f"invariants: {exception.__name__}: hostile response"] + report = e2e.write_report(tmp_path, META, [record]) + assert not report["contract_ok"] + assert json.loads((tmp_path / "report.json").read_text())["records"][0] == record + + +def test_failed_invariants_skip_properties(monkeypatch): + serve(monkeypatch, response()) + monkeypatch.setattr(e2e, "invariant_problems", lambda *args, **kwargs: ["bad invariant"]) + monkeypatch.setattr(e2e, "property_problems", forbid_properties) + record = e2e.judge("unused", ENDPOINT, probe(), SCHEMAS, RULES) + assert record["contract_problems"] == ["bad invariant"] + + +def test_original_and_properties_share_one_meter(monkeypatch): + serve(monkeypatch, response()) + meters = [] + + def invariants(*args, meter, **kwargs): + meters.append(meter) + meter.charge(4) + return [] + + def properties(*args, meter, **kwargs): + meters.append(meter) + meter.charge(5) + return [] + + monkeypatch.setattr(e2e, "invariant_problems", invariants) + monkeypatch.setattr(e2e, "property_problems", properties) + monkeypatch.setattr(e2e, "geometry_admission_problems", lambda *args, **kwargs: []) + meter = WorkMeter(limit=9) + record = e2e.judge("unused", ENDPOINT, probe(), SCHEMAS, RULES, meter=meter) + assert record["status"] == "pass" + assert record["verification_samples"] == 9 + assert meters == [meter, meter] + + +def test_meter_limit_is_one_specific_report_problem(monkeypatch, tmp_path): + serve(monkeypatch, response()) + + def invariants(*args, meter, **kwargs): + meter.charge(4) + return [] + + monkeypatch.setattr(e2e, "invariant_problems", invariants) + monkeypatch.setattr(e2e, "property_problems", forbid_properties) + monkeypatch.setattr(e2e, "geometry_admission_problems", lambda *args, **kwargs: []) + record = e2e.judge("unused", ENDPOINT, probe(), SCHEMAS, RULES, meter=WorkMeter(limit=3)) + assert record["problems"] == ["verification work limit exceeded (4 samples)"] + assert record["verification_samples"] == 4 + assert not e2e.write_report(tmp_path, META, [record])["contract_ok"] + + +def test_oversized_integer_is_an_admission_problem(monkeypatch, tmp_path): + result = response() + result["sweep"] = [ + {"wall_id": "w1", "start_ft": [0, 10**400], "outcome": "fail", "failing": [], "unsure": []} + ] + serve(monkeypatch, result) + monkeypatch.setattr(e2e, "property_problems", forbid_properties) + record = e2e.judge("unused", ENDPOINT, probe(), SCHEMAS, RULES) + assert record["status"] == "fail" + assert any("finite" in p for p in record["problems"]) + assert not any("OverflowError" in p for p in record["problems"]) + assert record["verification_samples"] == 0 + assert not e2e.write_report(tmp_path, META, [record])["contract_ok"] + + +def spot(wall_id): + return { + "outcome": "fail", + "wall_id": wall_id, + "segment": 0, + "span_ft": [1, 1 + RULES.width_ft], + "width_ft": RULES.width_ft, + "depth_ft": RULES.depth_ft, + "height_ft": RULES.battery_height_ft, + "footprint": [ + [1, 0], + [1 + RULES.width_ft, 0], + [1 + RULES.width_ft, RULES.depth_ft], + [1, RULES.depth_ft], + ], + "center": [1 + RULES.width_ft / 2, RULES.depth_ft / 2], + "along": [1, 0], + "outward": [0, 1], + "meter_offset_ft": [1 + RULES.width_ft / 2, RULES.depth_ft / 2], + "route_length_ft": None, + } + + +@pytest.mark.parametrize("field", ["sweep", "spot", "nearest_considered"]) +def test_unknown_wall_is_rejected_before_geometry(monkeypatch, tmp_path, field): + result = response() + if field == "sweep": + result[field] = [ + { + "wall_id": "missing", + "start_ft": [1, 2], + "outcome": "fail", + "failing": [], + "unsure": [], + } + ] + else: + result[field] = spot("missing") + serve(monkeypatch, result) + monkeypatch.setattr(e2e, "property_problems", forbid_properties) + record = e2e.judge("unused", ENDPOINT, probe(), SCHEMAS, RULES) + assert record["status"] == "fail" + assert any("unknown wall_id 'missing'" in p for p in record["problems"]) + assert record["verification_samples"] == 0 + assert not e2e.write_report(tmp_path, META, [record])["contract_ok"] + + +@pytest.mark.parametrize("failure", ["admission", "invariants"]) +def test_invalid_variant_never_reaches_comparisons(monkeypatch, failure): + original = response() + other = copy.deepcopy(original) + if failure == "admission": + other["nearest_considered"] = spot("missing") + serve(monkeypatch, original) + monkeypatch.setattr(e2e, "answer", lambda *args: (other, None)) + monkeypatch.setattr(e2e, "with_ground_short_of", lambda *args: None) + monkeypatch.setattr(e2e, "with_less_coverage", lambda *args: None) + monkeypatch.setattr(e2e, "with_more_error", lambda *args: copy.deepcopy(SCENE)) + monkeypatch.setattr(e2e, "tape_to_tap", lambda *args: None) + monkeypatch.setattr(e2e, "more_error_problems", forbid_properties) + monkeypatch.setattr(e2e, "less_coverage_problems", forbid_properties) + if failure == "admission": + monkeypatch.setattr(e2e, "invariant_problems", forbid_properties) + else: + monkeypatch.setattr(e2e, "invariant_problems", lambda *args, **kwargs: ["bad variant hash"]) + problems = e2e.property_problems("unused", ENDPOINT, probe(), original, SCHEMAS, RULES) + expected = "unknown wall_id" if failure == "admission" else "bad variant hash" + assert any(p.startswith("more error:") and expected in p for p in problems) + + +def test_variants_and_comparisons_charge_the_judge_meter(monkeypatch): + original = response() + serve(monkeypatch, original) + monkeypatch.setattr(e2e, "answer", lambda *args: (original, None)) + monkeypatch.setattr(e2e, "geometry_admission_problems", lambda *args, **kwargs: []) + monkeypatch.setattr(e2e, "with_ground_short_of", lambda *args: None) + monkeypatch.setattr(e2e, "with_less_coverage", lambda *args: None) + monkeypatch.setattr(e2e, "with_more_error", lambda *args: copy.deepcopy(SCENE)) + monkeypatch.setattr(e2e, "tape_to_tap", lambda *args: None) + received = [] + + def invariants(*args, meter, **kwargs): + received.append(meter) + meter.charge() + return [] + + def compare(*args, meter): + received.append(meter) + meter.charge(2) + return [] + + monkeypatch.setattr(e2e, "invariant_problems", invariants) + monkeypatch.setattr(e2e, "more_error_problems", compare) + meter = WorkMeter(limit=4) + problems = e2e.property_problems( + "unused", ENDPOINT, probe(), original, SCHEMAS, RULES, meter=meter + ) + assert not problems + assert received == [meter, meter, meter] + assert meter.samples == 4 + + +def test_unknown_rules_keep_rule_independent_properties(monkeypatch): + original = response() + changed = copy.deepcopy(original) + changed["summary"] = "Changed answer for the identical scene" + serve(monkeypatch, changed) + scene = copy.deepcopy(SCENE) + scene["coverage"] = {"observed": [{"band": "wall", "span_ft": [-30, 30]}]} + item = e2e.SceneInput("rule override", scene, json.dumps(scene).encode()) + variants = [] + checked_rules = [] + + def answer(url, endpoint, transformed): + variants.append(transformed.name.split("~")[-1]) + return original, None + + def invariants(*args, rules, **kwargs): + checked_rules.append(rules) + return [] + + def public_rule_transform(*args): + raise AssertionError("private override judged using a public-rule transform") + + monkeypatch.setattr(e2e, "answer", answer) + monkeypatch.setattr(e2e, "invariant_problems", invariants) + monkeypatch.setattr(e2e, "with_ground_short_of", public_rule_transform) + monkeypatch.setattr(e2e, "with_more_error", public_rule_transform) + monkeypatch.setattr(e2e, "tape_to_tap", public_rule_transform) + problems = e2e.property_problems( + "unused", ENDPOINT, item, original, SCHEMAS, RULES, rule_checks=False + ) + assert problems == ["sending the same scene twice gave different results"] + assert variants == ["mirrored", "no coverage", "less coverage"] + assert checked_rules == [None, None, None] diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 11b8a5322..640a36e61 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -6,11 +6,12 @@ import pytest from hsverify.resultcheck import ( - MAX_GEOMETRY_SAMPLES, MAX_SWEEP_SAMPLES, GroundPlan, Need, RuleSet, + WorkLimitExceeded, + WorkMeter, assumption_mismatches, battery_error, chain_ends_s, @@ -1593,18 +1594,15 @@ def starts(*args): assert len(calls) == 1 -def test_vet_many_distinct_admissible_runs_exceed_one_response_budget(monkeypatch): - def no_sampling(*args): - pytest.fail("over-budget response reached sampling") - - monkeypatch.setattr("hsverify.resultcheck.run_starts", no_sampling) +def test_vet_many_distinct_admissible_runs_exceed_one_response_budget(): scene = copy.deepcopy(SCENE) scene["walls"][0]["baseline"] = [[-2000, 0], [2000, 0]] runs = [pass_run(i / 1000, 1000 + i / 1000) | {"outcome": "unsure"} for i in range(5000)] answer = result(spot=False, sweep=runs) - assert invariant_problems(scene, answer, rules=RULES) == [ - f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples" - ] + meter = WorkMeter(1000) + with pytest.raises(WorkLimitExceeded): + invariant_problems(scene, answer, rules=RULES, meter=meter) + assert meter.samples == 1001 def test_vet_huge_spot_with_empty_sweep_stops_before_geometry(monkeypatch): @@ -1618,26 +1616,23 @@ def no_geometry(*args): assert problems and any("span_ft" in p for p in problems) -def test_vet_geometry_budget_counts_spot_ground_grid_before_iteration(monkeypatch): - def no_geometry(*args): - pytest.fail("over-budget spot reached the ground grid") - - monkeypatch.setattr(GroundPlan, "_samples", no_geometry) +def test_vet_unused_spot_grid_does_not_consume_predicted_work(): answer = result(sweep=[]) huge_reach = replace(RULES, needs={"gas_clearance": (Need("ground", 1e9),)}) - assert invariant_problems(SCENE, answer, rules=huge_reach) == [ - f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples" - ] + meter = WorkMeter(1000) + problems = invariant_problems(SCENE, answer, rules=huge_reach, meter=meter) + assert any("check gas_clearance passes" in p for p in problems) + assert meter.samples < 1000 -def test_vet_geometry_budget_counts_check_only_reported_width(monkeypatch): +def test_vet_check_only_width_does_not_predict_unused_geometry(): answer = result(spot=False, sweep=[]) answer["checks"].append( - {"id": "width", "threshold_ft": 1e9, "rule": {"key": "battery.width_ft"}} + {"id": "width", "outcome": "pass", "threshold_ft": 1e9, "rule": {"key": "battery.width_ft"}} ) - assert invariant_problems(SCENE, answer, rules=RULES) == [ - f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples" - ] + meter = WorkMeter(1000) + assert invariant_problems(SCENE, answer, rules=RULES, meter=meter) == [] + assert meter.samples < 1000 def test_vet_ground_samples_are_lazy_and_preserve_grid_edges_and_centres(): @@ -1646,19 +1641,18 @@ def test_vet_ground_samples_are_lazy_and_preserve_grid_edges_and_centres(): assert list(points) == pytest.approx([0, 0.025, 0.05, 0.075, 0.1]) -def test_vet_repeated_spot_checks_count_toward_the_shared_budget(monkeypatch): - def no_geometry(*args): - pytest.fail("repeated checks reached geometry sampling") - - monkeypatch.setattr(GroundPlan, "_samples", no_geometry) - answer = result(sweep=[], checks=[passing("ground_surface") for _ in range(1000)]) +def test_vet_repeated_spot_checks_reuse_geometry(): + one = result(sweep=[], checks=[passing("ground_surface")]) + many = result(sweep=[], checks=[passing("ground_surface") for _ in range(1000)]) broad = replace(RULES, needs={"ground_surface": (Need("ground", 10),)}) - assert invariant_problems(SCENE, answer, rules=broad) == [ - f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples" - ] + one_meter, many_meter = WorkMeter(100_000), WorkMeter(100_000) + assert coverage_problems(SCENE, many, broad, meter=many_meter) == coverage_problems( + SCENE, one, broad, meter=one_meter + ) + assert many_meter.samples == one_meter.samples -def test_vet_budget_includes_midpoint_direction_switches(monkeypatch): +def test_vet_runtime_samples_include_midpoint_direction_switches(): scene = copy.deepcopy(SCENE) scene["meter"]["plus_minus_ft"] = 8.8 scene["walls"] = [ @@ -1670,11 +1664,7 @@ def test_vet_budget_includes_midpoint_direction_switches(monkeypatch): RULES, width_ft=10, depth_ft=2, step_ft=0.1, needs={"ground_surface": (Need("ground", 0),)} ) answer = result(spot=False, sweep=[pass_run(0, 10)]) - - def no_sampling(*args): - pytest.fail("midpoint error exceeded budget after admission") - - monkeypatch.setattr("hsverify.resultcheck.run_starts", no_sampling) - assert invariant_problems(scene, answer, rules=broad) == [ - f"response geometry sample budget exceeds {MAX_GEOMETRY_SAMPLES} samples" - ] + meter = WorkMeter(1000) + with pytest.raises(WorkLimitExceeded): + invariant_problems(scene, answer, rules=broad, meter=meter) + assert meter.samples == 1001 diff --git a/verification/tests/test_runtime_meter.py b/verification/tests/test_runtime_meter.py new file mode 100644 index 000000000..991d8ec1c --- /dev/null +++ b/verification/tests/test_runtime_meter.py @@ -0,0 +1,181 @@ +import copy +import json +from dataclasses import replace +from pathlib import Path + +import pytest +from test_resultcheck import RULES, SCENE, check, result + +import hsverify.resultcheck as checks +from hsverify.resultcheck import ( + GroundPlan, + Need, + RuleSet, + WorkLimitExceeded, + WorkMeter, + finite, + geometry_admission_problems, + invariant_problems, + missing_evidence_problems, + reach_gaps, + schema_errors, +) + + +def test_oversized_integer_fails_admission_before_geometry(): + answer = result( + spot=False, sweep=[{"wall_id": "w1", "start_ft": [0, 10**400], "outcome": "fail"}] + ) + assert not finite(10**400) + meter = WorkMeter(0) + problems = invariant_problems(SCENE, answer, rules=RULES, meter=meter) + assert any("start_ft must be finite and ordered" in problem for problem in problems) + assert meter.samples == 0 + + +def test_unknown_nearest_wall_is_rejected_before_geometry(): + answer = result() + answer["nearest_considered"] = answer["spot"] | {"wall_id": "missing"} + meter = WorkMeter(0) + assert any( + "nearest_considered names unknown wall_id 'missing'" in problem + for problem in invariant_problems(SCENE, answer, rules=RULES, meter=meter) + ) + assert meter.samples == 0 + + +def test_identical_unsure_checks_do_not_repeat_geometry(): + unsure = check(outcome="unsure", measured=None, cause="unobserved") | {"id": "pool_clearance"} + answer = result(checks=[unsure], complete=False) + answer["spot"]["outcome"] = "unsure" + answer["reasons"] = [{"code": "unobserved_area"}] + answer["missing_evidence"] = [{"kind": "band", "band": "ground", "checks": ["pool_clearance"]}] + rules = replace(RULES, needs={"pool_clearance": (Need("ground", 10),)}) + meter = WorkMeter(100_000) + once = missing_evidence_problems(SCENE, answer, rules, meter=meter) + many = copy.deepcopy(answer) + many["checks"] *= 10_000 + duplicate_meter = WorkMeter(100_000) + assert missing_evidence_problems(SCENE, many, rules, meter=duplicate_meter) == once + assert duplicate_meter.samples == meter.samples + assert 0 < meter.samples < 100_000 + + +def test_reach_cache_includes_geometry_and_parameters(): + meter = WorkMeter(10_000) + first = reach_gaps(SCENE, RULES, "gas_clearance", "w1", 1, 3, meter=meter) + count = meter.samples + assert reach_gaps(SCENE, RULES, "gas_clearance", "w1", 1, 3, meter=meter) == first + assert meter.samples == count + reach_gaps(SCENE, RULES, "gas_clearance", "w1", 1, 4, meter=meter) + assert meter.samples > count + count = meter.samples + changed = copy.deepcopy(SCENE) + changed["coverage"]["observed"][1]["out_ft"] = 0.1 + reach_gaps(changed, RULES, "gas_clearance", "w1", 1, 3, meter=meter) + assert meter.samples > count + + +def test_ground_grid_stops_at_shared_runtime_limit(): + meter = WorkMeter(30) + plan = GroundPlan(SCENE, RULES, meter=meter) + with pytest.raises(WorkLimitExceeded) as caught: + plan.covers(2, 1, 3, 0, lambda _: 2) + assert caught.value.samples == meter.samples == 31 + assert str(caught.value) == "verification work limit exceeded (31 samples)" + + +def test_valid_wide_direct_coverage_uses_only_visited_work(): + rules = RuleSet.from_yaml((Path(__file__).parents[2] / "server" / "rules.yaml").read_text()) + scene = copy.deepcopy(SCENE) + scene["walls"][0]["baseline"] = [[-30, 0], [30, 0]] + scene["objects"] = [] + scene["ground"] = [{"type": "concrete", "polygon": [[-30, 0], [30, 0], [30, 20], [-30, 20]]}] + scene["coverage"]["observed"] = [ + {"band": band, "span_ft": [-30, 30], "out_ft": 20} + for band in ("ground", "wall", "facing", "overhead") + ] + answer = result(decision="pass") + answer["summary"] = "All public-rule checks pass with coverage beyond every required reach." + lo, hi = 2, 2 + rules.width_ft + answer["spot"].update( + span_ft=[lo, hi], + center=[(lo + hi) / 2, rules.depth_ft / 2], + meter_offset_ft=[(lo + hi) / 2, rules.depth_ft / 2], + footprint=[[lo, 0], [hi, 0], [hi, rules.depth_ft], [lo, rules.depth_ft]], + route_length_ft=2, + ) + answer["sweep"][0]["start_ft"] = [2, 12] + answer["route"] = { + "outcome": "pass", + "length_ft": 2, + "plus_minus_ft": 0, + "height_ft": rules.route_height_ft, + "polyline": [[0, 0], [2, 0]], + "detours": [], + "crossings": [], + } + answer["ends"] = { + "left": {"kind": "limit", "s_ft": -30, "point": [-30, 0]}, + "right": {"kind": "limit", "s_ft": 30, "point": [30, 0]}, + } + for kind, value in (("scene", scene), ("result", answer)): + schema = json.loads( + (Path(__file__).parents[2] / "server" / "schemas" / f"{kind}.schema.json").read_text() + ) + assert schema_errors(value, schema) == [] + assert len(json.dumps(answer).encode()) == 4420 + meter = WorkMeter(100_000) + assert geometry_admission_problems(scene, answer, rules) == [] + assert invariant_problems(scene, answer, rules=rules, meter=meter) == [] + assert 0 < meter.samples < 100_000 + + +def test_invariant_geometry_exception_names_its_check(monkeypatch): + def broken(*args, **kwargs): + raise RuntimeError("damaged geometry") + + monkeypatch.setattr("hsverify.resultcheck.coverage_problems", broken) + problems = invariant_problems(SCENE, result(), rules=RULES) + assert "coverage: RuntimeError: damaged geometry" in problems + + +def test_each_candidate_wall_piece_charges_the_shared_meter(monkeypatch): + monkeypatch.setattr(checks, "geometry_admission_problems", lambda *args, **kwargs: []) + monkeypatch.setattr( + checks, + "straight_pieces", + lambda *args, **kwargs: [(i * 10, i * 10 + 5) for i in range(1000)], + ) + answer = result( + spot=False, sweep=[{"wall_id": "w1", "start_ft": [9991, 9991], "outcome": "unsure"}] + ) + meter = WorkMeter(100) + with pytest.raises(WorkLimitExceeded): + checks.footprint_problems(SCENE, answer, RULES, meter=meter) + assert meter.samples == 101 + + +def test_finite_endpoints_with_overflowing_length_stop_admission(): + answer = result( + spot=False, sweep=[{"wall_id": "w1", "start_ft": [-1e308, 1e308], "outcome": "fail"}] + ) + meter = WorkMeter(0) + problems = invariant_problems(SCENE, answer, rules=RULES, meter=meter) + assert any("span length must be finite" in p for p in problems) + assert meter.samples == 0 + + +def test_equal_scene_copies_share_geometry_cache_keys(): + meter = WorkMeter() + assert meter.input_key(SCENE) == meter.input_key(copy.deepcopy(SCENE)) + assert isinstance(meter.input_key(SCENE), int) + before = meter.samples + for _ in range(10_000): + meter.input_key(SCENE) + assert meter.samples == before + + +def test_ground_samples_do_not_overflow_finite_span(): + points = GroundPlan._samples(0, 1e306) + assert all(finite(next(points)) for _ in range(1000)) From 67b10e568473df531a3295eb0802e4927e7cf566 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Wed, 14 Oct 2026 14:44:17 -0500 Subject: [PATCH 55/62] fix(verification): reject ambiguous capture scene paths --- verification/hsverify/e2e.py | 13 ++++++++++--- verification/tests/test_e2e.py | 23 +++++++++++++++++++---- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index 07458fde4..37406beb5 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -162,14 +162,21 @@ def read_bundle(path: Path) -> tuple[bytes, dict[str, bytes]]: declared = sum(i.file_size for i in infos) if declared > MAX_BUNDLE_BYTES: raise SystemExit(f"{path}: declares {declared} bytes, limit {MAX_BUNDLE_BYTES}") - infos = [i for i in infos if not i.is_dir() and not i.filename.startswith("__MACOSX/")] + capture_infos = [i for i in infos if not i.filename.startswith("__MACOSX/")] + infos = [i for i in capture_infos if not i.is_dir()] + # Mirror the public API layouts, but refuse competing scenes instead of + # preferring the root: the verifier must know which input it is testing. + candidates = [i for i in infos if i.filename.split("/")[-1] == "scene.json"] + if len({i.filename for i in candidates}) > 1: + raise SystemExit(f"{path}: ambiguous scene.json paths") scene_name = "scene.json" if not any(i.filename == scene_name for i in infos): # The public API also accepts a capture wrapped in one top-level folder. - tops = {i.filename.split("/", 1)[0] for i in infos} + tops = {i.filename.split("/", 1)[0] for i in capture_infos} if len(tops) == 1: (top,) = tops - scene_name = f"{top}/scene.json" + if all(i.filename.startswith(f"{top}/") for i in capture_infos): + scene_name = f"{top}/scene.json" scenes = [i for i in infos if i.filename == scene_name] if len(scenes) != 1: raise SystemExit( diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index 2f6bd5f9b..1691c42fa 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -608,23 +608,36 @@ def test_bundle_reads_one_wrapped_capture_and_ignores_macos_metadata(tmp_path): assert e2e.read_bundle(archive) == (b"canonical", {"frame.jpg": b"photo"}) -def test_bundle_root_scene_takes_precedence_over_wrapped_scene(tmp_path): +def test_bundle_accepts_public_api_finder_fixture_shape(tmp_path): + archive = tmp_path / "scene.zip" + with zipfile.ZipFile(archive, "w") as z: + z.writestr("capture-2026-09-26/scene.json", b"canonical") + z.writestr("__MACOSX/capture-2026-09-26/._scene.json", b"resource fork") + assert e2e.read_bundle(archive) == (b"canonical", {}) + + +@pytest.mark.parametrize("wrapped_scene", ["capture/scene.json", "a/b/scene.json"]) +def test_bundle_rejects_root_and_wrapped_scene_together(tmp_path, wrapped_scene): archive = tmp_path / "scene.zip" with zipfile.ZipFile(archive, "w") as z: z.writestr("scene.json", b"canonical") - z.writestr("capture/scene.json", b"other") - assert e2e.read_bundle(archive)[0] == b"canonical" + z.writestr(wrapped_scene, b"other") + with pytest.raises(SystemExit, match=r"ambiguous scene\.json"): + e2e.read_bundle(archive) @pytest.mark.parametrize( "names", [ ["old-scene.json"], + ["not-scene.json"], ["scene.json", "scene.json"], ["capture/scene.json", "capture/scene.json"], ["first/scene.json", "second/scene.json"], ["capture/scene.json", "other/frame.jpg"], ["capture/nested/scene.json"], + ["capture/scene.json", "other/"], + ["capture/scene.json", "capture"], ], ) def test_bundle_rejects_missing_or_duplicate_canonical_scene(tmp_path, names): @@ -632,7 +645,9 @@ def test_bundle_rejects_missing_or_duplicate_canonical_scene(tmp_path, names): with zipfile.ZipFile(archive, "w") as z: for name in names: z.writestr(name, b"{}") - with pytest.raises(SystemExit, match=r"exactly one canonical scene\.json"): + with pytest.raises( + SystemExit, match=r"exactly one canonical scene\.json|ambiguous scene\.json" + ): e2e.read_bundle(archive) From 45645f084f53ec39e51b8ca55c869df86a7a6bd1 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Wed, 14 Oct 2026 15:23:08 -0500 Subject: [PATCH 56/62] fix(verification): cache scene geometry and bound metadata work --- verification/hsverify/e2e.py | 24 +- verification/hsverify/resultcheck.py | 303 +- .../tests/fixtures/segmented-public-pass.json | 1383 +++++ .../fixtures/segmented-public-workload.json | 4565 +++++++++++++++++ verification/tests/segmented_wall.py | 27 + verification/tests/test_judge_scene_cache.py | 73 + verification/tests/test_metadata_bounds.py | 183 + verification/tests/test_runtime_meter.py | 8 +- verification/tests/test_scene_cache.py | 81 + 9 files changed, 6590 insertions(+), 57 deletions(-) create mode 100644 verification/tests/fixtures/segmented-public-pass.json create mode 100644 verification/tests/fixtures/segmented-public-workload.json create mode 100644 verification/tests/segmented_wall.py create mode 100644 verification/tests/test_judge_scene_cache.py create mode 100644 verification/tests/test_metadata_bounds.py create mode 100644 verification/tests/test_scene_cache.py diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index 37406beb5..45e539244 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -63,10 +63,12 @@ WorkLimitExceeded, WorkMeter, assumption_mismatches, + case_metadata_problems, comparable, coverage_blocks, expectation_problems, geometry_admission_problems, + ingestion_problems, invariant_problems, less_coverage_problems, mirror_scene, @@ -501,7 +503,11 @@ def judge( return record | {"status": "skipped", "problems": [], "skip_reason": item.skip_reason} if item.hostile: return record | {"hostile": True} | judge_hostile(url, endpoint, item, schemas, limit) - input_errors = schema_errors(item.scene, schemas["scene"]) + input_errors = ( + case_metadata_problems(item.rules_assumed, item.expect) + or ingestion_problems(item.scene, "scene") + or schema_errors(item.scene, schemas["scene"]) + ) if input_errors: return record | {"status": "bad input", "problems": input_errors[:20]} @@ -530,7 +536,12 @@ def check(name: str, function: Callable[[], list[str]]) -> list[str]: problems = check( "result schema", - lambda: [f"result schema: {e}" for e in schema_errors(result, schemas["result"])], + lambda: [ + f"result schema: {e}" + for e in ( + ingestion_problems(result, "result") or schema_errors(result, schemas["result"]) + ) + ], ) if problems: return record | {"status": "fail", "problems": problems[:20], "contract_problems": problems} @@ -608,6 +619,9 @@ def property_problems( problems: list[str] = [] def resend(name: str, scene: dict) -> dict | None: + if admission := ingestion_problems(scene, "scene"): + problems.extend(f"{name}: scene ingestion: {p}" for p in admission) + return None transformed = variant(item, name, scene) other, failure = answer(url, endpoint, transformed) if failure: @@ -615,7 +629,7 @@ def resend(name: str, scene: dict) -> dict | None: return None check_name = "result schema" try: - errors = schema_errors(other, schemas["result"]) + errors = ingestion_problems(other, "result") or schema_errors(other, schemas["result"]) if errors: problems.append(f"{name}: result schema: {errors[0]}") return None @@ -641,7 +655,9 @@ def resend(name: str, scene: dict) -> dict | None: # 1. Same input, same answer. status, payload, _ = post(url, endpoint, item) again = parse_result(payload) if status == 200 else None - if again is None or comparable(again) != comparable(result): + if again is not None and (admission := ingestion_problems(again, "result")): + problems.extend(f"identical resend: {p}" for p in admission) + elif again is None or comparable(again) != comparable(result): problems.append("sending the same scene twice gave different results") # 2. Mirrored left to right: same decision, same amount of each outcome along the wall. diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 8212ad087..8ca930ae9 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -52,15 +52,36 @@ EPS = 1e-6 NUMBER_TOL_FT = 1e-5 +# Geometry loops share WorkMeter. Metadata traversal, hashing, copying and interval +# sorting are bounded separately before schema validation: 50 walls, 500 baseline +# vertices per wall (C1's existing limits), 30,000 sweep runs, 16,000 checks and +# 16,000 requests, 1,000,000 numeric leaves, 2,000,000 JSON nodes, depth 32, and +# 4 MiB of string/key characters per document, and 64 case rule queries. +# Case queries may scan the bounded response; their count is fixed. Other metadata +# passes are linear or +# O(n log n); geometry comparisons and samples charge the meter. These operational +# limits retain the 30,000-run and 12,000-distinct-check regressions. They are not +# measured scan maxima; the node/numeric limits also bound otherwise uncapped C2 +# arrays and C1's combined polygons/keyframes before recursive library traversal. +MAX_RESULT_RUNS = 30_000 +MAX_RESULT_CHECKS = 16_000 +MAX_EVIDENCE_REQUESTS = 16_000 +MAX_NUMERIC_LEAVES = 1_000_000 +MAX_METADATA_NODES = 2_000_000 +MAX_METADATA_DEPTH = 32 +MAX_METADATA_CHARACTERS = 4 * 1024 * 1024 +MAX_SCHEMA_PROBLEMS = 100 +MAX_CASE_RULE_QUERIES = 64 # Bound verifier work even when a server span or rules step requests billions of samples. MAX_SWEEP_SAMPLES = 10_000 -# On 2026-10-14, 35 public fixture judgments without contract problems, including -# property variants, peaked at 80,788 units for pass-clear-side-wall. Another 468 -# distinct saved responses matched to current fixtures peaked at 392,115 invariant -# units for ETH3D facade. Allow 20 times that larger recorded peak for other scans -# and their property variants. This is an operational bound, not a calibration of -# measurement accuracy or evidence that the samples prove physical clearance. -MAX_WORK_SAMPLES = 20 * 392_115 +# On 2026-10-14, after caching scene derivatives and deferring ground fallback, +# 35 valid public fixture judgments including property variants peaked at 30,913 +# units. Twenty measurements of 20 four-foot walls at 1/4/12/13/20 subdivisions, +# original and mirrored, including full-judge resends, peaked at 87,427 units. +# Another 468 distinct saved responses peaked at 391,700 invariant units on ETH3D +# facade. Allow 20 times that largest measured valid workload for other scans and +# their variants. This is an operational margin, not physical-error calibration. +MAX_WORK_SAMPLES = 20 * 391_700 class WorkLimitExceeded(Exception): @@ -86,6 +107,14 @@ def charge(self, n: int = 1) -> None: if self.samples > self.limit: raise WorkLimitExceeded(self.samples) + def derived( + self, name: str, scene: dict, rules: RuleSet | None, build: Callable[[], Any] + ) -> Any: + key = (name, self.input_key(scene), self.input_key(rules)) + if key not in self.cache: + self.cache[key] = build() + return self.cache[key] + def input_key(self, value: Any) -> Any: # judge's scenes and rule sets are immutable during a response's checks. Keep the # source alive so a later transformed scene cannot reuse its object id. @@ -238,9 +267,90 @@ def reach(key: str) -> Need: ) +def ingestion_problems(instance: Any, kind: str | None = None) -> list[str]: + """Bound metadata before validation or copies, using a depth-sized iterator stack.""" + if kind is None and isinstance(instance, dict): + kind = "scene" if "walls" in instance else "result" + label = kind or "document" + if isinstance(instance, dict): + limits = ( + {"walls": 50} + if kind == "scene" + else { + "sweep": MAX_RESULT_RUNS, + "checks": MAX_RESULT_CHECKS, + "missing_evidence": MAX_EVIDENCE_REQUESTS, + } + ) + for field, limit in limits.items(): + items = instance.get(field) + if isinstance(items, (list, tuple)) and len(items) > limit: + return [f"{label} ingestion limit exceeded: {field} exceeds {limit} entries"] + if kind == "scene": + for wall in ( + instance.get("walls", []) if isinstance(instance.get("walls"), list) else [] + ): + points = wall.get("baseline") if isinstance(wall, dict) else None + if isinstance(points, (list, tuple)) and len(points) > 500: + return [f"{label} ingestion limit exceeded: baseline exceeds 500 vertices"] + + nodes = numeric = characters = 0 + stack = [iter((instance,))] + while stack: + try: + value = next(stack[-1]) + except StopIteration: + stack.pop() + continue + nodes += 1 + if nodes > MAX_METADATA_NODES: + return [f"{label} ingestion limit exceeded: more than {MAX_METADATA_NODES} JSON nodes"] + if isinstance(value, (int, float)) and not isinstance(value, bool): + numeric += 1 + if numeric > MAX_NUMERIC_LEAVES: + return [ + f"{label} ingestion limit exceeded: more than " + f"{MAX_NUMERIC_LEAVES} numeric leaves" + ] + elif isinstance(value, str): + characters += len(value) + elif isinstance(value, dict): + if len(value) > MAX_METADATA_NODES - nodes: + return [ + f"{label} ingestion limit exceeded: more than {MAX_METADATA_NODES} JSON nodes" + ] + characters += sum(len(key) for key in value if isinstance(key, str)) + if len(stack) > MAX_METADATA_DEPTH: + return [f"{label} ingestion limit exceeded: nesting exceeds {MAX_METADATA_DEPTH}"] + stack.append(iter(value.values())) + elif isinstance(value, (list, tuple)): + if len(value) > MAX_METADATA_NODES - nodes: + return [ + f"{label} ingestion limit exceeded: more than {MAX_METADATA_NODES} JSON nodes" + ] + if len(stack) > MAX_METADATA_DEPTH: + return [f"{label} ingestion limit exceeded: nesting exceeds {MAX_METADATA_DEPTH}"] + stack.append(iter(value)) + if characters > MAX_METADATA_CHARACTERS: + return [ + f"{label} ingestion limit exceeded: strings/keys exceed " + f"{MAX_METADATA_CHARACTERS} characters" + ] + return [] + + def schema_errors(instance: Any, schema: dict) -> list[str]: + properties = schema.get("properties", {}) + kind = "scene" if "walls" in properties else "result" if "sweep" in properties else None + if problems := ingestion_problems(instance, kind): + return problems validator = Draft202012Validator(schema) - errors = sorted(validator.iter_errors(instance), key=lambda e: list(e.absolute_path)) + # A malformed document may produce a diagnostic per node. Stop after enough + # diagnostics to fail it; valid documents still complete schema validation. + errors = sorted( + itertools.islice(validator.iter_errors(instance), MAX_SCHEMA_PROBLEMS), + key=lambda e: tuple(str(part) for part in e.absolute_path), + ) return [f"{'/'.join(map(str, e.absolute_path)) or '(root)'}: {e.message}" for e in errors] @@ -278,6 +388,9 @@ def observed( this category default at 6200098 (server/scene.py `observed_intervals`). None keeps the old reading, all the way up.""" meter = meter if meter is not None else WorkMeter() + key = ("observed", meter.input_key(scene), band, min_out_ft, beyond, wall_default_ft) + if key in meter.cache: + return meter.cache[key] spans = [] for entry in scene.get("coverage", {}).get("observed", []): meter.charge() @@ -303,7 +416,8 @@ def observed( merged[-1][1] = max(merged[-1][1], b) else: merged.append([a, b]) - return [(a, b) for a, b in merged] + meter.cache[key] = [(a, b) for a, b in merged] + return meter.cache[key] def covers( @@ -522,6 +636,9 @@ def straight_pieces( segments joined where the next one starts at the previous one's end and continues its line, within a wall or across two walls that meet.""" meter = meter if meter is not None else WorkMeter() + key = ("straight_pieces", meter.input_key(scene), meter.input_key(rules)) + if key in meter.cache: + return meter.cache[key] pieces: list[list] = [] # [s0, s1, start point, end point] for _, s0, s1, a, b in segments_s(scene, rules, meter=meter) or []: meter.charge() @@ -536,7 +653,8 @@ def straight_pieces( prev[1], prev[3] = s1, b continue pieces.append([s0, s1, a, b]) - return [(p[0], p[1]) for p in pieces] + meter.cache[key] = [(p[0], p[1]) for p in pieces] + return meter.cache[key] def footprint_problems( @@ -625,7 +743,7 @@ def spot_position_problems( f"[{fx:.2f}, {fz:.2f}]" ) lo, hi = spot["span_ft"] - plan = GroundPlan(scene, rules, meter=meter) + plan = ground_plan(scene, rules, meter) want = plan.point((lo + hi) / 2, (lo + hi) / 2, rules.depth_ft / 2) if want is not None and math.dist((cx, cz), want) > COLLINEAR_FT: problems.append( @@ -670,7 +788,7 @@ def battery_error( its checks (solver.py _errors): its wall's error at its far edge from the meter, plus the larger of the meter's error in plan and the battery's slide along the walls (slide).""" meter = meter if meter is not None else WorkMeter() - wall = next((w for w in scene["walls"] if w["id"] == wall_id), None) + wall = wall_lookup(scene, meter).get(wall_id) if wall is None: raise ValueError(f"the result names wall {wall_id!r}, which the scene does not have") return wall_error_at(wall, rules, max(abs(lo), abs(hi))) + max( @@ -684,7 +802,7 @@ def along_error( """The battery's error against spans measured along the walls: its wall's at its far edge plus its slide, without the meter's plan error (server 3baa338 _errors, along_err).""" meter = meter if meter is not None else WorkMeter() - wall = next(w for w in scene["walls"] if w["id"] == wall_id) + wall = wall_lookup(scene, meter)[wall_id] return wall_error_at(wall, rules, max(abs(lo), abs(hi))) + slide( scene, rules, lo, hi, meter=meter ) @@ -712,10 +830,13 @@ def slide( other wall between the meter and the battery's near edge, at its far end, times the same difference for that wall. 0 on the meter's own wall.""" meter = meter if meter is not None else WorkMeter() + key = ("slide", meter.input_key(scene), meter.input_key(rules), lo, hi) + if key in meter.cache: + return meter.cache[key] segments = segments_s(scene, rules, meter=meter) if not segments: return 0.0 - walls = {w["id"]: w for w in scene["walls"]} + walls = wall_lookup(scene, meter) def along(seg) -> Point: _, _, _, p, q = seg @@ -749,6 +870,7 @@ def turn(seg) -> float: if b - a <= EPS: continue total += wall_error_at(walls[seg[0]], rules, max(abs(a), abs(b))) * turn(seg) + meter.cache[key] = total return total @@ -796,7 +918,11 @@ def reach_gaps( error = battery_error(scene, rules, wall_id, lo, hi, meter=meter) reach = need.radius + error if need.band == "ground": - plan = GroundPlan(scene, rules, meter=meter) if rules.wall_join_ft is not None else None + plan = ( + (lambda: ground_plan(scene, rules, meter)) + if rules.wall_join_ft is not None + else None + ) gap = ground_gap(scene, lo, hi, rules.depth_ft, reach, plan, meter=meter) elif need.band == "wall": a, b = required_span(lo, hi, reach) @@ -831,17 +957,17 @@ def measured_band_gap( """The first stretch of [lo, hi] the facing or overhead band leaves unsettled: not observed, or observed short of `need` where no measurement covers it.""" meter = meter if meter is not None else WorkMeter() - meter.charge(len(scene.get("coverage", {}).get("observed", []))) - entries = [ - (*sorted(e["span_ft"]), reached(e)) - for e in scene.get("coverage", {}).get("observed", []) - if e["band"] == band - ] - measured = [ - tuple(sorted(m["span_ft"])) - for m in scene.get(MEASUREMENTS[band], []) - if m.get("wall_id", wall_id) == wall_id - ] + entries = band_entries(scene, band, meter) + measured = meter.derived( + f"measurements:{band}:{wall_id}", + scene, + None, + lambda: [ + tuple(sorted(m["span_ft"])) + for m in scene.get(MEASUREMENTS[band], []) + if m.get("wall_id", wall_id) == wall_id + ], + ) edges = {x for e in entries for x in e[:2]} | {x for m in measured for x in m} cuts = sorted({lo, hi} | {x for x in edges if lo < x < hi}) for p, q in itertools.pairwise(cuts): @@ -952,8 +1078,7 @@ def reported_dimension_problems(result: dict, rules: RuleSet | None) -> list[str for source, width in widths: if not finite(width) or width <= 0: message = f"{source} must be finite and positive" - if message not in problems: - problems.append(message) + problems.append(message) for check in result.get("checks", []): if check.get("rule", {}).get("key") == "route.corner_allowance_ft": allowance = check.get("threshold_ft") @@ -973,7 +1098,12 @@ def reported_dimension_problems(result: dict, rules: RuleSet | None) -> list[str and abs(first[field] - second[field]) > NUMBER_TOL_FT ): problems.append(f"spot and nearest_considered {field} disagree") - return problems + return _unique_messages(problems) + + +def _unique_messages(messages: list[str]) -> list[str]: + """Preserve diagnostic order without comparing each message to every predecessor.""" + return list(dict.fromkeys(messages)) def geometry_admission_problems( @@ -981,6 +1111,8 @@ def geometry_admission_problems( ) -> list[str]: """Validate response numbers and references before geometry. Work is counted as it runs.""" meter = meter if meter is not None else WorkMeter() + if problems := ingestion_problems(scene, "scene") + ingestion_problems(result, "result"): + return problems problems = reported_dimension_problems(result, rules) def numbers(value: Any, path: str) -> None: @@ -1157,19 +1289,14 @@ def ground_gap( hi: float, depth: float, radius: float, - plan: GroundPlan | None = None, + plan: GroundPlan | Callable[[], GroundPlan] | None = None, meter: WorkMeter | None = None, ) -> str | None: """The first stretch of ground seen less far out than a battery over [lo, hi] needs (see coverage_problems), or None. With `plan`, ground not seen in the battery's own wall's band still counts where it lies in another wall's observed band in plan (GroundPlan).""" meter = meter if meter is not None else WorkMeter() - meter.charge(len(scene.get("coverage", {}).get("observed", []))) - entries = [ - (*sorted(e["span_ft"]), e.get("out_ft", 0.0)) - for e in scene.get("coverage", {}).get("observed", []) - if e["band"] == "ground" - ] + entries = band_entries(scene, "ground", meter) a, b = lo - radius, hi + radius cuts = sorted({a, lo, hi, b} | {x for e in entries for x in e[:2] if a < x < b}) for p, q in itertools.pairwise(cuts): @@ -1187,6 +1314,8 @@ def need_at(s: float) -> float: meter.charge(len(entries)) seen = max((out for x, y, out in entries if x <= p + EPS and q - EPS <= y), default=None) if seen is None or seen + SEEN_GROWTH_FT < need: + if callable(plan): + plan = plan() if plan is not None and plan.covers((lo + hi) / 2, p, q, seen or 0.0, need_at): continue where = f"ground [{p:.2f}, {q:.2f}] observed" @@ -1241,10 +1370,11 @@ def missing_evidence_problems( return ["reason unobserved_area but missing_evidence is empty"] problems: list[str] = [] named = {(r.get("band"), c) for r in requests for c in r.get("checks", [])} + requested_ids = {check for _, check in named} problems += [ f"check {c} is unsure (unobserved) but no missing_evidence entry names it" for c in unseen - if not any(check == c for _, check in named) + if c not in requested_ids ] if rules is None or spot is None: return problems @@ -1268,6 +1398,15 @@ def missing_evidence_problems( Point = tuple[float, float] +def band_entries(scene: dict, band: str, meter: WorkMeter) -> list[tuple[float, float, float]]: + def build() -> list[tuple[float, float, float]]: + entries = scene.get("coverage", {}).get("observed", []) + meter.charge(len(entries)) + return [(*sorted(e["span_ft"]), reached(e)) for e in entries if e["band"] == band] + + return meter.derived(f"coverage_entries:{band}", scene, None, build) + + def segments_s( scene: dict, rules: RuleSet, meter: WorkMeter | None = None ) -> list[tuple[str, float, float, Point, Point]] | None: @@ -1279,6 +1418,9 @@ def segments_s( sweep.wall_join_ft. Narrower, the walls meet and s skips the space (server/scene.py at 7133283, `meets`).""" meter = meter if meter is not None else WorkMeter() + key = ("segments_s", meter.input_key(scene), meter.input_key(rules)) + if key in meter.cache: + return meter.cache[key] if rules.wall_join_ft is None: raise ValueError("laying out the walls needs the rules' sweep.wall_join_ft") walls = scene.get("walls", []) @@ -1309,7 +1451,26 @@ def segments_s( prev = (pts[-1], wall_error(wall, rules)) if meter_s is None: return None - return [(wid, a - meter_s[1], b - meter_s[1], p, q) for wid, a, b, p, q in raw] + merged: list[tuple[str, float, float, Point, Point]] = [] + # Only redundant vertices on the same straight wall are removed. Keep wall IDs, + # s distances and real turns, since wall errors and corner slides depend on them. + for wid, a, b, p, q in raw: + if b - a <= SERVER_EPS: + continue + if merged and merged[-1][0] == wid: + prev_wid, prev_a, _, start, end = merged[-1] + ux, uz = end[0] - start[0], end[1] - start[1] + vx, vz = q[0] - p[0], q[1] - p[1] + cross = abs(ux * vz - uz * vx) + if ( + cross <= SERVER_EPS * math.hypot(ux, uz) * math.hypot(vx, vz) + and ux * vx + uz * vz > 0 + ): + merged[-1] = (prev_wid, prev_a, b, start, q) + continue + merged.append((wid, a, b, p, q)) + meter.cache[key] = [(wid, a - meter_s[1], b - meter_s[1], p, q) for wid, a, b, p, q in merged] + return meter.cache[key] def wall_spans_s( @@ -1317,6 +1478,9 @@ def wall_spans_s( ) -> dict[str, tuple[float, float]] | None: """Each wall's stretch of s (see segments_s), or None without a meter wall.""" meter = meter if meter is not None else WorkMeter() + key = ("wall_spans_s", meter.input_key(scene), meter.input_key(rules)) + if key in meter.cache: + return meter.cache[key] segments = segments_s(scene, rules, meter=meter) if segments is None: return None @@ -1325,9 +1489,18 @@ def wall_spans_s( meter.charge() lo, hi = spans.get(wid, (a, b)) spans[wid] = (min(lo, a), max(hi, b)) + meter.cache[key] = spans return spans +def wall_lookup(scene: dict, meter: WorkMeter) -> dict[str, dict]: + return meter.derived("wall_lookup", scene, None, lambda: {w["id"]: w for w in scene["walls"]}) + + +def ground_plan(scene: dict, rules: RuleSet, meter: WorkMeter) -> GroundPlan: + return meter.derived("ground_plan", scene, rules, lambda: GroundPlan(scene, rules, meter=meter)) + + class GroundPlan: """The observed ground in plan: each entry's stretch of every segment it runs along, out to its `out_ft` on the segment's outward side (the baseline turned 90 degrees clockwise seen @@ -1341,18 +1514,29 @@ class GroundPlan: def __init__(self, scene: dict, rules: RuleSet, meter: WorkMeter | None = None) -> None: self.meter = meter if meter is not None else WorkMeter() self.segments = segments_s(scene, rules, meter=self.meter) or [] - self.strips = [] # (origin, along, outward, from s, to s, out) - for e in scene.get("coverage", {}).get("observed", []): - self.meter.charge() - if e["band"] != "ground": - continue - a, b = sorted(e["span_ft"]) - for _, s0, s1, p, q in self.segments: + self.scene = scene + self.rules = rules + + @property + def strips(self) -> list: + # Direct s-coverage and spot positioning need no ground polygons. Construct + # these strips only when an uncovered reach actually needs the plan fallback. + def build() -> list: + strips = [] # (origin, along, outward, from s, to s, out) + for e in self.scene.get("coverage", {}).get("observed", []): self.meter.charge() - if min(b, s1) - max(a, s0) > EPS and s1 - s0 > EPS: - along, outward = self._frame(p, q) - strip = (p, along, outward, max(a, s0) - s0, min(b, s1) - s0) - self.strips.append((*strip, e.get("out_ft", 0.0))) + if e["band"] != "ground": + continue + a, b = sorted(e["span_ft"]) + for _, s0, s1, p, q in self.segments: + self.meter.charge() + if min(b, s1) - max(a, s0) > EPS and s1 - s0 > EPS: + along, outward = self._frame(p, q) + strip = (p, along, outward, max(a, s0) - s0, min(b, s1) - s0) + strips.append((*strip, e.get("out_ft", 0.0))) + return strips + + return self.meter.derived("ground_strips", self.scene, self.rules, build) @staticmethod def _frame(p: Point, q: Point) -> tuple[Point, Point]: @@ -1450,10 +1634,14 @@ def chain_ends_s( ) -> tuple[float, float] | None: """s of the wall chain's left and right ends, or None without a meter wall.""" meter = meter if meter is not None else WorkMeter() + key = ("chain_ends_s", meter.input_key(scene), meter.input_key(rules)) + if key in meter.cache: + return meter.cache[key] spans = wall_spans_s(scene, rules, meter=meter) if spans is None: return None - return min(a for a, _ in spans.values()), max(b for _, b in spans.values()) + meter.cache[key] = min(a for a, _ in spans.values()), max(b for _, b in spans.values()) + return meter.cache[key] def declared_height_problems( @@ -1539,9 +1727,22 @@ def with_past_ends_asked( # --- Case expectations ----------------------------------------------------------------------- +def case_metadata_problems(assumed: dict, expect: dict | None = None) -> list[str]: + """Bound local case queries too: substring expectations scan the response once each.""" + expect = expect or {} + queries = len(assumed) + sum( + len(expect.get(field, [])) for field in ("sweep_runs", "checks", "start_outcomes") + ) + if queries > MAX_CASE_RULE_QUERIES: + return [f"case ingestion limit exceeded: more than {MAX_CASE_RULE_QUERIES} rule queries"] + return ingestion_problems({"assumed": assumed, "expect": expect}, "case") + + def assumption_mismatches( assumed: dict[str, float], result: dict, rules: RuleSet | None = None ) -> list[str]: + if problems := case_metadata_problems(assumed): + return problems out = [] configured = ( {"battery_width": rules.width_ft, "corner_allowance": rules.corner_allowance_ft} @@ -1564,6 +1765,8 @@ def assumption_mismatches( def expectation_problems(expect: dict, result: dict, meter: WorkMeter | None = None) -> list[str]: + if admission := case_metadata_problems({}, expect): + return admission problems: list[str] = [] decision = result["decision"] if "decision_in" in expect and decision not in expect["decision_in"]: diff --git a/verification/tests/fixtures/segmented-public-pass.json b/verification/tests/fixtures/segmented-public-pass.json new file mode 100644 index 000000000..4190d2822 --- /dev/null +++ b/verification/tests/fixtures/segmented-public-pass.json @@ -0,0 +1,1383 @@ +{ + "scene": { + "meter": { + "pos": [ + 0, + 4, + 0 + ], + "wall_id": "w11", + "plus_minus_ft": 0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -40.0, + 0 + ], + [ + -36.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w2", + "baseline": [ + [ + -36.0, + 0 + ], + [ + -32.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w3", + "baseline": [ + [ + -32.0, + 0 + ], + [ + -28.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w4", + "baseline": [ + [ + -28.0, + 0 + ], + [ + -24.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w5", + "baseline": [ + [ + -24.0, + 0 + ], + [ + -20.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w6", + "baseline": [ + [ + -20.0, + 0 + ], + [ + -16.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w7", + "baseline": [ + [ + -16.0, + 0 + ], + [ + -12.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w8", + "baseline": [ + [ + -12.0, + 0 + ], + [ + -8.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w9", + "baseline": [ + [ + -8.0, + 0 + ], + [ + -4.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w10", + "baseline": [ + [ + -4.0, + 0 + ], + [ + 0.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w11", + "baseline": [ + [ + 0.0, + 0 + ], + [ + 4.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w12", + "baseline": [ + [ + 4.0, + 0 + ], + [ + 8.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w13", + "baseline": [ + [ + 8.0, + 0 + ], + [ + 12.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w14", + "baseline": [ + [ + 12.0, + 0 + ], + [ + 16.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w15", + "baseline": [ + [ + 16.0, + 0 + ], + [ + 20.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w16", + "baseline": [ + [ + 20.0, + 0 + ], + [ + 24.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w17", + "baseline": [ + [ + 24.0, + 0 + ], + [ + 28.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w18", + "baseline": [ + [ + 28.0, + 0 + ], + [ + 32.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w19", + "baseline": [ + [ + 32.0, + 0 + ], + [ + 36.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w20", + "baseline": [ + [ + 36.0, + 0 + ], + [ + 40.0, + 0 + ] + ], + "height_ft": 9 + } + ], + "objects": [], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -40, + 0 + ], + [ + 40, + 0 + ], + [ + 40, + 30 + ], + [ + -40, + 30 + ] + ], + "plus_minus_ft": 0 + } + ], + "overheads": [ + { + "wall_id": "w1", + "span_ft": [ + -40, + -36 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w2", + "span_ft": [ + -36, + -32 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w3", + "span_ft": [ + -32, + -28 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w4", + "span_ft": [ + -28, + -24 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w5", + "span_ft": [ + -24, + -20 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w6", + "span_ft": [ + -20, + -16 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w7", + "span_ft": [ + -16, + -12 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w8", + "span_ft": [ + -12, + -8 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w9", + "span_ft": [ + -8, + -4 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w10", + "span_ft": [ + -4, + 0 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w11", + "span_ft": [ + 0, + 4 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w12", + "span_ft": [ + 4, + 8 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w13", + "span_ft": [ + 8, + 12 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w14", + "span_ft": [ + 12, + 16 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w15", + "span_ft": [ + 16, + 20 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w16", + "span_ft": [ + 20, + 24 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w17", + "span_ft": [ + 24, + 28 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w18", + "span_ft": [ + 28, + 32 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w19", + "span_ft": [ + 32, + 36 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w20", + "span_ft": [ + 36, + 40 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -40, + -36 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w2", + "span_ft": [ + -36, + -32 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w3", + "span_ft": [ + -32, + -28 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w4", + "span_ft": [ + -28, + -24 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w5", + "span_ft": [ + -24, + -20 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w6", + "span_ft": [ + -20, + -16 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w7", + "span_ft": [ + -16, + -12 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w8", + "span_ft": [ + -12, + -8 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w9", + "span_ft": [ + -8, + -4 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w10", + "span_ft": [ + -4, + 0 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w11", + "span_ft": [ + 0, + 4 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w12", + "span_ft": [ + 4, + 8 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w13", + "span_ft": [ + 8, + 12 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w14", + "span_ft": [ + 12, + 16 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w15", + "span_ft": [ + 16, + 20 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w16", + "span_ft": [ + 20, + 24 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w17", + "span_ft": [ + 24, + 28 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w18", + "span_ft": [ + 28, + 32 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w19", + "span_ft": [ + 32, + 36 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w20", + "span_ft": [ + 36, + 40 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -40, + 40 + ] + }, + { + "band": "ground", + "span_ft": [ + -40, + 40 + ], + "out_ft": 30 + }, + { + "band": "overhead", + "span_ft": [ + -40, + 40 + ] + }, + { + "band": "facing", + "span_ft": [ + -40, + 40 + ] + } + ] + } + }, + "result": { + "schema_version": "1.0", + "decision": "pass", + "summary": "The battery fits 3 ft 8 in left of the meter with a 2 ft 4 in cable run: every check passes. Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's.", + "reasons": [ + { + "code": "all_checks_pass", + "message": "A fully observed spot passes every check." + } + ], + "policy": { + "id": "demo", + "version": "2026-09-26", + "auto_approve": true, + "sources": [ + "public" + ], + "rules_sha256": "0df93f73b7c3176d33411888638189544a0848ffb51803219ab662a7ca51db66", + "notice": "Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's." + }, + "spot": { + "outcome": "pass", + "wall_id": "w10", + "segment": 0, + "span_ft": [ + -4.958333, + -2.375 + ], + "width_ft": 2.583333, + "depth_ft": 1.833333, + "height_ft": 3.291667, + "footprint": [ + [ + -4.958333, + 0.0 + ], + [ + -2.375, + 0.0 + ], + [ + -2.375, + 1.833333 + ], + [ + -4.958333, + 1.833333 + ] + ], + "center": [ + -3.666667, + 0.916667 + ], + "along": [ + 1.0, + 0.0 + ], + "outward": [ + -0.0, + 1.0 + ], + "meter_offset_ft": [ + -3.666667, + 0.916667 + ], + "route_length_ft": 2.375 + }, + "route": { + "outcome": "pass", + "length_ft": 2.375, + "plus_minus_ft": 1.093333, + "height_ft": 1.0, + "polyline": [ + [ + 0.0, + 0.0 + ], + [ + -2.375, + 0.0 + ] + ], + "detours": [], + "crossings": [] + }, + "checks": [ + { + "id": "wall_backing", + "label": "Flush against one straight wall", + "outcome": "pass", + "reason": "The whole footprint backs onto one straight, observed wall segment.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "battery.width_ft", + "source": "Base Core spec: 30.68 in wide, rounded up to 31 in (docs/04, What Base does today)", + "placeholder": false + } + }, + { + "id": "ground_surface", + "label": "Ground under the battery", + "outcome": "pass", + "reason": "The whole footprint stands on an allowed surface.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "ground.allowed", + "source": "Ground types from schemas/scene.schema.json; which surfaces are allowed is a demo choice", + "placeholder": true + } + }, + { + "id": "meter_working_space", + "label": "Clear of the meter's working space", + "outcome": "pass", + "reason": "The battery is 1 ft 1 in clear of the meter's 2 ft 6 in wide by 3 ft 0 in deep working space.", + "measured_ft": 1.125, + "plus_minus_ft": 1.093333, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": "meter", + "rule": { + "key": "meter_working_space.width_ft", + "source": "NEC 110.26(A)(2): 30 in wide working space", + "placeholder": false + } + }, + { + "id": "gas_clearance", + "label": "Distance from gas equipment", + "outcome": "pass", + "reason": "No gas meter or pipe within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.gas_ft", + "source": "Base help page: 3 ft from the gas meter; Austin Energy \u00a71.9; Texas Gas Service", + "placeholder": false + } + }, + { + "id": "ac_clearance", + "label": "Distance from AC units", + "outcome": "pass", + "reason": "No AC unit within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.ac_ft", + "source": "Base help page: 3 ft from AC units, fences and other batteries", + "placeholder": false + } + }, + { + "id": "drive_clearance", + "label": "Distance from the driveway", + "outcome": "pass", + "reason": "No drivable surface within 5 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 5.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.drive_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "pool_clearance", + "label": "Distance from a pool", + "outcome": "pass", + "reason": "No pool within 10 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 10.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.pool_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "opening_clearance", + "label": "Distance from doors and windows", + "outcome": "pass", + "reason": "No door or window within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.opening_ft", + "source": "IRC 2021 R328.4: 3 ft from doors and windows directly entering the dwelling. Applying it to garage doors (openings.types) is a demo choice the code does not require", + "placeholder": false + } + }, + { + "id": "wall_equipment_above", + "label": "No box or vent above the battery", + "outcome": "pass", + "reason": "Nothing is mounted on the wall above the battery.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.wall_equipment_ft", + "source": "No public value; 0 means only overlap fails", + "placeholder": true + } + }, + { + "id": "facing_gap", + "label": "Open space in front", + "outcome": "pass", + "reason": "The gap in front of the battery is 7 ft 2 in (\u00b1 0 ft 0 in), clear of the 3 ft 0 in rule.", + "measured_ft": 7.166667, + "plus_minus_ft": 0.0, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": "facing[8]", + "rule": { + "key": "facing.min_ft", + "source": "Base help page: 3 ft from fences", + "placeholder": false + } + }, + { + "id": "headroom", + "label": "Headroom above", + "outcome": "pass", + "reason": "The headroom is 9 ft 0 in (\u00b1 0 ft 0 in), clear of the 6 ft 6 in rule.", + "measured_ft": 9.0, + "plus_minus_ft": 0.0, + "threshold_ft": 6.5, + "comparison": "at_least", + "subject": "overheads[8]", + "rule": { + "key": "headroom.min_ft", + "source": "NEC 110.26(A)(3) working-space headroom, applied to the battery as a demo choice", + "placeholder": true + } + }, + { + "id": "route_path", + "label": "Cable route along the wall", + "outcome": "pass", + "reason": "The cable runs along continuous, observed wall with nothing blocking it.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "route.crossing", + "source": "Demo rule: the cable can't cross a door, a garage or a stretch with no wall", + "placeholder": false + } + }, + { + "id": "route_length", + "label": "Cable run length", + "outcome": "pass", + "reason": "The cable run is 2 ft 4 in (\u00b1 1 ft 1 in), within the 15 ft 0 in confident reach.", + "measured_ft": 2.375, + "plus_minus_ft": 1.093333, + "threshold_ft": 20.0, + "comparison": "at_most", + "subject": null, + "rule": { + "key": "route.max_ft", + "source": "Base help page: within 20 ft of the meter. Review past 15 ft 0 in: No public value", + "placeholder": true + }, + "review_threshold_ft": 15.0 + } + ], + "nearest_considered": null, + "missing_evidence": [], + "objects_not_used": [], + "ends": { + "left": { + "kind": "limit", + "s_ft": -40.0, + "point": [ + -40.0, + 0.0 + ], + "beyond_reach": true + }, + "right": { + "kind": "limit", + "s_ft": 40.0, + "point": [ + 40.0, + 0.0 + ], + "beyond_reach": true + } + }, + "sweep": [ + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -40.0, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.242064, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.183532, + -25.625 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -25.541667, + -25.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + -25.208333, + -21.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w6", + "segment": 0, + "start_ft": [ + -21.208333, + -17.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -17.208333, + -14.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -14.875, + -13.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w8", + "segment": 0, + "start_ft": [ + -13.208333, + -9.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w9", + "segment": 0, + "start_ft": [ + -9.208333, + -5.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -5.208333, + -4.958333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -4.875, + -3.125 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -3.041667, + -1.291667 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + -1.270833, + 0.458333 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 0.541667, + 2.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 2.375, + 2.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w12", + "segment": 0, + "start_ft": [ + 2.791667, + 6.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w13", + "segment": 0, + "start_ft": [ + 6.791667, + 10.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 10.791667, + 12.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 12.375, + 14.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w15", + "segment": 0, + "start_ft": [ + 14.791667, + 18.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + 18.791667, + 22.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 22.791667, + 22.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 23.041667, + 24.600198 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 24.65873, + 24.658731 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + 24.658731, + 37.416667 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + } + ], + "stats": { + "candidates": 815, + "pass": 312, + "unsure": 444, + "fail": 59, + "elapsed_ms": 0, + "input_sha256": "429ef7a1eeb84485c6d19036bfb71a589501b08206fb9a5a34c1458f5a8791cf" + } + } +} diff --git a/verification/tests/fixtures/segmented-public-workload.json b/verification/tests/fixtures/segmented-public-workload.json new file mode 100644 index 000000000..467ae56ab --- /dev/null +++ b/verification/tests/fixtures/segmented-public-workload.json @@ -0,0 +1,4565 @@ +{ + "surveyed wall": { + "schema_version": "1.0", + "decision": "pass", + "summary": "The battery fits 3 ft 8 in left of the meter with a 2 ft 4 in cable run: every check passes. Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's.", + "reasons": [ + { + "code": "all_checks_pass", + "message": "A fully observed spot passes every check." + } + ], + "policy": { + "id": "demo", + "version": "2026-09-26", + "auto_approve": true, + "sources": [ + "public" + ], + "rules_sha256": "0df93f73b7c3176d33411888638189544a0848ffb51803219ab662a7ca51db66", + "notice": "Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's." + }, + "spot": { + "outcome": "pass", + "wall_id": "w10", + "segment": 0, + "span_ft": [ + -4.958333, + -2.375 + ], + "width_ft": 2.583333, + "depth_ft": 1.833333, + "height_ft": 3.291667, + "footprint": [ + [ + -4.958333, + 0.0 + ], + [ + -2.375, + 0.0 + ], + [ + -2.375, + 1.833333 + ], + [ + -4.958333, + 1.833333 + ] + ], + "center": [ + -3.666667, + 0.916667 + ], + "along": [ + 1.0, + 0.0 + ], + "outward": [ + -0.0, + 1.0 + ], + "meter_offset_ft": [ + -3.666667, + 0.916667 + ], + "route_length_ft": 2.375 + }, + "route": { + "outcome": "pass", + "length_ft": 2.375, + "plus_minus_ft": 1.093333, + "height_ft": 1.0, + "polyline": [ + [ + 0.0, + 0.0 + ], + [ + -2.375, + 0.0 + ] + ], + "detours": [], + "crossings": [] + }, + "checks": [ + { + "id": "wall_backing", + "label": "Flush against one straight wall", + "outcome": "pass", + "reason": "The whole footprint backs onto one straight, observed wall segment.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "battery.width_ft", + "source": "Base Core spec: 30.68 in wide, rounded up to 31 in (docs/04, What Base does today)", + "placeholder": false + } + }, + { + "id": "ground_surface", + "label": "Ground under the battery", + "outcome": "pass", + "reason": "The whole footprint stands on an allowed surface.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "ground.allowed", + "source": "Ground types from schemas/scene.schema.json; which surfaces are allowed is a demo choice", + "placeholder": true + } + }, + { + "id": "meter_working_space", + "label": "Clear of the meter's working space", + "outcome": "pass", + "reason": "The battery is 1 ft 1 in clear of the meter's 2 ft 6 in wide by 3 ft 0 in deep working space.", + "measured_ft": 1.125, + "plus_minus_ft": 1.093333, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": "meter", + "rule": { + "key": "meter_working_space.width_ft", + "source": "NEC 110.26(A)(2): 30 in wide working space", + "placeholder": false + } + }, + { + "id": "gas_clearance", + "label": "Distance from gas equipment", + "outcome": "pass", + "reason": "No gas meter or pipe within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.gas_ft", + "source": "Base help page: 3 ft from the gas meter; Austin Energy \u00a71.9; Texas Gas Service", + "placeholder": false + } + }, + { + "id": "ac_clearance", + "label": "Distance from AC units", + "outcome": "pass", + "reason": "No AC unit within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.ac_ft", + "source": "Base help page: 3 ft from AC units, fences and other batteries", + "placeholder": false + } + }, + { + "id": "drive_clearance", + "label": "Distance from the driveway", + "outcome": "pass", + "reason": "No drivable surface within 5 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 5.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.drive_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "pool_clearance", + "label": "Distance from a pool", + "outcome": "pass", + "reason": "No pool within 10 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 10.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.pool_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "opening_clearance", + "label": "Distance from doors and windows", + "outcome": "pass", + "reason": "No door or window within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.opening_ft", + "source": "IRC 2021 R328.4: 3 ft from doors and windows directly entering the dwelling. Applying it to garage doors (openings.types) is a demo choice the code does not require", + "placeholder": false + } + }, + { + "id": "wall_equipment_above", + "label": "No box or vent above the battery", + "outcome": "pass", + "reason": "Nothing is mounted on the wall above the battery.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.wall_equipment_ft", + "source": "No public value; 0 means only overlap fails", + "placeholder": true + } + }, + { + "id": "facing_gap", + "label": "Open space in front", + "outcome": "pass", + "reason": "The gap in front of the battery is 7 ft 2 in (\u00b1 0 ft 0 in), clear of the 3 ft 0 in rule.", + "measured_ft": 7.166667, + "plus_minus_ft": 0.0, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": "facing[8]", + "rule": { + "key": "facing.min_ft", + "source": "Base help page: 3 ft from fences", + "placeholder": false + } + }, + { + "id": "headroom", + "label": "Headroom above", + "outcome": "pass", + "reason": "The headroom is 9 ft 0 in (\u00b1 0 ft 0 in), clear of the 6 ft 6 in rule.", + "measured_ft": 9.0, + "plus_minus_ft": 0.0, + "threshold_ft": 6.5, + "comparison": "at_least", + "subject": "overheads[8]", + "rule": { + "key": "headroom.min_ft", + "source": "NEC 110.26(A)(3) working-space headroom, applied to the battery as a demo choice", + "placeholder": true + } + }, + { + "id": "route_path", + "label": "Cable route along the wall", + "outcome": "pass", + "reason": "The cable runs along continuous, observed wall with nothing blocking it.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "route.crossing", + "source": "Demo rule: the cable can't cross a door, a garage or a stretch with no wall", + "placeholder": false + } + }, + { + "id": "route_length", + "label": "Cable run length", + "outcome": "pass", + "reason": "The cable run is 2 ft 4 in (\u00b1 1 ft 1 in), within the 15 ft 0 in confident reach.", + "measured_ft": 2.375, + "plus_minus_ft": 1.093333, + "threshold_ft": 20.0, + "comparison": "at_most", + "subject": null, + "rule": { + "key": "route.max_ft", + "source": "Base help page: within 20 ft of the meter. Review past 15 ft 0 in: No public value", + "placeholder": true + }, + "review_threshold_ft": 15.0 + } + ], + "nearest_considered": null, + "missing_evidence": [], + "objects_not_used": [], + "ends": { + "left": { + "kind": "limit", + "s_ft": -40.0, + "point": [ + -40.0, + 0.0 + ], + "beyond_reach": true + }, + "right": { + "kind": "limit", + "s_ft": 40.0, + "point": [ + 40.0, + 0.0 + ], + "beyond_reach": true + } + }, + "sweep": [ + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -40.0, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.242064, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.183532, + -25.625 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -25.541667, + -25.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + -25.208333, + -21.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w6", + "segment": 0, + "start_ft": [ + -21.208333, + -17.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -17.208333, + -14.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -14.875, + -13.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w8", + "segment": 0, + "start_ft": [ + -13.208333, + -9.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w9", + "segment": 0, + "start_ft": [ + -9.208333, + -5.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -5.208333, + -4.958333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -4.875, + -3.125 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -3.041667, + -1.291667 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + -1.270833, + 0.458333 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 0.541667, + 2.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 2.375, + 2.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w12", + "segment": 0, + "start_ft": [ + 2.791667, + 6.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w13", + "segment": 0, + "start_ft": [ + 6.791667, + 10.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 10.791667, + 12.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 12.375, + 14.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w15", + "segment": 0, + "start_ft": [ + 14.791667, + 18.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + 18.791667, + 22.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 22.791667, + 22.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 23.041667, + 24.600198 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 24.65873, + 24.658731 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + 24.658731, + 37.416667 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + } + ], + "stats": { + "candidates": 815, + "pass": 312, + "unsure": 444, + "fail": 59, + "elapsed_ms": 0, + "input_sha256": "d3a293f4c61b7fc3c40ac8275d32001c80425d0c774b155a06bc56724d77e71b" + } + }, + "surveyed wall~mirrored": { + "schema_version": "1.0", + "decision": "pass", + "summary": "The battery fits 3 ft 8 in left of the meter with a 2 ft 4 in cable run: every check passes. Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's.", + "reasons": [ + { + "code": "all_checks_pass", + "message": "A fully observed spot passes every check." + } + ], + "policy": { + "id": "demo", + "version": "2026-09-26", + "auto_approve": true, + "sources": [ + "public" + ], + "rules_sha256": "0df93f73b7c3176d33411888638189544a0848ffb51803219ab662a7ca51db66", + "notice": "Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's." + }, + "spot": { + "outcome": "pass", + "wall_id": "w11", + "segment": 0, + "span_ft": [ + -4.958333, + -2.375 + ], + "width_ft": 2.583333, + "depth_ft": 1.833333, + "height_ft": 3.291667, + "footprint": [ + [ + -4.958333, + 0.0 + ], + [ + -2.375, + 0.0 + ], + [ + -2.375, + 1.833333 + ], + [ + -4.958333, + 1.833333 + ] + ], + "center": [ + -3.666667, + 0.916667 + ], + "along": [ + 1.0, + 0.0 + ], + "outward": [ + -0.0, + 1.0 + ], + "meter_offset_ft": [ + -3.666667, + 0.916667 + ], + "route_length_ft": 2.375 + }, + "route": { + "outcome": "pass", + "length_ft": 2.375, + "plus_minus_ft": 1.093333, + "height_ft": 1.0, + "polyline": [ + [ + 0.0, + 0.0 + ], + [ + -2.375, + 0.0 + ] + ], + "detours": [], + "crossings": [] + }, + "checks": [ + { + "id": "wall_backing", + "label": "Flush against one straight wall", + "outcome": "pass", + "reason": "The whole footprint backs onto one straight, observed wall segment.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "battery.width_ft", + "source": "Base Core spec: 30.68 in wide, rounded up to 31 in (docs/04, What Base does today)", + "placeholder": false + } + }, + { + "id": "ground_surface", + "label": "Ground under the battery", + "outcome": "pass", + "reason": "The whole footprint stands on an allowed surface.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "ground.allowed", + "source": "Ground types from schemas/scene.schema.json; which surfaces are allowed is a demo choice", + "placeholder": true + } + }, + { + "id": "meter_working_space", + "label": "Clear of the meter's working space", + "outcome": "pass", + "reason": "The battery is 1 ft 1 in clear of the meter's 2 ft 6 in wide by 3 ft 0 in deep working space.", + "measured_ft": 1.125, + "plus_minus_ft": 1.093333, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": "meter", + "rule": { + "key": "meter_working_space.width_ft", + "source": "NEC 110.26(A)(2): 30 in wide working space", + "placeholder": false + } + }, + { + "id": "gas_clearance", + "label": "Distance from gas equipment", + "outcome": "pass", + "reason": "No gas meter or pipe within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.gas_ft", + "source": "Base help page: 3 ft from the gas meter; Austin Energy \u00a71.9; Texas Gas Service", + "placeholder": false + } + }, + { + "id": "ac_clearance", + "label": "Distance from AC units", + "outcome": "pass", + "reason": "No AC unit within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.ac_ft", + "source": "Base help page: 3 ft from AC units, fences and other batteries", + "placeholder": false + } + }, + { + "id": "drive_clearance", + "label": "Distance from the driveway", + "outcome": "pass", + "reason": "No drivable surface within 5 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 5.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.drive_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "pool_clearance", + "label": "Distance from a pool", + "outcome": "pass", + "reason": "No pool within 10 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 10.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.pool_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "opening_clearance", + "label": "Distance from doors and windows", + "outcome": "pass", + "reason": "No door or window within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.opening_ft", + "source": "IRC 2021 R328.4: 3 ft from doors and windows directly entering the dwelling. Applying it to garage doors (openings.types) is a demo choice the code does not require", + "placeholder": false + } + }, + { + "id": "wall_equipment_above", + "label": "No box or vent above the battery", + "outcome": "pass", + "reason": "Nothing is mounted on the wall above the battery.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.wall_equipment_ft", + "source": "No public value; 0 means only overlap fails", + "placeholder": true + } + }, + { + "id": "facing_gap", + "label": "Open space in front", + "outcome": "pass", + "reason": "The gap in front of the battery is 7 ft 2 in (\u00b1 0 ft 0 in), clear of the 3 ft 0 in rule.", + "measured_ft": 7.166667, + "plus_minus_ft": 0.0, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": "facing[10]", + "rule": { + "key": "facing.min_ft", + "source": "Base help page: 3 ft from fences", + "placeholder": false + } + }, + { + "id": "headroom", + "label": "Headroom above", + "outcome": "pass", + "reason": "The headroom is 9 ft 0 in (\u00b1 0 ft 0 in), clear of the 6 ft 6 in rule.", + "measured_ft": 9.0, + "plus_minus_ft": 0.0, + "threshold_ft": 6.5, + "comparison": "at_least", + "subject": "overheads[10]", + "rule": { + "key": "headroom.min_ft", + "source": "NEC 110.26(A)(3) working-space headroom, applied to the battery as a demo choice", + "placeholder": true + } + }, + { + "id": "route_path", + "label": "Cable route along the wall", + "outcome": "pass", + "reason": "The cable runs along continuous, observed wall with nothing blocking it.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "route.crossing", + "source": "Demo rule: the cable can't cross a door, a garage or a stretch with no wall", + "placeholder": false + } + }, + { + "id": "route_length", + "label": "Cable run length", + "outcome": "pass", + "reason": "The cable run is 2 ft 4 in (\u00b1 1 ft 1 in), within the 15 ft 0 in confident reach.", + "measured_ft": 2.375, + "plus_minus_ft": 1.093333, + "threshold_ft": 20.0, + "comparison": "at_most", + "subject": null, + "rule": { + "key": "route.max_ft", + "source": "Base help page: within 20 ft of the meter. Review past 15 ft 0 in: No public value", + "placeholder": true + }, + "review_threshold_ft": 15.0 + } + ], + "nearest_considered": null, + "missing_evidence": [], + "objects_not_used": [], + "ends": { + "left": { + "kind": "limit", + "s_ft": -40.0, + "point": [ + -40.0, + 0.0 + ], + "beyond_reach": true + }, + "right": { + "kind": "limit", + "s_ft": 40.0, + "point": [ + 40.0, + 0.0 + ], + "beyond_reach": true + } + }, + "sweep": [ + { + "wall_id": "w20", + "segment": 0, + "start_ft": [ + -40.0, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + -27.242064, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + -27.183532, + -25.625 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + -25.541667, + -25.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + -25.208333, + -21.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w15", + "segment": 0, + "start_ft": [ + -21.208333, + -17.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + -17.208333, + -14.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + -14.875, + -13.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w13", + "segment": 0, + "start_ft": [ + -13.208333, + -9.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w12", + "segment": 0, + "start_ft": [ + -9.208333, + -5.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + -5.208333, + -4.958333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + -4.875, + -3.125 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + -3.041667, + -1.291667 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -1.270833, + 0.458333 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + 0.541667, + 2.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + 2.375, + 2.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w9", + "segment": 0, + "start_ft": [ + 2.791667, + 6.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w8", + "segment": 0, + "start_ft": [ + 6.791667, + 10.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + 10.791667, + 12.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + 12.375, + 14.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w6", + "segment": 0, + "start_ft": [ + 14.791667, + 18.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + 18.791667, + 22.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + 22.791667, + 22.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + 23.041667, + 24.600198 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + 24.65873, + 24.658731 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w20", + "segment": 0, + "start_ft": [ + 24.658731, + 37.416667 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + } + ], + "stats": { + "candidates": 815, + "pass": 312, + "unsure": 444, + "fail": 59, + "elapsed_ms": 0, + "input_sha256": "4aee34ed47a051b645f3427c2ca421113b2b7441f8d70a4ac33a6e8a14b7d702" + } + }, + "surveyed wall~no coverage": { + "schema_version": "1.0", + "decision": "manual_review", + "summary": "More views are needed around the best spot, 3 ft 8 in left of the meter: 11 checks depend on areas the scan did not see. Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's.", + "reasons": [ + { + "code": "unsure_checks", + "checks": [ + "wall_backing", + "ground_surface", + "gas_clearance", + "ac_clearance", + "drive_clearance", + "pool_clearance", + "opening_clearance", + "wall_equipment_above", + "facing_gap", + "headroom", + "route_path" + ], + "message": "The best spot has checks nobody can settle from this scan: flush against one straight wall; ground under the battery; distance from gas equipment; distance from ac units; distance from the driveway; distance from a pool; distance from doors and windows; no box or vent above the battery; open space in front; headroom above; cable route along the wall." + }, + { + "code": "unobserved_area", + "checks": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ], + "message": "Part of the area the checks need was not seen." + } + ], + "policy": { + "id": "demo", + "version": "2026-09-26", + "auto_approve": true, + "sources": [ + "public" + ], + "rules_sha256": "0df93f73b7c3176d33411888638189544a0848ffb51803219ab662a7ca51db66", + "notice": "Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's." + }, + "spot": { + "outcome": "unsure", + "wall_id": "w10", + "segment": 0, + "span_ft": [ + -4.958333, + -2.375 + ], + "width_ft": 2.583333, + "depth_ft": 1.833333, + "height_ft": 3.291667, + "footprint": [ + [ + -4.958333, + 0.0 + ], + [ + -2.375, + 0.0 + ], + [ + -2.375, + 1.833333 + ], + [ + -4.958333, + 1.833333 + ] + ], + "center": [ + -3.666667, + 0.916667 + ], + "along": [ + 1.0, + 0.0 + ], + "outward": [ + -0.0, + 1.0 + ], + "meter_offset_ft": [ + -3.666667, + 0.916667 + ], + "route_length_ft": 2.375 + }, + "route": { + "outcome": "unsure", + "length_ft": 2.375, + "plus_minus_ft": 1.093333, + "height_ft": 1.0, + "polyline": [ + [ + 0.0, + 0.0 + ], + [ + -2.375, + 0.0 + ] + ], + "detours": [], + "crossings": [] + }, + "checks": [ + { + "id": "wall_backing", + "label": "Flush against one straight wall", + "outcome": "unsure", + "reason": "Part of the wall behind the battery was not seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "battery.width_ft", + "source": "Base Core spec: 30.68 in wide, rounded up to 31 in (docs/04, What Base does today)", + "placeholder": false + }, + "unsure_cause": "unobserved" + }, + { + "id": "ground_surface", + "label": "Ground under the battery", + "outcome": "unsure", + "reason": "The ground under the footprint was not seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "ground.allowed", + "source": "Ground types from schemas/scene.schema.json; which surfaces are allowed is a demo choice", + "placeholder": true + }, + "unsure_cause": "unobserved" + }, + { + "id": "meter_working_space", + "label": "Clear of the meter's working space", + "outcome": "pass", + "reason": "The battery is 1 ft 1 in clear of the meter's 2 ft 6 in wide by 3 ft 0 in deep working space.", + "measured_ft": 1.125, + "plus_minus_ft": 1.093333, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": "meter", + "rule": { + "key": "meter_working_space.width_ft", + "source": "NEC 110.26(A)(2): 30 in wide working space", + "placeholder": false + } + }, + { + "id": "gas_clearance", + "label": "Distance from gas equipment", + "outcome": "unsure", + "reason": "Not everything within 3 ft 0 in of the battery was seen, so a gas meter or pipe could hide there.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.gas_ft", + "source": "Base help page: 3 ft from the gas meter; Austin Energy \u00a71.9; Texas Gas Service", + "placeholder": false + }, + "unsure_cause": "unobserved" + }, + { + "id": "ac_clearance", + "label": "Distance from AC units", + "outcome": "unsure", + "reason": "Not everything within 3 ft 0 in of the battery was seen, so a AC unit could hide there.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.ac_ft", + "source": "Base help page: 3 ft from AC units, fences and other batteries", + "placeholder": false + }, + "unsure_cause": "unobserved" + }, + { + "id": "drive_clearance", + "label": "Distance from the driveway", + "outcome": "unsure", + "reason": "Not everything within 5 ft 0 in of the battery was seen, so a drivable surface could hide there.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 5.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.drive_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + }, + "unsure_cause": "unobserved" + }, + { + "id": "pool_clearance", + "label": "Distance from a pool", + "outcome": "unsure", + "reason": "Not everything within 10 ft 0 in of the battery was seen, so a pool could hide there.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 10.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.pool_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + }, + "unsure_cause": "unobserved" + }, + { + "id": "opening_clearance", + "label": "Distance from doors and windows", + "outcome": "unsure", + "reason": "Not everything within 3 ft 0 in of the battery was seen, so a door or window could hide there.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.opening_ft", + "source": "IRC 2021 R328.4: 3 ft from doors and windows directly entering the dwelling. Applying it to garage doors (openings.types) is a demo choice the code does not require", + "placeholder": false + }, + "unsure_cause": "unobserved" + }, + { + "id": "wall_equipment_above", + "label": "No box or vent above the battery", + "outcome": "unsure", + "reason": "The wall above the battery was not fully seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.wall_equipment_ft", + "source": "No public value; 0 means only overlap fails", + "placeholder": true + }, + "unsure_cause": "unobserved" + }, + { + "id": "facing_gap", + "label": "Open space in front", + "outcome": "unsure", + "reason": "The gap in front of the battery over the battery's stretch of wall was not measured everywhere.", + "measured_ft": 7.166667, + "plus_minus_ft": 0.0, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": "facing[8]", + "rule": { + "key": "facing.min_ft", + "source": "Base help page: 3 ft from fences", + "placeholder": false + }, + "unsure_cause": "unobserved" + }, + { + "id": "headroom", + "label": "Headroom above", + "outcome": "unsure", + "reason": "The headroom over the battery's stretch of wall was not measured everywhere.", + "measured_ft": 9.0, + "plus_minus_ft": 0.0, + "threshold_ft": 6.5, + "comparison": "at_least", + "subject": "overheads[8]", + "rule": { + "key": "headroom.min_ft", + "source": "NEC 110.26(A)(3) working-space headroom, applied to the battery as a demo choice", + "placeholder": true + }, + "unsure_cause": "unobserved" + }, + { + "id": "route_path", + "label": "Cable route along the wall", + "outcome": "unsure", + "reason": "Part of the wall the cable would run along was not seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "route.crossing", + "source": "Demo rule: the cable can't cross a door, a garage or a stretch with no wall", + "placeholder": false + }, + "unsure_cause": "unobserved" + }, + { + "id": "route_length", + "label": "Cable run length", + "outcome": "pass", + "reason": "The cable run is 2 ft 4 in (\u00b1 1 ft 1 in), within the 15 ft 0 in confident reach.", + "measured_ft": 2.375, + "plus_minus_ft": 1.093333, + "threshold_ft": 20.0, + "comparison": "at_most", + "subject": null, + "rule": { + "key": "route.max_ft", + "source": "Base help page: within 20 ft of the meter. Review past 15 ft 0 in: No public value", + "placeholder": true + }, + "review_threshold_ft": 15.0 + } + ], + "nearest_considered": null, + "missing_evidence": [ + { + "kind": "band", + "band": "wall", + "span_ft": [ + -9.056604, + 1.72327 + ], + "checks": [ + "gas_clearance", + "opening_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ], + "out_ft": 6.500001, + "message": "Show the wall from 9 ft 1 in left of the meter to 1 ft 9 in right of the meter, seen at least 6 ft 6 in up the wall." + }, + { + "kind": "band", + "band": "ground", + "span_ft": [ + -16.065046, + 8.731712 + ], + "checks": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "ground_surface", + "pool_clearance" + ], + "out_ft": 12.940046, + "message": "Show the ground in front of the wall from 16 ft 1 in left of the meter to 8 ft 9 in right of the meter, at least 12 ft 11 in out from the wall." + }, + { + "kind": "band", + "band": "facing", + "span_ft": [ + -4.958334, + -2.374999 + ], + "checks": [ + "facing_gap" + ], + "out_ft": 4.833334, + "message": "Show the gap in front of the wall from 4 ft 11 in left of the meter to 2 ft 4 in left of the meter, clear at least 4 ft 10 in out from the wall." + }, + { + "kind": "band", + "band": "overhead", + "span_ft": [ + -4.958334, + -2.374999 + ], + "checks": [ + "headroom" + ], + "out_ft": 6.500001, + "message": "Show the space overhead from 4 ft 11 in left of the meter to 2 ft 4 in left of the meter, clear at least 6 ft 6 in up." + } + ], + "objects_not_used": [], + "ends": { + "left": { + "kind": "unexplored", + "s_ft": -40.0, + "point": [ + -40.0, + 0.0 + ], + "beyond_reach": true + }, + "right": { + "kind": "unexplored", + "s_ft": 40.0, + "point": [ + 40.0, + 0.0 + ], + "beyond_reach": true + } + }, + "sweep": [ + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -40.0, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.242064, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.183532, + -25.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_length", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + -25.208333, + -21.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_length", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w6", + "segment": 0, + "start_ft": [ + -21.208333, + -17.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_length", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -17.208333, + -14.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_length", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -14.875, + -13.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w8", + "segment": 0, + "start_ft": [ + -13.208333, + -9.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w9", + "segment": 0, + "start_ft": [ + -9.208333, + -5.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -5.208333, + -4.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -4.875, + -3.125 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "meter_working_space", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -3.041667, + -2.604167 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -2.583333, + -1.291667 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + -1.270833, + 0.0 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 0.020833, + 0.458333 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 0.541667, + 2.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "meter_working_space", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 2.375, + 2.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w12", + "segment": 0, + "start_ft": [ + 2.791667, + 6.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w13", + "segment": 0, + "start_ft": [ + 6.791667, + 10.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 10.791667, + 12.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 12.375, + 14.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_length", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w15", + "segment": 0, + "start_ft": [ + 14.791667, + 18.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_length", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + 18.791667, + 22.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_length", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 22.791667, + 24.600198 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_length", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 24.65873, + 24.658731 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + 24.658731, + 37.416667 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + } + ], + "stats": { + "candidates": 815, + "pass": 0, + "unsure": 756, + "fail": 59, + "elapsed_ms": 0, + "input_sha256": "266238d71d75c0edf27c1b2e720c616f0315ee4154d6f57250a7268f90e5f09d" + } + }, + "surveyed wall~less coverage": { + "schema_version": "1.0", + "decision": "pass", + "summary": "The battery fits 3 ft 8 in left of the meter with a 2 ft 4 in cable run: every check passes. Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's.", + "reasons": [ + { + "code": "all_checks_pass", + "message": "A fully observed spot passes every check." + } + ], + "policy": { + "id": "demo", + "version": "2026-09-26", + "auto_approve": true, + "sources": [ + "public" + ], + "rules_sha256": "0df93f73b7c3176d33411888638189544a0848ffb51803219ab662a7ca51db66", + "notice": "Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's." + }, + "spot": { + "outcome": "pass", + "wall_id": "w10", + "segment": 0, + "span_ft": [ + -4.958333, + -2.375 + ], + "width_ft": 2.583333, + "depth_ft": 1.833333, + "height_ft": 3.291667, + "footprint": [ + [ + -4.958333, + 0.0 + ], + [ + -2.375, + 0.0 + ], + [ + -2.375, + 1.833333 + ], + [ + -4.958333, + 1.833333 + ] + ], + "center": [ + -3.666667, + 0.916667 + ], + "along": [ + 1.0, + 0.0 + ], + "outward": [ + -0.0, + 1.0 + ], + "meter_offset_ft": [ + -3.666667, + 0.916667 + ], + "route_length_ft": 2.375 + }, + "route": { + "outcome": "pass", + "length_ft": 2.375, + "plus_minus_ft": 1.093333, + "height_ft": 1.0, + "polyline": [ + [ + 0.0, + 0.0 + ], + [ + -2.375, + 0.0 + ] + ], + "detours": [], + "crossings": [] + }, + "checks": [ + { + "id": "wall_backing", + "label": "Flush against one straight wall", + "outcome": "pass", + "reason": "The whole footprint backs onto one straight, observed wall segment.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "battery.width_ft", + "source": "Base Core spec: 30.68 in wide, rounded up to 31 in (docs/04, What Base does today)", + "placeholder": false + } + }, + { + "id": "ground_surface", + "label": "Ground under the battery", + "outcome": "pass", + "reason": "The whole footprint stands on an allowed surface.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "ground.allowed", + "source": "Ground types from schemas/scene.schema.json; which surfaces are allowed is a demo choice", + "placeholder": true + } + }, + { + "id": "meter_working_space", + "label": "Clear of the meter's working space", + "outcome": "pass", + "reason": "The battery is 1 ft 1 in clear of the meter's 2 ft 6 in wide by 3 ft 0 in deep working space.", + "measured_ft": 1.125, + "plus_minus_ft": 1.093333, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": "meter", + "rule": { + "key": "meter_working_space.width_ft", + "source": "NEC 110.26(A)(2): 30 in wide working space", + "placeholder": false + } + }, + { + "id": "gas_clearance", + "label": "Distance from gas equipment", + "outcome": "pass", + "reason": "No gas meter or pipe within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.gas_ft", + "source": "Base help page: 3 ft from the gas meter; Austin Energy \u00a71.9; Texas Gas Service", + "placeholder": false + } + }, + { + "id": "ac_clearance", + "label": "Distance from AC units", + "outcome": "pass", + "reason": "No AC unit within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.ac_ft", + "source": "Base help page: 3 ft from AC units, fences and other batteries", + "placeholder": false + } + }, + { + "id": "drive_clearance", + "label": "Distance from the driveway", + "outcome": "pass", + "reason": "No drivable surface within 5 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 5.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.drive_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "pool_clearance", + "label": "Distance from a pool", + "outcome": "pass", + "reason": "No pool within 10 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 10.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.pool_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "opening_clearance", + "label": "Distance from doors and windows", + "outcome": "pass", + "reason": "No door or window within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.opening_ft", + "source": "IRC 2021 R328.4: 3 ft from doors and windows directly entering the dwelling. Applying it to garage doors (openings.types) is a demo choice the code does not require", + "placeholder": false + } + }, + { + "id": "wall_equipment_above", + "label": "No box or vent above the battery", + "outcome": "pass", + "reason": "Nothing is mounted on the wall above the battery.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.wall_equipment_ft", + "source": "No public value; 0 means only overlap fails", + "placeholder": true + } + }, + { + "id": "facing_gap", + "label": "Open space in front", + "outcome": "pass", + "reason": "The gap in front of the battery is 7 ft 2 in (\u00b1 0 ft 0 in), clear of the 3 ft 0 in rule.", + "measured_ft": 7.166667, + "plus_minus_ft": 0.0, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": "facing[8]", + "rule": { + "key": "facing.min_ft", + "source": "Base help page: 3 ft from fences", + "placeholder": false + } + }, + { + "id": "headroom", + "label": "Headroom above", + "outcome": "pass", + "reason": "The headroom is 9 ft 0 in (\u00b1 0 ft 0 in), clear of the 6 ft 6 in rule.", + "measured_ft": 9.0, + "plus_minus_ft": 0.0, + "threshold_ft": 6.5, + "comparison": "at_least", + "subject": "overheads[8]", + "rule": { + "key": "headroom.min_ft", + "source": "NEC 110.26(A)(3) working-space headroom, applied to the battery as a demo choice", + "placeholder": true + } + }, + { + "id": "route_path", + "label": "Cable route along the wall", + "outcome": "pass", + "reason": "The cable runs along continuous, observed wall with nothing blocking it.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "route.crossing", + "source": "Demo rule: the cable can't cross a door, a garage or a stretch with no wall", + "placeholder": false + } + }, + { + "id": "route_length", + "label": "Cable run length", + "outcome": "pass", + "reason": "The cable run is 2 ft 4 in (\u00b1 1 ft 1 in), within the 15 ft 0 in confident reach.", + "measured_ft": 2.375, + "plus_minus_ft": 1.093333, + "threshold_ft": 20.0, + "comparison": "at_most", + "subject": null, + "rule": { + "key": "route.max_ft", + "source": "Base help page: within 20 ft of the meter. Review past 15 ft 0 in: No public value", + "placeholder": true + }, + "review_threshold_ft": 15.0 + } + ], + "nearest_considered": null, + "missing_evidence": [], + "objects_not_used": [], + "ends": { + "left": { + "kind": "limit", + "s_ft": -40.0, + "point": [ + -40.0, + 0.0 + ], + "beyond_reach": true + }, + "right": { + "kind": "limit", + "s_ft": 40.0, + "point": [ + 40.0, + 0.0 + ], + "beyond_reach": true + } + }, + "sweep": [ + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -40.0, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.242064, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.183532, + -25.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + -25.208333, + -25.208333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + -25.125, + -21.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w6", + "segment": 0, + "start_ft": [ + -21.208333, + -17.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -17.208333, + -14.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -14.875, + -13.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w8", + "segment": 0, + "start_ft": [ + -13.208333, + -9.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w9", + "segment": 0, + "start_ft": [ + -9.208333, + -5.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -5.208333, + -4.958333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -4.875, + -3.125 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -3.041667, + -1.291667 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + -1.270833, + 0.458333 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 0.541667, + 2.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 2.375, + 2.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w12", + "segment": 0, + "start_ft": [ + 2.791667, + 6.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w13", + "segment": 0, + "start_ft": [ + 6.791667, + 10.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 10.791667, + 12.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 12.375, + 14.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w15", + "segment": 0, + "start_ft": [ + 14.791667, + 18.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + 18.791667, + 22.541667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + 22.625, + 22.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 22.791667, + 24.600198 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 24.65873, + 24.658731 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + 24.658731, + 37.416667 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + } + ], + "stats": { + "candidates": 815, + "pass": 312, + "unsure": 444, + "fail": 59, + "elapsed_ms": 0, + "input_sha256": "45f4346323d5517919839411fecc8c95bf07a62f2b08990f6b0b765e03fa9fc5" + } + }, + "surveyed wall~ground short of the largest clearance": { + "schema_version": "1.0", + "decision": "manual_review", + "summary": "More views are needed around the best spot, 3 ft 8 in left of the meter: 1 check depends on areas the scan did not see. Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's.", + "reasons": [ + { + "code": "unsure_checks", + "checks": [ + "pool_clearance" + ], + "message": "The best spot has checks nobody can settle from this scan: distance from a pool." + }, + { + "code": "unobserved_area", + "checks": [ + "pool_clearance" + ], + "message": "Part of the area the checks need was not seen." + } + ], + "policy": { + "id": "demo", + "version": "2026-09-26", + "auto_approve": true, + "sources": [ + "public" + ], + "rules_sha256": "0df93f73b7c3176d33411888638189544a0848ffb51803219ab662a7ca51db66", + "notice": "Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's." + }, + "spot": { + "outcome": "unsure", + "wall_id": "w10", + "segment": 0, + "span_ft": [ + -4.958333, + -2.375 + ], + "width_ft": 2.583333, + "depth_ft": 1.833333, + "height_ft": 3.291667, + "footprint": [ + [ + -4.958333, + 0.0 + ], + [ + -2.375, + 0.0 + ], + [ + -2.375, + 1.833333 + ], + [ + -4.958333, + 1.833333 + ] + ], + "center": [ + -3.666667, + 0.916667 + ], + "along": [ + 1.0, + 0.0 + ], + "outward": [ + -0.0, + 1.0 + ], + "meter_offset_ft": [ + -3.666667, + 0.916667 + ], + "route_length_ft": 2.375 + }, + "route": { + "outcome": "pass", + "length_ft": 2.375, + "plus_minus_ft": 1.093333, + "height_ft": 1.0, + "polyline": [ + [ + 0.0, + 0.0 + ], + [ + -2.375, + 0.0 + ] + ], + "detours": [], + "crossings": [] + }, + "checks": [ + { + "id": "wall_backing", + "label": "Flush against one straight wall", + "outcome": "pass", + "reason": "The whole footprint backs onto one straight, observed wall segment.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "battery.width_ft", + "source": "Base Core spec: 30.68 in wide, rounded up to 31 in (docs/04, What Base does today)", + "placeholder": false + } + }, + { + "id": "ground_surface", + "label": "Ground under the battery", + "outcome": "pass", + "reason": "The whole footprint stands on an allowed surface.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "ground.allowed", + "source": "Ground types from schemas/scene.schema.json; which surfaces are allowed is a demo choice", + "placeholder": true + } + }, + { + "id": "meter_working_space", + "label": "Clear of the meter's working space", + "outcome": "pass", + "reason": "The battery is 1 ft 1 in clear of the meter's 2 ft 6 in wide by 3 ft 0 in deep working space.", + "measured_ft": 1.125, + "plus_minus_ft": 1.093333, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": "meter", + "rule": { + "key": "meter_working_space.width_ft", + "source": "NEC 110.26(A)(2): 30 in wide working space", + "placeholder": false + } + }, + { + "id": "gas_clearance", + "label": "Distance from gas equipment", + "outcome": "pass", + "reason": "No gas meter or pipe within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.gas_ft", + "source": "Base help page: 3 ft from the gas meter; Austin Energy \u00a71.9; Texas Gas Service", + "placeholder": false + } + }, + { + "id": "ac_clearance", + "label": "Distance from AC units", + "outcome": "pass", + "reason": "No AC unit within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.ac_ft", + "source": "Base help page: 3 ft from AC units, fences and other batteries", + "placeholder": false + } + }, + { + "id": "drive_clearance", + "label": "Distance from the driveway", + "outcome": "pass", + "reason": "No drivable surface within 5 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 5.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.drive_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "pool_clearance", + "label": "Distance from a pool", + "outcome": "unsure", + "reason": "Not everything within 10 ft 0 in of the battery was seen, so a pool could hide there.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 10.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.pool_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + }, + "unsure_cause": "unobserved" + }, + { + "id": "opening_clearance", + "label": "Distance from doors and windows", + "outcome": "pass", + "reason": "No door or window within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.opening_ft", + "source": "IRC 2021 R328.4: 3 ft from doors and windows directly entering the dwelling. Applying it to garage doors (openings.types) is a demo choice the code does not require", + "placeholder": false + } + }, + { + "id": "wall_equipment_above", + "label": "No box or vent above the battery", + "outcome": "pass", + "reason": "Nothing is mounted on the wall above the battery.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.wall_equipment_ft", + "source": "No public value; 0 means only overlap fails", + "placeholder": true + } + }, + { + "id": "facing_gap", + "label": "Open space in front", + "outcome": "pass", + "reason": "The gap in front of the battery is 7 ft 2 in (\u00b1 0 ft 0 in), clear of the 3 ft 0 in rule.", + "measured_ft": 7.166667, + "plus_minus_ft": 0.0, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": "facing[8]", + "rule": { + "key": "facing.min_ft", + "source": "Base help page: 3 ft from fences", + "placeholder": false + } + }, + { + "id": "headroom", + "label": "Headroom above", + "outcome": "pass", + "reason": "The headroom is 9 ft 0 in (\u00b1 0 ft 0 in), clear of the 6 ft 6 in rule.", + "measured_ft": 9.0, + "plus_minus_ft": 0.0, + "threshold_ft": 6.5, + "comparison": "at_least", + "subject": "overheads[8]", + "rule": { + "key": "headroom.min_ft", + "source": "NEC 110.26(A)(3) working-space headroom, applied to the battery as a demo choice", + "placeholder": true + } + }, + { + "id": "route_path", + "label": "Cable route along the wall", + "outcome": "pass", + "reason": "The cable runs along continuous, observed wall with nothing blocking it.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "route.crossing", + "source": "Demo rule: the cable can't cross a door, a garage or a stretch with no wall", + "placeholder": false + } + }, + { + "id": "route_length", + "label": "Cable run length", + "outcome": "pass", + "reason": "The cable run is 2 ft 4 in (\u00b1 1 ft 1 in), within the 15 ft 0 in confident reach.", + "measured_ft": 2.375, + "plus_minus_ft": 1.093333, + "threshold_ft": 20.0, + "comparison": "at_most", + "subject": null, + "rule": { + "key": "route.max_ft", + "source": "Base help page: within 20 ft of the meter. Review past 15 ft 0 in: No public value", + "placeholder": true + }, + "review_threshold_ft": 15.0 + } + ], + "nearest_considered": null, + "missing_evidence": [ + { + "kind": "band", + "band": "ground", + "span_ft": [ + -9.961945, + 2.628612 + ], + "checks": [ + "pool_clearance" + ], + "out_ft": 12.940046, + "message": "Show the ground in front of the wall from 10 ft 0 in left of the meter to 2 ft 8 in right of the meter, at least 12 ft 11 in out from the wall." + } + ], + "objects_not_used": [], + "ends": { + "left": { + "kind": "limit", + "s_ft": -40.0, + "point": [ + -40.0, + 0.0 + ], + "beyond_reach": true + }, + "right": { + "kind": "limit", + "s_ft": 40.0, + "point": [ + 40.0, + 0.0 + ], + "beyond_reach": true + } + }, + "sweep": [ + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -40.0, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.242064, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.183532, + -25.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + -25.208333, + -21.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w6", + "segment": 0, + "start_ft": [ + -21.208333, + -17.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -17.208333, + -14.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -14.875, + -13.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w8", + "segment": 0, + "start_ft": [ + -13.208333, + -9.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w9", + "segment": 0, + "start_ft": [ + -9.208333, + -5.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -5.208333, + -4.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -4.875, + -3.125 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "pool_clearance" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -3.041667, + -1.291667 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + -1.270833, + 0.458333 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 0.541667, + 2.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "pool_clearance" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 2.375, + 2.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w12", + "segment": 0, + "start_ft": [ + 2.791667, + 6.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w13", + "segment": 0, + "start_ft": [ + 6.791667, + 10.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 10.791667, + 12.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 12.375, + 14.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w15", + "segment": 0, + "start_ft": [ + 14.791667, + 18.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + 18.791667, + 22.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 22.791667, + 24.600198 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 24.65873, + 24.658731 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + 24.658731, + 37.416667 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + } + ], + "stats": { + "candidates": 815, + "pass": 0, + "unsure": 756, + "fail": 59, + "elapsed_ms": 0, + "input_sha256": "11e367e90218bfe3b8663e25f78640d62daed9192d959bb8a599877aa7729684" + } + }, + "surveyed wall~more error": { + "schema_version": "1.0", + "decision": "manual_review", + "summary": "A person needs to check the best spot, 2 ft 6 in left of the meter: clear of the meter's working space; cable run length. Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's.", + "reasons": [ + { + "code": "unsure_checks", + "checks": [ + "meter_working_space", + "route_length" + ], + "message": "The best spot has checks nobody can settle from this scan: clear of the meter's working space; cable run length." + } + ], + "policy": { + "id": "demo", + "version": "2026-09-26", + "auto_approve": true, + "sources": [ + "public" + ], + "rules_sha256": "0df93f73b7c3176d33411888638189544a0848ffb51803219ab662a7ca51db66", + "notice": "Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's." + }, + "spot": { + "outcome": "unsure", + "wall_id": "w10", + "segment": 0, + "span_ft": [ + -3.833333, + -1.25 + ], + "width_ft": 2.583333, + "depth_ft": 1.833333, + "height_ft": 3.291667, + "footprint": [ + [ + -3.833333, + 0.0 + ], + [ + -1.25, + 0.0 + ], + [ + -1.25, + 1.833333 + ], + [ + -3.833333, + 1.833333 + ] + ], + "center": [ + -2.541667, + 0.916667 + ], + "along": [ + 1.0, + 0.0 + ], + "outward": [ + -0.0, + 1.0 + ], + "meter_offset_ft": [ + -2.541667, + 0.916667 + ], + "route_length_ft": 1.25 + }, + "route": { + "outcome": "unsure", + "length_ft": 1.25, + "plus_minus_ft": 14.1, + "height_ft": 1.0, + "polyline": [ + [ + 0.0, + 0.0 + ], + [ + -1.25, + 0.0 + ] + ], + "detours": [], + "crossings": [] + }, + "checks": [ + { + "id": "wall_backing", + "label": "Flush against one straight wall", + "outcome": "pass", + "reason": "The whole footprint backs onto one straight, observed wall segment.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "battery.width_ft", + "source": "Base Core spec: 30.68 in wide, rounded up to 31 in (docs/04, What Base does today)", + "placeholder": false + } + }, + { + "id": "ground_surface", + "label": "Ground under the battery", + "outcome": "pass", + "reason": "The whole footprint stands on an allowed surface.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "ground.allowed", + "source": "Ground types from schemas/scene.schema.json; which surfaces are allowed is a demo choice", + "placeholder": true + } + }, + { + "id": "meter_working_space", + "label": "Clear of the meter's working space", + "outcome": "unsure", + "reason": "The battery is within measurement error of the meter's 2 ft 6 in wide by 3 ft 0 in deep working space.", + "measured_ft": -0.0, + "plus_minus_ft": 14.1, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": "meter", + "rule": { + "key": "meter_working_space.width_ft", + "source": "NEC 110.26(A)(2): 30 in wide working space", + "placeholder": false + }, + "unsure_cause": "margin" + }, + { + "id": "gas_clearance", + "label": "Distance from gas equipment", + "outcome": "pass", + "reason": "No gas meter or pipe within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.gas_ft", + "source": "Base help page: 3 ft from the gas meter; Austin Energy \u00a71.9; Texas Gas Service", + "placeholder": false + } + }, + { + "id": "ac_clearance", + "label": "Distance from AC units", + "outcome": "pass", + "reason": "No AC unit within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.ac_ft", + "source": "Base help page: 3 ft from AC units, fences and other batteries", + "placeholder": false + } + }, + { + "id": "drive_clearance", + "label": "Distance from the driveway", + "outcome": "pass", + "reason": "No drivable surface within 5 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 5.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.drive_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "pool_clearance", + "label": "Distance from a pool", + "outcome": "pass", + "reason": "No pool within 10 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 10.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.pool_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "opening_clearance", + "label": "Distance from doors and windows", + "outcome": "pass", + "reason": "No door or window within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.opening_ft", + "source": "IRC 2021 R328.4: 3 ft from doors and windows directly entering the dwelling. Applying it to garage doors (openings.types) is a demo choice the code does not require", + "placeholder": false + } + }, + { + "id": "wall_equipment_above", + "label": "No box or vent above the battery", + "outcome": "pass", + "reason": "Nothing is mounted on the wall above the battery.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.wall_equipment_ft", + "source": "No public value; 0 means only overlap fails", + "placeholder": true + } + }, + { + "id": "facing_gap", + "label": "Open space in front", + "outcome": "pass", + "reason": "The gap in front of the battery is 7 ft 2 in (\u00b1 0 ft 0 in), clear of the 3 ft 0 in rule.", + "measured_ft": 7.166667, + "plus_minus_ft": 0.0, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": "facing[5]", + "rule": { + "key": "facing.min_ft", + "source": "Base help page: 3 ft from fences", + "placeholder": false + } + }, + { + "id": "headroom", + "label": "Headroom above", + "outcome": "pass", + "reason": "The headroom is 9 ft 0 in (\u00b1 0 ft 0 in), clear of the 6 ft 6 in rule.", + "measured_ft": 9.0, + "plus_minus_ft": 0.0, + "threshold_ft": 6.5, + "comparison": "at_least", + "subject": "overheads[5]", + "rule": { + "key": "headroom.min_ft", + "source": "NEC 110.26(A)(3) working-space headroom, applied to the battery as a demo choice", + "placeholder": true + } + }, + { + "id": "route_path", + "label": "Cable route along the wall", + "outcome": "pass", + "reason": "The cable runs along continuous, observed wall with nothing blocking it.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "route.crossing", + "source": "Demo rule: the cable can't cross a door, a garage or a stretch with no wall", + "placeholder": false + } + }, + { + "id": "route_length", + "label": "Cable run length", + "outcome": "unsure", + "reason": "The cable run is 1 ft 3 in (\u00b1 14 ft 1 in), within its error of the 15 ft 0 in confident reach or the 20 ft 0 in maximum: too close to call.", + "measured_ft": 1.25, + "plus_minus_ft": 14.1, + "threshold_ft": 20.0, + "comparison": "at_most", + "subject": null, + "rule": { + "key": "route.max_ft", + "source": "Base help page: within 20 ft of the meter. Review past 15 ft 0 in: No public value", + "placeholder": true + }, + "review_threshold_ft": 15.0, + "unsure_cause": "margin" + } + ], + "nearest_considered": null, + "missing_evidence": [], + "objects_not_used": [], + "ends": { + "left": { + "kind": "limit", + "s_ft": -40.0, + "point": [ + -40.0, + 0.0 + ], + "beyond_reach": true + }, + "right": { + "kind": "limit", + "s_ft": 40.0, + "point": [ + 40.0, + 0.0 + ], + "beyond_reach": true + } + }, + "sweep": [ + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -40.0, + -36.683334 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + }, + { + "wall_id": "w2", + "segment": 0, + "start_ft": [ + -36.683334, + -36.683334 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "ground_surface", + "opening_clearance", + "pool_clearance" + ] + }, + { + "wall_id": "w2", + "segment": 0, + "start_ft": [ + -36.683333, + -33.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "ground_surface", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w3", + "segment": 0, + "start_ft": [ + -33.208333, + -29.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "ground_surface", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -29.208333, + -26.429167 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "ground_surface", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -26.4, + -25.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + -25.208333, + -23.458333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + -23.375, + -21.458333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "drive_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + -21.375, + -21.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w6", + "segment": 0, + "start_ft": [ + -21.208333, + -17.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w6", + "segment": 0, + "start_ft": [ + -17.875, + -17.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -17.208333, + -16.458333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -16.375, + -13.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "route_length" + ] + }, + { + "wall_id": "w8", + "segment": 0, + "start_ft": [ + -13.208333, + -9.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "route_length" + ] + }, + { + "wall_id": "w9", + "segment": 0, + "start_ft": [ + -9.208333, + -5.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "route_length" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -5.208333, + -3.483333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "route_length" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -3.470833, + -1.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + -1.270833, + 0.8875 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 0.9, + 2.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "route_length" + ] + }, + { + "wall_id": "w12", + "segment": 0, + "start_ft": [ + 2.791667, + 6.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "route_length" + ] + }, + { + "wall_id": "w13", + "segment": 0, + "start_ft": [ + 6.791667, + 10.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "route_length" + ] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 10.791667, + 13.791667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "route_length" + ] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 13.875, + 14.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w15", + "segment": 0, + "start_ft": [ + 14.779167, + 15.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w15", + "segment": 0, + "start_ft": [ + 15.375, + 18.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + 18.791667, + 18.791667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + 18.875, + 20.791667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "drive_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + 20.875, + 22.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 22.791667, + 23.816667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 23.845833, + 26.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "ground_surface", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w18", + "segment": 0, + "start_ft": [ + 26.791667, + 30.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "ground_surface", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w19", + "segment": 0, + "start_ft": [ + 30.791667, + 34.1 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "ground_surface", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w19", + "segment": 0, + "start_ft": [ + 34.100001, + 34.100001 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "ground_surface", + "opening_clearance", + "pool_clearance" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + 34.100001, + 37.416667 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + } + ], + "stats": { + "candidates": 1095, + "pass": 0, + "unsure": 1093, + "fail": 2, + "elapsed_ms": 0, + "input_sha256": "89f16e3ba80ea5576ec49bca3db348b343d1a8f713417af402614a4ac1761505" + } + } +} diff --git a/verification/tests/segmented_wall.py b/verification/tests/segmented_wall.py new file mode 100644 index 000000000..a46904f2d --- /dev/null +++ b/verification/tests/segmented_wall.py @@ -0,0 +1,27 @@ +"""Equivalent surveyed straight walls for verifier representation regressions.""" + +import copy +import json +from pathlib import Path + + +def segmented_wall_scene(segments_per_wall: int) -> dict: + fixture = json.loads((Path(__file__).parent / "fixtures/wide-public-pass.json").read_text()) + scene = copy.deepcopy(fixture["scene"]) + scene["walls"] = [ + { + "id": f"w{i + 1}", + "baseline": [ + [-40 + 4 * i + 4 * j / segments_per_wall, 0] for j in range(segments_per_wall + 1) + ], + "height_ft": 9, + } + for i in range(20) + ] + scene["meter"]["wall_id"] = "w11" + for key in ("facing", "overheads"): + scene[key] = [ + dict(fixture["scene"][key][0], wall_id=f"w{i + 1}", span_ft=[-40 + 4 * i, -36 + 4 * i]) + for i in range(20) + ] + return scene diff --git a/verification/tests/test_judge_scene_cache.py b/verification/tests/test_judge_scene_cache.py new file mode 100644 index 000000000..b4e711770 --- /dev/null +++ b/verification/tests/test_judge_scene_cache.py @@ -0,0 +1,73 @@ +"""Recorded public responses exercise the complete judge without a server dependency.""" + +import copy +import hashlib +import json +from pathlib import Path + +import pytest +from segmented_wall import segmented_wall_scene + +from hsverify import e2e +from hsverify.resultcheck import MAX_RESULT_CHECKS, RuleSet, WorkMeter, ingestion_problems + +ROOT = Path(__file__).parents[2] +RULES = RuleSet.from_yaml((ROOT / "server/rules.yaml").read_text()) +SCHEMAS = { + kind: json.loads((ROOT / f"server/schemas/{kind}.schema.json").read_text()) + for kind in ("scene", "result") +} + + +def workload(monkeypatch, segments): + responses = json.loads( + (Path(__file__).parent / "fixtures/segmented-public-workload.json").read_text() + ) + calls = [] + + def post(url, endpoint, item): + calls.append(item.name) + answer = copy.deepcopy(responses[item.name]) + answer["stats"]["input_sha256"] = hashlib.sha256(item.raw).hexdigest() + return 200, json.dumps(answer).encode(), 0 + + monkeypatch.setattr(e2e, "post", post) + scene = segmented_wall_scene(segments) + item = e2e.SceneInput("surveyed wall", scene, json.dumps(scene).encode()) + meter = WorkMeter() + record = e2e.judge( + "unused", e2e.Endpoint("/place", "application/json"), item, SCHEMAS, RULES, meter=meter + ) + assert record["status"] == "pass", record["problems"] + assert calls.count("surveyed wall") == 2 + assert "surveyed wall~mirrored" in calls + return meter.samples + + +@pytest.mark.parametrize("segments", [13, 20]) +def test_full_judge_accepts_equivalent_segmented_walls(monkeypatch, segments): + baseline = workload(monkeypatch, 1) + actual = workload(monkeypatch, segments) + assert actual < baseline * 1.2 + + +def test_identical_resend_counts_are_checked_before_comparison(monkeypatch): + responses = json.loads( + (Path(__file__).parent / "fixtures/segmented-public-workload.json").read_text() + ) + original = copy.deepcopy(responses["surveyed wall"]) + oversized = original | {"checks": [None] * (MAX_RESULT_CHECKS + 1)} + replies = iter((original, oversized)) + scene = segmented_wall_scene(1) + item = e2e.SceneInput("surveyed wall", scene, json.dumps(scene).encode()) + original["stats"]["input_sha256"] = hashlib.sha256(item.raw).hexdigest() + monkeypatch.setattr(e2e, "post", lambda *args: (200, json.dumps(next(replies)).encode(), 0)) + # Other transforms are immaterial to the identical-resend admission control. + monkeypatch.setattr(e2e, "mirror_scene", lambda value: value) + monkeypatch.setattr(e2e, "answer", lambda *args: (None, "transform omitted by test")) + record = e2e.judge("unused", e2e.Endpoint("/place", "application/json"), item, SCHEMAS, RULES) + assert any( + "identical resend: result ingestion limit exceeded: checks" in problem + for problem in record["problems"] + ) + assert ingestion_problems(oversized, "result") diff --git a/verification/tests/test_metadata_bounds.py b/verification/tests/test_metadata_bounds.py new file mode 100644 index 000000000..e88cc657e --- /dev/null +++ b/verification/tests/test_metadata_bounds.py @@ -0,0 +1,183 @@ +"""Metadata admission and lookup costs for large, schema-shaped responses.""" + +import ast +import copy +import inspect +import json +from pathlib import Path + +import pytest +from test_resultcheck import RULES, SCENE, check, result + +import hsverify.resultcheck as checks + + +class ComparedName(str): + comparisons = 0 + + def __eq__(self, other): + type(self).comparisons += 1 + if type(self).comparisons > 100_000: + raise AssertionError("metadata lookup performed more than 100,000 comparisons") + return super().__eq__(other) + + __hash__ = str.__hash__ + + +def test_distinct_evidence_ids_use_linear_name_lookups(): + names = [ComparedName(f"clearance_{i}") for i in range(12_000)] + answer = result(spot=False, checks=[], complete=False) + answer["reasons"] = [{"code": "unobserved_area", "message": ""}] + answer["checks"] = [ + check(outcome="unsure", measured=None, cause="unobserved") | {"id": name} for name in names + ] + answer["missing_evidence"] = [ + {"kind": "band", "band": "ground", "checks": [name], "message": ""} for name in names + ] + ComparedName.comparisons = 0 + assert checks.missing_evidence_problems(SCENE, answer) == [] + assert ComparedName.comparisons < 4 * len(names) + + +def test_invalid_width_message_deduplication_is_linear(): + # Instrument only the diagnostic buffer. The old list-membership search calls + # ComparedName.__eq__ for every previous diagnostic; hash deduplication does not. + class DiagnosticBuffer(list): + def append(self, message): + super().append(ComparedName(message)) + + class InstrumentBuffer(ast.NodeTransformer): + def visit_Assign(self, node): + if any(isinstance(t, ast.Name) and t.id == "problems" for t in node.targets): + node.value = ast.Call( + func=ast.Name(id="DiagnosticBuffer", ctx=ast.Load()), args=[], keywords=[] + ) + return node + + source = ast.parse(inspect.getsource(checks.reported_dimension_problems)) + source = ast.fix_missing_locations(InstrumentBuffer().visit(source)) + scope = checks.reported_dimension_problems.__globals__ | {"DiagnosticBuffer": DiagnosticBuffer} + exec(compile(source, "", "exec"), scope) + answer = result(spot=False, checks=[], complete=False) + answer["checks"] = [ + check(threshold=-1) | {"id": f"width_{i}", "rule": {"key": "battery.width_ft"}} + for i in range(4_000) + ] + ComparedName.comparisons = 0 + problems = scope["reported_dimension_problems"](answer, RULES) + assert len(problems) == 4_000 + assert problems[0] == "width_0 must be finite and positive" + assert problems[-1] == "width_3999 must be finite and positive" + assert ComparedName.comparisons < 4 * len(problems) + + +@pytest.mark.parametrize( + ("field", "limit"), + [ + ("sweep", checks.MAX_RESULT_RUNS), + ("checks", checks.MAX_RESULT_CHECKS), + ("missing_evidence", checks.MAX_EVIDENCE_REQUESTS), + ], +) +def test_result_count_caps_accept_boundary_and_reject_next(field, limit): + answer = {field: [None] * limit} + assert checks.ingestion_problems(answer, "result") == [] + answer[field].append(None) + assert checks.ingestion_problems(answer, "result") == [ + f"result ingestion limit exceeded: {field} exceeds {limit} entries" + ] + + +@pytest.mark.parametrize("field", ["sweep", "checks", "missing_evidence"]) +def test_named_result_caps_allow_schema_valid_responses(field): + fixture = json.loads((Path(__file__).parent / "fixtures/wide-public-pass.json").read_text()) + answer = copy.deepcopy(fixture["result"]) + request = {"kind": "band", "message": "", "band": "ground", "checks": ["ground_surface"]} + first_check = answer["checks"][0] + first_check.update(label="", reason="", rule={"key": "", "source": "", "placeholder": False}) + entries = { + "sweep": (answer["sweep"][0], checks.MAX_RESULT_RUNS), + "checks": (first_check, checks.MAX_RESULT_CHECKS), + "missing_evidence": (request, checks.MAX_EVIDENCE_REQUESTS), + } + entry, limit = entries[field] + answer[field] = ( + answer[field] + [entry] * (limit - len(answer[field])) + if field == "checks" + else [entry] * limit + ) + schema = json.loads( + (Path(__file__).parents[2] / "server/schemas/result.schema.json").read_text() + ) + assert checks.schema_errors(answer, schema) == [] + rules = checks.RuleSet.from_yaml((Path(__file__).parents[2] / "server/rules.yaml").read_text()) + assert checks.invariant_problems(fixture["scene"], answer, rules=rules) == [] + + +def test_scene_geometry_counts_keep_existing_schema_caps(): + wall = {"id": "wall", "baseline": [[0, 0]] * 500} + scene = {"walls": [wall] * 50} + assert checks.ingestion_problems(scene, "scene") == [] + scene["walls"].append(wall) + assert "walls exceeds 50 entries" in checks.ingestion_problems(scene, "scene")[0] + assert ( + "baseline exceeds 500 vertices" + in checks.ingestion_problems({"walls": [wall | {"baseline": [[0, 0]] * 501}]}, "scene")[0] + ) + + +@pytest.mark.parametrize( + ("value", "message"), + [ + ([0] * checks.MAX_NUMERIC_LEAVES, "numeric leaves"), + ([None] * (checks.MAX_METADATA_NODES - 1), "JSON nodes"), + ("x" * checks.MAX_METADATA_CHARACTERS, "characters"), + ], +) +def test_generic_metadata_caps_accept_boundary_and_reject_next(value, message): + assert checks.ingestion_problems(value) == [] + over = value + ("x" if isinstance(value, str) else [value[0]]) + assert message in checks.ingestion_problems(over)[0] + + +def test_nesting_cap_precedes_recursive_schema_validation(monkeypatch): + nested = 0 + for _ in range(checks.MAX_METADATA_DEPTH): + nested = [nested] + assert checks.ingestion_problems(nested) == [] + + def unexpected_validator(*args): + pytest.fail("oversized metadata reached recursive schema validation") + + monkeypatch.setattr(checks, "Draft202012Validator", unexpected_validator) + assert checks.schema_errors([nested], {}) == [ + f"document ingestion limit exceeded: nesting exceeds {checks.MAX_METADATA_DEPTH}" + ] + + +def test_ingestion_failure_precedes_geometry(monkeypatch): + def unexpected_dimensions(*args): + pytest.fail("oversized metadata reached dimension validation") + + monkeypatch.setattr(checks, "reported_dimension_problems", unexpected_dimensions) + answer = {"checks": [None] * (checks.MAX_RESULT_CHECKS + 1)} + assert checks.geometry_admission_problems(SCENE, answer, RULES) == [ + f"result ingestion limit exceeded: checks exceeds {checks.MAX_RESULT_CHECKS} entries" + ] + + +def test_invalid_schema_stops_after_bounded_diagnostics(): + errors = checks.schema_errors([None] * 10_000, {"type": "array", "items": {"type": "number"}}) + assert len(errors) == checks.MAX_SCHEMA_PROBLEMS + + +def test_case_queries_have_a_fixed_scan_bound(): + at_cap = {f"rule_{i}": 0 for i in range(checks.MAX_CASE_RULE_QUERIES)} + assert checks.case_metadata_problems(at_cap) == [] + at_cap["one_more"] = 0 + assert checks.case_metadata_problems(at_cap) == [ + "case ingestion limit exceeded: more than 64 rule queries" + ] + assert checks.expectation_problems( + {"checks": [{"match": "gas"}] * (checks.MAX_CASE_RULE_QUERIES + 1)}, {} + ) == ["case ingestion limit exceeded: more than 64 rule queries"] diff --git a/verification/tests/test_runtime_meter.py b/verification/tests/test_runtime_meter.py index 43132a99f..cc7cef084 100644 --- a/verification/tests/test_runtime_meter.py +++ b/verification/tests/test_runtime_meter.py @@ -78,12 +78,14 @@ def test_reach_cache_includes_geometry_and_parameters(): def test_ground_grid_stops_at_shared_runtime_limit(): - meter = WorkMeter(30) + meter = WorkMeter() plan = GroundPlan(SCENE, RULES, meter=meter) + setup = meter.samples + meter.limit = setup + 30 with pytest.raises(WorkLimitExceeded) as caught: plan.covers(2, 1, 3, 0, lambda _: 2) - assert caught.value.samples == meter.samples == 31 - assert str(caught.value) == "verification work limit exceeded (31 samples)" + assert caught.value.samples == meter.samples == setup + 31 + assert str(caught.value) == f"verification work limit exceeded ({setup + 31} samples)" def test_valid_wide_direct_coverage_uses_only_visited_work(): diff --git a/verification/tests/test_scene_cache.py b/verification/tests/test_scene_cache.py new file mode 100644 index 000000000..af18ea263 --- /dev/null +++ b/verification/tests/test_scene_cache.py @@ -0,0 +1,81 @@ +import copy +import json +from dataclasses import replace +from pathlib import Path + +import pytest +from segmented_wall import segmented_wall_scene +from test_resultcheck import RULES, SCENE + +from hsverify.resultcheck import ( + RuleSet, + WorkMeter, + ground_plan, + invariant_problems, + schema_errors, + segments_s, + slide, + wall_spans_s, +) + + +@pytest.mark.parametrize("segments_per_wall", [13, 20]) +def test_public_pass_is_independent_of_redundant_baseline_vertices(segments_per_wall): + # Public solve(parse_scene(...), rules_from_dict(public YAML)) produced this + # 815-candidate, 26-run response. The geometry is unchanged by extra vertices. + fixture = json.loads( + (Path(__file__).parent / "fixtures/segmented-public-pass.json").read_text() + ) + rules = RuleSet.from_yaml((Path(__file__).parents[2] / "server/rules.yaml").read_text()) + answer = fixture["result"] + baseline_meter = WorkMeter() + assert invariant_problems(fixture["scene"], answer, rules=rules, meter=baseline_meter) == [] + scene = segmented_wall_scene(segments_per_wall) + for kind, value in (("scene", scene), ("result", answer)): + schema = json.loads( + (Path(__file__).parents[2] / f"server/schemas/{kind}.schema.json").read_text() + ) + assert schema_errors(value, schema) == [] + meter = WorkMeter() + assert invariant_problems(scene, answer, rules=rules, meter=meter) == [] + # Metadata grows with the input, but actual geometry work must not multiply + # with redundant baseline vertices. Leave room for the bounded input scan. + assert meter.samples < baseline_meter.samples * 1.2 + assert all(key[0] != "ground_strips" for key in meter.cache) + + +def test_scene_derivatives_are_built_once_per_judgment(): + scene = segmented_wall_scene(20) + meter = WorkMeter() + segments = segments_s(scene, RULES, meter=meter) + spans = wall_spans_s(scene, RULES, meter=meter) + plan = ground_plan(scene, RULES, meter) + once = meter.samples + assert segments_s(scene, RULES, meter=meter) is segments + assert wall_spans_s(scene, RULES, meter=meter) is spans + assert ground_plan(scene, RULES, meter) is plan + assert meter.samples == once + assert len(segments) == 20 + assert len({segment[0] for segment in segments}) == 20 + assert all(key[0] != "ground_strips" for key in meter.cache) + + +def test_collinear_merge_keeps_actual_turns_and_wall_error_sources(): + scene = copy.deepcopy(SCENE) + scene["walls"] = [ + {"id": "w1", "baseline": [[0, 0], [2, 0], [4, 0], [4, 4]], "plus_minus_ft": 0.1}, + {"id": "w2", "baseline": [[4, 4], [4, 6], [4, 8]], "plus_minus_ft": 0.2}, + ] + scene["meter"].update(pos=[0, 4, 0], wall_id="w1", plus_minus_ft=0) + rules = replace(RULES, wall_join_ft=0.1) + meter = WorkMeter() + segments = segments_s(scene, rules, meter=meter) + assert [(wid, a, b) for wid, a, b, _, _ in segments] == [ + ("w1", 0, 4), + ("w1", 4, 8), + ("w2", 8, 12), + ] + assert slide(scene, rules, 9, 10, meter=meter) == pytest.approx(0.1 * 2**0.5) + once = meter.samples + assert slide(scene, rules, 9, 10, meter=meter) == pytest.approx(0.1 * 2**0.5) + assert meter.samples == once From a7b3169fefce72375f868e51565d30b000889941 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Wed, 14 Oct 2026 15:37:38 -0500 Subject: [PATCH 57/62] fix(pipeline): verify reported rules and repair encrypted caches --- recon/recon/depth.py | 2 + recon/tests/test_review_debt.py | 21 +++- verification/hsverify/e2e.py | 87 +++++++++++++--- verification/hsverify/resultcheck.py | 112 +++++++++++++++----- verification/tests/test_generate_cases.py | 2 +- verification/tests/test_metadata_bounds.py | 5 + verification/tests/test_reported_rules.py | 114 +++++++++++++++++++++ verification/tests/test_resultcheck.py | 11 +- 8 files changed, 306 insertions(+), 48 deletions(-) create mode 100644 verification/tests/test_reported_rules.py diff --git a/recon/recon/depth.py b/recon/recon/depth.py index 70c49b2c2..7635a6afe 100644 --- a/recon/recon/depth.py +++ b/recon/recon/depth.py @@ -217,6 +217,8 @@ def _moge_locked(capture: Capture, work: Path) -> dict[str, Depth]: KeyError, EOFError, TypeError, + # A damaged ZIP encryption flag makes NumPy request a password. + RuntimeError, # ZIP corruption can name an unsupported compression method. NotImplementedError, zipfile.BadZipFile, diff --git a/recon/tests/test_review_debt.py b/recon/tests/test_review_debt.py index 372d6edf0..cda455e42 100644 --- a/recon/tests/test_review_debt.py +++ b/recon/tests/test_review_debt.py @@ -206,6 +206,18 @@ def unsupported_compression_npz() -> bytes: return bytes(raw) +def encrypted_flag_npz() -> bytes: + archive = io.BytesIO() + np.savez(archive, depth=np.ones((2, 2), np.float32)) + raw = bytearray(archive.getvalue()) + central = raw.index(b"PK\x01\x02") + # Corrupt only the directory's encryption bit. zipfile uses that bit when + # NumPy reads depth.npy, so a normal cache read now asks for a password. + flags = struct.unpack_from(" bytes: b"not an npz", corrupt_compressed_npz(), unsupported_compression_npz(), + encrypted_flag_npz(), + ], + ids=[ + "truncated_zip", + "not_npz", + "damaged_deflate", + "unsupported_compression", + "encrypted_flag", ], - ids=["truncated_zip", "not_npz", "damaged_deflate", "unsupported_compression"], ) def test_corrupt_moge_cache_is_recomputed(tmp_path, monkeypatch, bad): capture = _capture_named_scan(tmp_path, 10) diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index 45e539244..593cb9b5a 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -74,6 +74,8 @@ mirror_scene, more_error_problems, outcome_lengths, + public_assumption_warnings, + reported_rules, schema_errors, tape_to_tap, unobserved_checks, @@ -517,7 +519,7 @@ def judge( failure = [ f"HTTP {status} for a scene that validates: {payload[:500]!r}" if status is not None - else f"no answer within 60 s: {payload[:200]!r}" + else payload.decode(errors="replace")[:200] ] return record | {"status": "fail", "problems": failure, "contract_problems": failure} result = parse_result(payload) @@ -552,29 +554,53 @@ def check(name: str, function: Callable[[], list[str]]) -> list[str]: lambda: geometry_admission_problems(item.scene, result, rules, meter=meter), ) admitted = not contract + mismatches: list[str] = [] + warnings: list[str] = [] + active_rules = rules + expected: list[str] = [] + if admitted: + + def check_reported_rules() -> list[str]: + nonlocal active_rules + mismatches.extend(assumption_mismatches(item.rules_assumed, result)) + warnings.extend(public_assumption_warnings(item.rules_assumed, result, rules)) + active_rules = reported_rules(result, rules) + return [] + + contract += check("reported rules", check_reported_rules) + admitted = not contract + if warnings: + record["rule_checks_unverified"] = True if admitted: contract += check( "invariants", - lambda: invariant_problems(item.scene, result, sent=item.raw, rules=rules, meter=meter), + lambda: invariant_problems( + item.scene, + result, + sent=item.raw, + rules=None if warnings else active_rules, + meter=meter, + ), ) if not contract: contract += check( "properties", - lambda: property_problems(url, endpoint, item, result, schemas, rules, meter=meter), + lambda: property_problems( + url, + endpoint, + item, + result, + schemas, + active_rules, + meter=meter, + rule_checks=not warnings, + ), ) if item.real and ms > LATENCY_BUDGET_MS: contract.append(f"real-derived scene took {ms:.0f} ms, budget {LATENCY_BUDGET_MS:.0f}") - mismatches: list[str] = [] - expected: list[str] = [] exhausted = any(p.startswith("verification work limit exceeded (") for p in contract) if admitted and not exhausted: - - def check_assumptions() -> list[str]: - mismatches.extend(assumption_mismatches(item.rules_assumed, result, rules)) - return [] - - contract += check("assumptions", check_assumptions) - if item.expect and not mismatches: + if item.expect and not mismatches and not warnings: expected = check( "expectations", lambda: expectation_problems(item.expect, result, meter=meter) ) @@ -588,12 +614,20 @@ def summarize() -> list[str]: record["decision"] = result["decision"] if mismatches: record["assumption_mismatches"] = mismatches + if warnings: + record["assumption_warnings"] = warnings record["verification_samples"] = meter.samples record["contract_problems"] = contract record["expectation_problems"] = expected record["problems"] = contract + expected record["status"] = ( - "fail" if record["problems"] else "assumption mismatch" if mismatches else "pass" + "fail" + if record["problems"] + else "assumption mismatch" + if mismatches + else "assumptions unverified" + if warnings + else "pass" ) return record @@ -639,8 +673,9 @@ def resend(name: str, scene: dict) -> dict | None: problems.extend(f"{name}: {p}" for p in admission) return None check_name = "invariants" + variant_rules = reported_rules(other, rules) if rule_checks else None invariants = invariant_problems( - scene, other, sent=transformed.raw, rules=checked_rules, meter=meter + scene, other, sent=transformed.raw, rules=variant_rules, meter=meter ) if invariants: problems.extend(f"{name}: {p}" for p in invariants) @@ -876,7 +911,12 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: contract_problems = sum(len(r.get("contract_problems", [])) for r in records) killed = bool(meta["peak_memory"].get("server_killed")) all_passed = all(r["status"] == "pass" for r in records) and bool(checked) and not killed - contract_ok = bool(checked) and contract_problems == 0 and not killed + contract_ok = ( + bool(checked) + and contract_problems == 0 + and not killed + and not any(r.get("rule_checks_unverified") for r in checked) + ) executed = sorted( {r["name"] for r in records if r.get("hostile") and r.get("hostile_probe_executed") is True} ) @@ -935,7 +975,11 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: "| --- | --- | --- | --- | --- |", ] for r in records: - issues = "; ".join(r.get("problems", []) + r.get("assumption_mismatches", [])) + issues = "; ".join( + r.get("problems", []) + + r.get("assumption_mismatches", []) + + r.get("assumption_warnings", []) + ) issues = issues or r.get("skip_reason", "") lines.append( f"| {r['name']}{' (real)' if r.get('real') else ''} | {r['status']} | " @@ -1054,7 +1098,16 @@ def main(argv: list[str] | None = None) -> int: } report = write_report(out, meta, records) print(f"Report: {out / 'report.md'}") - return 0 if report["all_passed"] else 1 + if report["all_passed"]: + return 0 + # An unreported override is incomplete evidence, not a proven server failure. + if ( + not meta["peak_memory"].get("server_killed") + and all(r["status"] in ("pass", "assumptions unverified") for r in records) + and any(r["status"] == "assumptions unverified" for r in records) + ): + return 2 + return 1 if __name__ == "__main__": diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 8ca930ae9..07ffd4c77 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -31,7 +31,7 @@ import json import math from collections.abc import Callable, Iterator -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Any import yaml @@ -75,9 +75,9 @@ # Bound verifier work even when a server span or rules step requests billions of samples. MAX_SWEEP_SAMPLES = 10_000 # On 2026-10-14, after caching scene derivatives and deferring ground fallback, -# 35 valid public fixture judgments including property variants peaked at 30,913 +# 35 valid public fixture judgments including property variants peaked at 31,649 # units. Twenty measurements of 20 four-foot walls at 1/4/12/13/20 subdivisions, -# original and mirrored, including full-judge resends, peaked at 87,427 units. +# original and mirrored, including full-judge resends, peaked at 88,999 units. # Another 468 distinct saved responses peaked at 391,700 invariant units on ETH3D # facade. Allow 20 times that largest measured valid workload for other scans and # their variants. This is an operational margin, not physical-error calibration. @@ -144,7 +144,7 @@ def _input_key(value: Any, meter: WorkMeter | None = None) -> Any: if isinstance(value, (tuple, list)): return tuple(_input_key(item, meter) for item in value) if isinstance(value, RuleSet): - return _input_key(vars(value), meter) + return _input_key({k: v for k, v in vars(value).items() if k != "rules_sha256"}, meter) if isinstance(value, Need): return _input_key(vars(value), meter) return value @@ -204,6 +204,7 @@ class RuleSet: # the rules' headroom.min_ft: how high a wall view without out_ft reached headroom_ft: float | None = None corner_allowance_ft: float | None = None + rules_sha256: str | None = None @classmethod def from_yaml(cls, text: str) -> RuleSet: @@ -264,9 +265,33 @@ def reach(key: str) -> Need: value("battery", "height_ft"), headroom, value("route", "corner_allowance_ft"), + public_rules_fingerprint(data), ) +def public_rules_fingerprint(data: dict) -> str: + """Mirror public rules_from_dict's normalized JSON hash without importing the server.""" + + def normalize(value: Any) -> Any: + if isinstance(value, dict): + normalized = {key: normalize(item) for key, item in value.items()} + if "value" in normalized and "source" in normalized: + normalized["value"] = float(normalized["value"]) + normalized.setdefault("placeholder", False) + return normalized + if isinstance(value, list): + return [normalize(item) for item in value] + return value + + canonical = normalize(data) + canonical.get("policy", {}).setdefault("notice", None) + openings = canonical.get("openings", {}) + if openings.get("exempt_bottom_above_ft") is not None: + openings["exempt_bottom_above_ft"] = float(openings["exempt_bottom_above_ft"]) + payload = json.dumps(canonical, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode()).hexdigest() + + def ingestion_problems(instance: Any, kind: str | None = None) -> list[str]: """Bound metadata before validation or copies, using a depth-sized iterator stack.""" if kind is None and isinstance(instance, dict): @@ -1738,30 +1763,69 @@ def case_metadata_problems(assumed: dict, expect: dict | None = None) -> list[st return ingestion_problems({"assumed": assumed, "expect": expect}, "case") -def assumption_mismatches( - assumed: dict[str, float], result: dict, rules: RuleSet | None = None +def reported_assumption_values(key: str, result: dict) -> list[tuple[str, float]]: + """Rule values carried by C2 spots or check thresholds, without reading server files.""" + values = [] + if key == "battery_width": + for name in ("spot", "nearest_considered"): + spot = result.get(name) + if spot is not None and "width_ft" in spot: + values.append((f"{name}.width_ft", spot["width_ft"])) + rule_key = { + "battery_width": "battery.width_ft", + "corner_allowance": "route.corner_allowance_ft", + }.get(key) + for check in result.get("checks", []): + threshold = check.get("threshold_ft") + matches = check.get("rule", {}).get("key") == rule_key if rule_key else key in check["id"] + if matches and threshold is not None: + values.append((check["id"], threshold)) + return values + + +def reported_rules(result: dict, rules: RuleSet) -> RuleSet: + """Use reported geometry for invariants too, so public defaults cannot reject an override.""" + if problems := reported_dimension_problems(result, rules): + raise ValueError("; ".join(problems)) + changes = {} + for key, field in (("battery_width", "width_ft"), ("corner_allowance", "corner_allowance_ft")): + if values := reported_assumption_values(key, result): + changes[field] = values[0][1] + return replace(rules, **changes) + + +def assumption_mismatches(assumed: dict[str, float], result: dict) -> list[str]: + if problems := case_metadata_problems(assumed): + return problems + return [ + f"case assumes {key} threshold {value}, server's {source} uses {actual}" + for key, value in assumed.items() + for source, actual in reported_assumption_values(key, result) + if abs(actual - value) > EPS + ] + + +def public_assumption_warnings( + assumed: dict[str, float], result: dict, rules: RuleSet ) -> list[str]: if problems := case_metadata_problems(assumed): return problems - out = [] - configured = ( - {"battery_width": rules.width_ft, "corner_allowance": rules.corner_allowance_ft} - if rules - else {} + configured = {"battery_width": rules.width_ft, "corner_allowance": rules.corner_allowance_ft} + policy = result.get("policy", {}) + nonpublic = "private" in policy.get("sources", []) or ( + rules.rules_sha256 is not None + and policy.get("rules_sha256") is not None + and policy["rules_sha256"] != rules.rules_sha256 ) - for key, value in assumed.items(): - actual = configured.get(key) - if actual is not None and abs(actual - value) > EPS: - out.append(f"case assumes {key} {value}, rules use {actual}") - for key, value in assumed.items(): - for check in result.get("checks", []): - threshold = check.get("threshold_ft") - if key in check["id"] and threshold is not None and abs(threshold - value) > EPS: - out.append( - f"case assumes {key} threshold {value}, server's {check['id']} uses " - f"{check['threshold_ft']}" - ) - return out + return [ + f"case assumes {key} {value}, public rules use {configured[key]}; " + "assumed public rules; server may override; result does not report this value, " + "so case expectations are unverified" + for key, value in assumed.items() + if configured.get(key) is not None + and not reported_assumption_values(key, result) + and (nonpublic or abs(configured[key] - value) > EPS) + ] def expectation_problems(expect: dict, result: dict, meter: WorkMeter | None = None) -> list[str]: diff --git a/verification/tests/test_generate_cases.py b/verification/tests/test_generate_cases.py index 89228e128..3cab47a0d 100644 --- a/verification/tests/test_generate_cases.py +++ b/verification/tests/test_generate_cases.py @@ -101,7 +101,7 @@ def forbidden(*args): {"scene": {}, "result": {}}, rules, ) - assert record["status"] == "assumption mismatch" and record["assumption_mismatches"] + assert record["status"] == "assumptions unverified" and record["assumption_warnings"] assert record["expectation_problems"] == [] meta = { "peak_memory": {}, diff --git a/verification/tests/test_metadata_bounds.py b/verification/tests/test_metadata_bounds.py index e88cc657e..12fd16e4a 100644 --- a/verification/tests/test_metadata_bounds.py +++ b/verification/tests/test_metadata_bounds.py @@ -110,6 +110,11 @@ def test_named_result_caps_allow_schema_valid_responses(field): (Path(__file__).parents[2] / "server/schemas/result.schema.json").read_text() ) assert checks.schema_errors(answer, schema) == [] + if field == "sweep": + answer["spot"] = answer["route"] = None + answer["decision"] = "reject" + elif field == "missing_evidence": + answer["decision"] = "manual_review" rules = checks.RuleSet.from_yaml((Path(__file__).parents[2] / "server/rules.yaml").read_text()) assert checks.invariant_problems(fixture["scene"], answer, rules=rules) == [] diff --git a/verification/tests/test_reported_rules.py b/verification/tests/test_reported_rules.py new file mode 100644 index 000000000..f09e6cd39 --- /dev/null +++ b/verification/tests/test_reported_rules.py @@ -0,0 +1,114 @@ +"""Reported geometry and policy identity determine whether rule assumptions are verified.""" + +import copy +import hashlib +import json +from dataclasses import replace +from pathlib import Path + +import pytest +from segmented_wall import segmented_wall_scene +from test_judge_meter import META + +from hsverify import e2e +from hsverify.resultcheck import RuleSet, public_assumption_warnings + +ROOT = Path(__file__).parents[2] +RULES = RuleSet.from_yaml((ROOT / "server/rules.yaml").read_text()) +SCHEMAS = { + kind: json.loads((ROOT / f"server/schemas/{kind}.schema.json").read_text()) + for kind in ("scene", "result") +} + + +def test_public_fingerprint_matches_recorded_public_solver(): + fixture = json.loads( + (Path(__file__).parent / "fixtures/segmented-public-pass.json").read_text() + ) + assert RULES.rules_sha256 == fixture["result"]["policy"]["rules_sha256"] + + +@pytest.mark.parametrize("identity", ["private source", "different fingerprint"]) +def test_unreported_override_is_unverified_even_when_case_matches_public(identity): + answer = {"policy": {"sources": ["public"], "rules_sha256": RULES.rules_sha256}} + if identity == "private source": + answer["policy"]["sources"].append("private") + else: + answer["policy"]["rules_sha256"] = "f" * 64 + warnings = public_assumption_warnings({"battery_width": RULES.width_ft}, answer, RULES) + assert len(warnings) == 1 + assert "assumed public rules; server may override" in warnings[0] + + +@pytest.mark.parametrize("matches", [True, False]) +def test_judge_compares_reported_width_with_case_assumption(monkeypatch, matches): + answers = json.loads( + (Path(__file__).parent / "fixtures/segmented-public-workload.json").read_text() + ) + + def post(url, endpoint, item): + answer = copy.deepcopy(answers[item.name]) + answer["stats"]["input_sha256"] = hashlib.sha256(item.raw).hexdigest() + return 200, json.dumps(answer).encode(), 0 + + monkeypatch.setattr(e2e, "post", post) + scene = segmented_wall_scene(1) + item = e2e.SceneInput( + "surveyed wall", + scene, + json.dumps(scene).encode(), + rules_assumed={"battery_width": RULES.width_ft if matches else 4}, + ) + record = e2e.judge( + "unused", + e2e.Endpoint("/place", "application/json"), + item, + SCHEMAS, + replace(RULES, width_ft=4), + ) + assert record["status"] == ("pass" if matches else "assumption mismatch") + assert bool(record.get("assumption_mismatches")) is not matches + + +@pytest.mark.parametrize("identity", ["private source", "different fingerprint"]) +def test_unreported_private_width_cannot_claim_contract(monkeypatch, tmp_path, identity): + fixture = json.loads( + (Path(__file__).parent / "fixtures/segmented-public-pass.json").read_text() + ) + answer = copy.deepcopy(fixture["result"]) + answer.update(decision="reject", spot=None, route=None) + answer["reasons"] = [{"code": "policy_not_approved", "message": "Private rules reject"}] + answer["sweep"] = [ + { + "wall_id": "w1", + "start_ft": [1, 1], + "outcome": "fail", + "failing": ["route_length"], + "unsure": [], + } + ] + answer["stats"].update(candidates=1, fail=1, unsure=0, **{"pass": 0}) + for check in answer["checks"]: + check.update(outcome="fail", measured_ft=None, plus_minus_ft=None) + if identity == "private source": + answer["policy"]["sources"].append("private") + else: + answer["policy"]["rules_sha256"] = "f" * 64 + + def post(url, endpoint, item): + value = copy.deepcopy(answer) + value["stats"]["input_sha256"] = hashlib.sha256(item.raw).hexdigest() + return 200, json.dumps(value).encode(), 0 + + monkeypatch.setattr(e2e, "post", post) + scene = segmented_wall_scene(1) + item = e2e.SceneInput( + "private reject", + scene, + json.dumps(scene).encode(), + rules_assumed={"battery_width": RULES.width_ft}, + ) + record = e2e.judge("unused", e2e.Endpoint("/place", "application/json"), item, SCHEMAS, RULES) + assert record["status"] == "assumptions unverified", record["problems"] + assert record["rule_checks_unverified"] + assert not e2e.write_report(tmp_path, META, [record])["contract_ok"] diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 640a36e61..962b53541 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -1502,13 +1502,14 @@ def test_ground_trimming_keeps_omitted_reach_at_zero(): assert trimmed["coverage"]["observed"][0]["out_ft"] == 0 -def test_geometric_assumptions_compare_active_rules(): - rules = replace(RULES, corner_allowance_ft=0.5) - assert ( - len(assumption_mismatches({"battery_width": 100, "corner_allowance": 100}, {}, rules)) == 2 +def test_geometric_assumptions_compare_reported_rules(): + answer = result() + answer["checks"].append( + {"id": "route_corner", "threshold_ft": 0.5, "rule": {"key": "route.corner_allowance_ft"}} ) + assert len(assumption_mismatches({"battery_width": 100, "corner_allowance": 100}, answer)) == 2 assert ( - assumption_mismatches({"battery_width": rules.width_ft, "corner_allowance": 0.5}, {}, rules) + assumption_mismatches({"battery_width": RULES.width_ft, "corner_allowance": 0.5}, answer) == [] ) From db1e4c2f931711fc3bf12af7e49c3364e27d8bc8 Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:02:21 -0500 Subject: [PATCH 58/62] fix(verification): enforce public policy and validate case metadata Keep loaded dimensions when the response claims the tested public fingerprint, including transformed responses. Reject malformed local case fields without losing the report. Reviews: https://github.com/SamGu-NRX/BaseScanning/pull/240#discussion_r4276092143 https://github.com/SamGu-NRX/BaseScanning/pull/240#discussion_r4276060760 --- verification/hsverify/e2e.py | 11 +- verification/hsverify/resultcheck.py | 70 ++++++++- verification/tests/test_policy_admission.py | 163 ++++++++++++++++++++ verification/tests/test_reported_rules.py | 1 + 4 files changed, 241 insertions(+), 4 deletions(-) create mode 100644 verification/tests/test_policy_admission.py diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index 593cb9b5a..d9bf7e920 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -75,6 +75,7 @@ more_error_problems, outcome_lengths, public_assumption_warnings, + public_dimension_problems, reported_rules, schema_errors, tape_to_tap, @@ -569,6 +570,8 @@ def check_reported_rules() -> list[str]: contract += check("reported rules", check_reported_rules) admitted = not contract + if admitted: + contract += check("public dimensions", lambda: public_dimension_problems(result, rules)) if warnings: record["rule_checks_unverified"] = True if admitted: @@ -594,6 +597,7 @@ def check_reported_rules() -> list[str]: active_rules, meter=meter, rule_checks=not warnings, + public_rules=rules, ), ) if item.real and ms > LATENCY_BUDGET_MS: @@ -642,11 +646,13 @@ def property_problems( *, meter: WorkMeter | None = None, rule_checks: bool = True, + public_rules: RuleSet | None = None, ) -> list[str]: """The same scene resent unchanged and transformed. Every answer to a transformed scene must also satisfy the invariants, and the transforms that only lose information must not make the server surer.""" meter = meter if meter is not None else WorkMeter() + public_rules = public_rules if public_rules is not None else rules checked_rules = rules if rule_checks else None if admission := geometry_admission_problems(item.scene, result, checked_rules, meter=meter): return admission @@ -673,8 +679,9 @@ def resend(name: str, scene: dict) -> dict | None: problems.extend(f"{name}: {p}" for p in admission) return None check_name = "invariants" - variant_rules = reported_rules(other, rules) if rule_checks else None - invariants = invariant_problems( + variant_rules = reported_rules(other, public_rules) if rule_checks else None + dimensions = public_dimension_problems(other, public_rules) if rule_checks else [] + invariants = dimensions + invariant_problems( scene, other, sent=transformed.raw, rules=variant_rules, meter=meter ) if invariants: diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 07ffd4c77..1c030d6a0 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -1754,7 +1754,15 @@ def with_past_ends_asked( def case_metadata_problems(assumed: dict, expect: dict | None = None) -> list[str]: """Bound local case queries too: substring expectations scan the response once each.""" - expect = expect or {} + if not isinstance(assumed, dict): + return ["case rules_assumed must be an object"] + if expect is None: + expect = {} + if not isinstance(expect, dict): + return ["case expect must be an object"] + for field in ("sweep_runs", "checks", "start_outcomes"): + if field in expect and not isinstance(expect[field], list): + return [f"case expect.{field} must be an array"] queries = len(assumed) + sum( len(expect.get(field, [])) for field in ("sweep_runs", "checks", "start_outcomes") ) @@ -1783,10 +1791,68 @@ def reported_assumption_values(key: str, result: dict) -> list[tuple[str, float] return values +def reported_policy_overrides(result: dict, rules: RuleSet) -> bool: + """A reported fingerprint must identify a policy other than the loaded public rules.""" + policy = result.get("policy", {}) + fingerprint = policy.get("rules_sha256") + if not isinstance(fingerprint, str): + return False + if rules.rules_sha256 is not None: + return fingerprint != rules.rules_sha256 + return "private" in policy.get("sources", []) + + +def public_dimension_problems(result: dict, rules: RuleSet) -> list[str]: + if reported_policy_overrides(result, rules): + return [] + values = [] + for name in ("spot", "nearest_considered"): + spot = result.get(name) + if spot is not None: + for field, expected in ( + ("width_ft", rules.width_ft), + ("depth_ft", rules.depth_ft), + ("height_ft", rules.battery_height_ft), + ): + if field in spot and expected is not None: + values.append((f"{name}.{field}", spot[field], expected)) + for key, expected in ( + ("battery_width", rules.width_ft), + ("corner_allowance", rules.corner_allowance_ft), + ): + if expected is not None: + values.extend( + (source, actual, expected) + for source, actual in reported_assumption_values(key, result) + ) + if result.get("route") is not None and "height_ft" in result["route"]: + values.append(("route.height_ft", result["route"]["height_ft"], rules.route_height_ft)) + configured = { + "battery.depth_ft": rules.depth_ft, + "battery.height_ft": rules.battery_height_ft, + "route.height_ft": rules.route_height_ft, + } + for check in result.get("checks", []): + expected = configured.get(check.get("rule", {}).get("key")) + actual = check.get("threshold_ft") + if expected is not None and actual is not None: + values.append((check["id"], actual, expected)) + # C2 rounds dimensions to six decimals; preserve that representation tolerance. + return _unique_messages( + [ + f"{source} {actual} does not match loaded public rules {expected}" + for source, actual, expected in values + if abs(actual - expected) > NUMBER_TOL_FT + ] + ) + + def reported_rules(result: dict, rules: RuleSet) -> RuleSet: - """Use reported geometry for invariants too, so public defaults cannot reject an override.""" + """Use validated reported geometry only for an explicitly different policy.""" if problems := reported_dimension_problems(result, rules): raise ValueError("; ".join(problems)) + if not reported_policy_overrides(result, rules): + return rules changes = {} for key, field in (("battery_width", "width_ft"), ("corner_allowance", "corner_allowance_ft")): if values := reported_assumption_values(key, result): diff --git a/verification/tests/test_policy_admission.py b/verification/tests/test_policy_admission.py new file mode 100644 index 000000000..c09f9859f --- /dev/null +++ b/verification/tests/test_policy_admission.py @@ -0,0 +1,163 @@ +import copy +import hashlib +import json +from dataclasses import replace +from pathlib import Path + +import pytest +from segmented_wall import segmented_wall_scene +from test_judge_meter import META + +from hsverify import e2e +from hsverify.resultcheck import RuleSet, public_dimension_problems, reported_rules + +ROOT = Path(__file__).parents[2] +RULES = RuleSet.from_yaml((ROOT / "server/rules.yaml").read_text()) +SCHEMAS = { + kind: json.loads((ROOT / f"server/schemas/{kind}.schema.json").read_text()) + for kind in ("scene", "result") +} + + +def replies(): + return json.loads( + (Path(__file__).parent / "fixtures/segmented-public-workload.json").read_text() + ) + + +@pytest.mark.parametrize("field", ["width_ft", "depth_ft", "height_ft"]) +def test_matching_public_policy_keeps_loaded_dimensions(field): + answer = replies()["surveyed wall"] + answer["spot"].pop("footprint") + answer["spot"].pop("span_ft") + answer["spot"][field] = 1 + assert reported_rules(answer, RULES) is RULES + + +@pytest.mark.parametrize("wrong", [False, True]) +def test_public_policy_width_is_checked_without_case_assumptions(monkeypatch, wrong): + answers = replies() + if wrong: + spot = answers["surveyed wall"]["spot"] + lo = spot["span_ft"][0] + spot.update( + width_ft=1, + span_ft=[lo, lo + 1], + footprint=[[lo, 0], [lo + 1, 0], [lo + 1, RULES.depth_ft], [lo, RULES.depth_ft]], + center=[lo + 0.5, RULES.depth_ft / 2], + meter_offset_ft=[lo + 0.5, RULES.depth_ft / 2], + ) + seen_rules = [] + original = e2e.invariant_problems + + def invariant(*args, rules=None, **kwargs): + seen_rules.append(rules) + return original(*args, rules=rules, **kwargs) + + def post(url, endpoint, item): + answer = copy.deepcopy(answers[item.name]) + answer["stats"]["input_sha256"] = hashlib.sha256(item.raw).hexdigest() + return 200, json.dumps(answer).encode(), 0 + + monkeypatch.setattr(e2e, "post", post) + monkeypatch.setattr(e2e, "invariant_problems", invariant) + scene = segmented_wall_scene(1) + item = e2e.SceneInput("surveyed wall", scene, json.dumps(scene).encode()) + record = e2e.judge("unused", e2e.Endpoint("/place", "application/json"), item, SCHEMAS, RULES) + assert record["status"] == ("fail" if wrong else "pass"), record["problems"] + assert seen_rules[0] is RULES + if wrong: + assert any("width_ft" in p and "public rules" in p for p in record["contract_problems"]) + + +@pytest.mark.parametrize("sources", [["public", "private"], ["public"]]) +def test_different_policy_keeps_validated_reported_width(sources): + answer = replies()["surveyed wall"] + answer["policy"].update(sources=sources, rules_sha256="f" * 64) + loaded = replace(RULES, width_ft=4) + assert reported_rules(answer, loaded).width_ft == answer["spot"]["width_ft"] + + +def test_malformed_cases_are_reported_and_next_case_runs(monkeypatch, tmp_path): + scene = segmented_wall_scene(1) + cases = [] + for name, metadata in ( + ("null-assumptions", {"rules_assumed": None}), + ("null-checks", {"expect": {"checks": None}}), + ): + path = tmp_path / f"{name}.json" + path.write_text(json.dumps({"id": name, "scene": scene, "expect": {}, **metadata})) + cases.append(e2e.load_input(path)) + cases.append(e2e.SceneInput("surveyed wall", scene, json.dumps(scene).encode())) + answers = replies() + + def post(url, endpoint, item): + assert item.name.startswith("surveyed wall") + answer = copy.deepcopy(answers[item.name]) + answer["stats"]["input_sha256"] = hashlib.sha256(item.raw).hexdigest() + return 200, json.dumps(answer).encode(), 0 + + monkeypatch.setattr(e2e, "post", post) + records = [ + e2e.judge("unused", e2e.Endpoint("/place", "application/json"), item, SCHEMAS, RULES) + for item in cases + ] + assert [r["status"] for r in records] == ["bad input", "bad input", "pass"] + assert "rules_assumed" in records[0]["problems"][0] + assert "expect.checks" in records[1]["problems"][0] + (tmp_path / "report").mkdir() + e2e.write_report(tmp_path / "report", META, records) + saved = json.loads((tmp_path / "report/report.json").read_text()) + assert len(saved["records"]) == 3 + assert not saved["all_passed"] + + +@pytest.mark.parametrize("name", ["spot", "nearest_considered"]) +@pytest.mark.parametrize("field", ["width_ft", "depth_ft", "height_ft"]) +def test_every_public_spot_dimension_is_compared(name, field): + answer = {"policy": {"rules_sha256": RULES.rules_sha256}, name: {field: 1}} + assert any(f"{name}.{field}" in p for p in public_dimension_problems(answer, RULES)) + + +def test_public_corner_allowance_and_route_height_are_compared(): + answer = { + "policy": {"rules_sha256": RULES.rules_sha256}, + "route": {"height_ft": 100}, + "checks": [ + {"id": "corner", "rule": {"key": "route.corner_allowance_ft"}, "threshold_ft": 100} + ], + } + problems = public_dimension_problems(answer, RULES) + assert any("route.height_ft" in p for p in problems) + assert any("corner" in p for p in problems) + + +def test_private_marker_cannot_override_matching_fingerprint(): + answer = replies()["surveyed wall"] + answer["policy"]["sources"].append("private") + assert reported_rules(answer, RULES) is RULES + + +def test_missing_policy_cannot_override_loaded_dimensions(): + answer = replies()["surveyed wall"] + answer.pop("policy") + assert reported_rules(answer, RULES) is RULES + + +def test_transformed_public_response_uses_original_loaded_rules(monkeypatch): + answers = replies() + loaded = replace(RULES, width_ft=4) + + def post(url, endpoint, item): + answer = copy.deepcopy(answers[item.name]) + answer["stats"]["input_sha256"] = hashlib.sha256(item.raw).hexdigest() + if not item.name.endswith("mirrored"): + answer["policy"].update(sources=["public", "private"], rules_sha256="f" * 64) + return 200, json.dumps(answer).encode(), 0 + + monkeypatch.setattr(e2e, "post", post) + scene = segmented_wall_scene(1) + item = e2e.SceneInput("surveyed wall", scene, json.dumps(scene).encode()) + record = e2e.judge("unused", e2e.Endpoint("/place", "application/json"), item, SCHEMAS, loaded) + assert record["status"] == "fail" + assert any("mirrored" in p and "public rules" in p for p in record["contract_problems"]) diff --git a/verification/tests/test_reported_rules.py b/verification/tests/test_reported_rules.py index f09e6cd39..64442f0e2 100644 --- a/verification/tests/test_reported_rules.py +++ b/verification/tests/test_reported_rules.py @@ -49,6 +49,7 @@ def test_judge_compares_reported_width_with_case_assumption(monkeypatch, matches def post(url, endpoint, item): answer = copy.deepcopy(answers[item.name]) answer["stats"]["input_sha256"] = hashlib.sha256(item.raw).hexdigest() + answer["policy"].update(sources=["public", "private"], rules_sha256="f" * 64) return 200, json.dumps(answer).encode(), 0 monkeypatch.setattr(e2e, "post", post) From c557e3af387816f5ef5f3a430dacddf4311383ee Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:05:55 -0500 Subject: [PATCH 59/62] test(verification): identify private override fixtures explicitly --- verification/tests/test_e2e.py | 8 +++++++- verification/tests/test_resultcheck.py | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index 96cc59631..f1dad3a4b 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -1105,7 +1105,13 @@ def fake_judge(url, endpoint, probe, *args): def test_judge_distinguishes_reported_override_mismatch_and_unverified_defaults( tmp_path, monkeypatch, reported_width, contract_failure ): - answer = {"decision": "manual_review", "spot": None, "checks": [], "sweep": []} + answer = { + "decision": "manual_review", + "spot": None, + "checks": [], + "sweep": [], + "policy": {"sources": ["private"], "rules_sha256": "f" * 64}, + } if reported_width is not None: answer["checks"] = [ { diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 60cf16757..503b5ec01 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -1565,6 +1565,7 @@ def test_ground_trimming_keeps_omitted_reach_at_zero(): def test_geometric_assumptions_use_reported_override_instead_of_public_defaults(): public = replace(RULES, width_ft=4, corner_allowance_ft=10) answer = result() + answer["policy"] = {"sources": ["private"], "rules_sha256": "f" * 64} answer["checks"].append( { "id": "corner_allowance", @@ -1628,6 +1629,7 @@ def test_reported_width_sources_override_public_width(source): if source == "check" else {source: {"width_ft": RULES.width_ft}} ) + answer["policy"] = {"sources": ["private"], "rules_sha256": "f" * 64} assert reported_rules(answer, replace(RULES, width_ft=4)).width_ft == RULES.width_ft assert assumption_mismatches({"battery_width": RULES.width_ft}, answer) == [] @@ -1704,6 +1706,7 @@ def test_in_scan_sweep_with_tiny_rule_step_fails_loudly(): def test_reported_override_prevents_a_false_footprint_failure(): public = replace(RULES, width_ft=4) answer = result(spot=False, sweep=[pass_run(9, 9) | {"outcome": "unsure"}]) + answer["policy"] = {"sources": ["private"], "rules_sha256": "f" * 64} backing = next(c for c in answer["checks"] if c["id"] == "wall_backing") backing["threshold_ft"] = RULES.width_ft backing["rule"]["key"] = "battery.width_ft" @@ -1911,4 +1914,5 @@ def test_breadth_zero_or_positive_corner_allowance_is_valid(allowance): } ] } + answer["policy"] = {"sources": ["private"], "rules_sha256": "f" * 64} assert reported_rules(answer, RULES).corner_allowance_ft == allowance From c32ee975f10ba75df9eb5e88f01aa2743fa648d3 Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:26:39 -0500 Subject: [PATCH 60/62] fix(verification): bound diagnostics and harden numeric cache keys Aggregate expectation failures with bounded examples and cap serialized diagnostic text across the report. Use exact string keys instead of adversarial numeric hashes. Make the pre-end sampling count explicit while preserving sampled starts. Reviews: Astra resource addendum; https://github.com/SamGu-NRX/BaseScanning/pull/242#discussion_r4276295872 https://github.com/SamGu-NRX/BaseScanning/pull/242#discussion_r4276295884 --- verification/hsverify/diagnostics.py | 85 +++++++++ verification/hsverify/e2e.py | 6 + verification/hsverify/intervalindex.py | 34 +++- verification/hsverify/resultcheck.py | 180 +++++++++++++----- verification/tests/test_diagnostic_limits.py | 135 +++++++++++++ verification/tests/test_numeric_cache_keys.py | 72 +++++++ verification/tests/test_resultcheck.py | 3 +- .../tests/test_sweep_sampling_contract.py | 47 +++++ 8 files changed, 502 insertions(+), 60 deletions(-) create mode 100644 verification/hsverify/diagnostics.py create mode 100644 verification/tests/test_diagnostic_limits.py create mode 100644 verification/tests/test_numeric_cache_keys.py create mode 100644 verification/tests/test_sweep_sampling_contract.py diff --git a/verification/hsverify/diagnostics.py b/verification/hsverify/diagnostics.py new file mode 100644 index 000000000..ebdbec387 --- /dev/null +++ b/verification/hsverify/diagnostics.py @@ -0,0 +1,85 @@ +"""Bound diagnostic output independently of geometry work and input size.""" + +import json + +# Operational output limits, not measured scan maxima: 1,000 messages and 256 KiB +# leave a readable report without multiplying input-sized strings across cases. +# Reserve one message and 128 characters for the omitted-message count. +MAX_DIAGNOSTIC_COUNT = 1_000 +MAX_DIAGNOSTIC_CHARACTERS = 256 * 1024 +SUMMARY_RESERVE = 128 +MAX_EXPECTATION_EXAMPLES = 3 + + +class DiagnosticBudget: + def __init__(self) -> None: + self.count = 0 + self.characters = 0 + self.omitted = 0 + + def accept(self, message: str) -> bool: + prefix = "further problems omitted (" + if message.startswith(prefix) and message.endswith(")"): + count = message[len(prefix) : -1] + if len(count) <= 20 and count.isdecimal(): + self.omitted += int(count) + return False + if ( + self.omitted + or self.count >= MAX_DIAGNOSTIC_COUNT - 1 + or self.characters + len(message) > MAX_DIAGNOSTIC_CHARACTERS - SUMMARY_RESERVE + ): + self.omitted += 1 + return False + # JSON escapes control and non-ASCII characters; budget the saved text, + # not only the shorter Python string. The raw-length check above bounds + # this temporary encoding too. + characters = len(json.dumps(message)) + if self.characters + characters > MAX_DIAGNOSTIC_CHARACTERS - SUMMARY_RESERVE: + self.omitted += 1 + return False + self.count += 1 + self.characters += characters + return True + + def summary(self) -> list[str]: + return [f"further problems omitted ({self.omitted})"] if self.omitted else [] + + +class DiagnosticMessages: + def __init__(self) -> None: + self.budget = DiagnosticBudget() + self.messages: list[str] = [] + + def append(self, message: str, *values: object) -> None: + if self.budget.omitted: + self.budget.omitted += 1 + return + if values: + message = message.format(*values) + if self.budget.accept(message): + self.messages.append(message) + + def finish(self) -> list[str]: + return self.messages + self.budget.summary() + + +def bounded_report_records(records: list[dict]) -> tuple[list[dict], list[str]]: + """Use one budget across cases and duplicate diagnostic fields in the JSON report.""" + budget = DiagnosticBudget() + bounded = [] + for record in records: + copy = dict(record) + for field in ( + "contract_problems", + "expectation_problems", + "problems", + "assumption_mismatches", + "assumption_warnings", + ): + if field in copy: + copy[field] = [message for message in copy[field] if budget.accept(message)] + if copy.get("skip_reason") and not budget.accept(copy["skip_reason"]): + copy["skip_reason"] = "" + bounded.append(copy) + return bounded, budget.summary() diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index d9bf7e920..f5ebfa8b1 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -57,6 +57,7 @@ from hsverify import gitref, memory from hsverify.bundle_limits import zip_directory_limits +from hsverify.diagnostics import bounded_report_records from hsverify.memory import peak_rss_mb from hsverify.resultcheck import ( RuleSet, @@ -962,6 +963,10 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: "app_export_source": [r["source"] for r in exported], "records": records, } + records, omitted = bounded_report_records(records) + report["records"] = records + if omitted: + report["diagnostic_summary"] = omitted (out / "report.json").write_text(json.dumps(report, indent=2) + "\n") lines = [ f"# End-to-end run against `{meta['server_ref']}` at `{meta['server_sha'][:12]}`", @@ -992,6 +997,7 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: f"| {r['name']}{' (real)' if r.get('real') else ''} | {r['status']} | " f"{r.get('decision', '')} | {r.get('latency_ms', '')} | {issues.replace('|', '/')} |" ) + lines.extend(omitted) (out / "report.md").write_text("\n".join(lines) + "\n") return report diff --git a/verification/hsverify/intervalindex.py b/verification/hsverify/intervalindex.py index 0a2e596c9..ced4e9ddd 100644 --- a/verification/hsverify/intervalindex.py +++ b/verification/hsverify/intervalindex.py @@ -3,10 +3,24 @@ from __future__ import annotations import heapq +import itertools +import math from dataclasses import dataclass from typing import Protocol +def numeric_key(value: int | float) -> str: + """Hash numbers as exact strings so adversarial JSON integers cannot collide cheaply. + + Python hashes integers modulo a public prime. Rational form also keeps equal integer + and float inputs in one cache entry without rounding distinct large integers together. + """ + if isinstance(value, float) and not math.isfinite(value): + return repr(value) + numerator, denominator = value.as_integer_ratio() + return f"{numerator}/{denominator}" + + class Meter(Protocol): def charge(self, n: int = 1) -> None: ... @@ -61,8 +75,8 @@ def _build(self, runs: list[dict], eps: float) -> _WallIndex: _charge(self.meter) a, b = run["start_ft"] events.extend(((a - eps, 0, rank), (b + eps, 1, rank))) - starts.setdefault(a, run["outcome"]) - ends.setdefault(b, run["outcome"]) + starts.setdefault(numeric_key(a), (a, run["outcome"])) + ends.setdefault(numeric_key(b), (b, run["outcome"])) events.sort() active = [False] * len(runs) heap: list[int] = [] @@ -93,15 +107,15 @@ def first() -> str | None: _charge(self.meter) active[rank] = False after_endpoint.append(first()) - start_values, end_values = sorted(starts), sorted(ends) + start_entries, end_entries = sorted(starts.values()), sorted(ends.values()) return _WallIndex( endpoints, at_endpoint, after_endpoint, - start_values, - [starts[a] for a in start_values], - end_values, - [ends[b] for b in end_values], + [value for value, _ in start_entries], + [outcome for _, outcome in start_entries], + [value for value, _ in end_entries], + [outcome for _, outcome in end_entries], ) def outcome_at(self, wall_id: str, start_ft: float) -> str | None: @@ -129,16 +143,16 @@ def outcomes_near(self, wall_id: str, start_ft: float) -> tuple[set[str], str]: def probe_starts(*results: dict, meter: Meter | None = None) -> list[tuple[str, float]]: - points = set() + points = [] for result in results: for run in result.get("sweep", []): _charge(meter, 3) a, b = run["start_ft"] # Halving first avoids overflowing the sum of two admitted finite endpoints. - points.update( + points.extend( ((run["wall_id"], a), (run["wall_id"], b), (run["wall_id"], a / 2 + b / 2)) ) - return sorted(points) + return [point for point, _ in itertools.groupby(sorted(points))] def more_error_problems( diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 1c030d6a0..4798990c0 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -37,8 +37,10 @@ import yaml from jsonschema import Draft202012Validator +from .diagnostics import MAX_EXPECTATION_EXAMPLES, DiagnosticMessages from .intervalindex import ( SweepIndex, + numeric_key, ) from .intervalindex import ( less_coverage_problems as indexed_less_coverage_problems, @@ -57,6 +59,7 @@ # vertices per wall (C1's existing limits), 30,000 sweep runs, 16,000 checks and # 16,000 requests, 1,000,000 numeric leaves, 2,000,000 JSON nodes, depth 32, and # 4 MiB of string/key characters per document, and 64 case rule queries. +# Diagnostic output has a separate shared report budget in diagnostics.py. # Case queries may scan the bounded response; their count is fixed. Other metadata # passes are linear or # O(n log n); geometry comparisons and samples charge the meter. These operational @@ -147,6 +150,8 @@ def _input_key(value: Any, meter: WorkMeter | None = None) -> Any: return _input_key({k: v for k, v in vars(value).items() if k != "rules_sha256"}, meter) if isinstance(value, Need): return _input_key(vars(value), meter) + if isinstance(value, (int, float)) and not isinstance(value, bool): + return numeric_key(value).encode("ascii") return value @@ -413,7 +418,14 @@ def observed( this category default at 6200098 (server/scene.py `observed_intervals`). None keeps the old reading, all the way up.""" meter = meter if meter is not None else WorkMeter() - key = ("observed", meter.input_key(scene), band, min_out_ft, beyond, wall_default_ft) + key = ( + "observed", + meter.input_key(scene), + band, + numeric_key(min_out_ft), + beyond, + None if wall_default_ft is None else numeric_key(wall_default_ft), + ) if key in meter.cache: return meter.cache[key] spans = [] @@ -855,7 +867,13 @@ def slide( other wall between the meter and the battery's near edge, at its far end, times the same difference for that wall. 0 on the meter's own wall.""" meter = meter if meter is not None else WorkMeter() - key = ("slide", meter.input_key(scene), meter.input_key(rules), lo, hi) + key = ( + "slide", + meter.input_key(scene), + meter.input_key(rules), + numeric_key(lo), + numeric_key(hi), + ) if key in meter.cache: return meter.cache[key] segments = segments_s(scene, rules, meter=meter) @@ -929,7 +947,15 @@ def reach_gaps( the check's distance unless measured. """ meter = meter if meter is not None else WorkMeter() - key = ("reach_gaps", meter.input_key(scene), meter.input_key(rules), check, wall_id, lo, hi) + key = ( + "reach_gaps", + meter.input_key(scene), + meter.input_key(rules), + check, + wall_id, + numeric_key(lo), + numeric_key(hi), + ) if key in meter.cache: return meter.cache[key] gaps = [] @@ -993,8 +1019,11 @@ def measured_band_gap( if m.get("wall_id", wall_id) == wall_id ], ) - edges = {x for e in entries for x in e[:2]} | {x for m in measured for x in m} - cuts = sorted({lo, hi} | {x for x in edges if lo < x < hi}) + edges = itertools.chain((x for e in entries for x in e[:2]), (x for m in measured for x in m)) + cuts = [ + value + for value, _ in itertools.groupby(sorted([lo, hi, *(x for x in edges if lo < x < hi)])) + ] for p, q in itertools.pairwise(cuts): meter.charge() if q - p <= EPS: @@ -1019,7 +1048,7 @@ def measured_band_gap( def run_start_count(span: list[float], step_ft: float | None) -> int: - """Validate eagerly, then sample a run lazily: first, every step after it, and last. + """Count pre-end starts; run_starts appends the final endpoint separately. A sample, not the server's own list: the server also evaluates boundaries and midpoints that a run reports only as its two ends. Oversized runs fail rather than lose coverage @@ -1036,13 +1065,13 @@ def run_start_count(span: list[float], step_ft: float | None) -> int: count = max(int(a < b), math.ceil(steps - EPS)) if count + 1 > MAX_SWEEP_SAMPLES: raise ValueError(f"sweep sampling exceeds the {MAX_SWEEP_SAMPLES} sample limit") - return count + 1 + return count def run_starts(span: list[float], step_ft: float | None) -> Iterator[float]: count = run_start_count(span, step_ft) a, b = span - return itertools.chain((a + k * step_ft for k in range(count - 1)), (b,)) + return itertools.chain((a + k * step_ft for k in range(count)), (b,)) def deduplicate_result_runs(result: dict) -> dict: @@ -1323,7 +1352,12 @@ def ground_gap( meter = meter if meter is not None else WorkMeter() entries = band_entries(scene, "ground", meter) a, b = lo - radius, hi + radius - cuts = sorted({a, lo, hi, b} | {x for e in entries for x in e[:2] if a < x < b}) + cuts = [ + value + for value, _ in itertools.groupby( + sorted([a, lo, hi, b, *(x for e in entries for x in e[:2] if a < x < b)]) + ) + ] for p, q in itertools.pairwise(cuts): meter.charge() if q - p <= EPS: @@ -1897,17 +1931,17 @@ def public_assumption_warnings( def expectation_problems(expect: dict, result: dict, meter: WorkMeter | None = None) -> list[str]: if admission := case_metadata_problems({}, expect): return admission - problems: list[str] = [] + problems = DiagnosticMessages() decision = result["decision"] if "decision_in" in expect and decision not in expect["decision_in"]: - problems.append(f"decision {decision}, expected one of {expect['decision_in']}") + problems.append("decision {}, expected one of {}", decision, expect["decision_in"]) if decision in expect.get("decision_not", []): - problems.append(f"decision {decision} is ruled out for this case") + problems.append("decision {} is ruled out for this case", decision) if "spot" in expect: want, spot = expect["spot"], result.get("spot") if want is None and spot is not None: - problems.append(f"expected no spot, got {spot['wall_id']} {spot['span_ft']}") + problems.append("expected no spot, got {} {}", spot["wall_id"], spot["span_ft"]) if want is not None: if spot is None: problems.append("expected a spot, got none") @@ -1916,8 +1950,11 @@ def expectation_problems(expect: dict, result: dict, meter: WorkMeter | None = N s0, s1 = spot["span_ft"] if spot["wall_id"] != want["wall_id"] or s0 < a - 1 / 12 or s1 > b + 1 / 12: problems.append( - f"spot {spot['wall_id']} {spot['span_ft']} outside expected " - f"{want['wall_id']} {want['span_within']}" + "spot {} {} outside expected {} {}", + spot["wall_id"], + spot["span_ft"], + want["wall_id"], + want["span_within"], ) for rule in expect.get("sweep_runs", []): @@ -1928,43 +1965,84 @@ def expectation_problems(expect: dict, result: dict, meter: WorkMeter | None = N if r["wall_id"] == rule["wall_id"] and r["start_ft"][0] < b and r["start_ft"][1] > a ] if not runs and "outcome" in rule: - problems.append(f"no sweep run overlaps {rule['wall_id']} starts ({a}, {b})") + problems.append("no sweep run overlaps {} starts ({}, {})", rule["wall_id"], a, b) + failed = 0 + examples = [] for run in runs: - if "outcome" in rule and run["outcome"] != rule["outcome"]: - problems.append( - f"sweep {run['wall_id']} starts {run['start_ft']} is {run['outcome']}, " - f"expected {rule['outcome']}" + wrong = ( + ("outcome" in rule and run["outcome"] != rule["outcome"]) + or ("outcome_not" in rule and run["outcome"] == rule["outcome_not"]) + or any( + key in rule and not any(rule[key] in check for check in run[field]) + for field, key in (("failing", "failing_match"), ("unsure", "unsure_match")) ) - if "outcome_not" in rule and run["outcome"] == rule["outcome_not"]: - why = f" ({rule['reason']})" if "reason" in rule else "" - problems.append( - f"sweep {run['wall_id']} starts {run['start_ft']} is {run['outcome']}, " - f"which this case rules out{why}" - ) - for field, key in (("failing", "failing_match"), ("unsure", "unsure_match")): - if key in rule and not any(rule[key] in c for c in run[field]): - problems.append( - f"sweep {run['wall_id']} starts {run['start_ft']}: no {field} check " - f"matching {rule[key]!r} (has {run[field]})" + ) + if wrong: + failed += 1 + if len(examples) < MAX_EXPECTATION_EXAMPLES: + examples.append( + f"{run['wall_id']} starts {run['start_ft']} is {run['outcome']}" ) + if failed: + required = { + key: rule[key] + for key in ("outcome", "outcome_not", "failing_match", "unsure_match") + if key in rule + } + why = f"; reason: {rule['reason']}" if "reason" in rule else "" + problems.append( + "sweep expectation {} starts ({}, {}) failed for {} runs; " + "expected {}; examples: {}{}", + rule["wall_id"], + a, + b, + failed, + required, + "; ".join(examples), + why, + ) for rule in expect.get("checks", []): matching = [c for c in result.get("checks", []) if rule["match"] in c["id"]] if not matching: - problems.append(f"no check matching {rule['match']!r}") + problems.append("no check matching {!r}", rule["match"]) + failed = 0 + examples = [] for check in matching: - for field in ("outcome", "unsure_cause"): - if field in rule and check.get(field) != rule[field]: - problems.append( - f"check {check['id']}: {field} {check.get(field)}, expected {rule[field]}" + wrong = [ + field + for field in ("outcome", "unsure_cause") + if field in rule and check.get(field) != rule[field] + ] + # The public solver rounds measurements to six decimals. + wrong += [ + field + for field in ("measured_ft", "plus_minus_ft") + if field in rule + and (check.get(field) is None or abs(check[field] - rule[field]) > NUMBER_TOL_FT) + ] + if wrong: + failed += 1 + if len(examples) < MAX_EXPECTATION_EXAMPLES: + examples.append( + "; ".join( + f"check {check['id']}: {field} {check.get(field)}, " + f"expected {rule[field]}" + for field in wrong + ) ) - # Pinned numbers catch a server that measures from somewhere else (for example - # another battery depth) and lands on the right outcome by accident. The server - # rounds to 6 decimals. - for field in ("measured_ft", "plus_minus_ft"): - got = check.get(field) - if field in rule and (got is None or abs(got - rule[field]) > NUMBER_TOL_FT): - problems.append(f"check {check['id']}: {field} {got}, expected {rule[field]}") + if failed: + why = f"; reason: {rule['reason']}" if "reason" in rule else "" + if failed == 1: + problems.append("{}{}", examples[0], why) + else: + problems.append( + "check expectation {!r} failed for {} checks; examples: {}{}", + rule["match"], + failed, + "; ".join(examples), + why, + ) starts = expect.get("start_outcomes", []) index = SweepIndex(result, meter=meter) if starts else None @@ -1973,19 +2051,23 @@ def expectation_problems(expect: dict, result: dict, meter: WorkMeter | None = N if got != rule["outcome"]: why = f" ({rule['why']})" if "why" in rule else "" problems.append( - f"start {rule['wall_id']} {rule['start_ft']} is {got or 'not evaluated'}, " - f"expected {rule['outcome']}{why}" + "start {} {} is {}, expected {}{}", + rule["wall_id"], + rule["start_ft"], + got or "not evaluated", + rule["outcome"], + why, ) if "missing_evidence_empty" in expect: empty = not result.get("missing_evidence") if empty != expect["missing_evidence_empty"]: problems.append( - "missing_evidence " - + ("is empty" if empty else "is not empty") - + f", expected {'empty' if expect['missing_evidence_empty'] else 'requests'}" + "missing_evidence {}, expected {}", + "is empty" if empty else "is not empty", + "empty" if expect["missing_evidence_empty"] else "requests", ) - return problems + return problems.finish() # --- Transforms for black-box properties ----------------------------------------------------- diff --git a/verification/tests/test_diagnostic_limits.py b/verification/tests/test_diagnostic_limits.py new file mode 100644 index 000000000..7e3f16592 --- /dev/null +++ b/verification/tests/test_diagnostic_limits.py @@ -0,0 +1,135 @@ +import json + +import pytest +from test_judge_meter import META + +from hsverify import e2e +from hsverify.diagnostics import ( + MAX_DIAGNOSTIC_CHARACTERS, + MAX_DIAGNOSTIC_COUNT, + MAX_EXPECTATION_EXAMPLES, + DiagnosticMessages, +) +from hsverify.resultcheck import expectation_problems + + +def rejected_runs(count): + return { + "decision": "reject", + "sweep": [ + { + "wall_id": "w1", + "start_ft": [i, i + 1], + "outcome": "fail", + "failing": [], + "unsure": [], + } + for i in range(count) + ], + } + + +def test_repeated_sweep_failures_report_reason_once(tmp_path): + reason = "r" * 4_096 + expected = { + "sweep_runs": [ + {"wall_id": "w1", "start_ft": [0, 1001], "outcome_not": "fail", "reason": reason} + ] + } + problems = expectation_problems(expected, rejected_runs(1_000)) + assert len(problems) == 1 + assert "failed for 1000 runs" in problems[0] + assert problems[0].count(reason) == 1 + assert problems[0].count(" is fail") == MAX_EXPECTATION_EXAMPLES + records = [ + { + "name": "many runs", + "status": "fail", + "expectation_problems": problems, + "problems": problems, + } + ] + report = e2e.write_report(tmp_path, META, records) + assert len((tmp_path / "report.json").read_text()) < MAX_DIAGNOSTIC_CHARACTERS + assert len((tmp_path / "report.md").read_text()) < MAX_DIAGNOSTIC_CHARACTERS + assert not report["all_passed"] + + +@pytest.mark.parametrize( + "message", + ["bad", "x" * 4_096, "\0" * 4_000, "🦉" * 4_000], + ids=["count", "ascii", "controls", "unicode"], +) +def test_one_report_budget_covers_every_case_and_duplicate_field(tmp_path, message): + records = [ + { + "name": str(i), + "status": "fail", + "decision": "reject", + "problems": [message], + "contract_problems": [message], + } + for i in range(MAX_DIAGNOSTIC_COUNT) + ] + report = e2e.write_report(tmp_path, META, records) + messages = [ + m for r in report["records"] for key in ("problems", "contract_problems") for m in r[key] + ] + messages += report["diagnostic_summary"] + assert len(messages) <= MAX_DIAGNOSTIC_COUNT + assert sum(len(json.dumps(message)) for message in messages) <= MAX_DIAGNOSTIC_CHARACTERS + kept = len(messages) - 1 + assert report["diagnostic_summary"] == [f"further problems omitted ({2 * len(records) - kept})"] + assert (tmp_path / "report.md").read_text().count("further problems omitted (") == 1 + assert (tmp_path / "report.json").read_text().count("further problems omitted (") == 1 + assert report["contract_problem_count"] == len(records) + assert not report["contract_ok"] + assert all(r["problems"] == [message] for r in records) + + +def test_oversized_reason_is_omitted_before_accumulating_output(): + expected = { + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [0, 1001], + "outcome_not": "fail", + "reason": "r" * (3 * 1024 * 1024), + } + ] + } + assert expectation_problems(expected, rejected_runs(1_000)) == ["further problems omitted (1)"] + + +def test_repeated_check_failures_are_aggregated_per_expectation(): + result = { + "decision": "reject", + "checks": [{"id": f"check-{i}", "outcome": "fail"} for i in range(2_000)], + } + problems = expectation_problems({"checks": [{"match": "check", "outcome": "pass"}]}, result) + assert len(problems) <= MAX_DIAGNOSTIC_COUNT + assert sum(map(len, problems)) <= MAX_DIAGNOSTIC_CHARACTERS + assert len(problems) == 1 + assert "failed for 2000 checks" in problems[0] + assert problems[0].count("expected pass") == MAX_EXPECTATION_EXAMPLES + + +def test_report_coalesces_preexisting_omission_summaries(tmp_path): + collector = DiagnosticMessages() + collector.append("x" * MAX_DIAGNOSTIC_CHARACTERS) + records = [{"name": str(i), "status": "fail", "problems": collector.finish()} for i in range(3)] + report = e2e.write_report(tmp_path, META, records) + assert report["diagnostic_summary"] == ["further problems omitted (3)"] + assert json.dumps(report).count("further problems omitted (") == 1 + + +def test_diagnostic_budget_stops_formatting_after_exhaustion(): + messages = DiagnosticMessages() + messages.append("x" * MAX_DIAGNOSTIC_CHARACTERS) + + class ForbiddenFormat: + def __format__(self, spec): + raise AssertionError("formatting continued after diagnostic budget exhaustion") + + messages.append("{}", ForbiddenFormat()) + assert messages.finish() == ["further problems omitted (2)"] diff --git a/verification/tests/test_numeric_cache_keys.py b/verification/tests/test_numeric_cache_keys.py new file mode 100644 index 000000000..d9cc49c0f --- /dev/null +++ b/verification/tests/test_numeric_cache_keys.py @@ -0,0 +1,72 @@ +import sys + +import pytest + +from hsverify.intervalindex import SweepIndex, numeric_key, probe_starts +from hsverify.resultcheck import WorkMeter, observed + + +class ComparedInt(int): + """Count equality work while retaining the hash of the JSON integer it represents.""" + + comparisons = 0 + __hash__ = int.__hash__ + + def __eq__(self, other): + type(self).comparisons += 1 + if type(self).comparisons > 100_000: + raise AssertionError("numeric keys exceeded 100,000 equality comparisons") + return super().__eq__(other) + + +def values(colliding, count=2_000): + step = sys.hash_info.modulus if colliding else 1 + return [ComparedInt(step * i) for i in range(1, count + 1)] + + +def equality_work(operation, colliding): + numbers = values(colliding) + ComparedInt.comparisons = 0 + operation(numbers) + return ComparedInt.comparisons + + +def test_numeric_keys_preserve_exact_numeric_equivalence(): + assert numeric_key(1) == numeric_key(1.0) + assert numeric_key(0) == numeric_key(-0.0) + assert numeric_key(2**60) == numeric_key(float(2**60)) + assert numeric_key(2**53 + 1) != numeric_key(float(2**53 + 1)) + assert numeric_key(0.1) != numeric_key(1) + + +@pytest.mark.parametrize("operation", ["observed", "input", "index", "probes"]) +def test_modulus_multiple_response_numbers_do_not_multiply_equality_work(operation): + def exercise(numbers): + meter = WorkMeter() + if operation == "observed": + scene = {"coverage": {"observed": []}} + for number in numbers: + assert observed(scene, "ground", number, meter=meter) == [] + assert len(meter.cache) == len(numbers) + elif operation == "input": + for number in numbers: + meter.input_key({"out_ft": number}) + assert len(meter.input_values) == len(numbers) + else: + result = { + "sweep": [ + {"wall_id": "w1", "start_ft": [number, number], "outcome": "fail"} + for number in numbers + ] + } + if operation == "index": + index = SweepIndex(result) + assert len(index.walls["w1"].starts) == len(numbers) + else: + assert len(probe_starts(result)) >= len(numbers) + + ordinary = equality_work(exercise, False) + colliding = equality_work(exercise, True) + # Count comparisons rather than wall time: sorting/deduplication may do a small + # multiple of the ordinary workload, but a numeric hash table does millions. + assert colliding <= 4 * ordinary + 20 * 2_000 diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 962b53541..1467f8298 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -508,7 +508,8 @@ def test_outcome_not_rules_out_one_outcome_and_tolerates_no_runs(): ] } assert expectation_problems(bad, r) == [ - "sweep w1 starts [2.1, 3.0] is pass, which this case rules out (window)" + "sweep expectation w1 starts (0.5, 2.5) failed for 1 runs; " + "expected {'outcome_not': 'pass'}; examples: w1 starts [2.1, 3.0] is pass; reason: window" ] diff --git a/verification/tests/test_sweep_sampling_contract.py b/verification/tests/test_sweep_sampling_contract.py new file mode 100644 index 000000000..852eed588 --- /dev/null +++ b/verification/tests/test_sweep_sampling_contract.py @@ -0,0 +1,47 @@ +import copy +from dataclasses import replace + +import pytest +from test_resultcheck import RULES, SCENE, pass_run, result + +from hsverify import resultcheck as checks + + +@pytest.mark.parametrize( + "span,expected", [([0, 2.4], [0, 1, 2, 2.4]), ([0, 0.4], [0, 0.4]), ([2, 2], [2])] +) +def test_pre_end_count_plus_endpoint_matches_every_yield(span, expected): + samples = list(checks.run_starts(span, 1)) + assert samples == expected + assert len(samples) == checks.run_start_count(span, 1) + 1 + + +@pytest.mark.parametrize("span,failing_start", [([0, 2.4], 2), ([0, 0.4], 0)]) +def test_coverage_rejects_a_failure_at_the_required_pre_end_start(monkeypatch, span, failing_start): + rules = replace(RULES, needs={"ground_surface": ()}, step_ft=1) + scene = copy.deepcopy(SCENE) + seen = [] + + def gap(scene, rules, check, wall_id, lo, hi, meter=None): + seen.append(lo) + return [(None, "unobserved at required pre-end start")] if lo == failing_start else [] + + monkeypatch.setattr(checks, "reach_gaps", gap) + answer = result(spot=False, sweep=[pass_run(*span)]) + problems = checks.coverage_problems(scene, answer, rules) + assert seen == list(checks.run_starts(span, 1)) + assert any("unobserved at required pre-end start" in problem for problem in problems) + + +def test_coverage_charges_once_for_each_yielded_start(monkeypatch): + rules = replace(RULES, needs={"ground_surface": ()}, step_ft=1) + sampled = [] + + def gap(scene, rules, check, wall_id, lo, hi, meter=None): + sampled.append((lo, meter.samples)) + return [] + + monkeypatch.setattr(checks, "reach_gaps", gap) + checks.coverage_problems(SCENE, result(spot=False, sweep=[pass_run(0, 2.4)]), rules) + assert [start for start, _ in sampled] == [0, 1, 2, 2.4] + assert [count - sampled[0][1] for _, count in sampled] == [0, 1, 2, 3] From 1ed45b65b5ba8599c5204f919aee336f81375b13 Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:27:27 -0500 Subject: [PATCH 61/62] fix(recon): reject free-space rays blocked by fused geometry Use the regular visibility path stop-short tolerance before crediting a camera ray. Preserve support for the target surface itself and require two unobstructed views. Review: https://github.com/SamGu-NRX/BaseScanning/pull/242#discussion_r4276295897 --- recon/recon/coverage.py | 10 +++++-- recon/tests/test_coverage_views.py | 44 +++++++++++++++++++++++++++--- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/recon/recon/coverage.py b/recon/recon/coverage.py index a9b35533f..ffff18b72 100644 --- a/recon/recon/coverage.py +++ b/recon/recon/coverage.py @@ -164,8 +164,14 @@ def seen_through(self, points: np.ndarray) -> np.ndarray: du = np.clip((u[n_i] * scale).astype(np.int64), 0, dw - 1) dv = np.clip((v[n_i] * scale).astype(np.int64), 0, dh - 1) measured = depth_map[dv, du] - reached = np.isfinite(measured) & (measured >= z[n_i] - tolerance(dist[n_i])) - out[n_i[reached], j] = True + tol = tolerance(dist[n_i]) + reached = np.isfinite(measured) & (measured >= z[n_i] - tol) + keep = n_i[reached] + if len(keep): + # A depth map may miss an obstruction retained by fusion. Ignore only the + # sample's own surface, using the same stop-short distance as seen_by. + blocked = self.vol.blocked(self.C[j], points[keep], tol[reached] + self.voxel) + out[keep[~blocked], j] = True return out diff --git a/recon/tests/test_coverage_views.py b/recon/tests/test_coverage_views.py index 306ece85e..a1a764321 100644 --- a/recon/tests/test_coverage_views.py +++ b/recon/tests/test_coverage_views.py @@ -8,17 +8,19 @@ from recon import coverage from recon.capture import Frame from recon.depth import Depth +from recon.fusion import Volume from recon.geometry import WallFrame class ClearVolume: voxel = 0.05 + blocked = Volume.blocked def observed(self, points): - return np.ones(len(points), bool) + return np.ones(points.shape[:-1], bool) def occupied(self, points): - return np.zeros(len(points), bool) + return np.zeros(points.shape[:-1], bool) def views(count): @@ -127,8 +129,8 @@ def __init__(self, band): def occupied(self, points): if self.band == "facing": # A thin hit in the first outward slab, too small to become the wall front. - return np.isclose(points[:, 2], 0.1) & np.isclose(points[:, 1], 1.0) - return np.isclose(points[:, 1], 0.3) & np.isclose(points[:, 2], 0.3) + return np.isclose(points[..., 2], 0.1) & np.isclose(points[..., 1], 1.0) + return np.isclose(points[..., 1], 0.3) & np.isclose(points[..., 2], 0.3) def free_space_from_positions(raw, xs): @@ -186,3 +188,37 @@ def test_empty_volume_needs_two_views_before_exporting_clear_space(xs, supported else: assert cov.facing_clear[0] == cov.overhead_clear[0] == 0.0 assert entries == [] + + +@pytest.mark.parametrize( + ("obstruction_z", "visible"), + [(2.0, False), (None, True), (0.0, True)], + ids=["fused_obstruction", "clear_ray", "supported_surface"], +) +def test_free_space_rays_respect_fused_obstructions_but_not_the_sample_surface( + obstruction_z, visible +): + frames, depths = views(2) + for frame, x in zip(frames, [-0.2, 0.2], strict=True): + frame.cam_to_world[:3, :3] = np.eye(3) + frame.cam_to_world[:3, 3] = [x, 1.0, 4.0] + depths[frame.id].depth[:] = 4.0 + shape = (21, 31, 51) + tsdf = np.ones(shape, np.float32) + if obstruction_z is not None: + tsdf[:, :, round(obstruction_z / 0.1)] = -1.0 + raw = Volume( + origin=np.array([-1.0, 0.0, 0.0]), + voxel=0.1, + shape=shape, + tsdf=tsdf, + weight=np.ones(shape, np.float32), + color=np.zeros((*shape, 3), np.float32), + color_n=np.zeros(shape, np.float32), + ) + view = coverage.SeenFromTwo(raw, frames, depths) + points = np.array([[0.0, 1.0, 0.0]]) + # Both depth maps put the sample on a surface. Fusion independently retains a + # nearer obstacle in the failing case; a hit on the sample itself remains supported. + np.testing.assert_array_equal(view.seen_through(points), [[visible, visible]]) + np.testing.assert_array_equal(view.observed(points), [visible]) From 1428d04732e5bef97629eecc2d7620abe0e6a342 Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:37:48 -0500 Subject: [PATCH 62/62] fix(verification): preserve Markdown explanations within output budget Prioritize the Markdown-visible problems array before duplicate JSON classifications. Regressions retain the first explanation for 1,000 contract or expectation failures without exceeding the shared diagnostic budget. Review: https://github.com/SamGu-NRX/BaseScanning/pull/240#discussion_r4276493597 --- verification/hsverify/diagnostics.py | 3 +- verification/tests/test_diagnostic_limits.py | 29 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/verification/hsverify/diagnostics.py b/verification/hsverify/diagnostics.py index ebdbec387..3c81ed8ee 100644 --- a/verification/hsverify/diagnostics.py +++ b/verification/hsverify/diagnostics.py @@ -70,10 +70,11 @@ def bounded_report_records(records: list[dict]) -> tuple[list[dict], list[str]]: bounded = [] for record in records: copy = dict(record) + # Markdown renders problems; retain its explanations before their JSON duplicates. for field in ( + "problems", "contract_problems", "expectation_problems", - "problems", "assumption_mismatches", "assumption_warnings", ): diff --git a/verification/tests/test_diagnostic_limits.py b/verification/tests/test_diagnostic_limits.py index 7e3f16592..0e0ca7e1f 100644 --- a/verification/tests/test_diagnostic_limits.py +++ b/verification/tests/test_diagnostic_limits.py @@ -133,3 +133,32 @@ def __format__(self, spec): messages.append("{}", ForbiddenFormat()) assert messages.finish() == ["further problems omitted (2)"] + + +@pytest.mark.parametrize("field", ["contract_problems", "expectation_problems"]) +def test_markdown_keeps_first_explanation_when_duplicate_fields_fill_budget(tmp_path, field): + problems = [f"explanation {i}" for i in range(1_000)] + record = { + "name": "many failures", + "status": "fail", + "decision": "reject", + "problems": problems, + field: problems, + } + report = e2e.write_report(tmp_path, META, [record]) + row = next( + line + for line in (tmp_path / "report.md").read_text().splitlines() + if line.startswith("| many failures |") + ) + assert "explanation 0" in row + retained = [ + message + for item in report["records"] + for key in ("problems", field) + for message in item[key] + ] + retained += report["diagnostic_summary"] + assert len(retained) <= MAX_DIAGNOSTIC_COUNT + assert sum(len(json.dumps(message)) for message in retained) <= MAX_DIAGNOSTIC_CHARACTERS + assert not report["all_passed"]