diff --git a/Makefile b/Makefile index 52b807c3c..f5c2f605d 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,21 @@ # Suite targets follow their CI jobs where one exists. The viewer has no CI job yet. # `make check` runs server, web and ios, then scoring, measure-lab, evals, recon -# meter-closeup and the integration viewer where the branch has them. +# meter-closeup, verification and the integration viewer where the branch has them. # tests/makefile.sh checks this dispatch with stub suites. XCODEGEN ?= xcodegen # Must match XCODEGEN_VERSION in .github/workflows/ios.yml; other versions write a different project. XCODEGEN_VERSION := 2.46.0 -# The file each experiment suite's target needs. These suites arrive in separate pull requests, so +# The file each optional suite's target needs. These suites arrive in separate pull requests, so # `check` includes one only when its file exists and the Makefile works before and after each -# merge. Once all five are on main, list them in `check` directly so a deleted folder fails. +# merge. Once all seven are on main, list them in `check` directly so a deleted folder fails. SCORING := experiments/scoring/pyproject.toml MEASURE_LAB := experiments/measure-lab/Makefile EVALS := experiments/evals/Makefile RECON := recon/Makefile METER_CLOSEUP := experiments/meter-closeup/Makefile +VERIFICATION := verification/Makefile INTEGRATION_VIEWER := tools/integration-viewer/package.json # $(call present,FILE,TARGET) is TARGET when FILE exists, otherwise nothing. @@ -22,7 +23,7 @@ present = $(if $(wildcard $(1)),$(2)) # $(call require,FILE,PR): stop before any command runs when the suite's file is missing. require = @test -f $(1) || { echo "make $@: $(1) is missing; the suite arrives with pull request $(2)." >&2; exit 1; } -.PHONY: check server web ios ios-project smoke scoring measure-lab evals recon meter-closeup integration-viewer +.PHONY: check server web ios ios-project smoke scoring measure-lab evals recon meter-closeup verification integration-viewer check: server web ios \ $(call present,$(SCORING),scoring) \ @@ -30,6 +31,7 @@ check: server web ios \ $(call present,$(EVALS),evals) \ $(call present,$(RECON),recon) \ $(call present,$(METER_CLOSEUP),meter-closeup) \ + $(call present,$(VERIFICATION),verification) \ $(call present,$(INTEGRATION_VIEWER),integration-viewer) server: @@ -94,6 +96,12 @@ meter-closeup: cd experiments/meter-closeup && uv sync --locked $(MAKE) -C experiments/meter-closeup check +# Match verification CI's locked sync so a stale lockfile cannot pass the local check. +verification: + $(call require,$(VERIFICATION),#13) + cd verification && uv sync --locked + $(MAKE) -C verification test + integration-viewer: $(call require,$(INTEGRATION_VIEWER),#124) cd tools/integration-viewer && node --test "test/*.test.js" diff --git a/README.md b/README.md index 1e412e34a..d0038a46f 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,8 @@ The criteria (e.g. the distance the battery must be from the gas meter) exist in Below, one wall makes the whole trip, from photos to a checked spot. The models build the wall and name what's on it. The last step, the rule checks, is that deterministic evaluation. + + An animated drawing of one wall in six steps. Photos arrive, each with the phone's position. Points are matched between them. A ruler marks feet out from the meter. Dots fill in the wall and the ground, while the left side nobody filmed stays hatched and reads Not seen. The window, meter, AC unit and gravel get labels. Last, a battery appears 12 ft right of the meter with its cable, beside a card where wall behind it, ground under it, open space in front and a 13.6 ft cable run all pass. @@ -75,6 +77,8 @@ Here's where each piece runs. The phone uploads the capture to Google Cloud Stor ## Unseen? Then Unsure + + A wall with the left side hazed over. Three checks read Not seen. The haze sweeps away, the checks turn Unsure, then settle: wall and ground pass, clear space fails, and the spot reads Not here. diff --git a/docs/how-it-works.html b/docs/how-it-works.html index c4eec8c2c..4bb714da2 100644 --- a/docs/how-it-works.html +++ b/docs/how-it-works.html @@ -11,6 +11,10 @@ --sky:#e2edf8; --wall:#e8dcc4; --eave:#6b5a4a; --ground:#c9dcb0; --bush:#6f9a52; --tape:#f3cf4b; --tape-ink:#3d3106; --phone:#2a78d6; --server:#eb6834; --fog:#8d8d86; --battery:#2f3e4e; + /* Text in the phone and server colours. The fills above are 4.0:1 and 2.9:1 on --bg, too + faint for small text; these keep the hue with OKLCH lightness lowered until text clears + 4.5:1 on --bg (4.63 and 4.61) and on --card (5.06 and 5.03). */ + --phone-ink:#1e6ecb; --server-ink:#c44400; --meterbox:#6f7680; --meter:#a9d3f2; --gas:#b9b3a6; --glass:#bfe0f5; --zone:rgba(208,59,59,.14); --zone-line:#d03b3b; --pass:#107a2a; --fail:#b83232; --fail-ink:#a12b2b; --unsure:#fab219; @@ -24,6 +28,8 @@ --sky:#1c2734; --wall:#4a4033; --eave:#8a7562; --ground:#2f4128; --bush:#5d8a45; --tape:#7a6016; --tape-ink:#fff3c4; --phone:#3987e5; --server:#e8683a; --fog:#6e6e68; --battery:#cfd8e3; + /* The dark fills already read as text: 4.69:1 and 5.26:1 on --card. */ + --phone-ink:#3987e5; --server-ink:#e8683a; --meterbox:#8f97a3; --meter:#5c8fb8; --gas:#7d786d; --glass:#6f9fbf; --zone:rgba(230,103,103,.22); --zone-line:#e66767; --pass:#107a2a; --fail:#b83232; --fail-ink:#f28b8b; --unsure:#fab219; @@ -42,7 +48,7 @@ .prose{max-width:58ch} .meta{font-family:var(--sans);color:var(--muted);font-size:15px} .lede{font-size:20px;line-height:1.45;max-width:60ch} - a{color:var(--phone);text-underline-offset:3px} + a{color:var(--phone-ink);text-underline-offset:3px} a:focus-visible,summary:focus-visible{outline:2px solid var(--phone);outline-offset:3px;border-radius:2px} code{font-family:var(--mono);font-size:.86em;background:var(--code-bg);padding:1px 5px;border-radius:3px;overflow-wrap:anywhere} ul,ol{padding-left:1.3em} @@ -99,8 +105,8 @@ .lab.s{font-size:16px} .lab.b{font-weight:600} /* Colour utilities come after .lab and .t so they win at equal specificity. */ - .lab.blue,.t.blue{fill:var(--phone)} - .lab.orange,.t.orange{fill:var(--server)} + .lab.blue,.t.blue{fill:var(--phone-ink)} + .lab.orange,.t.orange{fill:var(--server-ink)} .lab.red,.t.red{fill:var(--fail-ink)} .lab.mu,.t.mu{fill:var(--muted)} .tp{font-family:var(--sans);font-size:15px;fill:var(--tape-ink);font-weight:600} @@ -715,43 +721,48 @@

Unseen means unsure

Coverage is a fixed zone

-

Not the whole house, just a zone around the meter. The app always knows exactly what's left.

+

Not the whole house. The zone is every spot the cable can reach, plus everything the checks on those spots look at. The rules fix it before the scan starts, so the app always knows exactly what's left.

The zone a scan must cover, seen from above - A plan view of the house with the electric meter on its front wall. A dashed blue rectangle marks the zone the checks need: the wall 20 feet either side of the meter and the ground in front of it out to 13 feet. Everything outside it is grey haze labelled not needed. Inside, a bush stands in front of the wall, and the strip of wall behind it is unseen, so checks there are UNSURE. A dotted path shows the phone walking along the zone. - + A plan view of the house with the electric meter on its front wall. An orange band along the wall marks the candidate battery spots, 20 feet of cable either side of the meter. A dashed blue outline marks the zone the checks need: that band grown by how far each check reaches, so it runs past the cable's reach at both ends and out over the ground in front, with rounded front corners. A bracket under it reads needed: all of it, past the cable's reach too. Everything outside is washed out and labelled no check looks here. Inside the zone, a bush stands in front of the wall, and the strip of wall behind it is unseen, so checks there are UNSURE. A dotted path shows the phone walking through the zone. + - - - house, seen from above - + + + house, seen from above + + + + + candidate spots: 20 ft of cable either side + + + + + - + - cable reach: 20 ft either side - - - - - not needed - - + needed: all of it, past the cable's reach too + no check looks here - meter + meter - - - wall behind the bush: - unseen, so UNSURE + + + wall behind the bush: + unseen, so UNSURE - - - needed: all of it - ground out to 13 ft + + -
The zone: wall within 20 ft of cable each side, ground out to 6 to 13 ft, and headroom above. It reaches a little past 20 ft for clearances at the edge.
+
Each check reaches out from the spot it tests: 3 ft along the wall for gas meters, AC units, doors and windows, 5 ft for a driveway and 10 ft for a pool (both placeholders, with no public value), and up for headroom and wall equipment. So the zone runs past the 20 ft of cable at both ends, and over the ground out to 13 ft. Any part of it nobody saw keeps its checks UNSURE.

LiDAR helps but isn't required

diff --git a/docs/readme/rebuild-dark-still.webp b/docs/readme/rebuild-dark-still.webp new file mode 100644 index 000000000..5e9a0a431 Binary files /dev/null and b/docs/readme/rebuild-dark-still.webp differ diff --git a/docs/readme/rebuild-light-still.webp b/docs/readme/rebuild-light-still.webp new file mode 100644 index 000000000..4b3479148 Binary files /dev/null and b/docs/readme/rebuild-light-still.webp differ diff --git a/docs/readme/unseen-dark-still.webp b/docs/readme/unseen-dark-still.webp new file mode 100644 index 000000000..8050be4e6 Binary files /dev/null and b/docs/readme/unseen-dark-still.webp differ diff --git a/docs/readme/unseen-light-still.webp b/docs/readme/unseen-light-still.webp new file mode 100644 index 000000000..be34045ee Binary files /dev/null and b/docs/readme/unseen-light-still.webp differ diff --git a/experiments/drift-anatomy/METHODS.md b/experiments/drift-anatomy/METHODS.md index 7fba32f7e..346076b6b 100644 --- a/experiments/drift-anatomy/METHODS.md +++ b/experiments/drift-anatomy/METHODS.md @@ -8,8 +8,8 @@ The server allows 0.16 ft of error per foot along the wall. That allowance domin - **A1, return to the meter.** When a walk comes back within 0.3 m of an earlier point more than 30 s later, ARKit's closure residual r is the error in the displacement between the two visits. Does r bound the largest error between the visits? - **A2, scale.** Is ARKit's scale one number per walk? This measures the spread of scale over sliding 5 m windows within a walk. -- **A3, the wall plane.** What share of the error runs across the direction of travel, which a wall plane would remove, compared with along it? -- **A4, a range to the meter.** A second phone left at the meter ranges the walking phone by UWB (σ 10 cm, simulated) at every window end. What is the p90 left over? +- **A3, along and across travel.** What share of the error runs across the direction of travel, compared with along it? A wall plane removes the across component only when travel is parallel to that wall; these walks do not annotate that orientation. +- **A4, a range to the meter.** A second phone left at the meter ranges the walking phone by UWB (σ 10 cm, simulated) at every window end. What is the p90 left over? Each window is one session that starts at the meter: the walking phone leaves the meter at the window's start, the second phone stays there, and the range runs from that start to the window's end. Sliding the start along a walk samples many meter placements; the error depends on the path walked from the meter, not on where in the dataset the meter sits. ## Pass criteria, fixed before the run @@ -20,13 +20,13 @@ The server allows 0.16 ft of error per foot along the wall. That allowance domin ## Result -All four fail. Tables: [results/drift_anatomy.md](results/drift_anatomy.md). +A1, A2 and A4 fail their numeric criteria. A3 fails the 60% criterion for a travel-axis proxy; this is not evidence that a wall-plane experiment failed. Wall-plane benefit remains untested without wall annotations. Tables: [results/drift_anatomy.md](results/drift_anatomy.md). - **A1.** 30 of 56 MARViN loops (54%). ADVIO has no revisit under 0.3 m while ARKit tracks. Scale and heading error cancel on return, so r misses them. - **A2.** 5.7% SD on MARViN, 19% on ADVIO: drop. A few outlier windows inflate MARViN's SD; its robust SD is 1.8%. -- **A3.** At 20 ft, along is 0.92 of the p90 on MARViN and 0.76 on ADVIO. +- **A3.** At 20 ft, along is 0.92 of the p90 on MARViN and 0.76 on ADVIO. The straight-path check selects windows by reference-path deviation from their start-to-end line: within 0.25 m, along is 0.98 of the p90 on MARViN and 0.75 on ADVIO; within 0.5 m, 0.95 and 0.75. These untuned, non-pre-registered filters select straightness only. With no wall annotations, they cannot identify wall-parallel travel or establish a wall-plane constraint's benefit. - **A4.** MARViN goes from 10.4 and 13.6 in to 7.7 and 8.5 in. At σ 10 cm the range's own p90 is 6.5 in, so the bar was nearly out of reach. At 5 cm: 4.9 and 6.1 in. **Run:** `uv run python run.py`. Needs the evals checkout at 190ed33 (`EVALS_HARNESS`) and MARViN and ADVIO in `~/house-scanning-data`. -**What it changes:** the error worth attacking runs along travel. A range to the meter reduces it, a wall plane wouldn't, and r can't bound a session. +**What it changes:** the error worth attacking runs along travel. A range to the meter reduces it, and r cannot bound a session. These unannotated walks do not establish whether a wall-plane constraint helps. diff --git a/experiments/drift-anatomy/README.md b/experiments/drift-anatomy/README.md index e745d756f..c5c68944f 100644 --- a/experiments/drift-anatomy/README.md +++ b/experiments/drift-anatomy/README.md @@ -2,7 +2,7 @@ ## Question -The server allows 0.16 ft of error per foot along the wall, which dominates a check's error bar beyond a few feet. Which part of ARKit's error is worth attacking? A1 asks whether returning to the meter bounds it, A2 whether scale is one number per walk, A3 how much runs across travel, where a wall plane would help, and A4 what a UWB range to the meter leaves. The data is MARViN (iPhone 14 Pro Max) and ADVIO (iPhone 6s); [METHODS.md](METHODS.md) has the details. +The server allows 0.16 ft of error per foot along the wall, which dominates a check's error bar beyond a few feet. Which part of ARKit's error is worth attacking? A1 asks whether returning to the meter bounds it, A2 whether scale is one number per walk, A3 how much runs across travel, and A4 what a UWB range to the meter leaves. Each window stands for a session that starts at the meter, so in A4 the second phone sits at the window's start. The data is MARViN (iPhone 14 Pro Max) and ADVIO (iPhone 6s); [METHODS.md](METHODS.md) has the details. ## Pass criteria, fixed before the run (397507f) @@ -13,13 +13,13 @@ The server allows 0.16 ft of error per foot along the wall, which dominates a ch ## Result -All four fail ([tables](results/drift_anatomy.md)). +A1, A2 and A4 fail their numeric criteria. A3 fails the 60% criterion for a travel-axis proxy; the unannotated walks cannot test wall-plane benefit ([tables](results/drift_anatomy.md)). - **A1.** The closure residual bounds the peak error in 30 of 56 loops. Scale and heading error cancel on return. - **A2.** Scale spread within a walk is 1.8% robust SD (5.7% SD) on MARViN. -- **A3.** At 20 ft, the along-travel p90 is 9.6 of 10.4 in. +- **A3.** At 20 ft, the along-travel p90 is 9.6 of 10.4 in. In the straight-path check, along is 0.95 to 0.98 of the total on MARViN (added after review). The walks have no wall annotations, so this does not establish wall-parallel travel or the benefit of a wall-plane constraint. - **A4.** UWB cuts 20 ft from 10.4 to 7.7 in at σ 10 cm, and to 4.9 in at σ 5 cm. **Run:** `uv run python run.py`: 20 s. Needs the evals checkout at 190ed33 and MARViN and ADVIO in `~/house-scanning-data`. -**What it changes:** the error worth attacking runs along travel. Measuring a span inside one photo avoids it. A wall plane, a return to the meter or one ruler at the meter don't. +**What it changes:** the error worth attacking runs along travel. Measuring a span inside one photo avoids it. The return-to-meter and one-ruler criteria fail. These unannotated walks do not establish whether a wall-plane constraint helps. diff --git a/experiments/drift-anatomy/metrics.py b/experiments/drift-anatomy/metrics.py index 1107729a4..af2d72e8d 100644 --- a/experiments/drift-anatomy/metrics.py +++ b/experiments/drift-anatomy/metrics.py @@ -167,6 +167,25 @@ def along_across(err: np.ndarray, ref_disp: np.ndarray) -> tuple[np.ndarray, np. return along, across +def chord_deviation(ref: np.ndarray, i: np.ndarray, j: np.ndarray) -> np.ndarray: + """Largest horizontal distance from each window's start-to-end line. + + This measures path straightness; it carries no information about wall orientation. + """ + out = np.empty(len(i)) + for k, (a, b) in enumerate(zip(i, j, strict=True)): + start = ref[a, [0, 2]] + end = at(ref, np.array([b]))[0, [0, 2]] + pts = np.vstack([ref[a : int(np.floor(b)) + 1, [0, 2]], end]) - start + chord = end - start + n = np.linalg.norm(chord) + if n == 0: + raise ValueError(f"window {a}..{b} ends where it starts and has no travel line") + u = chord / n + out[k] = np.abs(u[1] * pts[:, 0] - u[0] * pts[:, 1]).max() + return out + + # A4: a range to the meter diff --git a/experiments/drift-anatomy/results/drift_anatomy.json b/experiments/drift-anatomy/results/drift_anatomy.json index 73d0a8375..4491e4e2d 100644 --- a/experiments/drift-anatomy/results/drift_anatomy.json +++ b/experiments/drift-anatomy/results/drift_anatomy.json @@ -1357,30 +1357,78 @@ "p90_along_in": 5.93244053666383, "p90_across_in": 2.383144858716218, "p90_horizontal_in": 6.356644425002893, + "along_share_of_total_p90": 0.9332660661857188, "median_horizontal_in": 2.6181875496018105, "p90_vertical_in": 1.9654002913094888, "p90_3d_in": 6.796782660764225, - "along_share_of_total_p90": 0.9332660661857188 + "straight_path": { + "0.25": { + "windows": 3601, + "p90_along_in": 5.826966988094975, + "p90_across_in": 2.111570110459862, + "p90_horizontal_in": 6.152786774170161, + "along_share_of_total_p90": 0.9470451686310665 + }, + "0.50": { + "windows": 4489, + "p90_along_in": 5.877316441113764, + "p90_across_in": 2.256740731176376, + "p90_horizontal_in": 6.281388835683555, + "along_share_of_total_p90": 0.9356714884016857 + } + } }, "20": { "windows": 4956, "p90_along_in": 9.614029066697613, "p90_across_in": 3.8201651149007376, "p90_horizontal_in": 10.416259256830854, + "along_share_of_total_p90": 0.9229828895044881, "median_horizontal_in": 4.395873486077264, "p90_vertical_in": 2.9551007554751285, "p90_3d_in": 10.87021034634674, - "along_share_of_total_p90": 0.9229828895044881 + "straight_path": { + "0.25": { + "windows": 1416, + "p90_along_in": 9.604249774006092, + "p90_across_in": 2.6576477424629545, + "p90_horizontal_in": 9.791322826235511, + "along_share_of_total_p90": 0.9808939960872128 + }, + "0.50": { + "windows": 2735, + "p90_along_in": 9.735246181368389, + "p90_across_in": 2.9849409636746014, + "p90_horizontal_in": 10.254954545799945, + "along_share_of_total_p90": 0.9493212415412987 + } + } }, "30": { "windows": 4866, "p90_along_in": 12.50959372814818, "p90_across_in": 5.464582640836947, "p90_horizontal_in": 13.628585737497145, + "along_share_of_total_p90": 0.917893754282206, "median_horizontal_in": 6.016112751925093, "p90_vertical_in": 3.6388989773814653, "p90_3d_in": 14.139117730080716, - "along_share_of_total_p90": 0.917893754282206 + "straight_path": { + "0.25": { + "windows": 556, + "p90_along_in": 11.52265233012735, + "p90_across_in": 3.0105234637262224, + "p90_horizontal_in": 11.707672299015773, + "along_share_of_total_p90": 0.9841966904980782 + }, + "0.50": { + "windows": 1506, + "p90_along_in": 12.273519598588699, + "p90_across_in": 3.6974588667790194, + "p90_horizontal_in": 12.509892612405189, + "along_share_of_total_p90": 0.981105112478576 + } + } } }, "advio": { @@ -1389,30 +1437,78 @@ "p90_along_in": 40.16740160527824, "p90_across_in": 41.217205108912495, "p90_horizontal_in": 55.603313400377814, + "along_share_of_total_p90": 0.7223922307659694, "median_horizontal_in": 26.202438486533016, "p90_vertical_in": 3.697033252704125, "p90_3d_in": 55.7048750882303, - "along_share_of_total_p90": 0.7223922307659694 + "straight_path": { + "0.25": { + "windows": 1831, + "p90_along_in": 38.817104640899515, + "p90_across_in": 40.40824203776524, + "p90_horizontal_in": 54.051646176045516, + "along_share_of_total_p90": 0.7181484263119888 + }, + "0.50": { + "windows": 1905, + "p90_along_in": 39.71706496159873, + "p90_across_in": 40.95345003105173, + "p90_horizontal_in": 55.31558133196048, + "along_share_of_total_p90": 0.7180086334670921 + } + } }, "20": { "windows": 1903, "p90_along_in": 72.05520337403567, "p90_across_in": 77.9870905225656, "p90_horizontal_in": 94.28551696290192, + "along_share_of_total_p90": 0.76422345334742, "median_horizontal_in": 50.231545101074694, "p90_vertical_in": 6.375715770353499, "p90_3d_in": 94.60404389848367, - "along_share_of_total_p90": 0.76422345334742 + "straight_path": { + "0.25": { + "windows": 1572, + "p90_along_in": 66.36786396952603, + "p90_across_in": 73.94467380934859, + "p90_horizontal_in": 88.94005061866382, + "along_share_of_total_p90": 0.7462089745606566 + }, + "0.50": { + "windows": 1720, + "p90_along_in": 70.01436124445905, + "p90_across_in": 77.12953122417956, + "p90_horizontal_in": 93.0226402125133, + "along_share_of_total_p90": 0.7526593642634624 + } + } }, "30": { "windows": 1889, "p90_along_in": 97.37446063335635, "p90_across_in": 110.39001216955775, "p90_horizontal_in": 132.53687138322834, + "along_share_of_total_p90": 0.7346971421394095, "median_horizontal_in": 73.64854529941651, "p90_vertical_in": 9.174807119748339, "p90_3d_in": 132.57989402774612, - "along_share_of_total_p90": 0.7346971421394095 + "straight_path": { + "0.25": { + "windows": 1421, + "p90_along_in": 93.12387571665461, + "p90_across_in": 102.7530290645741, + "p90_horizontal_in": 124.45301489257115, + "along_share_of_total_p90": 0.7482653256495223 + }, + "0.50": { + "windows": 1549, + "p90_along_in": 99.08655440233662, + "p90_across_in": 107.2678362551467, + "p90_horizontal_in": 129.17438173809424, + "along_share_of_total_p90": 0.7670758943769378 + } + } } } }, diff --git a/experiments/drift-anatomy/results/drift_anatomy.md b/experiments/drift-anatomy/results/drift_anatomy.md index bece27d8d..475a4c8e0 100644 --- a/experiments/drift-anatomy/results/drift_anatomy.md +++ b/experiments/drift-anatomy/results/drift_anatomy.md @@ -132,9 +132,28 @@ The evals' windows: from every image (MARViN) or every 0.5 s (ADVIO) until the r Criterion at 20 ft: along p90 at most 0.60 of the horizontal total p90. MARViN 0.92, FAIL; ADVIO 0.76, FAIL. +### Straight-path check + +The windows below have reference paths within the given distance of their own start-to-end line. This checks the along/across split on straight paths. The walks have no wall annotations, so it cannot identify wall-parallel travel or measure the benefit of a wall-plane constraint. The widths are untuned and not pre-registered. + +| Data | Walked | Max chord deviation | Windows | Along | Across | Horizontal total | Along / total | +| --- | --- | --- | --- | --- | --- | --- | --- | +| MARViN | 10 ft | 0.25 m | 3601 | 5.8 | 2.1 | 6.2 | 0.95 | +| MARViN | 10 ft | 0.50 m | 4489 | 5.9 | 2.3 | 6.3 | 0.94 | +| MARViN | 20 ft | 0.25 m | 1416 | 9.6 | 2.7 | 9.8 | 0.98 | +| MARViN | 20 ft | 0.50 m | 2735 | 9.7 | 3.0 | 10.3 | 0.95 | +| MARViN | 30 ft | 0.25 m | 556 | 11.5 | 3.0 | 11.7 | 0.98 | +| MARViN | 30 ft | 0.50 m | 1506 | 12.3 | 3.7 | 12.5 | 0.98 | +| ADVIO | 10 ft | 0.25 m | 1831 | 38.8 | 40.4 | 54.1 | 0.72 | +| ADVIO | 10 ft | 0.50 m | 1905 | 39.7 | 41.0 | 55.3 | 0.72 | +| ADVIO | 20 ft | 0.25 m | 1572 | 66.4 | 73.9 | 88.9 | 0.75 | +| ADVIO | 20 ft | 0.50 m | 1720 | 70.0 | 77.1 | 93.0 | 0.75 | +| ADVIO | 30 ft | 0.25 m | 1421 | 93.1 | 102.8 | 124.5 | 0.75 | +| ADVIO | 30 ft | 0.50 m | 1549 | 99.1 | 107.3 | 129.2 | 0.77 | + ## A4, a range to the meter -A simulated range from each window's start to its end: the reference's 3-D distance plus N(0, sigma), 20 draws per window (seed 0). ARKit's end moves radially in the horizontal plane about the start, keeping its height, onto the range. p90 horizontal error, inches. sigma 0.10 m is graded; 0.05 and 0.20 m are not. +Each window is a session that starts at the meter, where the second phone stays. A simulated range from each window's start to its end: the reference's 3-D distance plus N(0, sigma), 20 draws per window (seed 0). ARKit's end moves radially in the horizontal plane about the start, keeping its height, onto the range. p90 horizontal error, inches. sigma 0.10 m is graded; 0.05 and 0.20 m are not. | Data | Walked | Before | After, 0.05 m | After, 0.10 m | After, 0.20 m | | --- | --- | --- | --- | --- | --- | diff --git a/experiments/drift-anatomy/run.py b/experiments/drift-anatomy/run.py index 51556068c..d472d8d9c 100644 --- a/experiments/drift-anatomy/run.py +++ b/experiments/drift-anatomy/run.py @@ -17,6 +17,7 @@ from metrics import ( along_across, at, + chord_deviation, find_loops, local_scale, loop_errors, @@ -56,6 +57,9 @@ RANGE_SIGMAS_M = (0.05, 0.10, 0.20) GRADED_SIGMA_M = 0.10 RANGE_DRAWS = 20 +# Chord deviation selects straight paths; without wall annotations it cannot select paths +# parallel to a wall. The widths are untuned and not pre-registered. +STRAIGHT_PATH_M = (0.25, 0.5) SEED = 0 @@ -236,15 +240,31 @@ def a2(walks: list[Walk]) -> dict: # A3 and A4 -def window_errors(ws: list[Walk], ft: int, window_pairs) -> tuple[np.ndarray, np.ndarray]: - """ARKit's displacement turned by the heading offset at each window's start, and the - reference's, pooled over walks; the evals' windows (every start sample, exactly ft of path).""" - est, ref = [], [] +def window_errors( + ws: list[Walk], ft: int, window_pairs +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """ARKit's displacement turned by the heading offset at each window's start, the reference's, + and how far the reference path strays from its start-to-end line, pooled over walks; the + evals' windows (every start sample, exactly ft of path).""" + est, ref, dev = [], [], [] for w in ws: i, j = window_pairs(w.ref, ft * FEET, w.start_step) est.append(turn(at(w.ark, j) - w.ark[i], w.delta[i])) ref.append(at(w.ref, j) - w.ref[i]) - return np.concatenate(est), np.concatenate(ref) + dev.append(chord_deviation(w.ref, i, j)) + return np.concatenate(est), np.concatenate(ref), np.concatenate(dev) + + +def split_p90(err: np.ndarray, ref: np.ndarray) -> dict: + along, across = along_across(err, ref) + horiz = np.hypot(along, across) + return { + "windows": len(err), + "p90_along_in": p90(np.abs(along)) / INCH, + "p90_across_in": p90(np.abs(across)) / INCH, + "p90_horizontal_in": p90(horiz) / INCH, + "along_share_of_total_p90": p90(np.abs(along)) / p90(horiz), + } def a3_a4(walks: list[Walk]) -> tuple[dict, dict]: @@ -254,19 +274,21 @@ def a3_a4(walks: list[Walk]) -> tuple[dict, dict]: for dataset, ws in by_dataset(walks).items(): a3[dataset], a4[dataset] = {}, {} for ft in WINDOW_FT: - est, ref = window_errors(ws, ft, drift.window_pairs) + est, ref, dev = window_errors(ws, ft, drift.window_pairs) err = est - ref along, across = along_across(err, ref) horiz = np.hypot(along, across) a3[dataset][ft] = { - "windows": len(err), - "p90_along_in": p90(np.abs(along)) / INCH, - "p90_across_in": p90(np.abs(across)) / INCH, - "p90_horizontal_in": p90(horiz) / INCH, + **split_p90(err, ref), "median_horizontal_in": float(np.median(horiz)) / INCH, "p90_vertical_in": p90(np.abs(err[:, 1])) / INCH, "p90_3d_in": p90(np.linalg.norm(err, axis=1)) / INCH, - "along_share_of_total_p90": p90(np.abs(along)) / p90(horiz), + "straight_path": { + f"{m:.2f}": split_p90(err[dev <= m], ref[dev <= m]) + if (dev <= m).any() + else None + for m in STRAIGHT_PATH_M + }, } true_range = np.linalg.norm(ref, axis=1) a4[dataset][ft] = {"before_p90_in": p90(horiz) / INCH} @@ -457,10 +479,33 @@ def markdown(meta: dict, r1: dict, r2: dict, r3: dict, r4: dict, r5: dict) -> st for d, x in r3.items() ) + ".", + "", + "### Straight-path check", + "", + "The windows below have reference paths within the given distance of their own " + "start-to-end line. This checks the along/across split on straight paths. The walks have " + "no wall annotations, so it cannot identify wall-parallel travel or measure the benefit " + "of a wall-plane constraint. The widths are untuned and not pre-registered.", + "", + "| Data | Walked | Max chord deviation | Windows | Along | Across | Horizontal total | Along / total |", + "| --- | --- | --- | --- | --- | --- | --- | --- |", + ] + for d, x in r3.items(): + for ft, y in x.items(): + for m, z in y["straight_path"].items(): + if z is None: + L.append(f"| {names[d]} | {ft} ft | {m} m | 0 | | | | |") + continue + L.append( + f"| {names[d]} | {ft} ft | {m} m | {z['windows']} | {z['p90_along_in']:.1f} | " + f"{z['p90_across_in']:.1f} | {z['p90_horizontal_in']:.1f} | " + f"{z['along_share_of_total_p90']:.2f} |" + ) + L += [ "", "## A4, a range to the meter", "", - f"A simulated range from each window's start to its end: the reference's 3-D distance plus " + f"Each window is a session that starts at the meter, where the second phone stays. A simulated range from each window's start to its end: the reference's 3-D distance plus " f"N(0, sigma), {RANGE_DRAWS} draws per window (seed {SEED}). ARKit's end moves radially in " "the horizontal plane about the start, keeping its height, onto the range. p90 horizontal " "error, inches. sigma 0.10 m is graded; 0.05 and 0.20 m are not.", diff --git a/experiments/drift-anatomy/tests/test_metrics.py b/experiments/drift-anatomy/tests/test_metrics.py index 3b49b968e..6a607e373 100644 --- a/experiments/drift-anatomy/tests/test_metrics.py +++ b/experiments/drift-anatomy/tests/test_metrics.py @@ -6,6 +6,7 @@ from metrics import ( along_across, at, + chord_deviation, find_loops, local_scale, loop_errors, @@ -189,6 +190,21 @@ def test_along_across_is_pythagorean_and_turn_invariant(): np.testing.assert_allclose(c2, across, atol=1e-12) +def test_chord_deviation_is_zero_on_a_straight_walk_and_the_corner_offset_on_a_turn(): + lap = square_lap(side_m=15.0) # 1 m samples: corners at 15, 30, 45 + i, j = np.array([0, 10]), np.array([12.5, 20.0]) + dev = chord_deviation(lap, i, j) + assert dev[0] == pytest.approx(0.0, abs=1e-12) # 12.5 m along the first side + # From (10, 0) round the corner to (15, 5): the corner is 5 / sqrt(2) off the chord. + assert dev[1] == pytest.approx(5 / np.sqrt(2), abs=1e-12) + + +def test_chord_deviation_refuses_a_window_with_no_travel(): + p = np.c_[np.zeros(5), np.ones(5), np.zeros(5)] + with pytest.raises(ValueError, match="no travel line"): + chord_deviation(p, np.array([0]), np.array([3.0])) + + # Range to the meter diff --git a/experiments/drift-anatomy/tests/test_report.py b/experiments/drift-anatomy/tests/test_report.py new file mode 100644 index 000000000..7c3ae974e --- /dev/null +++ b/experiments/drift-anatomy/tests/test_report.py @@ -0,0 +1,32 @@ +"""The cached run must distinguish path straightness from wall orientation.""" + +import json +from pathlib import Path + +from run import markdown + + +def test_straight_path_report_qualifies_the_filter(): + def numeric_keys(pairs): + def key(value): + try: + return int(value) if value.isdigit() else float(value) + except ValueError: + return value + + return { + key(k): ({f"{m:.2f}": z for m, z in v.items()} if k == "straight_path" else v) + for k, v in pairs + } + + report = json.loads( + (Path(__file__).parents[1] / "results/drift_anatomy.json").read_text(), + object_pairs_hook=numeric_keys, + ) + rendered = markdown(*(report[k] for k in ("meta", "a1", "a2", "a3", "a4", "a5"))) + assert "### Straight-path check" in rendered + assert "Max chord deviation" in rendered + assert "cannot identify wall-parallel travel" in rendered + assert "measure the benefit of a wall-plane constraint" in rendered + assert "stand in for a walk along a wall" not in rendered + assert rendered == (Path(__file__).parents[1] / "results/drift_anatomy.md").read_text() diff --git a/experiments/edge-geometry/METHODS.md b/experiments/edge-geometry/METHODS.md index cacfc3c06..c984b0091 100644 --- a/experiments/edge-geometry/METHODS.md +++ b/experiments/edge-geometry/METHODS.md @@ -37,6 +37,8 @@ Electro: 3,000 edge points on 21 laser wall planes, 2,258 in-plane and 742 proud Added after the run: M2 with the true pixel still gives 11.5. The MoGe-2 wall plane misses the edge point by 10.0 at p90, where the laser plane misses it by 1.4. A snap started at the true pixel moves 8.8 px on vertical edges, so the snap's pixel error is ill-posed for these points. M3 matches within 2 px give 2.9 with exact poses and 8.9 with `modern_assumed`. Facade has 866 points but no wall within 6 m, so only M0 runs there. +Added after review: M3 pairs each point with the narrowest qualifying ray angle (at least 5°), which may triangulate worse than a wider pair. `uv run python run.py --pair=widest` reruns it with the widest angle ([results/edge_geometry_widest_pair.md](results/edge_geometry_widest_pair.md)). On electro it is no better: with `modern_assumed` poses M3's 3-D p90 is 158.7 in against 154.6, and 41% of points match either way. With exact poses the wide pair is worse, 156.4 in against 112.5, because fewer of its matches land within 2 px (35% against 49%). P4 fails under both, so the narrow pair is not what makes M3 look bad. The single-photo tables came out identical to the committed ones. + Run: `uv run python run.py`. It takes 2 minutes and 0.9 GB. It needs the evals checkout at `190ed33` (`EDGE_EVALS_DIR`) and its ETH3D, MoGe-2 and pose caches. What it changes: the wall plane's depth error times tan(view angle) sets the edge error, so get the edge photo within 30° of head-on. A snapped tap can't beat the plane's error, and patch matching at edges fails on more than half the points. diff --git a/experiments/edge-geometry/README.md b/experiments/edge-geometry/README.md index 36d3563d8..7cc614a41 100644 --- a/experiments/edge-geometry/README.md +++ b/experiments/edge-geometry/README.md @@ -19,7 +19,7 @@ P1 passes; P2 to P4 fail ([tables](results/edge_geometry.md)). - **Angle.** In-plane edges are 0.9 in p90 along the wall within 15° of face-on and 24 in beyond 45°. - **Ray to plane (M1).** 6.7 in overall, 0.5 in within 15°. - **The app's plane (M1-anchor).** Its yaw error is 2.2° median and 6.7° p90. -- **M2 and M3.** Tap, snap and median gives 12.6 in, because the learned plane's own error dominates. Two-photo matching finds 41% of points. +- **M2 and M3.** Tap, snap and median gives 12.6 in, because the learned plane's own error dominates. Two-photo matching finds 41% of points. Pairing each point with the widest photo pair instead of the narrowest is no better (added after review). **Run:** `uv run python run.py`: 2 min, 0.9 GB. Needs the evals checkout at 190ed33 and its ETH3D and MoGe-2 caches. diff --git a/experiments/edge-geometry/imagematch.py b/experiments/edge-geometry/imagematch.py index 5fb90a19d..757850f10 100644 --- a/experiments/edge-geometry/imagematch.py +++ b/experiments/edge-geometry/imagematch.py @@ -87,7 +87,9 @@ def epipolar_match( fail = (np.full(2, np.nan), -1.0) if not (half <= uv1[0] <= w1 - 1 - half and half <= uv1[1] <= h1 - 1 - half): return fail - template = cv2.getRectSubPix(img1.astype(np.float32), (patch, patch), (float(uv1[0]), float(uv1[1]))) + template = cv2.getRectSubPix( + img1.astype(np.float32), (patch, patch), (float(uv1[0]), float(uv1[1])) + ) # The depths whose projection lands inside img2, in front of it: an interval along the ray, # so its two ends bound a straight segment in img2. @@ -109,12 +111,20 @@ def epipolar_match( samples = ends[0] + np.linspace(0.0, 1.0, n)[:, None] * (ends[1] - ends[0]) offs = np.arange(patch, dtype=np.float32) - half - map_x = (samples[:, 0, None, None] + offs[None, None, :]).astype(np.float32) - map_y = (samples[:, 1, None, None] + offs[None, :, None]).astype(np.float32) - map_x = np.broadcast_to(map_x, (n, patch, patch)).reshape(n * patch, patch) - map_y = np.broadcast_to(map_y, (n, patch, patch)).reshape(n * patch, patch) - patches = cv2.remap(img2.astype(np.float32), map_x, map_y, cv2.INTER_LINEAR) - score = _ncc(template, patches.reshape(n, patch, patch)) + image2 = img2.astype(np.float32) + # cv2.remap takes at most SHRT_MAX rows, and a wide photo pair's segment can cross most of + # img2, so the patches are cut in chunks that fit; shorter segments are one chunk, as before. + chunk = (np.iinfo(np.int16).max - 1) // patch + patches = [] + for a in range(0, n, chunk): + part = samples[a : a + chunk] + m = len(part) + map_x = (part[:, 0, None, None] + offs[None, None, :]).astype(np.float32) + map_y = (part[:, 1, None, None] + offs[None, :, None]).astype(np.float32) + map_x = np.broadcast_to(map_x, (m, patch, patch)).reshape(m * patch, patch) + map_y = np.broadcast_to(map_y, (m, patch, patch)).reshape(m * patch, patch) + patches.append(cv2.remap(image2, map_x, map_y, cv2.INTER_LINEAR).reshape(m, patch, patch)) + score = _ncc(template, np.concatenate(patches)) i = int(np.argmax(score)) off = 0.0 if 0 < i < n - 1: diff --git a/experiments/edge-geometry/results/edge_geometry_widest_pair.json b/experiments/edge-geometry/results/edge_geometry_widest_pair.json new file mode 100644 index 000000000..bb3fddb62 --- /dev/null +++ b/experiments/edge-geometry/results/edge_geometry_widest_pair.json @@ -0,0 +1,4170 @@ +{ + "m3_pairing": "widest", + "evals_commit": "190ed339f41924ae0244e0b88c5ebe647ddd9eb4", + "evals_dir": "~/Programming Projects/review-debt/worktrees/evals-190ed33/experiments/evals", + "parameters": { + "NEAR_M": 6.0, + "TOLERANCE": 0.04, + "VERTICAL_MAX_UP": 0.3, + "WIDTH": 1024, + "SCENES": [ + "electro", + "facade" + ], + "MIN_POINTS_TO_REPORT": 200, + "MAX_POINTS": 3000, + "PLANE_TOL_M": 0.03, + "MAX_TILT_DEG": 17.0, + "RANSAC_POINTS": 400000, + "JOIN_M": 0.6, + "MEMBER_REACH_M": 1.0, + "IN_PLANE_M": 0.05, + "EDGE_RADIUS_M": 0.15, + "VERTICAL_EDGE_DEG": 30.0, + "MIN_WALL_PX": 50, + "ANCHOR_ALONG_M": [ + 1.0, + 3.0 + ], + "ANCHOR_TARGET_M": 2.0, + "PATCH_RADIUS_M": 0.3, + "MIN_PATCH_PX": 20, + "ANCHOR_TRIES": 5, + "TAP_SIGMA_PX": 10.0, + "SNAP_RADIUS_PX": 25, + "SNAP_ROWS": 9, + "WORLD_PLANE_PX_PER_PHOTO": 3000, + "MIN_BASELINE_M": 0.3, + "MIN_RAY_ANGLE_DEG": 5.0, + "NCC_MIN": 0.8, + "NCC_PATCH": 15, + "DEPTH_RANGE_M": [ + 0.5, + 10.0 + ], + "SEED": 20260926 + }, + "scenes": { + "electro": { + "meta": { + "photos": 45, + "wall_planes": 21, + "edge_points_within_6m": 27436, + "edge_points_on_a_wall": 25492, + "points_sampled": 3000, + "in_plane": 2258, + "proud": 742, + "vertical_edges": 730, + "up_vs_camera_up_deg": 5.21, + "wall_tilt_deg_max": 3.14, + "oracle_scale_median": 0.9961415148011799, + "seconds": 53 + }, + "single": { + "m0": { + "all": { + "all angles": { + "along": { + "n": 7871, + "points": 2994, + "median_in": 2.13, + "p90_in": 15.64 + }, + "out": { + "n": 7871, + "points": 2994, + "median_in": 3.11, + "p90_in": 19.17 + }, + "3d": { + "n": 7871, + "points": 2994, + "median_in": 5.72, + "p90_in": 38.62 + }, + "missing": 23 + }, + "0-15": { + "along": { + "n": 647, + "points": 345, + "median_in": 0.19, + "p90_in": 1.35 + }, + "out": { + "n": 647, + "points": 345, + "median_in": 4.83, + "p90_in": 14.83 + }, + "3d": { + "n": 647, + "points": 345, + "median_in": 4.9, + "p90_in": 14.93 + }, + "missing": 0 + }, + "15-30": { + "along": { + "n": 1483, + "points": 912, + "median_in": 0.73, + "p90_in": 4.59 + }, + "out": { + "n": 1483, + "points": 912, + "median_in": 3.76, + "p90_in": 15.26 + }, + "3d": { + "n": 1483, + "points": 912, + "median_in": 4.15, + "p90_in": 16.98 + }, + "missing": 2 + }, + "30-45": { + "along": { + "n": 2460, + "points": 1430, + "median_in": 2.02, + "p90_in": 8.89 + }, + "out": { + "n": 2460, + "points": 1430, + "median_in": 4.17, + "p90_in": 20.37 + }, + "3d": { + "n": 2460, + "points": 1430, + "median_in": 5.25, + "p90_in": 24.91 + }, + "missing": 14 + }, + ">45": { + "along": { + "n": 3281, + "points": 1593, + "median_in": 5.09, + "p90_in": 24.36 + }, + "out": { + "n": 3281, + "points": 1593, + "median_in": 1.72, + "p90_in": 23.7 + }, + "3d": { + "n": 3281, + "points": 1593, + "median_in": 7.5, + "p90_in": 66.17 + }, + "missing": 7 + } + }, + "in_plane": { + "all angles": { + "along": { + "n": 6067, + "points": 2258, + "median_in": 2.0, + "p90_in": 14.56 + }, + "out": { + "n": 6067, + "points": 2258, + "median_in": 2.78, + "p90_in": 16.17 + }, + "3d": { + "n": 6067, + "points": 2258, + "median_in": 5.33, + "p90_in": 31.73 + }, + "missing": 9 + }, + "0-15": { + "along": { + "n": 547, + "points": 281, + "median_in": 0.19, + "p90_in": 0.91 + }, + "out": { + "n": 547, + "points": 281, + "median_in": 4.83, + "p90_in": 14.63 + }, + "3d": { + "n": 547, + "points": 281, + "median_in": 4.9, + "p90_in": 14.78 + }, + "missing": 0 + }, + "15-30": { + "along": { + "n": 1067, + "points": 708, + "median_in": 0.67, + "p90_in": 4.47 + }, + "out": { + "n": 1067, + "points": 708, + "median_in": 3.36, + "p90_in": 16.86 + }, + "3d": { + "n": 1067, + "points": 708, + "median_in": 3.64, + "p90_in": 18.19 + }, + "missing": 2 + }, + "30-45": { + "along": { + "n": 1835, + "points": 1077, + "median_in": 1.9, + "p90_in": 7.84 + }, + "out": { + "n": 1835, + "points": 1077, + "median_in": 3.85, + "p90_in": 15.95 + }, + "3d": { + "n": 1835, + "points": 1077, + "median_in": 4.91, + "p90_in": 19.86 + }, + "missing": 5 + }, + ">45": { + "along": { + "n": 2618, + "points": 1211, + "median_in": 4.78, + "p90_in": 24.06 + }, + "out": { + "n": 2618, + "points": 1211, + "median_in": 1.52, + "p90_in": 17.68 + }, + "3d": { + "n": 2618, + "points": 1211, + "median_in": 6.63, + "p90_in": 62.06 + }, + "missing": 2 + } + }, + "proud": { + "all angles": { + "along": { + "n": 1804, + "points": 736, + "median_in": 2.45, + "p90_in": 18.03 + }, + "out": { + "n": 1804, + "points": 736, + "median_in": 4.17, + "p90_in": 25.02 + }, + "3d": { + "n": 1804, + "points": 736, + "median_in": 7.34, + "p90_in": 57.26 + }, + "missing": 14 + }, + "0-15": { + "along": { + "n": 100, + "points": 64, + "median_in": 0.35, + "p90_in": 2.73 + }, + "out": { + "n": 100, + "points": 64, + "median_in": 4.85, + "p90_in": 16.17 + }, + "3d": { + "n": 100, + "points": 64, + "median_in": 4.93, + "p90_in": 16.4 + }, + "missing": 0 + }, + "15-30": { + "along": { + "n": 416, + "points": 204, + "median_in": 0.93, + "p90_in": 4.76 + }, + "out": { + "n": 416, + "points": 204, + "median_in": 4.22, + "p90_in": 12.73 + }, + "3d": { + "n": 416, + "points": 204, + "median_in": 4.6, + "p90_in": 14.1 + }, + "missing": 0 + }, + "30-45": { + "along": { + "n": 625, + "points": 353, + "median_in": 2.4, + "p90_in": 12.12 + }, + "out": { + "n": 625, + "points": 353, + "median_in": 5.44, + "p90_in": 24.81 + }, + "3d": { + "n": 625, + "points": 353, + "median_in": 6.87, + "p90_in": 30.17 + }, + "missing": 9 + }, + ">45": { + "along": { + "n": 663, + "points": 382, + "median_in": 6.76, + "p90_in": 27.06 + }, + "out": { + "n": 663, + "points": 382, + "median_in": 3.14, + "p90_in": 48.07 + }, + "3d": { + "n": 663, + "points": 382, + "median_in": 19.91, + "p90_in": 116.19 + }, + "missing": 5 + } + } + }, + "m1": { + "all": { + "all angles": { + "along": { + "n": 6257, + "points": 2712, + "median_in": 1.26, + "p90_in": 7.25 + }, + "out": { + "n": 6257, + "points": 2712, + "median_in": 3.11, + "p90_in": 10.49 + }, + "3d": { + "n": 6257, + "points": 2712, + "median_in": 4.19, + "p90_in": 15.1 + }, + "missing": 1637 + }, + "0-15": { + "along": { + "n": 615, + "points": 342, + "median_in": 0.16, + "p90_in": 0.58 + }, + "out": { + "n": 615, + "points": 342, + "median_in": 4.16, + "p90_in": 11.43 + }, + "3d": { + "n": 615, + "points": 342, + "median_in": 4.17, + "p90_in": 11.57 + }, + "missing": 32 + }, + "15-30": { + "along": { + "n": 1406, + "points": 898, + "median_in": 0.53, + "p90_in": 2.75 + }, + "out": { + "n": 1406, + "points": 898, + "median_in": 2.88, + "p90_in": 10.29 + }, + "3d": { + "n": 1406, + "points": 898, + "median_in": 3.17, + "p90_in": 11.11 + }, + "missing": 79 + }, + "30-45": { + "along": { + "n": 2358, + "points": 1376, + "median_in": 1.63, + "p90_in": 6.32 + }, + "out": { + "n": 2358, + "points": 1376, + "median_in": 3.31, + "p90_in": 8.87 + }, + "3d": { + "n": 2358, + "points": 1376, + "median_in": 4.16, + "p90_in": 11.35 + }, + "missing": 116 + }, + ">45": { + "along": { + "n": 1878, + "points": 1219, + "median_in": 3.49, + "p90_in": 21.69 + }, + "out": { + "n": 1878, + "points": 1219, + "median_in": 2.56, + "p90_in": 16.32 + }, + "3d": { + "n": 1878, + "points": 1219, + "median_in": 5.33, + "p90_in": 45.01 + }, + "missing": 1410 + } + }, + "in_plane": { + "all angles": { + "along": { + "n": 4920, + "points": 2097, + "median_in": 1.13, + "p90_in": 6.65 + }, + "out": { + "n": 4920, + "points": 2097, + "median_in": 2.7, + "p90_in": 9.16 + }, + "3d": { + "n": 4920, + "points": 2097, + "median_in": 3.77, + "p90_in": 12.0 + }, + "missing": 1156 + }, + "0-15": { + "along": { + "n": 516, + "points": 278, + "median_in": 0.13, + "p90_in": 0.45 + }, + "out": { + "n": 516, + "points": 278, + "median_in": 4.25, + "p90_in": 11.52 + }, + "3d": { + "n": 516, + "points": 278, + "median_in": 4.27, + "p90_in": 11.65 + }, + "missing": 31 + }, + "15-30": { + "along": { + "n": 1041, + "points": 699, + "median_in": 0.42, + "p90_in": 2.3 + }, + "out": { + "n": 1041, + "points": 699, + "median_in": 2.44, + "p90_in": 9.78 + }, + "3d": { + "n": 1041, + "points": 699, + "median_in": 2.7, + "p90_in": 10.67 + }, + "missing": 28 + }, + "30-45": { + "along": { + "n": 1787, + "points": 1044, + "median_in": 1.52, + "p90_in": 5.76 + }, + "out": { + "n": 1787, + "points": 1044, + "median_in": 3.01, + "p90_in": 7.64 + }, + "3d": { + "n": 1787, + "points": 1044, + "median_in": 3.74, + "p90_in": 10.11 + }, + "missing": 53 + }, + ">45": { + "along": { + "n": 1576, + "points": 998, + "median_in": 2.89, + "p90_in": 15.02 + }, + "out": { + "n": 1576, + "points": 998, + "median_in": 1.89, + "p90_in": 8.92 + }, + "3d": { + "n": 1576, + "points": 998, + "median_in": 4.49, + "p90_in": 36.17 + }, + "missing": 1044 + } + }, + "proud": { + "all angles": { + "along": { + "n": 1337, + "points": 615, + "median_in": 2.01, + "p90_in": 14.46 + }, + "out": { + "n": 1337, + "points": 615, + "median_in": 5.0, + "p90_in": 19.02 + }, + "3d": { + "n": 1337, + "points": 615, + "median_in": 6.29, + "p90_in": 27.46 + }, + "missing": 481 + }, + "0-15": { + "along": { + "n": 99, + "points": 64, + "median_in": 0.28, + "p90_in": 1.05 + }, + "out": { + "n": 99, + "points": 64, + "median_in": 3.82, + "p90_in": 7.23 + }, + "3d": { + "n": 99, + "points": 64, + "median_in": 3.88, + "p90_in": 7.34 + }, + "missing": 1 + }, + "15-30": { + "along": { + "n": 365, + "points": 199, + "median_in": 0.99, + "p90_in": 3.54 + }, + "out": { + "n": 365, + "points": 199, + "median_in": 4.62, + "p90_in": 11.83 + }, + "3d": { + "n": 365, + "points": 199, + "median_in": 5.02, + "p90_in": 12.83 + }, + "missing": 51 + }, + "30-45": { + "along": { + "n": 571, + "points": 332, + "median_in": 2.23, + "p90_in": 8.69 + }, + "out": { + "n": 571, + "points": 332, + "median_in": 4.54, + "p90_in": 15.93 + }, + "3d": { + "n": 571, + "points": 332, + "median_in": 5.83, + "p90_in": 20.66 + }, + "missing": 63 + }, + ">45": { + "along": { + "n": 302, + "points": 221, + "median_in": 12.09, + "p90_in": 31.95 + }, + "out": { + "n": 302, + "points": 221, + "median_in": 9.57, + "p90_in": 29.5 + }, + "3d": { + "n": 302, + "points": 221, + "median_in": 19.18, + "p90_in": 77.71 + }, + "missing": 366 + } + } + }, + "m1a": { + "all": { + "all angles": { + "along": { + "n": 3411, + "points": 1707, + "median_in": 2.04, + "p90_in": 12.18 + }, + "out": { + "n": 3411, + "points": 1707, + "median_in": 3.82, + "p90_in": 17.86 + }, + "3d": { + "n": 3411, + "points": 1707, + "median_in": 5.11, + "p90_in": 27.19 + }, + "missing": 4483 + }, + "0-15": { + "along": { + "n": 301, + "points": 199, + "median_in": 0.15, + "p90_in": 0.57 + }, + "out": { + "n": 301, + "points": 199, + "median_in": 3.72, + "p90_in": 12.65 + }, + "3d": { + "n": 301, + "points": 199, + "median_in": 3.77, + "p90_in": 12.77 + }, + "missing": 346 + }, + "15-30": { + "along": { + "n": 821, + "points": 592, + "median_in": 1.13, + "p90_in": 5.55 + }, + "out": { + "n": 821, + "points": 592, + "median_in": 3.92, + "p90_in": 14.03 + }, + "3d": { + "n": 821, + "points": 592, + "median_in": 4.26, + "p90_in": 15.65 + }, + "missing": 664 + }, + "30-45": { + "along": { + "n": 1411, + "points": 924, + "median_in": 2.16, + "p90_in": 10.46 + }, + "out": { + "n": 1411, + "points": 924, + "median_in": 4.28, + "p90_in": 19.67 + }, + "3d": { + "n": 1411, + "points": 924, + "median_in": 5.39, + "p90_in": 26.69 + }, + "missing": 1063 + }, + ">45": { + "along": { + "n": 878, + "points": 633, + "median_in": 5.12, + "p90_in": 27.12 + }, + "out": { + "n": 878, + "points": 633, + "median_in": 3.03, + "p90_in": 18.81 + }, + "3d": { + "n": 878, + "points": 633, + "median_in": 6.57, + "p90_in": 36.78 + }, + "missing": 2410 + } + }, + "in_plane": { + "all angles": { + "along": { + "n": 2755, + "points": 1360, + "median_in": 1.93, + "p90_in": 10.43 + }, + "out": { + "n": 2755, + "points": 1360, + "median_in": 3.38, + "p90_in": 13.62 + }, + "3d": { + "n": 2755, + "points": 1360, + "median_in": 4.66, + "p90_in": 21.04 + }, + "missing": 3321 + }, + "0-15": { + "along": { + "n": 278, + "points": 181, + "median_in": 0.15, + "p90_in": 0.5 + }, + "out": { + "n": 278, + "points": 181, + "median_in": 3.5, + "p90_in": 11.78 + }, + "3d": { + "n": 278, + "points": 181, + "median_in": 3.52, + "p90_in": 12.04 + }, + "missing": 269 + }, + "15-30": { + "along": { + "n": 633, + "points": 478, + "median_in": 1.02, + "p90_in": 5.83 + }, + "out": { + "n": 633, + "points": 478, + "median_in": 3.49, + "p90_in": 13.65 + }, + "3d": { + "n": 633, + "points": 478, + "median_in": 3.88, + "p90_in": 15.34 + }, + "missing": 436 + }, + "30-45": { + "along": { + "n": 1088, + "points": 727, + "median_in": 2.21, + "p90_in": 8.58 + }, + "out": { + "n": 1088, + "points": 727, + "median_in": 4.06, + "p90_in": 16.15 + }, + "3d": { + "n": 1088, + "points": 727, + "median_in": 5.05, + "p90_in": 19.5 + }, + "missing": 752 + }, + ">45": { + "along": { + "n": 756, + "points": 526, + "median_in": 4.54, + "p90_in": 27.69 + }, + "out": { + "n": 756, + "points": 526, + "median_in": 2.76, + "p90_in": 12.21 + }, + "3d": { + "n": 756, + "points": 526, + "median_in": 5.68, + "p90_in": 36.52 + }, + "missing": 1864 + } + }, + "proud": { + "all angles": { + "along": { + "n": 656, + "points": 347, + "median_in": 2.48, + "p90_in": 15.62 + }, + "out": { + "n": 656, + "points": 347, + "median_in": 6.29, + "p90_in": 42.42 + }, + "3d": { + "n": 656, + "points": 347, + "median_in": 7.99, + "p90_in": 64.13 + }, + "missing": 1162 + }, + "0-15": { + "along": { + "n": 23, + "points": 18, + "median_in": 2.13, + "p90_in": 154.09 + }, + "out": { + "n": 23, + "points": 18, + "median_in": 12.74, + "p90_in": 3149.65 + }, + "3d": { + "n": 23, + "points": 18, + "median_in": 12.98, + "p90_in": 3164.05 + }, + "missing": 77 + }, + "15-30": { + "along": { + "n": 188, + "points": 114, + "median_in": 1.69, + "p90_in": 5.06 + }, + "out": { + "n": 188, + "points": 114, + "median_in": 7.13, + "p90_in": 14.6 + }, + "3d": { + "n": 188, + "points": 114, + "median_in": 7.77, + "p90_in": 16.43 + }, + "missing": 228 + }, + "30-45": { + "along": { + "n": 323, + "points": 197, + "median_in": 1.99, + "p90_in": 15.69 + }, + "out": { + "n": 323, + "points": 197, + "median_in": 4.69, + "p90_in": 63.53 + }, + "3d": { + "n": 323, + "points": 197, + "median_in": 5.84, + "p90_in": 79.7 + }, + "missing": 311 + }, + ">45": { + "along": { + "n": 122, + "points": 107, + "median_in": 9.51, + "p90_in": 17.91 + }, + "out": { + "n": 122, + "points": 107, + "median_in": 7.42, + "p90_in": 41.02 + }, + "3d": { + "n": 122, + "points": 107, + "median_in": 14.5, + "p90_in": 63.92 + }, + "missing": 546 + } + } + }, + "m1a_yaw": { + "observations": 3411, + "abs_psi_deg": { + "median": 2.15, + "p90": 6.72 + }, + "abs_s_m_median": 2.0, + "all_added_after_the_run": { + "sign_agreement_where_prediction_over_1cm": 0.7, + "median_error_over_prediction_where_over_1cm": 0.73, + "view_angle_under_60": { + "n": 3080, + "slope": 0.36, + "r2": 0.01 + }, + "spearman_abs_error_vs_tan_angle_alone": 0.5 + }, + "all": { + "n": 3411, + "signed_first_order_slope": 0.05, + "signed_first_order_r2": 0.02, + "signed_r2_with_slope_1": -4.58, + "abs_error_over_s_psi_tan_median": 0.94, + "abs_error_over_s_psi_tan_n": 2811, + "spearman_abs_error_vs_s_psi_tan": 0.63, + "abs_error_p90_in": 12.18, + "residual_after_first_order_p90_in": 10.9 + }, + "in_plane_added_after_the_run": { + "sign_agreement_where_prediction_over_1cm": 0.74, + "median_error_over_prediction_where_over_1cm": 0.83, + "view_angle_under_60": { + "n": 2455, + "slope": 0.29, + "r2": 0.02 + }, + "spearman_abs_error_vs_tan_angle_alone": 0.54 + }, + "in_plane": { + "n": 2755, + "signed_first_order_slope": 0.05, + "signed_first_order_r2": 0.02, + "signed_r2_with_slope_1": -12.61, + "abs_error_over_s_psi_tan_median": 0.94, + "abs_error_over_s_psi_tan_n": 2231, + "spearman_abs_error_vs_s_psi_tan": 0.67, + "abs_error_p90_in": 10.43, + "residual_after_first_order_p90_in": 9.41 + } + }, + "added_after_the_run_edge_point_to_plane": { + "laser_plane_in_plane_edges": { + "n": 6076, + "points": 2258, + "median_in": 0.4, + "p90_in": 1.38 + }, + "m1_plane_in_plane_edges": { + "n": 4920, + "points": 2097, + "median_in": 2.54, + "p90_in": 9.81 + } + } + }, + "multi": { + "exact": { + "all": { + "M2": { + "span_along": { + "n": 1819, + "points": 1819, + "median_in": 4.01, + "p90_in": 14.46, + "p90_ci95_in": [ + 13.85, + 15.72 + ] + }, + "span_3d": { + "n": 1819, + "points": 1819, + "median_in": 6.39, + "p90_in": 23.93 + }, + "frame_along": { + "n": 2617, + "points": 2617, + "median_in": 3.37, + "p90_in": 16.39 + }, + "frame_3d": { + "n": 2617, + "points": 2617, + "median_in": 6.08, + "p90_in": 27.83 + } + }, + "M2_added_after_the_run": { + "span_along_tap_no_snap": { + "n": 1819, + "points": 1819, + "median_in": 2.8, + "p90_in": 12.97 + }, + "span_along_true_pixel": { + "n": 1819, + "points": 1819, + "median_in": 2.55, + "p90_in": 12.57 + }, + "frame_along_true_pixel": { + "n": 2617, + "points": 2617, + "median_in": 1.31, + "p90_in": 13.11 + }, + "edge_point_to_fused_plane": { + "n": 2617, + "points": 2617, + "median_in": 2.69, + "p90_in": 13.87 + }, + "span_along_by_median_view_angle": { + "0-15": { + "n": 191, + "points": 191, + "median_in": 2.22, + "p90_in": 5.69 + }, + "15-30": { + "n": 343, + "points": 343, + "median_in": 3.28, + "p90_in": 11.66 + }, + "30-45": { + "n": 959, + "points": 959, + "median_in": 4.16, + "p90_in": 12.33 + }, + ">45": { + "n": 326, + "points": 326, + "median_in": 7.53, + "p90_in": 38.75 + } + }, + "span_along_true_pixel_by_median_view_angle": { + "0-15": { + "n": 191, + "points": 191, + "median_in": 0.17, + "p90_in": 4.53 + }, + "15-30": { + "n": 343, + "points": 343, + "median_in": 1.28, + "p90_in": 12.04 + }, + "30-45": { + "n": 959, + "points": 959, + "median_in": 2.58, + "p90_in": 11.79 + }, + ">45": { + "n": 326, + "points": 326, + "median_in": 5.95, + "p90_in": 51.18 + } + } + }, + "M3": { + "pairs": 2489, + "match_rate": 0.441, + "match_px": { + "n": 1097, + "median": 6.05, + "p90": 186.45 + }, + "frame_3d": { + "n": 1097, + "points": 1097, + "median_in": 4.92, + "p90_in": 101.76 + }, + "frame_along": { + "n": 1097, + "points": 1097, + "median_in": 1.43, + "p90_in": 49.88 + }, + "span_along": { + "n": 439, + "points": 439, + "median_in": 9.32, + "p90_in": 59.52, + "p90_ci95_in": [ + 54.35, + 67.41 + ] + }, + "span_3d": { + "n": 439, + "points": 439, + "median_in": 43.87, + "p90_in": 156.35, + "p90_ci95_in": [ + 134.29, + 161.35 + ] + }, + "anchor_matched": 439, + "added_after_the_run_frame_3d_where_match_within_2px": { + "n": 381, + "points": 381, + "median_in": 0.55, + "p90_in": 1.84 + }, + "added_after_the_run_share_of_matches_within_2px": 0.347 + } + }, + "in_plane": { + "M2": { + "span_along": { + "n": 1452, + "points": 1452, + "median_in": 3.74, + "p90_in": 13.02, + "p90_ci95_in": [ + 12.06, + 14.18 + ] + }, + "span_3d": { + "n": 1452, + "points": 1452, + "median_in": 5.78, + "p90_in": 22.02 + }, + "frame_along": { + "n": 2028, + "points": 2028, + "median_in": 3.26, + "p90_in": 14.39 + }, + "frame_3d": { + "n": 2028, + "points": 2028, + "median_in": 5.85, + "p90_in": 19.3 + } + }, + "M2_added_after_the_run": { + "span_along_tap_no_snap": { + "n": 1452, + "points": 1452, + "median_in": 2.55, + "p90_in": 10.89 + }, + "span_along_true_pixel": { + "n": 1452, + "points": 1452, + "median_in": 2.15, + "p90_in": 10.93 + }, + "frame_along_true_pixel": { + "n": 2028, + "points": 2028, + "median_in": 1.23, + "p90_in": 10.19 + }, + "edge_point_to_fused_plane": { + "n": 2028, + "points": 2028, + "median_in": 2.26, + "p90_in": 10.04 + }, + "span_along_by_median_view_angle": { + "0-15": { + "n": 178, + "points": 178, + "median_in": 2.29, + "p90_in": 5.63 + }, + "15-30": { + "n": 245, + "points": 245, + "median_in": 2.68, + "p90_in": 6.78 + }, + "30-45": { + "n": 780, + "points": 780, + "median_in": 4.19, + "p90_in": 11.6 + }, + ">45": { + "n": 249, + "points": 249, + "median_in": 6.09, + "p90_in": 29.1 + } + }, + "span_along_true_pixel_by_median_view_angle": { + "0-15": { + "n": 178, + "points": 178, + "median_in": 0.15, + "p90_in": 4.27 + }, + "15-30": { + "n": 245, + "points": 245, + "median_in": 1.12, + "p90_in": 3.63 + }, + "30-45": { + "n": 780, + "points": 780, + "median_in": 2.54, + "p90_in": 10.5 + }, + ">45": { + "n": 249, + "points": 249, + "median_in": 5.06, + "p90_in": 13.8 + } + } + }, + "M3": { + "pairs": 1951, + "match_rate": 0.463, + "match_px": { + "n": 903, + "median": 6.44, + "p90": 185.54 + }, + "frame_3d": { + "n": 903, + "points": 903, + "median_in": 5.16, + "p90_in": 102.72 + }, + "frame_along": { + "n": 903, + "points": 903, + "median_in": 1.55, + "p90_in": 55.5 + }, + "span_along": { + "n": 368, + "points": 368, + "median_in": 9.16, + "p90_in": 61.2, + "p90_ci95_in": [ + 55.94, + 69.41 + ] + }, + "span_3d": { + "n": 368, + "points": 368, + "median_in": 47.15, + "p90_in": 157.49, + "p90_ci95_in": [ + 132.58, + 162.28 + ] + }, + "anchor_matched": 368, + "added_after_the_run_frame_3d_where_match_within_2px": { + "n": 296, + "points": 296, + "median_in": 0.57, + "p90_in": 1.98 + }, + "added_after_the_run_share_of_matches_within_2px": 0.328 + } + }, + "proud": { + "M2": { + "span_along": { + "n": 367, + "points": 367, + "median_in": 5.46, + "p90_in": 18.1, + "p90_ci95_in": [ + 17.13, + 20.83 + ] + }, + "span_3d": { + "n": 367, + "points": 367, + "median_in": 11.71, + "p90_in": 43.59 + }, + "frame_along": { + "n": 589, + "points": 589, + "median_in": 3.95, + "p90_in": 24.87 + }, + "frame_3d": { + "n": 589, + "points": 589, + "median_in": 7.52, + "p90_in": 44.47 + } + }, + "M2_added_after_the_run": { + "span_along_tap_no_snap": { + "n": 367, + "points": 367, + "median_in": 4.91, + "p90_in": 17.96 + }, + "span_along_true_pixel": { + "n": 367, + "points": 367, + "median_in": 5.41, + "p90_in": 17.03 + }, + "frame_along_true_pixel": { + "n": 589, + "points": 589, + "median_in": 2.21, + "p90_in": 27.38 + }, + "edge_point_to_fused_plane": { + "n": 589, + "points": 589, + "median_in": 4.91, + "p90_in": 31.59 + }, + "span_along_by_median_view_angle": { + "0-15": { + "n": 13, + "points": 13, + "median_in": 1.56, + "p90_in": 7.34 + }, + "15-30": { + "n": 98, + "points": 98, + "median_in": 7.3, + "p90_in": 14.5 + }, + "30-45": { + "n": 179, + "points": 179, + "median_in": 4.14, + "p90_in": 15.66 + }, + ">45": { + "n": 77, + "points": 77, + "median_in": 13.7, + "p90_in": 103.07 + } + }, + "span_along_true_pixel_by_median_view_angle": { + "0-15": { + "n": 13, + "points": 13, + "median_in": 4.53, + "p90_in": 5.42 + }, + "15-30": { + "n": 98, + "points": 98, + "median_in": 5.98, + "p90_in": 13.97 + }, + "30-45": { + "n": 179, + "points": 179, + "median_in": 3.58, + "p90_in": 15.8 + }, + ">45": { + "n": 77, + "points": 77, + "median_in": 10.15, + "p90_in": 122.8 + } + } + }, + "M3": { + "pairs": 538, + "match_rate": 0.361, + "match_px": { + "n": 194, + "median": 2.77, + "p90": 191.71 + }, + "frame_3d": { + "n": 194, + "points": 194, + "median_in": 2.16, + "p90_in": 97.51 + }, + "frame_along": { + "n": 194, + "points": 194, + "median_in": 0.7, + "p90_in": 28.16 + }, + "span_along": { + "n": 71, + "points": 71, + "median_in": 10.2, + "p90_in": 41.75, + "p90_ci95_in": [ + 36.09, + 117.05 + ] + }, + "span_3d": { + "n": 71, + "points": 71, + "median_in": 37.37, + "p90_in": 136.05, + "p90_ci95_in": [ + 120.08, + 158.91 + ] + }, + "anchor_matched": 71, + "added_after_the_run_frame_3d_where_match_within_2px": { + "n": 85, + "points": 85, + "median_in": 0.49, + "p90_in": 1.57 + }, + "added_after_the_run_share_of_matches_within_2px": 0.438 + } + }, + "in_plane_vertical": { + "M2": { + "span_along": { + "n": 381, + "points": 381, + "median_in": 3.25, + "p90_in": 15.43, + "p90_ci95_in": [ + 13.66, + 16.83 + ] + }, + "span_3d": { + "n": 381, + "points": 381, + "median_in": 5.92, + "p90_in": 24.14 + }, + "frame_along": { + "n": 529, + "points": 529, + "median_in": 2.96, + "p90_in": 12.86 + }, + "frame_3d": { + "n": 529, + "points": 529, + "median_in": 5.94, + "p90_in": 20.94 + } + }, + "M2_added_after_the_run": { + "span_along_tap_no_snap": { + "n": 381, + "points": 381, + "median_in": 2.91, + "p90_in": 11.88 + }, + "span_along_true_pixel": { + "n": 381, + "points": 381, + "median_in": 2.62, + "p90_in": 11.38 + }, + "frame_along_true_pixel": { + "n": 529, + "points": 529, + "median_in": 1.61, + "p90_in": 10.26 + }, + "edge_point_to_fused_plane": { + "n": 529, + "points": 529, + "median_in": 2.63, + "p90_in": 13.28 + }, + "span_along_by_median_view_angle": { + "0-15": { + "n": 43, + "points": 43, + "median_in": 1.47, + "p90_in": 3.45 + }, + "15-30": { + "n": 83, + "points": 83, + "median_in": 3.25, + "p90_in": 8.93 + }, + "30-45": { + "n": 165, + "points": 165, + "median_in": 5.12, + "p90_in": 13.6 + }, + ">45": { + "n": 90, + "points": 90, + "median_in": 4.17, + "p90_in": 23.98 + } + }, + "span_along_true_pixel_by_median_view_angle": { + "0-15": { + "n": 43, + "points": 43, + "median_in": 0.6, + "p90_in": 2.6 + }, + "15-30": { + "n": 83, + "points": 83, + "median_in": 1.99, + "p90_in": 6.95 + }, + "30-45": { + "n": 165, + "points": 165, + "median_in": 2.7, + "p90_in": 12.55 + }, + ">45": { + "n": 90, + "points": 90, + "median_in": 5.23, + "p90_in": 11.84 + } + } + }, + "M3": { + "pairs": 509, + "match_rate": 0.505, + "match_px": { + "n": 257, + "median": 24.35, + "p90": 200.48 + }, + "frame_3d": { + "n": 257, + "points": 257, + "median_in": 20.71, + "p90_in": 108.95 + }, + "frame_along": { + "n": 257, + "points": 257, + "median_in": 2.11, + "p90_in": 72.83 + }, + "span_along": { + "n": 106, + "points": 106, + "median_in": 7.05, + "p90_in": 64.42, + "p90_ci95_in": [ + 48.03, + 90.2 + ] + }, + "span_3d": { + "n": 106, + "points": 106, + "median_in": 67.39, + "p90_in": 154.53, + "p90_ci95_in": [ + 109.0, + 212.49 + ] + }, + "anchor_matched": 106, + "added_after_the_run_frame_3d_where_match_within_2px": { + "n": 67, + "points": 67, + "median_in": 0.77, + "p90_in": 2.34 + }, + "added_after_the_run_share_of_matches_within_2px": 0.261 + } + } + }, + "modern_assumed": { + "all": { + "M2": { + "span_along": { + "n": 9095, + "points": 1819, + "median_in": 3.92, + "p90_in": 14.33, + "p90_ci95_in": [ + 13.42, + 15.32 + ] + }, + "span_3d": { + "n": 9095, + "points": 1819, + "median_in": 6.52, + "p90_in": 23.25 + }, + "frame_along": { + "n": 13085, + "points": 2617, + "median_in": 4.24, + "p90_in": 17.4 + }, + "frame_3d": { + "n": 13085, + "points": 2617, + "median_in": 6.93, + "p90_in": 28.87 + } + }, + "M2_added_after_the_run": { + "span_along_tap_no_snap": { + "n": 9090, + "points": 1818, + "median_in": 2.98, + "p90_in": 13.5 + }, + "span_along_true_pixel": { + "n": 9090, + "points": 1818, + "median_in": 2.87, + "p90_in": 12.74 + }, + "frame_along_true_pixel": { + "n": 13080, + "points": 2616, + "median_in": 1.66, + "p90_in": 14.48 + }, + "edge_point_to_fused_plane": { + "n": 13085, + "points": 2617, + "median_in": 2.79, + "p90_in": 13.8 + }, + "span_along_by_median_view_angle": { + "0-15": { + "n": 955, + "points": 191, + "median_in": 2.57, + "p90_in": 6.1 + }, + "15-30": { + "n": 1715, + "points": 343, + "median_in": 3.2, + "p90_in": 11.31 + }, + "30-45": { + "n": 4795, + "points": 959, + "median_in": 4.1, + "p90_in": 12.39 + }, + ">45": { + "n": 1630, + "points": 326, + "median_in": 6.28, + "p90_in": 37.39 + } + }, + "span_along_true_pixel_by_median_view_angle": { + "0-15": { + "n": 955, + "points": 191, + "median_in": 0.46, + "p90_in": 5.26 + }, + "15-30": { + "n": 1715, + "points": 343, + "median_in": 1.35, + "p90_in": 10.79 + }, + "30-45": { + "n": 4795, + "points": 959, + "median_in": 2.88, + "p90_in": 12.15 + }, + ">45": { + "n": 1625, + "points": 325, + "median_in": 6.34, + "p90_in": 46.74 + } + } + }, + "M3": { + "pairs": 12445, + "match_rate": 0.409, + "match_px": { + "n": 5089, + "median": 11.05, + "p90": 190.03 + }, + "frame_3d": { + "n": 5089, + "points": 1162, + "median_in": 9.93, + "p90_in": 102.85 + }, + "frame_along": { + "n": 5089, + "points": 1162, + "median_in": 3.47, + "p90_in": 54.49 + }, + "span_along": { + "n": 1663, + "points": 453, + "median_in": 13.14, + "p90_in": 64.34, + "p90_ci95_in": [ + 58.63, + 75.75 + ] + }, + "span_3d": { + "n": 1663, + "points": 453, + "median_in": 51.37, + "p90_in": 158.68, + "p90_ci95_in": [ + 146.22, + 169.18 + ] + }, + "anchor_matched": 1663, + "added_after_the_run_frame_3d_where_match_within_2px": { + "n": 789, + "points": 371, + "median_in": 3.51, + "p90_in": 6.76 + }, + "added_after_the_run_share_of_matches_within_2px": 0.155 + } + }, + "in_plane": { + "M2": { + "span_along": { + "n": 7260, + "points": 1452, + "median_in": 3.63, + "p90_in": 12.55, + "p90_ci95_in": [ + 11.74, + 13.77 + ] + }, + "span_3d": { + "n": 7260, + "points": 1452, + "median_in": 5.87, + "p90_in": 21.37 + }, + "frame_along": { + "n": 10140, + "points": 2028, + "median_in": 4.32, + "p90_in": 16.03 + }, + "frame_3d": { + "n": 10140, + "points": 2028, + "median_in": 6.74, + "p90_in": 20.76 + } + }, + "M2_added_after_the_run": { + "span_along_tap_no_snap": { + "n": 7260, + "points": 1452, + "median_in": 2.68, + "p90_in": 11.52 + }, + "span_along_true_pixel": { + "n": 7260, + "points": 1452, + "median_in": 2.47, + "p90_in": 11.53 + }, + "frame_along_true_pixel": { + "n": 10140, + "points": 2028, + "median_in": 1.7, + "p90_in": 12.76 + }, + "edge_point_to_fused_plane": { + "n": 10140, + "points": 2028, + "median_in": 2.26, + "p90_in": 10.01 + }, + "span_along_by_median_view_angle": { + "0-15": { + "n": 890, + "points": 178, + "median_in": 2.6, + "p90_in": 6.0 + }, + "15-30": { + "n": 1225, + "points": 245, + "median_in": 2.72, + "p90_in": 6.89 + }, + "30-45": { + "n": 3900, + "points": 780, + "median_in": 4.04, + "p90_in": 11.78 + }, + ">45": { + "n": 1245, + "points": 249, + "median_in": 5.25, + "p90_in": 26.92 + } + }, + "span_along_true_pixel_by_median_view_angle": { + "0-15": { + "n": 890, + "points": 178, + "median_in": 0.43, + "p90_in": 4.88 + }, + "15-30": { + "n": 1225, + "points": 245, + "median_in": 1.17, + "p90_in": 3.77 + }, + "30-45": { + "n": 3900, + "points": 780, + "median_in": 2.83, + "p90_in": 10.95 + }, + ">45": { + "n": 1245, + "points": 249, + "median_in": 5.23, + "p90_in": 16.1 + } + } + }, + "M3": { + "pairs": 9755, + "match_rate": 0.426, + "match_px": { + "n": 4155, + "median": 11.91, + "p90": 189.43 + }, + "frame_3d": { + "n": 4155, + "points": 945, + "median_in": 10.72, + "p90_in": 104.22 + }, + "frame_along": { + "n": 4155, + "points": 945, + "median_in": 3.7, + "p90_in": 60.1 + }, + "span_along": { + "n": 1392, + "points": 381, + "median_in": 11.88, + "p90_in": 67.45, + "p90_ci95_in": [ + 59.33, + 79.32 + ] + }, + "span_3d": { + "n": 1392, + "points": 381, + "median_in": 52.94, + "p90_in": 163.91, + "p90_ci95_in": [ + 144.62, + 172.97 + ] + }, + "anchor_matched": 1392, + "added_after_the_run_frame_3d_where_match_within_2px": { + "n": 561, + "points": 276, + "median_in": 3.51, + "p90_in": 7.34 + }, + "added_after_the_run_share_of_matches_within_2px": 0.135 + } + }, + "proud": { + "M2": { + "span_along": { + "n": 1835, + "points": 367, + "median_in": 5.98, + "p90_in": 18.22, + "p90_ci95_in": [ + 16.42, + 21.24 + ] + }, + "span_3d": { + "n": 1835, + "points": 367, + "median_in": 12.2, + "p90_in": 45.64 + }, + "frame_along": { + "n": 2945, + "points": 589, + "median_in": 4.01, + "p90_in": 26.32 + }, + "frame_3d": { + "n": 2945, + "points": 589, + "median_in": 7.98, + "p90_in": 51.54 + } + }, + "M2_added_after_the_run": { + "span_along_tap_no_snap": { + "n": 1830, + "points": 366, + "median_in": 5.53, + "p90_in": 18.13 + }, + "span_along_true_pixel": { + "n": 1830, + "points": 366, + "median_in": 5.54, + "p90_in": 17.16 + }, + "frame_along_true_pixel": { + "n": 2940, + "points": 588, + "median_in": 1.6, + "p90_in": 28.32 + }, + "edge_point_to_fused_plane": { + "n": 2945, + "points": 589, + "median_in": 5.38, + "p90_in": 30.69 + }, + "span_along_by_median_view_angle": { + "0-15": { + "n": 65, + "points": 13, + "median_in": 2.06, + "p90_in": 9.88 + }, + "15-30": { + "n": 490, + "points": 98, + "median_in": 8.12, + "p90_in": 13.94 + }, + "30-45": { + "n": 895, + "points": 179, + "median_in": 4.6, + "p90_in": 15.9 + }, + ">45": { + "n": 385, + "points": 77, + "median_in": 13.6, + "p90_in": 63.84 + } + }, + "span_along_true_pixel_by_median_view_angle": { + "0-15": { + "n": 65, + "points": 13, + "median_in": 5.07, + "p90_in": 7.76 + }, + "15-30": { + "n": 490, + "points": 98, + "median_in": 7.29, + "p90_in": 13.36 + }, + "30-45": { + "n": 895, + "points": 179, + "median_in": 4.06, + "p90_in": 15.56 + }, + ">45": { + "n": 380, + "points": 76, + "median_in": 10.0, + "p90_in": 73.08 + } + } + }, + "M3": { + "pairs": 2690, + "match_rate": 0.347, + "match_px": { + "n": 934, + "median": 4.46, + "p90": 195.18 + }, + "frame_3d": { + "n": 934, + "points": 217, + "median_in": 5.83, + "p90_in": 97.29 + }, + "frame_along": { + "n": 934, + "points": 217, + "median_in": 2.13, + "p90_in": 30.92 + }, + "span_along": { + "n": 271, + "points": 72, + "median_in": 15.65, + "p90_in": 48.62, + "p90_ci95_in": [ + 39.13, + 74.07 + ] + }, + "span_3d": { + "n": 271, + "points": 72, + "median_in": 46.4, + "p90_in": 152.3, + "p90_ci95_in": [ + 122.03, + 158.71 + ] + }, + "anchor_matched": 271, + "added_after_the_run_frame_3d_where_match_within_2px": { + "n": 228, + "points": 95, + "median_in": 3.48, + "p90_in": 5.51 + }, + "added_after_the_run_share_of_matches_within_2px": 0.244 + } + }, + "in_plane_vertical": { + "M2": { + "span_along": { + "n": 1905, + "points": 381, + "median_in": 3.36, + "p90_in": 15.22, + "p90_ci95_in": [ + 13.42, + 16.5 + ] + }, + "span_3d": { + "n": 1905, + "points": 381, + "median_in": 6.45, + "p90_in": 22.94 + }, + "frame_along": { + "n": 2645, + "points": 529, + "median_in": 3.63, + "p90_in": 15.53 + }, + "frame_3d": { + "n": 2645, + "points": 529, + "median_in": 6.65, + "p90_in": 22.96 + } + }, + "M2_added_after_the_run": { + "span_along_tap_no_snap": { + "n": 1905, + "points": 381, + "median_in": 3.17, + "p90_in": 14.45 + }, + "span_along_true_pixel": { + "n": 1905, + "points": 381, + "median_in": 2.97, + "p90_in": 12.6 + }, + "frame_along_true_pixel": { + "n": 2645, + "points": 529, + "median_in": 2.47, + "p90_in": 14.19 + }, + "edge_point_to_fused_plane": { + "n": 2645, + "points": 529, + "median_in": 2.58, + "p90_in": 13.44 + }, + "span_along_by_median_view_angle": { + "0-15": { + "n": 215, + "points": 43, + "median_in": 1.55, + "p90_in": 3.68 + }, + "15-30": { + "n": 415, + "points": 83, + "median_in": 2.97, + "p90_in": 9.15 + }, + "30-45": { + "n": 825, + "points": 165, + "median_in": 4.59, + "p90_in": 13.03 + }, + ">45": { + "n": 450, + "points": 90, + "median_in": 5.83, + "p90_in": 22.66 + } + }, + "span_along_true_pixel_by_median_view_angle": { + "0-15": { + "n": 215, + "points": 43, + "median_in": 0.99, + "p90_in": 2.93 + }, + "15-30": { + "n": 415, + "points": 83, + "median_in": 1.79, + "p90_in": 6.92 + }, + "30-45": { + "n": 825, + "points": 165, + "median_in": 3.01, + "p90_in": 13.04 + }, + ">45": { + "n": 450, + "points": 90, + "median_in": 7.77, + "p90_in": 14.15 + } + } + }, + "M3": { + "pairs": 2545, + "match_rate": 0.496, + "match_px": { + "n": 1262, + "median": 32.46, + "p90": 208.58 + }, + "frame_3d": { + "n": 1262, + "points": 271, + "median_in": 21.49, + "p90_in": 110.39 + }, + "frame_along": { + "n": 1262, + "points": 271, + "median_in": 3.85, + "p90_in": 72.17 + }, + "span_along": { + "n": 474, + "points": 115, + "median_in": 8.73, + "p90_in": 77.67, + "p90_ci95_in": [ + 53.34, + 93.21 + ] + }, + "span_3d": { + "n": 474, + "points": 115, + "median_in": 62.4, + "p90_in": 152.39, + "p90_ci95_in": [ + 129.39, + 211.36 + ] + }, + "anchor_matched": 474, + "added_after_the_run_frame_3d_where_match_within_2px": { + "n": 167, + "points": 69, + "median_in": 3.03, + "p90_in": 7.9 + }, + "added_after_the_run_share_of_matches_within_2px": 0.132 + } + } + }, + "tap_px": { + "tap_abs_du": { + "n": 12581, + "median": 6.73, + "p90": 16.47 + }, + "snap_abs_du": { + "n": 12581, + "median": 11.34, + "p90": 27.79 + }, + "snap_within_1px": 0.058, + "added_after_the_run_snap_from_true_pixel_abs_du": { + "n": 12581, + "median": 11.66, + "p90": 23.29 + }, + "added_after_the_run_vertical_in_plane_edges": { + "tap_px": { + "n": 2511, + "median": 6.6, + "p90": 16.28 + }, + "snap_px": { + "n": 2511, + "median": 8.85, + "p90": 24.59 + }, + "snap_from_true_px": { + "n": 2511, + "median": 8.8, + "p90": 21.87 + } + } + }, + "plans": { + "points_with_2_photos_in_a_group": 2641, + "photos_per_point_median": 5.0, + "points_with_a_second_photo": 2489 + } + }, + "criteria": { + "P1": { + "m0_along_p90_in_0_15": 1.35, + "m0_along_p90_in_over_30": 19.8, + "pass": true + }, + "P2": { + "m1_along_p90_in_in_plane": 6.65, + "pass": false + }, + "P3": { + "m2_span_along_p90_in_in_plane_modern": 12.55, + "pass": false + }, + "P4": { + "m3_span_3d_p90_in_modern": 158.68, + "match_rate": 0.409, + "pass": false + }, + "kill": { + "m0_along_p90_in_per_bin": [ + 1.35, + 4.59, + 8.89, + 24.36 + ], + "triggered": false + } + } + }, + "facade": { + "meta": { + "photos": 76, + "wall_planes": 2, + "edge_points_within_6m": 4734, + "edge_points_on_a_wall": 866, + "points_sampled": 866, + "in_plane": 12, + "proud": 854, + "vertical_edges": 19, + "up_vs_camera_up_deg": 10.52, + "wall_tilt_deg_max": 1.05, + "oracle_scale_median": 1.0735974441681835, + "seconds": 7 + }, + "single": { + "m0": { + "all": { + "all angles": { + "along": { + "n": 1446, + "points": 866, + "median_in": 13.89, + "p90_in": 29.41 + }, + "out": { + "n": 1446, + "points": 866, + "median_in": 7.3, + "p90_in": 12.66 + }, + "3d": { + "n": 1446, + "points": 866, + "median_in": 16.03, + "p90_in": 31.68 + }, + "missing": 0 + }, + "0-15": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 0 + }, + "15-30": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 0 + }, + "30-45": { + "along": { + "n": 19, + "points": 19, + "median_in": 4.89, + "p90_in": 17.16 + }, + "out": { + "n": 19, + "points": 19, + "median_in": 5.41, + "p90_in": 17.57 + }, + "3d": { + "n": 19, + "points": 19, + "median_in": 7.36, + "p90_in": 24.58 + }, + "missing": 0 + }, + ">45": { + "along": { + "n": 1427, + "points": 859, + "median_in": 13.97, + "p90_in": 29.65 + }, + "out": { + "n": 1427, + "points": 859, + "median_in": 7.32, + "p90_in": 12.55 + }, + "3d": { + "n": 1427, + "points": 859, + "median_in": 16.16, + "p90_in": 31.92 + }, + "missing": 0 + } + }, + "in_plane": { + "all angles": { + "along": { + "n": 49, + "points": 12, + "median_in": 15.03, + "p90_in": 31.74 + }, + "out": { + "n": 49, + "points": 12, + "median_in": 7.58, + "p90_in": 20.86 + }, + "3d": { + "n": 49, + "points": 12, + "median_in": 18.21, + "p90_in": 36.22 + }, + "missing": 0 + }, + "0-15": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 0 + }, + "15-30": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 0 + }, + "30-45": { + "along": { + "n": 8, + "points": 8, + "median_in": 5.23, + "p90_in": 18.45 + }, + "out": { + "n": 8, + "points": 8, + "median_in": 5.71, + "p90_in": 19.29 + }, + "3d": { + "n": 8, + "points": 8, + "median_in": 7.82, + "p90_in": 26.72 + }, + "missing": 0 + }, + ">45": { + "along": { + "n": 41, + "points": 11, + "median_in": 15.1, + "p90_in": 31.94 + }, + "out": { + "n": 41, + "points": 11, + "median_in": 7.63, + "p90_in": 20.72 + }, + "3d": { + "n": 41, + "points": 11, + "median_in": 18.22, + "p90_in": 36.89 + }, + "missing": 0 + } + }, + "proud": { + "all angles": { + "along": { + "n": 1397, + "points": 854, + "median_in": 13.67, + "p90_in": 29.35 + }, + "out": { + "n": 1397, + "points": 854, + "median_in": 7.2, + "p90_in": 12.5 + }, + "3d": { + "n": 1397, + "points": 854, + "median_in": 15.89, + "p90_in": 31.46 + }, + "missing": 0 + }, + "0-15": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 0 + }, + "15-30": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 0 + }, + "30-45": { + "along": { + "n": 11, + "points": 11, + "median_in": 4.66, + "p90_in": 5.43 + }, + "out": { + "n": 11, + "points": 11, + "median_in": 5.25, + "p90_in": 5.92 + }, + "3d": { + "n": 11, + "points": 11, + "median_in": 7.08, + "p90_in": 8.11 + }, + "missing": 0 + }, + ">45": { + "along": { + "n": 1386, + "points": 848, + "median_in": 13.81, + "p90_in": 29.41 + }, + "out": { + "n": 1386, + "points": 848, + "median_in": 7.25, + "p90_in": 12.5 + }, + "3d": { + "n": 1386, + "points": 848, + "median_in": 15.97, + "p90_in": 31.52 + }, + "missing": 0 + } + } + }, + "m1": { + "all": { + "all angles": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 1446 + }, + "0-15": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 0 + }, + "15-30": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 0 + }, + "30-45": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 19 + }, + ">45": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 1427 + } + }, + "in_plane": { + "all angles": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 49 + }, + "0-15": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 0 + }, + "15-30": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 0 + }, + "30-45": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 8 + }, + ">45": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 41 + } + }, + "proud": { + "all angles": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 1397 + }, + "0-15": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 0 + }, + "15-30": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 0 + }, + "30-45": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 11 + }, + ">45": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 1386 + } + } + }, + "m1a": { + "all": { + "all angles": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 1446 + }, + "0-15": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 0 + }, + "15-30": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 0 + }, + "30-45": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 19 + }, + ">45": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 1427 + } + }, + "in_plane": { + "all angles": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 49 + }, + "0-15": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 0 + }, + "15-30": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 0 + }, + "30-45": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 8 + }, + ">45": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 41 + } + }, + "proud": { + "all angles": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 1397 + }, + "0-15": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 0 + }, + "15-30": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 0 + }, + "30-45": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 11 + }, + ">45": { + "along": { + "n": 0, + "points": 0 + }, + "out": { + "n": 0, + "points": 0 + }, + "3d": { + "n": 0, + "points": 0 + }, + "missing": 1386 + } + } + }, + "m1a_yaw": { + "observations": 0 + } + }, + "multi": { + "exact": { + "all": { + "M2": { + "span_along": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "span_3d": { + "n": 0, + "points": 0 + }, + "frame_along": { + "n": 0, + "points": 0 + }, + "frame_3d": { + "n": 0, + "points": 0 + } + }, + "M2_added_after_the_run": { + "span_along_tap_no_snap": { + "n": 0, + "points": 0 + }, + "span_along_true_pixel": { + "n": 0, + "points": 0 + }, + "frame_along_true_pixel": { + "n": 0, + "points": 0 + }, + "edge_point_to_fused_plane": { + "n": 0, + "points": 0 + }, + "span_along_by_median_view_angle": { + "0-15": { + "n": 0, + "points": 0 + }, + "15-30": { + "n": 0, + "points": 0 + }, + "30-45": { + "n": 0, + "points": 0 + }, + ">45": { + "n": 0, + "points": 0 + } + }, + "span_along_true_pixel_by_median_view_angle": { + "0-15": { + "n": 0, + "points": 0 + }, + "15-30": { + "n": 0, + "points": 0 + }, + "30-45": { + "n": 0, + "points": 0 + }, + ">45": { + "n": 0, + "points": 0 + } + } + }, + "M3": { + "pairs": 603, + "match_rate": 0.138, + "match_px": { + "n": 83, + "median": 1.42, + "p90": 28.68 + }, + "frame_3d": { + "n": 83, + "points": 83, + "median_in": 1.43, + "p90_in": 29.45 + }, + "frame_along": { + "n": 83, + "points": 83, + "median_in": 1.2, + "p90_in": 22.06 + }, + "span_along": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "span_3d": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "anchor_matched": 0, + "added_after_the_run_frame_3d_where_match_within_2px": { + "n": 51, + "points": 51, + "median_in": 0.78, + "p90_in": 1.76 + }, + "added_after_the_run_share_of_matches_within_2px": 0.614 + } + }, + "in_plane": { + "M2": { + "span_along": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "span_3d": { + "n": 0, + "points": 0 + }, + "frame_along": { + "n": 0, + "points": 0 + }, + "frame_3d": { + "n": 0, + "points": 0 + } + }, + "M2_added_after_the_run": { + "span_along_tap_no_snap": { + "n": 0, + "points": 0 + }, + "span_along_true_pixel": { + "n": 0, + "points": 0 + }, + "frame_along_true_pixel": { + "n": 0, + "points": 0 + }, + "edge_point_to_fused_plane": { + "n": 0, + "points": 0 + }, + "span_along_by_median_view_angle": { + "0-15": { + "n": 0, + "points": 0 + }, + "15-30": { + "n": 0, + "points": 0 + }, + "30-45": { + "n": 0, + "points": 0 + }, + ">45": { + "n": 0, + "points": 0 + } + }, + "span_along_true_pixel_by_median_view_angle": { + "0-15": { + "n": 0, + "points": 0 + }, + "15-30": { + "n": 0, + "points": 0 + }, + "30-45": { + "n": 0, + "points": 0 + }, + ">45": { + "n": 0, + "points": 0 + } + } + }, + "M3": { + "pairs": 12, + "match_rate": 0.0, + "match_px": { + "n": 0 + }, + "frame_3d": { + "n": 0, + "points": 0 + }, + "frame_along": { + "n": 0, + "points": 0 + }, + "span_along": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "span_3d": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "anchor_matched": 0, + "added_after_the_run_frame_3d_where_match_within_2px": { + "n": 0, + "points": 0 + }, + "added_after_the_run_share_of_matches_within_2px": NaN + } + }, + "proud": { + "M2": { + "span_along": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "span_3d": { + "n": 0, + "points": 0 + }, + "frame_along": { + "n": 0, + "points": 0 + }, + "frame_3d": { + "n": 0, + "points": 0 + } + }, + "M2_added_after_the_run": { + "span_along_tap_no_snap": { + "n": 0, + "points": 0 + }, + "span_along_true_pixel": { + "n": 0, + "points": 0 + }, + "frame_along_true_pixel": { + "n": 0, + "points": 0 + }, + "edge_point_to_fused_plane": { + "n": 0, + "points": 0 + }, + "span_along_by_median_view_angle": { + "0-15": { + "n": 0, + "points": 0 + }, + "15-30": { + "n": 0, + "points": 0 + }, + "30-45": { + "n": 0, + "points": 0 + }, + ">45": { + "n": 0, + "points": 0 + } + }, + "span_along_true_pixel_by_median_view_angle": { + "0-15": { + "n": 0, + "points": 0 + }, + "15-30": { + "n": 0, + "points": 0 + }, + "30-45": { + "n": 0, + "points": 0 + }, + ">45": { + "n": 0, + "points": 0 + } + } + }, + "M3": { + "pairs": 591, + "match_rate": 0.14, + "match_px": { + "n": 83, + "median": 1.42, + "p90": 28.68 + }, + "frame_3d": { + "n": 83, + "points": 83, + "median_in": 1.43, + "p90_in": 29.45 + }, + "frame_along": { + "n": 83, + "points": 83, + "median_in": 1.2, + "p90_in": 22.06 + }, + "span_along": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "span_3d": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "anchor_matched": 0, + "added_after_the_run_frame_3d_where_match_within_2px": { + "n": 51, + "points": 51, + "median_in": 0.78, + "p90_in": 1.76 + }, + "added_after_the_run_share_of_matches_within_2px": 0.614 + } + }, + "in_plane_vertical": { + "M2": { + "span_along": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "span_3d": { + "n": 0, + "points": 0 + }, + "frame_along": { + "n": 0, + "points": 0 + }, + "frame_3d": { + "n": 0, + "points": 0 + } + }, + "M2_added_after_the_run": { + "span_along_tap_no_snap": { + "n": 0, + "points": 0 + }, + "span_along_true_pixel": { + "n": 0, + "points": 0 + }, + "frame_along_true_pixel": { + "n": 0, + "points": 0 + }, + "edge_point_to_fused_plane": { + "n": 0, + "points": 0 + }, + "span_along_by_median_view_angle": { + "0-15": { + "n": 0, + "points": 0 + }, + "15-30": { + "n": 0, + "points": 0 + }, + "30-45": { + "n": 0, + "points": 0 + }, + ">45": { + "n": 0, + "points": 0 + } + }, + "span_along_true_pixel_by_median_view_angle": { + "0-15": { + "n": 0, + "points": 0 + }, + "15-30": { + "n": 0, + "points": 0 + }, + "30-45": { + "n": 0, + "points": 0 + }, + ">45": { + "n": 0, + "points": 0 + } + } + }, + "M3": { + "pairs": 0, + "match_rate": 0.0, + "match_px": { + "n": 0 + }, + "frame_3d": { + "n": 0, + "points": 0 + }, + "frame_along": { + "n": 0, + "points": 0 + }, + "span_along": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "span_3d": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "anchor_matched": 0, + "added_after_the_run_frame_3d_where_match_within_2px": { + "n": 0, + "points": 0 + }, + "added_after_the_run_share_of_matches_within_2px": NaN + } + } + }, + "modern_assumed": { + "all": { + "M2": { + "span_along": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "span_3d": { + "n": 0, + "points": 0 + }, + "frame_along": { + "n": 0, + "points": 0 + }, + "frame_3d": { + "n": 0, + "points": 0 + } + }, + "M2_added_after_the_run": { + "span_along_tap_no_snap": { + "n": 0, + "points": 0 + }, + "span_along_true_pixel": { + "n": 0, + "points": 0 + }, + "frame_along_true_pixel": { + "n": 0, + "points": 0 + }, + "edge_point_to_fused_plane": { + "n": 0, + "points": 0 + }, + "span_along_by_median_view_angle": { + "0-15": { + "n": 0, + "points": 0 + }, + "15-30": { + "n": 0, + "points": 0 + }, + "30-45": { + "n": 0, + "points": 0 + }, + ">45": { + "n": 0, + "points": 0 + } + }, + "span_along_true_pixel_by_median_view_angle": { + "0-15": { + "n": 0, + "points": 0 + }, + "15-30": { + "n": 0, + "points": 0 + }, + "30-45": { + "n": 0, + "points": 0 + }, + ">45": { + "n": 0, + "points": 0 + } + } + }, + "M3": { + "pairs": 3015, + "match_rate": 0.102, + "match_px": { + "n": 308, + "median": 5.12, + "p90": 35.04 + }, + "frame_3d": { + "n": 308, + "points": 105, + "median_in": 6.22, + "p90_in": 33.48 + }, + "frame_along": { + "n": 308, + "points": 105, + "median_in": 4.91, + "p90_in": 25.57 + }, + "span_along": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "span_3d": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "anchor_matched": 0, + "added_after_the_run_frame_3d_where_match_within_2px": { + "n": 62, + "points": 42, + "median_in": 3.71, + "p90_in": 11.14 + }, + "added_after_the_run_share_of_matches_within_2px": 0.201 + } + }, + "in_plane": { + "M2": { + "span_along": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "span_3d": { + "n": 0, + "points": 0 + }, + "frame_along": { + "n": 0, + "points": 0 + }, + "frame_3d": { + "n": 0, + "points": 0 + } + }, + "M2_added_after_the_run": { + "span_along_tap_no_snap": { + "n": 0, + "points": 0 + }, + "span_along_true_pixel": { + "n": 0, + "points": 0 + }, + "frame_along_true_pixel": { + "n": 0, + "points": 0 + }, + "edge_point_to_fused_plane": { + "n": 0, + "points": 0 + }, + "span_along_by_median_view_angle": { + "0-15": { + "n": 0, + "points": 0 + }, + "15-30": { + "n": 0, + "points": 0 + }, + "30-45": { + "n": 0, + "points": 0 + }, + ">45": { + "n": 0, + "points": 0 + } + }, + "span_along_true_pixel_by_median_view_angle": { + "0-15": { + "n": 0, + "points": 0 + }, + "15-30": { + "n": 0, + "points": 0 + }, + "30-45": { + "n": 0, + "points": 0 + }, + ">45": { + "n": 0, + "points": 0 + } + } + }, + "M3": { + "pairs": 60, + "match_rate": 0.0, + "match_px": { + "n": 0 + }, + "frame_3d": { + "n": 0, + "points": 0 + }, + "frame_along": { + "n": 0, + "points": 0 + }, + "span_along": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "span_3d": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "anchor_matched": 0, + "added_after_the_run_frame_3d_where_match_within_2px": { + "n": 0, + "points": 0 + }, + "added_after_the_run_share_of_matches_within_2px": NaN + } + }, + "proud": { + "M2": { + "span_along": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "span_3d": { + "n": 0, + "points": 0 + }, + "frame_along": { + "n": 0, + "points": 0 + }, + "frame_3d": { + "n": 0, + "points": 0 + } + }, + "M2_added_after_the_run": { + "span_along_tap_no_snap": { + "n": 0, + "points": 0 + }, + "span_along_true_pixel": { + "n": 0, + "points": 0 + }, + "frame_along_true_pixel": { + "n": 0, + "points": 0 + }, + "edge_point_to_fused_plane": { + "n": 0, + "points": 0 + }, + "span_along_by_median_view_angle": { + "0-15": { + "n": 0, + "points": 0 + }, + "15-30": { + "n": 0, + "points": 0 + }, + "30-45": { + "n": 0, + "points": 0 + }, + ">45": { + "n": 0, + "points": 0 + } + }, + "span_along_true_pixel_by_median_view_angle": { + "0-15": { + "n": 0, + "points": 0 + }, + "15-30": { + "n": 0, + "points": 0 + }, + "30-45": { + "n": 0, + "points": 0 + }, + ">45": { + "n": 0, + "points": 0 + } + } + }, + "M3": { + "pairs": 2955, + "match_rate": 0.104, + "match_px": { + "n": 308, + "median": 5.12, + "p90": 35.04 + }, + "frame_3d": { + "n": 308, + "points": 105, + "median_in": 6.22, + "p90_in": 33.48 + }, + "frame_along": { + "n": 308, + "points": 105, + "median_in": 4.91, + "p90_in": 25.57 + }, + "span_along": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "span_3d": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "anchor_matched": 0, + "added_after_the_run_frame_3d_where_match_within_2px": { + "n": 62, + "points": 42, + "median_in": 3.71, + "p90_in": 11.14 + }, + "added_after_the_run_share_of_matches_within_2px": 0.201 + } + }, + "in_plane_vertical": { + "M2": { + "span_along": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "span_3d": { + "n": 0, + "points": 0 + }, + "frame_along": { + "n": 0, + "points": 0 + }, + "frame_3d": { + "n": 0, + "points": 0 + } + }, + "M2_added_after_the_run": { + "span_along_tap_no_snap": { + "n": 0, + "points": 0 + }, + "span_along_true_pixel": { + "n": 0, + "points": 0 + }, + "frame_along_true_pixel": { + "n": 0, + "points": 0 + }, + "edge_point_to_fused_plane": { + "n": 0, + "points": 0 + }, + "span_along_by_median_view_angle": { + "0-15": { + "n": 0, + "points": 0 + }, + "15-30": { + "n": 0, + "points": 0 + }, + "30-45": { + "n": 0, + "points": 0 + }, + ">45": { + "n": 0, + "points": 0 + } + }, + "span_along_true_pixel_by_median_view_angle": { + "0-15": { + "n": 0, + "points": 0 + }, + "15-30": { + "n": 0, + "points": 0 + }, + "30-45": { + "n": 0, + "points": 0 + }, + ">45": { + "n": 0, + "points": 0 + } + } + }, + "M3": { + "pairs": 0, + "match_rate": 0.0, + "match_px": { + "n": 0 + }, + "frame_3d": { + "n": 0, + "points": 0 + }, + "frame_along": { + "n": 0, + "points": 0 + }, + "span_along": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "span_3d": { + "n": 0, + "points": 0, + "p90_ci95_in": null + }, + "anchor_matched": 0, + "added_after_the_run_frame_3d_where_match_within_2px": { + "n": 0, + "points": 0 + }, + "added_after_the_run_share_of_matches_within_2px": NaN + } + } + }, + "tap_px": { + "tap_abs_du": { + "n": 3030, + "median": 6.75, + "p90": 16.18 + }, + "snap_abs_du": { + "n": 3030, + "median": 12.78, + "p90": 28.05 + }, + "snap_within_1px": 0.037, + "added_after_the_run_snap_from_true_pixel_abs_du": { + "n": 3030, + "median": 13.12, + "p90": 23.39 + }, + "added_after_the_run_vertical_in_plane_edges": { + "tap_px": { + "n": 0 + }, + "snap_px": { + "n": 0 + }, + "snap_from_true_px": { + "n": 0 + } + } + }, + "plans": { + "points_with_2_photos_in_a_group": 606, + "photos_per_point_median": 5.0, + "points_with_a_second_photo": 603 + } + }, + "criteria": { + "P1": { + "m0_along_p90_in_0_15": NaN, + "m0_along_p90_in_over_30": 29.41, + "pass": false + }, + "P2": { + "m1_along_p90_in_in_plane": NaN, + "pass": false + }, + "P3": { + "m2_span_along_p90_in_in_plane_modern": NaN, + "pass": false + }, + "P4": { + "m3_span_3d_p90_in_modern": NaN, + "match_rate": 0.102, + "pass": false + }, + "kill": { + "m0_along_p90_in_per_bin": [ + NaN, + NaN, + 17.16, + 29.65 + ], + "triggered": false + } + } + } + } +} diff --git a/experiments/edge-geometry/results/edge_geometry_widest_pair.md b/experiments/edge-geometry/results/edge_geometry_widest_pair.md new file mode 100644 index 000000000..3ad019172 --- /dev/null +++ b/experiments/edge-geometry/results/edge_geometry_widest_pair.md @@ -0,0 +1,231 @@ +# Edge geometry (generated by `uv run python run.py`) + +Evals harness commit `190ed339f41924ae0244e0b88c5ebe647ddd9eb4`. Errors in inches, median / p90 (points). M3 pairs each point's reference photo with the widest qualifying ray angle. + +## electro + +`{"photos": 45, "wall_planes": 21, "edge_points_within_6m": 27436, "edge_points_on_a_wall": 25492, "points_sampled": 3000, "in_plane": 2258, "proud": 742, "vertical_edges": 730, "up_vs_camera_up_deg": 5.21, "wall_tilt_deg_max": 3.14, "oracle_scale_median": 0.9961415148011799, "seconds": 53}` + +### M0, depth at the pixel + +| Edges | View angle | Along the wall | Out of plane | 3-D | +| --- | --- | --- | --- | --- | +| all | all angles | 2.1 / 15.6 (2994) | 3.1 / 19.2 (2994) | 5.7 / 38.6 (2994) | +| all | 0-15 | 0.2 / 1.4 (345) | 4.8 / 14.8 (345) | 4.9 / 14.9 (345) | +| all | 15-30 | 0.7 / 4.6 (912) | 3.8 / 15.3 (912) | 4.2 / 17.0 (912) | +| all | 30-45 | 2.0 / 8.9 (1430) | 4.2 / 20.4 (1430) | 5.2 / 24.9 (1430) | +| all | >45 | 5.1 / 24.4 (1593) | 1.7 / 23.7 (1593) | 7.5 / 66.2 (1593) | +| in_plane | all angles | 2.0 / 14.6 (2258) | 2.8 / 16.2 (2258) | 5.3 / 31.7 (2258) | +| in_plane | 0-15 | 0.2 / 0.9 (281) | 4.8 / 14.6 (281) | 4.9 / 14.8 (281) | +| in_plane | 15-30 | 0.7 / 4.5 (708) | 3.4 / 16.9 (708) | 3.6 / 18.2 (708) | +| in_plane | 30-45 | 1.9 / 7.8 (1077) | 3.9 / 15.9 (1077) | 4.9 / 19.9 (1077) | +| in_plane | >45 | 4.8 / 24.1 (1211) | 1.5 / 17.7 (1211) | 6.6 / 62.1 (1211) | +| proud | all angles | 2.5 / 18.0 (736) | 4.2 / 25.0 (736) | 7.3 / 57.3 (736) | +| proud | 0-15 | 0.3 / 2.7 (64) | 4.8 / 16.2 (64) | 4.9 / 16.4 (64) | +| proud | 15-30 | 0.9 / 4.8 (204) | 4.2 / 12.7 (204) | 4.6 / 14.1 (204) | +| proud | 30-45 | 2.4 / 12.1 (353) | 5.4 / 24.8 (353) | 6.9 / 30.2 (353) | +| proud | >45 | 6.8 / 27.1 (382) | 3.1 / 48.1 (382) | 19.9 / 116.2 (382) | + +### M1, ray to this photo's wall plane + +| Edges | View angle | Along the wall | Out of plane | 3-D | +| --- | --- | --- | --- | --- | +| all | all angles | 1.3 / 7.2 (2712) | 3.1 / 10.5 (2712) | 4.2 / 15.1 (2712) | +| all | 0-15 | 0.2 / 0.6 (342) | 4.2 / 11.4 (342) | 4.2 / 11.6 (342) | +| all | 15-30 | 0.5 / 2.8 (898) | 2.9 / 10.3 (898) | 3.2 / 11.1 (898) | +| all | 30-45 | 1.6 / 6.3 (1376) | 3.3 / 8.9 (1376) | 4.2 / 11.3 (1376) | +| all | >45 | 3.5 / 21.7 (1219) | 2.6 / 16.3 (1219) | 5.3 / 45.0 (1219) | +| in_plane | all angles | 1.1 / 6.7 (2097) | 2.7 / 9.2 (2097) | 3.8 / 12.0 (2097) | +| in_plane | 0-15 | 0.1 / 0.5 (278) | 4.2 / 11.5 (278) | 4.3 / 11.7 (278) | +| in_plane | 15-30 | 0.4 / 2.3 (699) | 2.4 / 9.8 (699) | 2.7 / 10.7 (699) | +| in_plane | 30-45 | 1.5 / 5.8 (1044) | 3.0 / 7.6 (1044) | 3.7 / 10.1 (1044) | +| in_plane | >45 | 2.9 / 15.0 (998) | 1.9 / 8.9 (998) | 4.5 / 36.2 (998) | +| proud | all angles | 2.0 / 14.5 (615) | 5.0 / 19.0 (615) | 6.3 / 27.5 (615) | +| proud | 0-15 | 0.3 / 1.1 (64) | 3.8 / 7.2 (64) | 3.9 / 7.3 (64) | +| proud | 15-30 | 1.0 / 3.5 (199) | 4.6 / 11.8 (199) | 5.0 / 12.8 (199) | +| proud | 30-45 | 2.2 / 8.7 (332) | 4.5 / 15.9 (332) | 5.8 / 20.7 (332) | +| proud | >45 | 12.1 / 31.9 (221) | 9.6 / 29.5 (221) | 19.2 / 77.7 (221) | + +### M1-anchor, ray to a 0.3 m patch plane 1-3 m away + +| Edges | View angle | Along the wall | Out of plane | 3-D | +| --- | --- | --- | --- | --- | +| all | all angles | 2.0 / 12.2 (1707) | 3.8 / 17.9 (1707) | 5.1 / 27.2 (1707) | +| all | 0-15 | 0.1 / 0.6 (199) | 3.7 / 12.7 (199) | 3.8 / 12.8 (199) | +| all | 15-30 | 1.1 / 5.5 (592) | 3.9 / 14.0 (592) | 4.3 / 15.7 (592) | +| all | 30-45 | 2.2 / 10.5 (924) | 4.3 / 19.7 (924) | 5.4 / 26.7 (924) | +| all | >45 | 5.1 / 27.1 (633) | 3.0 / 18.8 (633) | 6.6 / 36.8 (633) | +| in_plane | all angles | 1.9 / 10.4 (1360) | 3.4 / 13.6 (1360) | 4.7 / 21.0 (1360) | +| in_plane | 0-15 | 0.1 / 0.5 (181) | 3.5 / 11.8 (181) | 3.5 / 12.0 (181) | +| in_plane | 15-30 | 1.0 / 5.8 (478) | 3.5 / 13.7 (478) | 3.9 / 15.3 (478) | +| in_plane | 30-45 | 2.2 / 8.6 (727) | 4.1 / 16.1 (727) | 5.0 / 19.5 (727) | +| in_plane | >45 | 4.5 / 27.7 (526) | 2.8 / 12.2 (526) | 5.7 / 36.5 (526) | +| proud | all angles | 2.5 / 15.6 (347) | 6.3 / 42.4 (347) | 8.0 / 64.1 (347) | +| proud | 0-15 | 2.1 / 154.1 (18) | 12.7 / 3149.7 (18) | 13.0 / 3164.1 (18) | +| proud | 15-30 | 1.7 / 5.1 (114) | 7.1 / 14.6 (114) | 7.8 / 16.4 (114) | +| proud | 30-45 | 2.0 / 15.7 (197) | 4.7 / 63.5 (197) | 5.8 / 79.7 (197) | +| proud | >45 | 9.5 / 17.9 (107) | 7.4 / 41.0 (107) | 14.5 / 63.9 (107) | + +### M1-anchor yaw + +`{"observations": 3411, "abs_psi_deg": {"median": 2.15, "p90": 6.72}, "abs_s_m_median": 2.0, "all_added_after_the_run": {"sign_agreement_where_prediction_over_1cm": 0.7, "median_error_over_prediction_where_over_1cm": 0.73, "view_angle_under_60": {"n": 3080, "slope": 0.36, "r2": 0.01}, "spearman_abs_error_vs_tan_angle_alone": 0.5}, "all": {"n": 3411, "signed_first_order_slope": 0.05, "signed_first_order_r2": 0.02, "signed_r2_with_slope_1": -4.58, "abs_error_over_s_psi_tan_median": 0.94, "abs_error_over_s_psi_tan_n": 2811, "spearman_abs_error_vs_s_psi_tan": 0.63, "abs_error_p90_in": 12.18, "residual_after_first_order_p90_in": 10.9}, "in_plane_added_after_the_run": {"sign_agreement_where_prediction_over_1cm": 0.74, "median_error_over_prediction_where_over_1cm": 0.83, "view_angle_under_60": {"n": 2455, "slope": 0.29, "r2": 0.02}, "spearman_abs_error_vs_tan_angle_alone": 0.54}, "in_plane": {"n": 2755, "signed_first_order_slope": 0.05, "signed_first_order_r2": 0.02, "signed_r2_with_slope_1": -12.61, "abs_error_over_s_psi_tan_median": 0.94, "abs_error_over_s_psi_tan_n": 2231, "spearman_abs_error_vs_s_psi_tan": 0.67, "abs_error_p90_in": 10.43, "residual_after_first_order_p90_in": 9.41}}` + +### M2 and M3 + +| Poses | Edges | M2 span along | M2 p90 95% CI | M3 pairs | M3 match | M3 match px | M3 span along | M3 span 3-D | M3 3-D p90 95% CI | M3 frame 3-D | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| exact | all | 4.0 / 14.5 (1819) | [13.85, 15.72] | 2489 | 44% | 6.05 / 186.45 | 9.3 / 59.5 (439) | 43.9 / 156.3 (439) | [134.29, 161.35] | 4.9 / 101.8 (1097) | +| exact | in_plane | 3.7 / 13.0 (1452) | [12.06, 14.18] | 1951 | 46% | 6.44 / 185.54 | 9.2 / 61.2 (368) | 47.1 / 157.5 (368) | [132.58, 162.28] | 5.2 / 102.7 (903) | +| exact | proud | 5.5 / 18.1 (367) | [17.13, 20.83] | 538 | 36% | 2.77 / 191.71 | 10.2 / 41.8 (71) | 37.4 / 136.1 (71) | [120.08, 158.91] | 2.2 / 97.5 (194) | +| exact | in_plane_vertical | 3.2 / 15.4 (381) | [13.66, 16.83] | 509 | 50% | 24.35 / 200.48 | 7.0 / 64.4 (106) | 67.4 / 154.5 (106) | [109.0, 212.49] | 20.7 / 109.0 (257) | +| modern_assumed | all | 3.9 / 14.3 (1819) | [13.42, 15.32] | 12445 | 41% | 11.05 / 190.03 | 13.1 / 64.3 (453) | 51.4 / 158.7 (453) | [146.22, 169.18] | 9.9 / 102.8 (1162) | +| modern_assumed | in_plane | 3.6 / 12.6 (1452) | [11.74, 13.77] | 9755 | 43% | 11.91 / 189.43 | 11.9 / 67.5 (381) | 52.9 / 163.9 (381) | [144.62, 172.97] | 10.7 / 104.2 (945) | +| modern_assumed | proud | 6.0 / 18.2 (367) | [16.42, 21.24] | 2690 | 35% | 4.46 / 195.18 | 15.7 / 48.6 (72) | 46.4 / 152.3 (72) | [122.03, 158.71] | 5.8 / 97.3 (217) | +| modern_assumed | in_plane_vertical | 3.4 / 15.2 (381) | [13.42, 16.5] | 2545 | 50% | 32.46 / 208.58 | 8.7 / 77.7 (115) | 62.4 / 152.4 (115) | [129.39, 211.36] | 21.5 / 110.4 (271) | + +M3 match px is in pixels, not inches. + +### Added after the run + +| Poses | Edges | M2 with the true pixel | M2 tap, no snap | Edge point to fused MoGe-2 plane | M3 3-D where the match is within 2 px | Matches within 2 px | +| --- | --- | --- | --- | --- | --- | --- | +| exact | all | 2.5 / 12.6 (1819) | 2.8 / 13.0 (1819) | 2.7 / 13.9 (2617) | 0.6 / 1.8 (381) | 0.347 | +| exact | in_plane | 2.1 / 10.9 (1452) | 2.5 / 10.9 (1452) | 2.3 / 10.0 (2028) | 0.6 / 2.0 (296) | 0.328 | +| exact | proud | 5.4 / 17.0 (367) | 4.9 / 18.0 (367) | 4.9 / 31.6 (589) | 0.5 / 1.6 (85) | 0.438 | +| exact | in_plane_vertical | 2.6 / 11.4 (381) | 2.9 / 11.9 (381) | 2.6 / 13.3 (529) | 0.8 / 2.3 (67) | 0.261 | +| modern_assumed | all | 2.9 / 12.7 (1818) | 3.0 / 13.5 (1818) | 2.8 / 13.8 (2617) | 3.5 / 6.8 (371) | 0.155 | +| modern_assumed | in_plane | 2.5 / 11.5 (1452) | 2.7 / 11.5 (1452) | 2.3 / 10.0 (2028) | 3.5 / 7.3 (276) | 0.135 | +| modern_assumed | proud | 5.5 / 17.2 (366) | 5.5 / 18.1 (366) | 5.4 / 30.7 (589) | 3.5 / 5.5 (95) | 0.244 | +| modern_assumed | in_plane_vertical | 3.0 / 12.6 (381) | 3.2 / 14.4 (381) | 2.6 / 13.4 (529) | 3.0 / 7.9 (69) | 0.132 | + +M2 along-wall span error by the point's median view angle (snap, then true pixel): + +- exact, in-plane, snapped: 0-15 2.3 / 5.6 (178), 15-30 2.7 / 6.8 (245), 30-45 4.2 / 11.6 (780), >45 6.1 / 29.1 (249) +- exact, in-plane, true pixel: 0-15 0.1 / 4.3 (178), 15-30 1.1 / 3.6 (245), 30-45 2.5 / 10.5 (780), >45 5.1 / 13.8 (249) +- modern_assumed, in-plane, snapped: 0-15 2.6 / 6.0 (178), 15-30 2.7 / 6.9 (245), 30-45 4.0 / 11.8 (780), >45 5.2 / 26.9 (249) +- modern_assumed, in-plane, true pixel: 0-15 0.4 / 4.9 (178), 15-30 1.2 / 3.8 (245), 30-45 2.8 / 10.9 (780), >45 5.2 / 16.1 (249) + +Edge point to plane (single photo): `{"laser_plane_in_plane_edges": {"n": 6076, "points": 2258, "median_in": 0.4, "p90_in": 1.38}, "m1_plane_in_plane_edges": {"n": 4920, "points": 2097, "median_in": 2.54, "p90_in": 9.81}}` + + +Tap and snap, horizontal pixel error: `{"tap_abs_du": {"n": 12581, "median": 6.73, "p90": 16.47}, "snap_abs_du": {"n": 12581, "median": 11.34, "p90": 27.79}, "snap_within_1px": 0.058, "added_after_the_run_snap_from_true_pixel_abs_du": {"n": 12581, "median": 11.66, "p90": 23.29}, "added_after_the_run_vertical_in_plane_edges": {"tap_px": {"n": 2511, "median": 6.6, "p90": 16.28}, "snap_px": {"n": 2511, "median": 8.85, "p90": 24.59}, "snap_from_true_px": {"n": 2511, "median": 8.8, "p90": 21.87}}}` + +`{"points_with_2_photos_in_a_group": 2641, "photos_per_point_median": 5.0, "points_with_a_second_photo": 2489}` + +### Criteria + +`{"P1": {"m0_along_p90_in_0_15": 1.35, "m0_along_p90_in_over_30": 19.8, "pass": true}, "P2": {"m1_along_p90_in_in_plane": 6.65, "pass": false}, "P3": {"m2_span_along_p90_in_in_plane_modern": 12.55, "pass": false}, "P4": {"m3_span_3d_p90_in_modern": 158.68, "match_rate": 0.409, "pass": false}, "kill": {"m0_along_p90_in_per_bin": [1.35, 4.59, 8.89, 24.36], "triggered": false}}` + +## facade + +`{"photos": 76, "wall_planes": 2, "edge_points_within_6m": 4734, "edge_points_on_a_wall": 866, "points_sampled": 866, "in_plane": 12, "proud": 854, "vertical_edges": 19, "up_vs_camera_up_deg": 10.52, "wall_tilt_deg_max": 1.05, "oracle_scale_median": 1.0735974441681835, "seconds": 7}` + +### M0, depth at the pixel + +| Edges | View angle | Along the wall | Out of plane | 3-D | +| --- | --- | --- | --- | --- | +| all | all angles | 13.9 / 29.4 (866) | 7.3 / 12.7 (866) | 16.0 / 31.7 (866) | +| all | 0-15 | n/a | n/a | n/a | +| all | 15-30 | n/a | n/a | n/a | +| all | 30-45 | 4.9 / 17.2 (19) | 5.4 / 17.6 (19) | 7.4 / 24.6 (19) | +| all | >45 | 14.0 / 29.6 (859) | 7.3 / 12.6 (859) | 16.2 / 31.9 (859) | +| in_plane | all angles | 15.0 / 31.7 (12) | 7.6 / 20.9 (12) | 18.2 / 36.2 (12) | +| in_plane | 0-15 | n/a | n/a | n/a | +| in_plane | 15-30 | n/a | n/a | n/a | +| in_plane | 30-45 | 5.2 / 18.4 (8) | 5.7 / 19.3 (8) | 7.8 / 26.7 (8) | +| in_plane | >45 | 15.1 / 31.9 (11) | 7.6 / 20.7 (11) | 18.2 / 36.9 (11) | +| proud | all angles | 13.7 / 29.4 (854) | 7.2 / 12.5 (854) | 15.9 / 31.5 (854) | +| proud | 0-15 | n/a | n/a | n/a | +| proud | 15-30 | n/a | n/a | n/a | +| proud | 30-45 | 4.7 / 5.4 (11) | 5.2 / 5.9 (11) | 7.1 / 8.1 (11) | +| proud | >45 | 13.8 / 29.4 (848) | 7.2 / 12.5 (848) | 16.0 / 31.5 (848) | + +### M1, ray to this photo's wall plane + +| Edges | View angle | Along the wall | Out of plane | 3-D | +| --- | --- | --- | --- | --- | +| all | all angles | n/a | n/a | n/a | +| all | 0-15 | n/a | n/a | n/a | +| all | 15-30 | n/a | n/a | n/a | +| all | 30-45 | n/a | n/a | n/a | +| all | >45 | n/a | n/a | n/a | +| in_plane | all angles | n/a | n/a | n/a | +| in_plane | 0-15 | n/a | n/a | n/a | +| in_plane | 15-30 | n/a | n/a | n/a | +| in_plane | 30-45 | n/a | n/a | n/a | +| in_plane | >45 | n/a | n/a | n/a | +| proud | all angles | n/a | n/a | n/a | +| proud | 0-15 | n/a | n/a | n/a | +| proud | 15-30 | n/a | n/a | n/a | +| proud | 30-45 | n/a | n/a | n/a | +| proud | >45 | n/a | n/a | n/a | + +### M1-anchor, ray to a 0.3 m patch plane 1-3 m away + +| Edges | View angle | Along the wall | Out of plane | 3-D | +| --- | --- | --- | --- | --- | +| all | all angles | n/a | n/a | n/a | +| all | 0-15 | n/a | n/a | n/a | +| all | 15-30 | n/a | n/a | n/a | +| all | 30-45 | n/a | n/a | n/a | +| all | >45 | n/a | n/a | n/a | +| in_plane | all angles | n/a | n/a | n/a | +| in_plane | 0-15 | n/a | n/a | n/a | +| in_plane | 15-30 | n/a | n/a | n/a | +| in_plane | 30-45 | n/a | n/a | n/a | +| in_plane | >45 | n/a | n/a | n/a | +| proud | all angles | n/a | n/a | n/a | +| proud | 0-15 | n/a | n/a | n/a | +| proud | 15-30 | n/a | n/a | n/a | +| proud | 30-45 | n/a | n/a | n/a | +| proud | >45 | n/a | n/a | n/a | + +### M1-anchor yaw + +`{"observations": 0}` + +### M2 and M3 + +| Poses | Edges | M2 span along | M2 p90 95% CI | M3 pairs | M3 match | M3 match px | M3 span along | M3 span 3-D | M3 3-D p90 95% CI | M3 frame 3-D | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| exact | all | n/a | None | 603 | 14% | 1.42 / 28.68 | n/a | n/a | None | 1.4 / 29.4 (83) | +| exact | in_plane | n/a | None | 12 | 0% | n/a / n/a | n/a | n/a | None | n/a | +| exact | proud | n/a | None | 591 | 14% | 1.42 / 28.68 | n/a | n/a | None | 1.4 / 29.4 (83) | +| exact | in_plane_vertical | n/a | None | 0 | 0% | n/a / n/a | n/a | n/a | None | n/a | +| modern_assumed | all | n/a | None | 3015 | 10% | 5.12 / 35.04 | n/a | n/a | None | 6.2 / 33.5 (105) | +| modern_assumed | in_plane | n/a | None | 60 | 0% | n/a / n/a | n/a | n/a | None | n/a | +| modern_assumed | proud | n/a | None | 2955 | 10% | 5.12 / 35.04 | n/a | n/a | None | 6.2 / 33.5 (105) | +| modern_assumed | in_plane_vertical | n/a | None | 0 | 0% | n/a / n/a | n/a | n/a | None | n/a | + +M3 match px is in pixels, not inches. + +### Added after the run + +| Poses | Edges | M2 with the true pixel | M2 tap, no snap | Edge point to fused MoGe-2 plane | M3 3-D where the match is within 2 px | Matches within 2 px | +| --- | --- | --- | --- | --- | --- | --- | +| exact | all | n/a | n/a | n/a | 0.8 / 1.8 (51) | 0.614 | +| exact | in_plane | n/a | n/a | n/a | n/a | nan | +| exact | proud | n/a | n/a | n/a | 0.8 / 1.8 (51) | 0.614 | +| exact | in_plane_vertical | n/a | n/a | n/a | n/a | nan | +| modern_assumed | all | n/a | n/a | n/a | 3.7 / 11.1 (42) | 0.201 | +| modern_assumed | in_plane | n/a | n/a | n/a | n/a | nan | +| modern_assumed | proud | n/a | n/a | n/a | 3.7 / 11.1 (42) | 0.201 | +| modern_assumed | in_plane_vertical | n/a | n/a | n/a | n/a | nan | + +M2 along-wall span error by the point's median view angle (snap, then true pixel): + +- exact, in-plane, snapped: 0-15 n/a, 15-30 n/a, 30-45 n/a, >45 n/a +- exact, in-plane, true pixel: 0-15 n/a, 15-30 n/a, 30-45 n/a, >45 n/a +- modern_assumed, in-plane, snapped: 0-15 n/a, 15-30 n/a, 30-45 n/a, >45 n/a +- modern_assumed, in-plane, true pixel: 0-15 n/a, 15-30 n/a, 30-45 n/a, >45 n/a + +Edge point to plane (single photo): `null` + + +Tap and snap, horizontal pixel error: `{"tap_abs_du": {"n": 3030, "median": 6.75, "p90": 16.18}, "snap_abs_du": {"n": 3030, "median": 12.78, "p90": 28.05}, "snap_within_1px": 0.037, "added_after_the_run_snap_from_true_pixel_abs_du": {"n": 3030, "median": 13.12, "p90": 23.39}, "added_after_the_run_vertical_in_plane_edges": {"tap_px": {"n": 0}, "snap_px": {"n": 0}, "snap_from_true_px": {"n": 0}}}` + +`{"points_with_2_photos_in_a_group": 606, "photos_per_point_median": 5.0, "points_with_a_second_photo": 603}` + +### Criteria + +`{"P1": {"m0_along_p90_in_0_15": NaN, "m0_along_p90_in_over_30": 29.41, "pass": false}, "P2": {"m1_along_p90_in_in_plane": NaN, "pass": false}, "P3": {"m2_span_along_p90_in_in_plane_modern": NaN, "pass": false}, "P4": {"m3_span_3d_p90_in_modern": NaN, "match_rate": 0.102, "pass": false}, "kill": {"m0_along_p90_in_per_bin": [NaN, NaN, 17.16, 29.65], "triggered": false}}` diff --git a/experiments/edge-geometry/run.py b/experiments/edge-geometry/run.py index 2d62dc943..1ce009e33 100644 --- a/experiments/edge-geometry/run.py +++ b/experiments/edge-geometry/run.py @@ -2,6 +2,7 @@ whether a ray to the wall plane, a snapped tap or a second photo places an edge better. uv run python run.py # writes results/edge_geometry.json and results/edge_geometry.md + uv run python run.py --pair=widest # M3 with the widest pair: results/edge_geometry_widest_pair.* Reads the evals harness read-only (EDGE_EVALS_DIR, default below) and its caches under ~/house-scanning-data/evals: the ETH3D scenes after `evals.recon prepare`, MoGe-2 predictions @@ -81,6 +82,10 @@ WORLD_PLANE_PX_PER_PHOTO = 3000 MIN_BASELINE_M = 0.3 MIN_RAY_ANGLE_DEG = 5.0 +# M3's second photo, pre-registered: the narrowest ray angle at or above MIN_RAY_ANGLE_DEG, which +# keeps the two patches most alike for matching. A narrow pair triangulates worst, so after review +# `--pair=widest` reruns M3 with the widest qualifying angle instead, into separate results files. +PAIRING = "widest" if "--pair=widest" in sys.argv else "narrowest" NCC_MIN = 0.8 NCC_PATCH = 15 DEPTH_RANGE_M = (0.5, 10.0) @@ -305,7 +310,17 @@ def run_scene(scene: str) -> dict: E_all = cands[edge_ids].astype(np.float64) plane_e, sd_e = assign_edges(E_all, cands, plane_of, planes) assigned = np.flatnonzero(plane_e >= 0) - pick = assigned[np.unique(np.linspace(0, len(assigned) - 1, min(MAX_POINTS, len(assigned))).round().astype(int))] if len(assigned) else assigned + pick = ( + assigned[ + np.unique( + np.linspace(0, len(assigned) - 1, min(MAX_POINTS, len(assigned))) + .round() + .astype(int) + ) + ] + if len(assigned) + else assigned + ) ids = edge_ids[pick] P = E_all[pick] plane = plane_e[pick] @@ -321,7 +336,9 @@ def run_scene(scene: str) -> dict: "proud": int((~in_plane).sum()), "vertical_edges": int(vertical_edge.sum()), "up_vs_camera_up_deg": round(float(np.degrees(np.arccos(up @ camera_up))), 2), - "wall_tilt_deg_max": round(float(max(np.degrees(np.arcsin(abs(n @ up))) for n, _ in planes)), 2), + "wall_tilt_deg_max": round( + float(max(np.degrees(np.arcsin(abs(n @ up))) for n, _ in planes)), 2 + ), "oracle_scale_median": float(np.median([d.scale for d in vd.values()])), } print(f"{scene}: {meta}") @@ -351,9 +368,21 @@ def run_scene(scene: str) -> dict: def single_photo_methods(ids, P, plane, planes, frames, names, vd, obs, up) -> dict: """M0, M1 and M1-anchor on every photo where the point is an edge.""" - out: dict[str, list] = {k: [] for k in ( - "j", "angle", "m0", "m1", "m1_offset", "m1a", "psi", "s", "pred_signed", "pred_mag", - )} + out: dict[str, list] = { + k: [] + for k in ( + "j", + "angle", + "m0", + "m1", + "m1_offset", + "m1a", + "psi", + "s", + "pred_signed", + "pred_mag", + ) + } fits: dict[tuple[str, int], tuple[np.ndarray, float]] = {} for j, vi, row in zip(obs["j"], obs["view"], obs["row"], strict=True): d = vd[names[vi]] @@ -451,13 +480,16 @@ def multi_photo_methods(scene, ids, P, plane, planes, frames, names, vd, obs, up ref = min(flagged or range(len(members)), key=lambda i: dist[i]) rn = members[ref][0] second = None - best_angle = np.inf + best_angle = np.inf if PAIRING == "narrowest" else -np.inf for n, r in members: if n == rn or np.linalg.norm(vd[n].C - vd[rn].C) < MIN_BASELINE_M: continue a, b = p - vd[rn].C, p - vd[n].C - ang = np.degrees(np.arccos(np.clip(a @ b / np.linalg.norm(a) / np.linalg.norm(b), -1, 1))) - if MIN_RAY_ANGLE_DEG <= ang < best_angle: + ang = np.degrees( + np.arccos(np.clip(a @ b / np.linalg.norm(a) / np.linalg.norm(b), -1, 1)) + ) + better = ang < best_angle if PAIRING == "narrowest" else ang > best_angle + if ang >= MIN_RAY_ANGLE_DEG and better: second, best_angle = (n, r), ang plan[j] = (g, members, ref, second) @@ -480,7 +512,14 @@ def multi_photo_methods(scene, ids, P, plane, planes, frames, names, vd, obs, up snap_true_err.append(abs(from_true[0] - d.uv[r][0])) tap_j.append(j) m2_angle = { - j: float(np.median([geom.view_angle_deg((P[j] - vd[n].C)[None], planes[int(plane[j])][0])[0] for n, _ in members])) + j: float( + np.median( + [ + geom.view_angle_deg((P[j] - vd[n].C)[None], planes[int(plane[j])][0])[0] + for n, _ in members + ] + ) + ) for j, (_g, members, _ref, _second) in plan.items() } @@ -493,20 +532,39 @@ def world_plane(key, g, k, gp): wall = vd[m].walls.get(k) if m in vd else None if wall is None: continue - sel = np.linspace(0, len(wall.cam) - 1, min(WORLD_PLANE_PX_PER_PHOTO, len(wall.cam))).astype(int) + sel = np.linspace( + 0, len(wall.cam) - 1, min(WORLD_PLANE_PX_PER_PHOTO, len(wall.cam)) + ).astype(int) T = gp[g][m] pts.append(wall.cam[sel] @ T[:3, :3].T + T[:3, 3]) n_pts = sum(len(x) for x in pts) - world_planes[key] = geom.fit_plane(np.concatenate(pts)) if n_pts >= MIN_WALL_PX else None + world_planes[key] = ( + geom.fit_plane(np.concatenate(pts)) if n_pts >= MIN_WALL_PX else None + ) return world_planes[key] out = {} for (setting, draw), gp in poses.items(): - rec: dict[str, list] = {k: [] for k in ( - "j", "draw", "m2_frame", "m2_span", "m2_span_tap", "m2_span_true", "m2_frame_true", - "m2_angle", "m2_offset", "m3_pair", "m3_ncc", "m3_px", "m3_frame", - "m3_span", "m3_anchor", - )} + rec: dict[str, list] = { + k: [] + for k in ( + "j", + "draw", + "m2_frame", + "m2_span", + "m2_span_tap", + "m2_span_true", + "m2_frame_true", + "m2_angle", + "m2_offset", + "m3_pair", + "m3_ncc", + "m3_px", + "m3_frame", + "m3_span", + "m3_anchor", + ) + } for j, (g, members, ref, second) in plan.items(): k = int(plane[j]) F, p = frames[k], P[j] @@ -649,7 +707,9 @@ def summarise_single(s: dict) -> dict: return out yaw: dict = { "observations": int(ok.sum()), - "abs_psi_deg": {q: round(float(np.percentile(psi, p)), 2) for q, p in (("median", 50), ("p90", 90))}, + "abs_psi_deg": { + q: round(float(np.percentile(psi, p)), 2) for q, p in (("median", 50), ("p90", 90)) + }, "abs_s_m_median": round(float(np.median(np.abs(s["s"][ok]))), 2), } for cat, m in (("all", np.ones(len(err), bool)), ("in_plane", ip)): @@ -667,10 +727,20 @@ def summarise_single(s: dict) -> dict: slope60 = float(pl @ el / (pl @ pl)) r2_60 = 1 - float(np.sum((el - slope60 * pl) ** 2) / np.sum((el - el.mean()) ** 2)) yaw[cat + "_added_after_the_run"] = { - "sign_agreement_where_prediction_over_1cm": round(float(np.mean(np.sign(e[big1]) == np.sign(pr[big1]))), 2), - "median_error_over_prediction_where_over_1cm": round(float(np.median(e[big1] / pr[big1])), 2), - "view_angle_under_60": {"n": int(low.sum()), "slope": round(slope60, 2), "r2": round(r2_60, 2)}, - "spearman_abs_error_vs_tan_angle_alone": round(spearman(np.abs(e), np.tan(np.radians(ang[m]))), 2), + "sign_agreement_where_prediction_over_1cm": round( + float(np.mean(np.sign(e[big1]) == np.sign(pr[big1]))), 2 + ), + "median_error_over_prediction_where_over_1cm": round( + float(np.median(e[big1] / pr[big1])), 2 + ), + "view_angle_under_60": { + "n": int(low.sum()), + "slope": round(slope60, 2), + "r2": round(r2_60, 2), + }, + "spearman_abs_error_vs_tan_angle_alone": round( + spearman(np.abs(e), np.tan(np.radians(ang[m]))), 2 + ), } yaw[cat] = { "n": int(m.sum()), @@ -754,10 +824,14 @@ def summarise_multi(m: dict) -> dict: | {"p90_ci95_in": stats.bootstrap_p90(m3_span["3d"], j)}, "anchor_matched": int(r["m3_anchor"][c].sum()), "added_after_the_run_frame_3d_where_match_within_2px": stats.summary( - m3_frame["3d"][matched & (r["m3_px"][c] <= 2.0)], j[matched & (r["m3_px"][c] <= 2.0)] + m3_frame["3d"][matched & (r["m3_px"][c] <= 2.0)], + j[matched & (r["m3_px"][c] <= 2.0)], ), "added_after_the_run_share_of_matches_within_2px": round( - float(np.mean(r["m3_px"][c][matched] <= 2.0)) if matched.any() else float("nan"), 3 + float(np.mean(r["m3_px"][c][matched] <= 2.0)) + if matched.any() + else float("nan"), + 3, ), }, } @@ -780,20 +854,28 @@ def summarise_multi(m: dict) -> dict: def pct(x: np.ndarray) -> dict: if not len(x): return {"n": 0} - return {"n": len(x), "median": round(float(np.median(x)), 2), "p90": round(float(np.percentile(x, 90)), 2)} + return { + "n": len(x), + "median": round(float(np.median(x)), 2), + "p90": round(float(np.percentile(x, 90)), 2), + } def main() -> None: RESULTS.mkdir(exist_ok=True) + suffix = "" if PAIRING == "narrowest" else "_widest_pair" report: dict = { + "m3_pairing": PAIRING, "evals_commit": evals_commit(), "evals_dir": str(EVALS_DIR).replace(str(Path.home()), "~"), - "parameters": {k: v for k, v in globals().items() if k.isupper() and isinstance(v, (int, float, tuple))}, + "parameters": { + k: v for k, v in globals().items() if k.isupper() and isinstance(v, (int, float, tuple)) + }, "scenes": {}, } # `--from-raw` re-summarises the per-point records of the last full run (data/, git-ignored). for scene in SCENES: - raw = HERE / "data" / f"raw_{scene}.pkl" + raw = HERE / "data" / f"raw_{scene}{suffix}.pkl" if "--from-raw" in sys.argv: res = pickle.loads(raw.read_bytes()) else: @@ -804,12 +886,18 @@ def main() -> None: report["scenes"][scene] = res continue single, multi = res["single"], res["multi"] - summary = {"meta": res["meta"], "single": summarise_single(single), "multi": summarise_multi(multi)} + summary = { + "meta": res["meta"], + "single": summarise_single(single), + "multi": summarise_multi(multi), + } summary["criteria"] = grade(single, multi) report["scenes"][scene] = summary - (RESULTS / "edge_geometry.json").write_text(json.dumps(report, indent=1, default=float) + "\n") - (RESULTS / "edge_geometry.md").write_text(markdown(report)) - print((RESULTS / "edge_geometry.md").read_text()) + (RESULTS / f"edge_geometry{suffix}.json").write_text( + json.dumps(report, indent=1, default=float) + "\n" + ) + (RESULTS / f"edge_geometry{suffix}.md").write_text(markdown(report)) + print((RESULTS / f"edge_geometry{suffix}.md").read_text()) def p90_in(x: np.ndarray) -> float: @@ -832,11 +920,22 @@ def grade(single: dict, multi: dict) -> dict: rate = float(matched.sum() / max(pair.sum(), 1)) m3 = p90_in(np.linalg.norm(ma["m3_span"], axis=1)) return { - "P1": {"m0_along_p90_in_0_15": near, "m0_along_p90_in_over_30": wide, "pass": bool(near <= 0.5 * wide)}, + "P1": { + "m0_along_p90_in_0_15": near, + "m0_along_p90_in_over_30": wide, + "pass": bool(near <= 0.5 * wide), + }, "P2": {"m1_along_p90_in_in_plane": m1, "pass": bool(m1 <= 4.0)}, "P3": {"m2_span_along_p90_in_in_plane_modern": m2, "pass": bool(m2 <= 2.0)}, - "P4": {"m3_span_3d_p90_in_modern": m3, "match_rate": round(rate, 3), "pass": bool(m3 <= 2.0 and rate >= 0.7)}, - "kill": {"m0_along_p90_in_per_bin": per_bin, "triggered": bool(all(x <= 4.0 for x in per_bin))}, + "P4": { + "m3_span_3d_p90_in_modern": m3, + "match_rate": round(rate, 3), + "pass": bool(m3 <= 2.0 and rate >= 0.7), + }, + "kill": { + "m0_along_p90_in_per_bin": per_bin, + "triggered": bool(all(x <= 4.0 for x in per_bin)), + }, } @@ -850,7 +949,9 @@ def markdown(report: dict) -> str: L = [ "# Edge geometry (generated by `uv run python run.py`)", "", - f"Evals harness commit `{report['evals_commit']}`. Errors in inches, median / p90 (points).", + f"Evals harness commit `{report['evals_commit']}`. Errors in inches, median / p90 (points). " + f"M3 pairs each point's reference photo with the {report.get('m3_pairing', 'narrowest')} " + f"qualifying ray angle.", ] for scene, r in report["scenes"].items(): L += ["", f"## {scene}", "", f"`{json.dumps(r['meta'])}`"] @@ -858,14 +959,32 @@ def markdown(report: dict) -> str: L += ["", f"Skipped: {r['skipped']}."] continue s = r["single"] - for method, title in (("m0", "M0, depth at the pixel"), ("m1", "M1, ray to this photo's wall plane"), ("m1a", "M1-anchor, ray to a 0.3 m patch plane 1-3 m away")): - L += ["", f"### {title}", "", "| Edges | View angle | Along the wall | Out of plane | 3-D |", "| --- | --- | --- | --- | --- |"] + for method, title in ( + ("m0", "M0, depth at the pixel"), + ("m1", "M1, ray to this photo's wall plane"), + ("m1a", "M1-anchor, ray to a 0.3 m patch plane 1-3 m away"), + ): + L += [ + "", + f"### {title}", + "", + "| Edges | View angle | Along the wall | Out of plane | 3-D |", + "| --- | --- | --- | --- | --- |", + ] for cat in ("all", "in_plane", "proud"): for label, row in s[method][cat].items(): - L.append(f"| {cat} | {label} | {fmt(row['along'])} | {fmt(row['out'])} | {fmt(row['3d'])} |") + L.append( + f"| {cat} | {label} | {fmt(row['along'])} | {fmt(row['out'])} | {fmt(row['3d'])} |" + ) L += ["", "### M1-anchor yaw", "", f"`{json.dumps(s['m1a_yaw'])}`"] m = r["multi"] - L += ["", "### M2 and M3", "", "| Poses | Edges | M2 span along | M2 p90 95% CI | M3 pairs | M3 match | M3 match px | M3 span along | M3 span 3-D | M3 3-D p90 95% CI | M3 frame 3-D |", "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"] + L += [ + "", + "### M2 and M3", + "", + "| Poses | Edges | M2 span along | M2 p90 95% CI | M3 pairs | M3 match | M3 match px | M3 span along | M3 span 3-D | M3 3-D p90 95% CI | M3 frame 3-D |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |", + ] for setting in ("exact", "modern_assumed"): for cat, row in m[setting].items(): m2, m3 = row["M2"], row["M3"] @@ -874,7 +993,10 @@ def markdown(report: dict) -> str: f"| {m3['match_px'].get('median', 'n/a')} / {m3['match_px'].get('p90', 'n/a')} | {fmt(m3['span_along'])} | {fmt(m3['span_3d'])} | {m3['span_3d'].get('p90_ci95_in')} | {fmt(m3['frame_3d'])} |" ) L += ["", "M3 match px is in pixels, not inches.", "", "### Added after the run", ""] - L += ["| Poses | Edges | M2 with the true pixel | M2 tap, no snap | Edge point to fused MoGe-2 plane | M3 3-D where the match is within 2 px | Matches within 2 px |", "| --- | --- | --- | --- | --- | --- | --- |"] + L += [ + "| Poses | Edges | M2 with the true pixel | M2 tap, no snap | Edge point to fused MoGe-2 plane | M3 3-D where the match is within 2 px | Matches within 2 px |", + "| --- | --- | --- | --- | --- | --- | --- |", + ] for setting in ("exact", "modern_assumed"): for cat, row in m[setting].items(): d2, m3 = row["M2_added_after_the_run"], row["M3"] @@ -882,14 +1004,34 @@ def markdown(report: dict) -> str: f"| {setting} | {cat} | {fmt(d2['span_along_true_pixel'])} | {fmt(d2['span_along_tap_no_snap'])} | {fmt(d2['edge_point_to_fused_plane'])} " f"| {fmt(m3['added_after_the_run_frame_3d_where_match_within_2px'])} | {m3['added_after_the_run_share_of_matches_within_2px']} |" ) - L += ["", "M2 along-wall span error by the point's median view angle (snap, then true pixel):", ""] + L += [ + "", + "M2 along-wall span error by the point's median view angle (snap, then true pixel):", + "", + ] for setting in ("exact", "modern_assumed"): d2 = m[setting]["in_plane"]["M2_added_after_the_run"] - snapped = ", ".join(f"{b} {fmt(x)}" for b, x in d2["span_along_by_median_view_angle"].items()) - true = ", ".join(f"{b} {fmt(x)}" for b, x in d2["span_along_true_pixel_by_median_view_angle"].items()) - L += [f"- {setting}, in-plane, snapped: {snapped}", f"- {setting}, in-plane, true pixel: {true}"] - L += ["", f"Edge point to plane (single photo): `{json.dumps(s.get('added_after_the_run_edge_point_to_plane'))}`"] - L += ["", "", f"Tap and snap, horizontal pixel error: `{json.dumps(m['tap_px'])}`", "", f"`{json.dumps(m['plans'])}`"] + snapped = ", ".join( + f"{b} {fmt(x)}" for b, x in d2["span_along_by_median_view_angle"].items() + ) + true = ", ".join( + f"{b} {fmt(x)}" for b, x in d2["span_along_true_pixel_by_median_view_angle"].items() + ) + L += [ + f"- {setting}, in-plane, snapped: {snapped}", + f"- {setting}, in-plane, true pixel: {true}", + ] + L += [ + "", + f"Edge point to plane (single photo): `{json.dumps(s.get('added_after_the_run_edge_point_to_plane'))}`", + ] + L += [ + "", + "", + f"Tap and snap, horizontal pixel error: `{json.dumps(m['tap_px'])}`", + "", + f"`{json.dumps(m['plans'])}`", + ] L += ["", "### Criteria", "", f"`{json.dumps(r['criteria'])}`"] return "\n".join(L) + "\n" diff --git a/experiments/edge-geometry/tests/test_geom.py b/experiments/edge-geometry/tests/test_geom.py index e13fd1ffd..1df32fece 100644 --- a/experiments/edge-geometry/tests/test_geom.py +++ b/experiments/edge-geometry/tests/test_geom.py @@ -54,7 +54,14 @@ def test_view_angle_and_bins(): rays = np.array([[0.0, -1.0, 0.0], [np.sin(np.radians(20)), np.cos(np.radians(20)), 0.0]]) np.testing.assert_allclose(view_angle_deg(rays, n), [0.0, 20.0], atol=1e-9) assert list(angle_bin([0.0, 14.999, 15.0, 29.9, 30.0, 45.0, 89.0, 90.0])) == [ - 0, 0, 1, 1, 2, 3, 3, 3, + 0, + 0, + 1, + 1, + 2, + 3, + 3, + 3, ] with pytest.raises(ValueError): angle_bin([91.0]) @@ -103,14 +110,21 @@ def test_midpoint_triangulation(): def test_midpoint_of_skew_rays_is_the_middle_of_their_common_perpendicular(): # x-axis ray from (-1, 0, 0) and a ray along -y from (0, 1, 1): closest at (0,0,0) and (0,0,1). got, angle = midpoint_triangulate( - np.array([-1.0, 0, 0]), np.array([1.0, 0, 0]), np.array([0, 1.0, 1.0]), np.array([0, -1.0, 0]) + np.array([-1.0, 0, 0]), + np.array([1.0, 0, 0]), + np.array([0, 1.0, 1.0]), + np.array([0, -1.0, 0]), ) np.testing.assert_allclose(got, [0.0, 0.0, 0.5], atol=1e-12) assert angle == pytest.approx(90.0) def test_midpoint_triangulation_rejects_parallel_and_behind(): - got, _ = midpoint_triangulate(np.zeros(3), np.array([0, 0, 1.0]), np.ones(3), np.array([0, 0, 1.0])) + got, _ = midpoint_triangulate( + np.zeros(3), np.array([0, 0, 1.0]), np.ones(3), np.array([0, 0, 1.0]) + ) assert np.isnan(got).all() - got, _ = midpoint_triangulate(np.zeros(3), np.array([0, 0, -1.0]), np.array([1.0, 0, 0]), np.array([-1.0, 0, 1.0])) + got, _ = midpoint_triangulate( + np.zeros(3), np.array([0, 0, -1.0]), np.array([1.0, 0, 0]), np.array([-1.0, 0, 1.0]) + ) assert np.isnan(got).all() diff --git a/experiments/edge-geometry/tests/test_imagematch.py b/experiments/edge-geometry/tests/test_imagematch.py index d6f1eac72..129aeb4f1 100644 --- a/experiments/edge-geometry/tests/test_imagematch.py +++ b/experiments/edge-geometry/tests/test_imagematch.py @@ -87,3 +87,19 @@ def test_epipolar_search_fails_cleanly_when_the_template_leaves_the_image(): uv2, score = epipolar_match(img, img, np.array([3.0, 3.0]), K, T, K, T) assert np.isnan(uv2).all() assert score == -1.0 + + +def test_epipolar_search_longer_than_one_remap_still_matches(): + # A 0.05 px step makes the segment's patches overflow cv2.remap's SHRT_MAX rows, as a wide + # photo pair's long segment does at the normal step. + rng = np.random.default_rng(3) + texture = cv2.GaussianBlur(rng.random((3000, 3000)).astype(np.float32), (0, 0), 3) + T1 = look_at(np.array([0.0, 0.0, 0.0]), np.array([0.0, 0.0, 3.0])) + T2 = look_at(np.array([0.4, 0.05, 0.1]), np.array([0.1, 0.0, 3.0])) + img1, img2 = render_plane(T1, texture, 0.002, 3.0), render_plane(T2, texture, 0.002, 3.0) + uv1 = np.array([300.3, 250.6]) + o1, d1 = pixel_ray(K, T1, uv1) + truth = project(K, T2, (o1 + (3.0 - o1[2]) / d1[2] * d1)[None])[0][0] + uv2, score = epipolar_match(img1, img2, uv1, K, T1, K, T2, step_px=0.05) + assert score > 0.95 + assert np.linalg.norm(uv2 - truth) < 0.3 diff --git a/experiments/live-dots/Sources/LiveDots/DotRenderer.swift b/experiments/live-dots/Sources/LiveDots/DotRenderer.swift index bb7ca2926..bb7439980 100644 --- a/experiments/live-dots/Sources/LiveDots/DotRenderer.swift +++ b/experiments/live-dots/Sources/LiveDots/DotRenderer.swift @@ -3,24 +3,6 @@ import Metal import MetalKit import simd -/// What one frame shows: which keyframe, when on the playback clock, and the toggles. -struct FrameRequest: Equatable { - var mode: CaptureMode - var keyframe: Int - /// Playback seconds; animations are evaluated at this time. - var time: Float - var reduceMotion: Bool - var fog: FogStyle - var scheme: DotScheme = .hologram - /// Seconds on a clock that keeps running while playback is paused, for the fog's drift. - var ambientTime: Float = 0 - /// Seconds since the previous frame, for the fog's lag. - var frameInterval: Float = 1 / 60 - - /// A scrub or a still shows its keyframe settled; the renderer passes a far-future time. - var isSettled: Bool { time > 1e4 } -} - /// Draws a keyframe: the camera image, the optional fog veil, then the dots as additive point /// sprites from one vertex buffer. The buffer is rebuilt only when the keyframe or mode changes; /// between keyframes the shader animates births and evidence from the playback time. @@ -60,6 +42,7 @@ final class DotRenderer { private var linkBuffer: MTLBuffer? private var linkVertexCount = 0 private var cachedKey: (mode: CaptureMode, keyframe: Int, scheme: DotScheme)? + private var fogContinuity = FogContinuity() /// Replaces the timeline's sprites, for the 6,000-dot benchmark. var spriteOverride: [DotSprite]? { didSet { cachedKey = nil } @@ -120,11 +103,13 @@ final class DotRenderer { let projection = ScreenProjection(keyframe: keyframe, viewSize: pixelSize) var clip = projection.clipMatrix + let resetFog = fogContinuity.consume(request) if request.fog == .on { fog.encodeMask( into: commandBuffer, splats: splatBuffer, splatCount: splatCount, clip: clip, pixelsPerMetre: keyframe.intrinsics.x * projection.scale * Float(FogLayer.maskWidth) / pixelSize.x, - time: request.time, dt: request.frameInterval, reduceMotion: request.reduceMotion, reset: request.isSettled) + time: request.time, dt: request.frameInterval, reduceMotion: request.reduceMotion, + reset: resetFog) } guard let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: pass) else { return } diff --git a/experiments/live-dots/Sources/LiveDots/Exporter.swift b/experiments/live-dots/Sources/LiveDots/Exporter.swift index 2e50429d8..8c398a5cd 100644 --- a/experiments/live-dots/Sources/LiveDots/Exporter.swift +++ b/experiments/live-dots/Sources/LiveDots/Exporter.swift @@ -51,14 +51,24 @@ enum Exporter { } /// One keyframe as a PNG. Without `at`, the keyframe settled: every animation finished and - /// the fog lag caught up. With `at`, the moment `at` seconds after the keyframe appears, - /// reached by playing the replay from the start at 30 fps, so the fog's lag is what the - /// video shows. + /// the fog lag caught up. With `at`, the moment `at` seconds after the keyframe appears, before + /// the next one replaces it, reached by playing the replay from the start at 30 fps, so the + /// fog's lag is what the video shows. static func exportStill(keyframe number: Int, at offset: Float?, to path: String, options: Command.RenderOptions) throws { let data = try ReplayData.loadNow(folder: FixtureLocator.folder(options.fixture)) guard number <= data.keyframeCount else { throw Failure(description: "--still \(number) is past the last keyframe, \(data.keyframeCount)") } + // Stills are named and compared by keyframe, so an offset that plays into the next + // keyframe would label one keyframe's image with another's number, and one past the end + // of playback shows a moment the replay never reaches. + let shown = Schedule.onScreen(number - 1, count: data.keyframeCount) + if let offset, offset >= shown { + let next = number < data.keyframeCount ? " or --still \(number + 1)" : "" + throw Failure(description: String( + format: "--at %g is past keyframe %d, which is on screen for %g s; use a smaller --at", + offset, number, shown) + next) + } let renderer = try DotRenderer(data: data) let target = try OffscreenTarget(renderer: renderer) let state = data.timeline(options.mode).states[number - 1] diff --git a/experiments/live-dots/Sources/LiveDots/FogLayer.swift b/experiments/live-dots/Sources/LiveDots/FogLayer.swift index 974a49111..a5e63f447 100644 --- a/experiments/live-dots/Sources/LiveDots/FogLayer.swift +++ b/experiments/live-dots/Sources/LiveDots/FogLayer.swift @@ -2,15 +2,6 @@ import LiveDotsCore import Metal import simd -/// How the unmeasured world is shown. -enum FogStyle: String, CaseIterable { - /// The fog layer: one continuous haze that dissolves where dots arrive. - case on - /// The old comparison: a flat 35% veil over 30 cm wall cells no dot has reached. - case veil - case off -} - /// The fog layer's GPU state: the 96 x 208 reveal mask, its blur, the lagged copy that carries /// from frame to frame, and each keyframe's depth for the depth term. The shaders and what they /// borrow are described at the fog section of Shaders.swift. @@ -86,7 +77,7 @@ final class FogLayer { } /// Splats, blurs and eases the reveal mask. `reset` snaps the lag to the new mask (a scrub, a - /// still, the first frame). + /// mode switch, the fog turned back on, a still); the first frame always snaps. func encodeMask( into commandBuffer: MTLCommandBuffer, splats: MTLBuffer?, splatCount: Int, clip: simd_float4x4, pixelsPerMetre: Float, time: Float, dt: Float, reduceMotion: Bool, reset: Bool diff --git a/experiments/live-dots/Sources/LiveDots/OffscreenTarget.swift b/experiments/live-dots/Sources/LiveDots/OffscreenTarget.swift index ded73f408..fabafd73e 100644 --- a/experiments/live-dots/Sources/LiveDots/OffscreenTarget.swift +++ b/experiments/live-dots/Sources/LiveDots/OffscreenTarget.swift @@ -1,3 +1,4 @@ +import LiveDotsCore import CoreGraphics import Metal diff --git a/experiments/live-dots/Sources/LiveDots/ReplayPlayer.swift b/experiments/live-dots/Sources/LiveDots/ReplayPlayer.swift index 0b606dbd1..8ecf5cb14 100644 --- a/experiments/live-dots/Sources/LiveDots/ReplayPlayer.swift +++ b/experiments/live-dots/Sources/LiveDots/ReplayPlayer.swift @@ -13,65 +13,38 @@ final class ReplayPlayer { var fog: FogStyle = .on private(set) var isPlaying = false private(set) var keyframe = 0 - /// Playback seconds. Keyframe k is on screen from k / 4 to (k + 1) / 4. - @ObservationIgnored private(set) var playhead: Float = 0 - /// After a scrub the frame shows its settled state instead of replaying births. - @ObservationIgnored private var settled = false - @ObservationIgnored private var lastTick: CFTimeInterval? - @ObservationIgnored private var lastFrame: CFTimeInterval? - @ObservationIgnored private var ambient: Float = 0 - @ObservationIgnored private var frameInterval: Float = 1 / 60 - - /// Playback runs this long past the last keyframe so its births finish. - private var end: Float { Schedule.end(count: keyframeCount) + Tuning.birthDuration } + var playhead: Float { playback.playhead } + @ObservationIgnored private var playback: ReplayPlayback init(keyframeCount: Int, reduceMotion: Bool) { self.keyframeCount = keyframeCount + self.playback = ReplayPlayback(keyframeCount: keyframeCount) self.reduceMotion = reduceMotion } var request: FrameRequest { - FrameRequest( - mode: mode, keyframe: keyframe, time: settled ? 1e5 : playhead, - reduceMotion: reduceMotion, fog: fog, scheme: scheme, ambientTime: ambient, frameInterval: frameInterval) + playback.frameRequest(mode: mode, reduceMotion: reduceMotion, fog: fog, scheme: scheme) } func togglePlayback() { - if isPlaying { - isPlaying = false - return - } - if playhead >= end - 1e-3 || (keyframe == keyframeCount - 1 && settled) { seek(to: 0, settle: false) } - settled = false - lastTick = nil - isPlaying = true + playback.togglePlayback() + updateControls() } /// Jumps to a keyframe and shows it settled. Scrubbing while playing keeps playing from there. func scrub(to index: Int) { - seek(to: index, settle: !isPlaying) - } - - private func seek(to index: Int, settle: Bool) { - keyframe = min(max(index, 0), keyframeCount - 1) - playhead = Schedule.start(of: keyframe) - settled = settle - lastTick = nil + playback.scrub(to: index) + updateControls() } /// Advances the clock; the Metal view calls this once per display frame. func tick(now: CFTimeInterval) { - if let lastFrame { frameInterval = min(Float(now - lastFrame), 0.1) } - lastFrame = now - ambient += frameInterval - guard isPlaying else { return } - if let lastTick { playhead += Float(now - lastTick) } - lastTick = now - let index = Schedule.keyframe(at: playhead, count: keyframeCount) - if index != keyframe { keyframe = index } - if playhead >= end { - playhead = end - isPlaying = false - } + playback.tick(now: now) + updateControls() + } + + private func updateControls() { + if isPlaying != playback.isPlaying { isPlaying = playback.isPlaying } + if keyframe != playback.keyframe { keyframe = playback.keyframe } } } diff --git a/experiments/live-dots/Sources/LiveDots/Shaders.swift b/experiments/live-dots/Sources/LiveDots/Shaders.swift index 528c732f1..463efa25e 100644 --- a/experiments/live-dots/Sources/LiveDots/Shaders.swift +++ b/experiments/live-dots/Sources/LiveDots/Shaders.swift @@ -86,8 +86,9 @@ enum Shaders { constant float3 emberHalo = float3(0xFF, 0xC9, 0x78) / 255.0f; // Constellation hides flat dots, so a dot that turns edge is born then, not at its voxel's birth. + // Feature points never become edges, and their edge time of 1e6 s would hide them for good. static float visibleBirth(Sprite s, int scheme) { - return scheme == 1 ? max(s.b.x, s.c.x) : s.b.x; + return scheme == 1 && s.a.w < 0.5f ? max(s.b.x, s.c.x) : s.b.x; } struct DotOut { diff --git a/experiments/live-dots/Sources/LiveDotsCore/DepthMap.swift b/experiments/live-dots/Sources/LiveDotsCore/DepthMap.swift index 6fbb8ce9d..78a79704f 100644 --- a/experiments/live-dots/Sources/LiveDotsCore/DepthMap.swift +++ b/experiments/live-dots/Sources/LiveDotsCore/DepthMap.swift @@ -43,10 +43,19 @@ public struct DepthMap: Sendable { /// Depth at the pixel containing continuous coordinate (u, v), or nil outside the map. public func depth(atU u: Float, v: Float) -> Float? { + index(atU: u, v: v).map { meters[$0] } + } + + /// Confidence at the pixel containing continuous coordinate (u, v), or nil outside the map. + public func confidence(atU u: Float, v: Float) -> UInt8? { + index(atU: u, v: v).map { confidence[$0] } + } + + private func index(atU u: Float, v: Float) -> Int? { guard u >= 0, v >= 0 else { return nil } let i = Int(u), j = Int(v) guard i < width, j < height else { return nil } - return meters[j * width + i] + return j * width + i } } diff --git a/experiments/live-dots/Sources/LiveDotsCore/DotField.swift b/experiments/live-dots/Sources/LiveDotsCore/DotField.swift index 33836f984..324d16f49 100644 --- a/experiments/live-dots/Sources/LiveDotsCore/DotField.swift +++ b/experiments/live-dots/Sources/LiveDotsCore/DotField.swift @@ -13,8 +13,9 @@ // (a) a face neighbour's normal differs by more than 35 degrees (the wall-to-ground corner, // the bin's corners); // (b) it has surface along one side and seen-empty space along another (the wall's top against -// the sky, the bin's silhouette). Seen-empty means a depth ray passed the neighbour with -// 10 cm to spare, so the edge of what has been scanned so far is never an edge; or +// the sky, the bin's silhouette). Seen-empty means a depth ray of medium or high confidence +// passed the neighbour with 10 cm to spare, so the edge of what has been scanned so far is +// never an edge; or // (c) the colour gradient at its samples reaches 0.2 (painted outlines). The gradient is read // from a pyramid level whose pixel spans about one voxel, and ignored near a depth // silhouette, where the colour step is occlusion rather than paint. On the fixture @@ -226,7 +227,9 @@ public struct VoxelField: Sendable { /// Tests the empty in-plane neighbours of occupied voxels against this keyframe's depth: a /// neighbour is seen-empty when the ray through it hit nothing or hit something farther by /// `Tuning.freeSpaceMargin`. A neighbour outside the view or behind something stays unknown, - /// which is why the edge of what has been scanned so far never counts as a boundary. + /// which is why the edge of what has been scanned so far never counts as a boundary. So does + /// one whose depth pixel is below `Tuning.minimumConfidence`: fusion ignores that pixel, and + /// free space is never forgotten, so one bad zero or far reading would leave a lasting edge. mutating func carveFreeSpace(_ frame: FrameInput) { let depth = frame.depth let worldToCamera = frame.keyframe.cameraToWorld.inverse @@ -239,7 +242,9 @@ public struct VoxelField: Sendable { let camera = CameraMath.transform(worldToCamera, centre(of: neighbour)) guard let pixel = CameraMath.project(camera, intrinsics: depth.intrinsics), pixel.depth >= Tuning.depthRange.lowerBound, - let measured = depth.depth(atU: pixel.u, v: pixel.v) + let measured = depth.depth(atU: pixel.u, v: pixel.v), + let confidence = depth.confidence(atU: pixel.u, v: pixel.v), + confidence >= Tuning.minimumConfidence else { continue } if measured == 0 || measured > pixel.depth + Tuning.freeSpaceMargin { found.insert(neighbour) } } diff --git a/experiments/live-dots/Sources/LiveDotsCore/FogContinuity.swift b/experiments/live-dots/Sources/LiveDotsCore/FogContinuity.swift new file mode 100644 index 000000000..256efb089 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/FogContinuity.swift @@ -0,0 +1,21 @@ +/// Keeps fog easing within one continuous playback, never across a seek or a settled preview. +public struct FogContinuity { + private var previous: FrameRequest? + + public init() {} + + /// Returns whether the renderer must replace its lagged mask with the current frame's mask. + public mutating func consume(_ request: FrameRequest) -> Bool { + guard request.fog == .on else { + previous = nil + return false + } + defer { previous = request } + guard let previous else { return true } + return request.isSettled || previous.isSettled + || request.seekGeneration != previous.seekGeneration + || request.mode != previous.mode + || !(0...1).contains(request.keyframe - previous.keyframe) + || request.time < previous.time + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/FrameRequest.swift b/experiments/live-dots/Sources/LiveDotsCore/FrameRequest.swift new file mode 100644 index 000000000..ef5bb63d1 --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/FrameRequest.swift @@ -0,0 +1,43 @@ +/// How the unmeasured world is shown. +public enum FogStyle: String, CaseIterable, Sendable { + /// The fog layer: one continuous haze that dissolves where dots arrive. + case on + /// The old comparison: a flat 35% veil over 30 cm wall cells no dot has reached. + case veil + case off +} + +/// What one frame shows: which keyframe, when on the playback clock, and the toggles. +public struct FrameRequest: Equatable, Sendable { + public var mode: CaptureMode + public var keyframe: Int + /// Playback seconds; animations are evaluated at this time. + public var time: Float + public var reduceMotion: Bool + public var fog: FogStyle + public var scheme: DotScheme = .hologram + /// Seconds on a clock that keeps running while playback is paused, for the fog's drift. + public var ambientTime: Float = 0 + /// Seconds since the previous frame, for the fog's lag. + public var frameInterval: Float = 1 / 60 + + /// Changes on explicit seeks and on resuming from a settled scrub. + public var seekGeneration: UInt64 = 0 + + public init(mode: CaptureMode, keyframe: Int, time: Float, reduceMotion: Bool, fog: FogStyle, + scheme: DotScheme = .hologram, ambientTime: Float = 0, frameInterval: Float = 1 / 60, + seekGeneration: UInt64 = 0) { + self.mode = mode + self.keyframe = keyframe + self.time = time + self.reduceMotion = reduceMotion + self.fog = fog + self.scheme = scheme + self.ambientTime = ambientTime + self.frameInterval = frameInterval + self.seekGeneration = seekGeneration + } + + /// A scrub or a still shows its keyframe settled; the renderer passes a far-future time. + public var isSettled: Bool { time > 1e4 } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/ReplayPlayback.swift b/experiments/live-dots/Sources/LiveDotsCore/ReplayPlayback.swift new file mode 100644 index 000000000..cb46a15da --- /dev/null +++ b/experiments/live-dots/Sources/LiveDotsCore/ReplayPlayback.swift @@ -0,0 +1,70 @@ +/// The replay clock shared by the interactive player and its continuity tests. +public struct ReplayPlayback { + public let keyframeCount: Int + public private(set) var isPlaying = false + public private(set) var keyframe = 0 + public private(set) var playhead: Float = 0 + public private(set) var seekGeneration: UInt64 = 0 + private var settled = false + private var lastTick: Double? + private var lastFrame: Double? + private var ambient: Float = 0 + private var frameInterval: Float = 1 / 60 + + private var end: Float { Schedule.playbackEnd(count: keyframeCount) } + + public init(keyframeCount: Int) { + precondition(keyframeCount > 0, "replay playback needs at least one keyframe") + self.keyframeCount = keyframeCount + } + + public func frameRequest(mode: CaptureMode, reduceMotion: Bool, fog: FogStyle, + scheme: DotScheme = .hologram) -> FrameRequest { + FrameRequest( + mode: mode, keyframe: keyframe, time: settled ? 1e5 : playhead, + reduceMotion: reduceMotion, fog: fog, scheme: scheme, ambientTime: ambient, + frameInterval: frameInterval, seekGeneration: seekGeneration) + } + + public mutating func togglePlayback() { + if isPlaying { + isPlaying = false + return + } + if playhead >= end - 1e-3 || (keyframe == keyframeCount - 1 && settled) { + seek(to: 0, settle: false) + } else if settled { + // Births restart at the scrubbed keyframe's start, not at its settled preview time. + seekGeneration &+= 1 + } + settled = false + lastTick = nil + isPlaying = true + } + + public mutating func scrub(to index: Int) { + seek(to: index, settle: !isPlaying) + } + + private mutating func seek(to index: Int, settle: Bool) { + keyframe = min(max(index, 0), keyframeCount - 1) + playhead = Schedule.start(of: keyframe) + settled = settle + seekGeneration &+= 1 + lastTick = nil + } + + public mutating func tick(now: Double) { + if let lastFrame { frameInterval = min(Float(now - lastFrame), 0.1) } + lastFrame = now + ambient += frameInterval + guard isPlaying else { return } + if let lastTick { playhead += Float(now - lastTick) } + lastTick = now + keyframe = Schedule.keyframe(at: playhead, count: keyframeCount) + if playhead >= end { + playhead = end + isPlaying = false + } + } +} diff --git a/experiments/live-dots/Sources/LiveDotsCore/Schedule.swift b/experiments/live-dots/Sources/LiveDotsCore/Schedule.swift index 2637afe10..ee329e9fb 100644 --- a/experiments/live-dots/Sources/LiveDotsCore/Schedule.swift +++ b/experiments/live-dots/Sources/LiveDotsCore/Schedule.swift @@ -23,4 +23,16 @@ public enum Schedule { public static func end(count: Int) -> Float { start(of: count) } + + /// When playback stops: past the last keyframe's slot, long enough for its births to finish. + public static func playbackEnd(count: Int) -> Float { + end(count: count) + Tuning.birthDuration + } + + /// Seconds keyframe `index` stays on screen: until the next keyframe appears, or for the last + /// one, until playback stops. `keyframe(at:count:)` holds the last keyframe forever, so it + /// cannot bound an offset into that one. + public static func onScreen(_ index: Int, count: Int) -> Float { + (index + 1 < count ? start(of: index + 1) : playbackEnd(count: count)) - start(of: index) + } } diff --git a/experiments/live-dots/Tests/LiveDotsCoreTests/FieldRuleTests.swift b/experiments/live-dots/Tests/LiveDotsCoreTests/FieldRuleTests.swift index 0bb451250..13f2d20bd 100644 --- a/experiments/live-dots/Tests/LiveDotsCoreTests/FieldRuleTests.swift +++ b/experiments/live-dots/Tests/LiveDotsCoreTests/FieldRuleTests.swift @@ -45,6 +45,47 @@ struct FieldRuleTests { #expect(!field.free.contains(VoxelKey(4, 4, 0))) } + /// One keyframe square on to a flat wall at z = 0 from 2.5 m, with depth columns 39 and 40 + /// (the strip over voxel column x = 0) replaced by `stripDepth` at `stripConfidence`. + func wallWithStrip(stripDepth: Float, stripConfidence: UInt8) -> VoxelField { + let width = 80, height = 60 + var meters = [Float](repeating: 2.5, count: width * height) + var confidence = [UInt8](repeating: 2, count: width * height) + for j in 0..(100, 100, 40, 30) + let depth = DepthMap(meters: meters, confidence: confidence, width: width, height: height, intrinsics: intrinsics) + let pose = simd_float4x4(columns: (SIMD4(1, 0, 0, 0), SIMD4(0, 1, 0, 0), SIMD4(0, 0, 1, 0), SIMD4(0, 0, 2.5, 1))) + let keyframe = Keyframe( + id: "k", imagePath: "k.jpg", width: width, height: height, intrinsics: intrinsics, + cameraToWorld: pose, timestamp: 0, depth: nil) + let image = RGBImage(width: width, height: height, rgba: [UInt8](repeating: 128, count: width * height * 4)) + var field = VoxelField() + field.integrate(FrameInput(index: 0, keyframe: keyframe, depth: depth, gradient: GradientPyramid(image: image))) + return field + } + + @Test func `a confident hole in the wall carves free space and makes boundary edges`() { + let field = wallWithStrip(stripDepth: 0, stripConfidence: 2) + #expect(field.voxels[VoxelKey(0, 0, 0)] == nil) + #expect(field.free.contains(VoxelKey(0, 0, 0))) + #expect(field.voxels[VoxelKey(1, 0, 0)]?.isEdge == true) + #expect(field.voxels[VoxelKey(-1, 0, 0)]?.isEdge == true) + } + + @Test(arguments: [Float(0), 9]) + func `low-confidence depth carves no free space`(stripDepth: Float) { + let field = wallWithStrip(stripDepth: stripDepth, stripConfidence: 0) + #expect(field.voxels[VoxelKey(0, 0, 0)] == nil, "fusion skips the strip") + #expect(!field.free.contains(VoxelKey(0, 0, 0))) + #expect(field.voxels.values.allSatisfy { !$0.isEdge }, "a flat wall with no confident hole has no edges") + } + @Test func `the gradient threshold makes an image edge`() { var field = VoxelField() for x in Int32(0)..<3 { diff --git a/experiments/live-dots/Tests/LiveDotsCoreTests/ReplayPlaybackTests.swift b/experiments/live-dots/Tests/LiveDotsCoreTests/ReplayPlaybackTests.swift new file mode 100644 index 000000000..c002be63e --- /dev/null +++ b/experiments/live-dots/Tests/LiveDotsCoreTests/ReplayPlaybackTests.swift @@ -0,0 +1,177 @@ +import Testing +@testable import LiveDotsCore + +struct ReplayPlaybackTests { + private func request(_ playback: ReplayPlayback, reduceMotion: Bool = false) -> FrameRequest { + playback.frameRequest(mode: .lidar, reduceMotion: reduceMotion, fog: .on) + } + + @Test func `a seek within the playing keyframe resets the fog once`() { + var playback = ReplayPlayback(keyframeCount: 10) + var continuity = FogContinuity() + playback.togglePlayback() + playback.tick(now: 0) + var resetsFog = continuity.consume(request(playback)) + #expect(resetsFog) + playback.tick(now: 0.2) + let before = request(playback) + resetsFog = continuity.consume(before) + #expect(!resetsFog) + playback.scrub(to: 0) + let after = request(playback) + #expect(after.keyframe == before.keyframe) + #expect(after.time == 0 && !after.isSettled) + #expect(after.seekGeneration == before.seekGeneration + 1) + resetsFog = continuity.consume(after) + #expect(resetsFog) + playback.tick(now: 0.3) + resetsFog = continuity.consume(request(playback)) + #expect(!resetsFog) + } + + @Test func `natural playback eases across same and adjacent frames but explicit seeks reset`() { + var playback = ReplayPlayback(keyframeCount: 10) + var continuity = FogContinuity() + playback.togglePlayback() + playback.tick(now: 0) + var resetsFog = continuity.consume(request(playback)) + #expect(resetsFog) + playback.tick(now: 0.1) + resetsFog = continuity.consume(request(playback)) + #expect(!resetsFog) + playback.tick(now: 0.3) + #expect(playback.keyframe == 1) + resetsFog = continuity.consume(request(playback)) + #expect(!resetsFog) + playback.scrub(to: 2) + resetsFog = continuity.consume(request(playback)) + #expect(resetsFog) + playback.scrub(to: 1) + resetsFog = continuity.consume(request(playback)) + #expect(resetsFog) + } + + @Test(arguments: [false, true]) func `resuming a settled scrub resets at the birth time`(reduceMotion: Bool) { + var playback = ReplayPlayback(keyframeCount: 10) + var continuity = FogContinuity() + playback.scrub(to: 2) + let preview = request(playback, reduceMotion: reduceMotion) + #expect(preview.isSettled) + var resetsFog = continuity.consume(preview) + #expect(resetsFog) + playback.togglePlayback() + let resumed = request(playback, reduceMotion: reduceMotion) + #expect(resumed.time == Schedule.start(of: 2) && !resumed.isSettled) + #expect(resumed.seekGeneration == preview.seekGeneration + 1) + #expect(resumed.reduceMotion == reduceMotion) + resetsFog = continuity.consume(resumed) + #expect(resetsFog) + resetsFog = continuity.consume(resumed) + #expect(!resetsFog) + } + + @Test func `scrub and resume between rendered frames still invalidate the old mask`() { + var playback = ReplayPlayback(keyframeCount: 10) + var continuity = FogContinuity() + playback.togglePlayback() + playback.tick(now: 0) + var resetsFog = continuity.consume(request(playback)) + #expect(resetsFog) + playback.tick(now: 0.2) + resetsFog = continuity.consume(request(playback)) + #expect(!resetsFog) + playback.togglePlayback() + playback.scrub(to: 0) + playback.togglePlayback() + resetsFog = continuity.consume(request(playback)) + #expect(resetsFog) + } + + @Test func `pause and resume without a scrub preserve the playhead and fog easing`() { + var playback = ReplayPlayback(keyframeCount: 10) + var continuity = FogContinuity() + playback.togglePlayback() + playback.tick(now: 0) + var resetsFog = continuity.consume(request(playback)) + #expect(resetsFog) + playback.tick(now: 0.2) + let before = request(playback) + resetsFog = continuity.consume(before) + #expect(!resetsFog) + playback.togglePlayback() + playback.tick(now: 5) + playback.togglePlayback() + playback.tick(now: 10) + let resumed = request(playback) + #expect(resumed.time == before.time) + #expect(resumed.seekGeneration == before.seekGeneration) + resetsFog = continuity.consume(resumed) + #expect(!resetsFog) + playback.tick(now: 10.02) + #expect(abs(playback.playhead - before.time - 0.02) < 1e-5) + } + + @Test func `restarting completed playback seeks to the first frame`() { + var playback = ReplayPlayback(keyframeCount: 3) + var continuity = FogContinuity() + playback.togglePlayback() + playback.tick(now: 0) + playback.tick(now: 10) + #expect(!playback.isPlaying) + let final = request(playback) + var resetsFog = continuity.consume(final) + #expect(resetsFog) + playback.togglePlayback() + let restarted = request(playback) + #expect(restarted.keyframe == 0 && restarted.time == 0) + #expect(restarted.seekGeneration == final.seekGeneration + 1) + resetsFog = continuity.consume(restarted) + #expect(resetsFog) + } + + @Test func `mode changes fog reentry and settled previews invalidate continuity`() { + var playback = ReplayPlayback(keyframeCount: 10) + var continuity = FogContinuity() + playback.togglePlayback() + var frame = request(playback) + var resetsFog = continuity.consume(frame) + #expect(resetsFog) + resetsFog = continuity.consume(frame) + #expect(!resetsFog) + frame.mode = .noLidar + resetsFog = continuity.consume(frame) + #expect(resetsFog) + resetsFog = continuity.consume(frame) + #expect(!resetsFog) + frame.fog = .off + resetsFog = continuity.consume(frame) + #expect(!resetsFog) + frame.fog = .on + resetsFog = continuity.consume(frame) + #expect(resetsFog) + frame.time = 1e5 + resetsFog = continuity.consume(frame) + #expect(resetsFog) + frame.time = 0 + resetsFog = continuity.consume(frame) + #expect(resetsFog) + resetsFog = continuity.consume(frame) + #expect(!resetsFog) + } + + @Test func `offscreen clock rewinds and nonadjacent jumps reset without a player`() { + var continuity = FogContinuity() + var frame = FrameRequest(mode: .lidar, keyframe: 0, time: 0.2, reduceMotion: false, fog: .on) + var resetsFog = continuity.consume(frame) + #expect(resetsFog) + frame.time = 0.1 + resetsFog = continuity.consume(frame) + #expect(resetsFog) + frame.keyframe = 3 + frame.time = Schedule.start(of: 3) + resetsFog = continuity.consume(frame) + #expect(resetsFog) + resetsFog = continuity.consume(frame) + #expect(!resetsFog) + } +} diff --git a/experiments/live-dots/Tests/LiveDotsCoreTests/ScheduleTests.swift b/experiments/live-dots/Tests/LiveDotsCoreTests/ScheduleTests.swift new file mode 100644 index 000000000..192eec18a --- /dev/null +++ b/experiments/live-dots/Tests/LiveDotsCoreTests/ScheduleTests.swift @@ -0,0 +1,25 @@ +import Testing +@testable import LiveDotsCore + +struct ScheduleTests { + func close(_ a: Float, _ b: Float) -> Bool { abs(a - b) < 1e-5 } + + @Test func `a keyframe is on screen until the next one appears`() { + #expect(close(Schedule.onScreen(0, count: 31), 0.25)) + // The hold keyframe stays for the ring fill and the bracket flash. + #expect(close(Schedule.onScreen(Schedule.holdIndex, count: 31), Schedule.holdDuration + Schedule.flashDuration)) + for index in 0..<30 { + let shown = Schedule.onScreen(index, count: 31) + #expect(Schedule.keyframe(at: Schedule.start(of: index) + shown - 1e-3, count: 31) == index) + #expect(Schedule.keyframe(at: Schedule.start(of: index) + shown + 1e-3, count: 31) == index + 1) + } + } + + @Test func `the last keyframe is on screen only until playback stops`() { + // keyframe(at:) holds the last keyframe at any time, so it cannot bound an offset into it. + #expect(Schedule.keyframe(at: 1e5, count: 31) == 30) + let shown = Schedule.onScreen(30, count: 31) + #expect(close(Schedule.start(of: 30) + shown, Schedule.playbackEnd(count: 31))) + #expect(close(shown, 0.25 + Tuning.birthDuration)) + } +} diff --git a/experiments/meter-closeup/METHODS.md b/experiments/meter-closeup/METHODS.md index 666b52599..43109205d 100644 --- a/experiments/meter-closeup/METHODS.md +++ b/experiments/meter-closeup/METHODS.md @@ -17,7 +17,7 @@ Two AI models, not people, read every photo, so every number is only as good as - **Loose** (73 photos, used for the headline): reader 1's number equals any number reader 2 listed, or one contains the other and the shorter has at least 6 characters, and neither reader doubted a character. - **Strict** (62 photos): both readers' main numbers are identical and both were sure. Reading and ranking results are reported under both rules. -To confirm the labels by hand, run `make review` and open `~/house-scanning-data/meter-closeup/review.html`. Each photo shows whole, beside the agreed number. Press K to keep the number, or F to type a correction. Then run `uv run python -m meter_eval.review ingest ` and rebuild with `uv run python -m meter_eval.labels`. A kept number counts as agreed under both rules; a corrected one does not until someone keeps it. Ingest merges new answers into earlier ones, and a rebuilt page shows each earlier correction in place of the readers' number, so Keep confirms the correction even in a new browser. +To confirm the labels by hand, run `make review` and open `~/house-scanning-data/meter-closeup/review.html`. Each photo shows whole, beside the agreed number. Press K to keep the number, or F to type a correction. Then run `uv run python -m meter_eval.review ingest ` and rebuild with `uv run python -m meter_eval.labels`. A kept number counts as agreed under both rules; a corrected one does not until someone keeps it. A Keep of the readers' number saved in the browser stays only while the page shows that number, so after a label correction that photo opens again; a correction typed there, and a Keep of that correction, stay. Ingest merges new answers into earlier ones, and a rebuilt page shows each earlier correction in place of the readers' number, so Keep confirms the correction even in a new browser. ## Keeping meter numbers out of the repository diff --git a/experiments/meter-closeup/src/meter_eval/labels.py b/experiments/meter-closeup/src/meter_eval/labels.py index d148175ac..d2150cc98 100644 --- a/experiments/meter-closeup/src/meter_eval/labels.py +++ b/experiments/meter-closeup/src/meter_eval/labels.py @@ -134,7 +134,19 @@ def build() -> list[dict]: "usable": usable, "notes": scrub(r1["notes"]), } - verdict = human.get(image_id, {}).get("verdict", "") + answer = human.get(image_id, {}) + verdict = answer.get("verdict", "") + # A bare Keep confirms only the label shown at review, not a later reader update. + shown = "" if number == "NONE" else number + shown_known = answer.get("shown_known") == "yes" or ( + "shown_known" not in answer and bool(answer.get("shown")) + ) + if ( + verdict == "keep" + and not answer.get("number") + and (not shown_known or answer.get("shown") != shown) + ): + verdict = "" # A person's number replaces the readers': typed with "fix", or confirmed with "keep" # after a fix. A "keep" with no number confirms the readers' own number. if verdict in ("fix", "keep") and human[image_id].get("number"): diff --git a/experiments/meter-closeup/src/meter_eval/match.py b/experiments/meter-closeup/src/meter_eval/match.py index 13aaf85f2..86a9bb354 100644 --- a/experiments/meter-closeup/src/meter_eval/match.py +++ b/experiments/meter-closeup/src/meter_eval/match.py @@ -10,6 +10,7 @@ import functools import hashlib import hmac +import itertools import os import re from collections.abc import Sequence @@ -64,11 +65,28 @@ def digest(normalized: str) -> str: return hmac.new(key(), normalized.encode(), hashlib.sha256).hexdigest() +# Boxes in one row join only across a horizontal gap of at most this many character widths +# (a box's width over its non-space characters, averaged over the two boxes). It is measured in +# the image's own x units, so it needs no aspect ratio. On the cached Vision reads of the 75 +# labelled photos (clean and locate runs, every configuration), a limit of 2 to 6 widths leaves +# every number match as it was and 1.5 loses three; 3 keeps a margin. The limit does split +# unrelated fields there (a model code from a serial on the same row), but no read depended on +# that join, so these photos show no false read it prevents. +MAX_JOIN_GAP_CHARS = 3.0 + + +def _char_width(line: dict) -> float: + return line["box"][2] / max(1, len(line["text"].replace(" ", ""))) + + def rows_of_text(lines: Sequence[dict]) -> list[tuple[str, list[list[float]]]]: - """Each line's (text, [box]), plus lines sharing a row joined left to right. + """Each line's (text, [box]), plus neighbouring lines in one row joined left to right. Vision sometimes splits one printed number into two observations ("123 45" and "678"). - Two boxes share a row when their vertical overlap is at least half the shorter box. + Two boxes share a row when their vertical overlap is at least half the shorter box, and + they join only when the gap between them is at most MAX_JOIN_GAP_CHARS character widths, + so digits from an unrelated field further along the same row are not read as part of the + number. """ texts = [(line["text"], [line["box"]]) for line in lines] rows: list[list[dict]] = [] @@ -83,9 +101,17 @@ def rows_of_text(lines: Sequence[dict]) -> list[tuple[str, list[list[float]]]]: else: rows.append([line]) for row in rows: - if len(row) > 1: - ordered = sorted(row, key=lambda item: item["box"][0]) - texts.append((" ".join(item["text"] for item in ordered), [i["box"] for i in ordered])) + ordered = sorted(row, key=lambda item: item["box"][0]) + chains = [[ordered[0]]] + for left, right in itertools.pairwise(ordered): + gap = right["box"][0] - (left["box"][0] + left["box"][2]) + if gap <= MAX_JOIN_GAP_CHARS * (_char_width(left) + _char_width(right)) / 2: + chains[-1].append(right) + else: + chains.append([right]) + for chain in chains: + if len(chain) > 1: + texts.append((" ".join(i["text"] for i in chain), [i["box"] for i in chain])) return texts diff --git a/experiments/meter-closeup/src/meter_eval/review.py b/experiments/meter-closeup/src/meter_eval/review.py index 6eb71c770..61c6e2a0d 100644 --- a/experiments/meter-closeup/src/meter_eval/review.py +++ b/experiments/meter-closeup/src/meter_eval/review.py @@ -96,30 +96,50 @@ def read_answers(path) -> dict[str, dict]: rows = list(csv.DictReader(handle)) if rows and (missing := {"id", "verdict", "number"} - set(rows[0])): raise SystemExit(f"{path} lacks columns {sorted(missing)}; export it from review.html") - return {r["id"]: {"verdict": r["verdict"], "number": r["number"]} for r in rows if r["verdict"]} + return { + r["id"]: { + "verdict": r["verdict"], + "number": r["number"], + **( + {"shown": r["shown"]} + if r.get("shown_known") == "yes" or ("shown_known" not in r and r.get("shown")) + else {} + ), + } + for r in rows + if r["verdict"] + } def merge(earlier: dict[str, dict], new: dict[str, dict]) -> dict[str, dict]: """New answers replace earlier ones photo by photo; photos not answered again keep theirs. - A Keep that carries no number confirms what the page showed, which is the earlier - correction when there is one, so it keeps that number rather than dropping it. + A bare Keep confirms an earlier correction only when its recorded label matches. + An unbound legacy Keep retains the earlier correction without upgrading its verdict. """ merged = dict(earlier) for image_id, answer in new.items(): before = earlier.get(image_id, {}) if answer["verdict"] == "keep" and not answer["number"] and before.get("number"): - answer = {"verdict": "keep", "number": before["number"]} + if "shown" not in answer: + # An unbound legacy Keep cannot confirm a new number or erase a prior correction. + answer = before + elif answer["shown"] == before["number"]: + answer = {**answer, "number": before["number"]} merged[image_id] = answer return merged def ingest(path: str) -> None: merged = merge(read_answers(HUMAN_LABELS), read_answers(Path(path))) + # CSV empty cells cannot distinguish a reviewed empty label from missing legacy metadata. with HUMAN_LABELS.open("w", newline="") as handle: writer = csv.writer(handle) - writer.writerow(["id", "verdict", "number"]) - writer.writerows([i, a["verdict"], a["number"]] for i, a in sorted(merged.items())) + writer.writerow(["id", "verdict", "number", "shown", "shown_known"]) + writer.writerows( + [i, a["verdict"], a["number"], a.get("shown", ""), "yes" if "shown" in a else ""] + for i, a in sorted(merged.items()) + ) fixed = [i for i, a in merged.items() if a["verdict"] == "fix"] print(f"{len(merged)} answers, {len(fixed)} corrections -> {HUMAN_LABELS}") for image_id in sorted(fixed): @@ -222,12 +242,22 @@ def ingest(path: str) -> None: ', page, re.DOTALL).group(1)) + return row, items + + _, items = reload() + stored = {"m01": {**keep_answer(None, items[0]), "shown": items[0]["number"]}} + rows = on_page("answerRows(items, stored)", items=items, stored=stored) + assert rows == [ + ["id", "verdict", "number", "shown", "shown_known"], + ["m01", "keep", "", "1111111", "yes"], + ] + if legacy_csv: + rows = [row[:3] for row in rows] + download = tmp_path / "answers.csv" + with download.open("w", newline="") as handle: + csv.writer(handle).writerows(rows) + review.ingest(str(download)) + imported = review.read_answers(human)["m01"] + assert imported.get("shown", "") == ("" if legacy_csv else "1111111") + if not legacy_csv: + before, items = reload() + assert before["human_check"] == "keep" + assert on_page("startingAnswers(items, {})", items=items) == {"m01": imported} + + write(tmp_path / "labels_reader1.csv", [{**reader, "meter_number": new_number}]) + after, items = reload() + assert items[0]["number"] == ("" if new_number == "NONE" else new_number) + assert after.get("human_check", "") == "" + assert after.get("number_agreed", "no") == after.get("number_agreed_strict", "no") == "no" + # Both a fresh browser and the stale original browser must reopen the changed label. + assert on_page("startingAnswers(items, {})", items=items) == {} + assert on_page("startingAnswers(items, stored)", items=items, stored=stored) == {} + assert on_page("answerRows(items, startingAnswers(items, stored))", items=items, stored=stored)[ + 1 + ] == [ + "m01", + "", + "", + "", + "", + ] + + +def test_a_keep_for_a_different_shown_number_does_not_inherit_an_earlier_correction(): + old = {"m01": {"verdict": "fix", "number": "1111111"}} + new = {"m01": {"verdict": "keep", "number": "", "shown": "2222222"}} + assert review.merge(old, new) == new + + +@pytest.mark.parametrize("legacy_csv", [False, True]) +def test_csv_distinguishes_a_reviewed_empty_label_from_missing_binding( + tmp_path, monkeypatch, legacy_csv +): + items = [{"id": "m01", "number": "", "corrected": False, "answer": None}] + kept = {"m01": {"verdict": "keep", "number": "", "shown": ""}} + rows = on_page("answerRows(items, kept)", items=items, kept=kept) + if legacy_csv: + rows = [row[:3] for row in rows] + download = tmp_path / "download.csv" + with download.open("w", newline="") as handle: + csv.writer(handle).writerows(rows) + human = tmp_path / "human.csv" + monkeypatch.setattr(review, "HUMAN_LABELS", human) + review.ingest(str(download)) + imported = review.read_answers(human) + items[0]["answer"] = imported["m01"] + if legacy_csv: + assert "shown" not in imported["m01"] + assert on_page("startingAnswers(items, {})", items=items) == {} + else: + assert imported == kept + assert on_page("startingAnswers(items, {})", items=items) == kept + # Writing and re-ingesting the human file must not invent a binding on a second pass. + review.ingest(str(human)) + assert review.read_answers(human) == imported + + +def test_a_bound_keep_of_the_prior_correction_retains_its_label_binding(): + old = {"m01": {"verdict": "fix", "number": "1111111"}} + new = {"m01": {"verdict": "keep", "number": "", "shown": "1111111"}} + assert review.merge(old, new) == { + "m01": {"verdict": "keep", "number": "1111111", "shown": "1111111"} + } diff --git a/experiments/research-handoff/capture-admission/admission.py b/experiments/research-handoff/capture-admission/admission.py index 8ed9e022c..b603daa04 100644 --- a/experiments/research-handoff/capture-admission/admission.py +++ b/experiments/research-handoff/capture-admission/admission.py @@ -1,4 +1,5 @@ """Synthetic capture-admission reference; upstream assessment is NOT verified here.""" + from __future__ import annotations import hashlib @@ -31,15 +32,27 @@ class ReturnedStill: def encoded(self) -> str: if not valid_id(self.frame_id) or not valid_id(self.epoch): raise ValueError("Returned-frame identity/epoch must be nonempty strings") - if type(self.timestamp) not in (int, float) or not math.isfinite(self.timestamp) or self.timestamp < 0: + if ( + type(self.timestamp) not in (int, float) + or not math.isfinite(self.timestamp) + or self.timestamp < 0 + ): raise ValueError("Returned-frame identity/epoch/finite timestamp required") - if not isinstance(self.tracking, str) or self.tracking not in {"normal", "limited", "unavailable"}: + if not isinstance(self.tracking, str) or self.tracking not in { + "normal", + "limited", + "unavailable", + }: raise ValueError("Unknown tracking state") if type(self.usable) is not bool: raise ValueError("Assessment must explicitly state usable") if not isinstance(self.supports, tuple) or any( - not isinstance(pair, tuple) or len(pair) != 2 or not valid_id(pair[0]) - or type(pair[1]) is not int or pair[1] < 1 for pair in self.supports + not isinstance(pair, tuple) + or len(pair) != 2 + or not valid_id(pair[0]) + or type(pair[1]) is not int + or pair[1] < 1 + for pair in self.supports ): raise ValueError("Invalid assessed requirement revision") d = asdict(self) @@ -103,46 +116,84 @@ def set_epoch(self, epoch: str): """Start a fresh coordinate epoch. Re-registration into old maps is not modeled.""" if not valid_id(epoch): raise ValueError("Epoch is required") + def op(): - current = self.db.execute("SELECT value FROM settings WHERE key='epoch'").fetchone() + current = self.db.execute( + "SELECT value FROM settings WHERE key='epoch'" + ).fetchone() if current and current[0] == epoch: return if self.db.execute("SELECT 1 FROM epochs WHERE id=?", (epoch,)).fetchone(): - raise Conflict("Epoch ID cannot be reused without explicit registration") + raise Conflict( + "Epoch ID cannot be reused without explicit registration" + ) self.db.execute("INSERT INTO epochs VALUES(?)", (epoch,)) - self.db.execute("INSERT OR REPLACE INTO settings VALUES('epoch',?)", (epoch,)) + self.db.execute( + "INSERT OR REPLACE INTO settings VALUES('epoch',?)", (epoch,) + ) + self._transaction(op) def require(self, key: str, revision: int, kind: str): - if not valid_id(key) or type(revision) is not int or revision < 1 or not isinstance(kind, str) or kind not in {"detail", "spatial"}: + if ( + not valid_id(key) + or type(revision) is not int + or revision < 1 + or not isinstance(kind, str) + or kind not in {"detail", "spatial"} + ): raise ValueError("Invalid requirement") + def op(): - old = self.db.execute("SELECT revision,kind FROM requirements WHERE id=?", (key,)).fetchone() + old = self.db.execute( + "SELECT revision,kind FROM requirements WHERE id=?", (key,) + ).fetchone() if old and (revision < old[0] or (revision == old[0] and kind != old[1])): raise Conflict("A changed requirement needs a newer revision") - self.db.execute("INSERT OR REPLACE INTO requirements VALUES(?,?,?)", (key, revision, kind)) + self.db.execute( + "INSERT OR REPLACE INTO requirements VALUES(?,?,?)", + (key, revision, kind), + ) + self._transaction(op) def request(self, capture_id: str, target: str, preview_frame: str): if not all(valid_id(x) for x in (capture_id, target, preview_frame)): raise ValueError("Capture and preview IDs required") + def op(): - task = self.db.execute("SELECT revision FROM requirements WHERE id=?", (target,)).fetchone() - epoch = self.db.execute("SELECT value FROM settings WHERE key='epoch'").fetchone() + task = self.db.execute( + "SELECT revision FROM requirements WHERE id=?", (target,) + ).fetchone() + epoch = self.db.execute( + "SELECT value FROM settings WHERE key='epoch'" + ).fetchone() if not task or not epoch: raise ValueError("Current requirement and epoch required") identity = (target, task[0], preview_frame, epoch[0]) - old = self.db.execute("SELECT target,revision,preview_frame,epoch FROM requests WHERE id=?", (capture_id,)).fetchone() + old = self.db.execute( + "SELECT target,revision,preview_frame,epoch FROM requests WHERE id=?", + (capture_id,), + ).fetchone() if old: if old != identity: raise Conflict("Capture ID was reused for different intent") return "duplicate_request" - self.db.execute("INSERT INTO requests(id,target,revision,preview_frame,epoch,status) VALUES(?,?,?,?,?,'pending')", (capture_id, *identity)) + self.db.execute( + "INSERT INTO requests(id,target,revision,preview_frame,epoch,status) VALUES(?,?,?,?,?,'pending')", + (capture_id, *identity), + ) return "requested" + return self._transaction(op) - def receive(self, capture_id: str, payload: bytes, still: ReturnedStill, - before_commit: Callable[[], None] | None = None): + def receive( + self, + capture_id: str, + payload: bytes, + still: ReturnedStill, + before_commit: Callable[[], None] | None = None, + ): """Fault hook is an experiment seam. Payload is opaque; no quality/OCR is inferred.""" if not valid_id(capture_id) or not isinstance(still, ReturnedStill): raise ValueError("Capture ID and returned-still record required") @@ -150,8 +201,12 @@ def receive(self, capture_id: str, payload: bytes, still: ReturnedStill, raise ValueError("Nonempty returned bytes required") metadata = still.encoded() digest = hashlib.sha256(payload).hexdigest() + def op(): - prior = self.db.execute("SELECT status,still_metadata,asset_hash FROM requests WHERE id=?", (capture_id,)).fetchone() + prior = self.db.execute( + "SELECT status,still_metadata,asset_hash FROM requests WHERE id=?", + (capture_id,), + ).fetchone() if not prior: raise Conflict("Unknown capture callback") status, old_metadata, old_hash = prior @@ -160,64 +215,109 @@ def op(): raise Conflict("Conflicting callback for completed capture") return "duplicate_callback" if status != "pending": - raise Conflict("Native capture was interrupted; callback requires explicit recovery") + raise Conflict( + "Native capture was interrupted; callback requires explicit recovery" + ) new_status = "admitted" if still.usable else "rejected" # Retain source bytes for either decision in this tiny audit prototype. - self.db.execute("INSERT OR IGNORE INTO assets(hash,payload) VALUES(?,?)", (digest, payload)) - self.db.execute("UPDATE requests SET status=?,still_metadata=?,asset_hash=? WHERE id=?", (new_status, metadata, digest, capture_id)) + self.db.execute( + "INSERT OR IGNORE INTO assets(hash,payload) VALUES(?,?)", + (digest, payload), + ) + self.db.execute( + "UPDATE requests SET status=?,still_metadata=?,asset_hash=? WHERE id=?", + (new_status, metadata, digest, capture_id), + ) if still.usable: for key, revision in set(still.supports): # An unrecognized assessment must not retroactively satisfy a # requirement introduced later under the same identifier. - self.db.execute("""INSERT INTO support + self.db.execute( + """INSERT INTO support SELECT ?,id,revision FROM requirements WHERE id=? AND revision=? AND (kind='detail' OR (kind='spatial' AND ?='normal' AND ?=(SELECT value FROM settings WHERE key='epoch')))""", - (capture_id, key, revision, still.tracking, still.epoch)) + (capture_id, key, revision, still.tracking, still.epoch), + ) if before_commit: before_commit() return new_status + return self._transaction(op) def recover_orphaned_requests(self): """Called only after the app establishes no corresponding native operations survive.""" - return self.db.execute("UPDATE requests SET status='interrupted' WHERE status='pending'").rowcount + return self.db.execute( + "UPDATE requests SET status='interrupted' WHERE status='pending'" + ).rowcount def acknowledge(self, digest: str, byte_count: int): if not valid_id(digest): raise Conflict("Asset hash must be a string") + def op(): - asset = self.db.execute("SELECT length(payload) FROM assets WHERE hash=?", (digest,)).fetchone() + asset = self.db.execute( + "SELECT length(payload) FROM assets WHERE hash=?", (digest,) + ).fetchone() if type(byte_count) is not int or not asset or byte_count != asset[0]: raise Conflict("Server acceptance does not identify a committed asset") - self.db.execute("UPDATE assets SET server_accepted=1 WHERE hash=?", (digest,)) + self.db.execute( + "UPDATE assets SET server_accepted=1 WHERE hash=?", (digest,) + ) + self._transaction(op) def progress(self): return self._read_view(self._progress) def _progress(self): - epoch = self.db.execute("SELECT value FROM settings WHERE key='epoch'").fetchone() + epoch = self.db.execute( + "SELECT value FROM settings WHERE key='epoch'" + ).fetchone() result = {} - for key, revision, kind in self.db.execute("SELECT id,revision,kind FROM requirements ORDER BY id").fetchall(): - candidates = self.db.execute("""SELECT r.asset_hash,r.still_metadata FROM support s + for key, revision, kind in self.db.execute( + "SELECT id,revision,kind FROM requirements ORDER BY id" + ).fetchall(): + candidates = self.db.execute( + """SELECT r.asset_hash,r.still_metadata FROM support s JOIN requests r ON r.id=s.capture_id JOIN assets a ON a.hash=r.asset_hash - WHERE s.requirement_id=? AND s.revision=? AND r.status='admitted'""", (key, revision)).fetchall() + WHERE s.requirement_id=? AND s.revision=? AND r.status='admitted'""", + (key, revision), + ).fetchall() assets = set() for digest, encoded in candidates: frame = json.loads(encoded) - if kind == "spatial" and (not epoch or frame["epoch"] != epoch[0] or frame["tracking"] != "normal"): + if kind == "spatial" and ( + not epoch + or frame["epoch"] != epoch[0] + or frame["tracking"] != "normal" + ): continue assets.add(digest) - result[key] = {"revision": revision, "kind": kind, "captured": bool(assets), - "unique_assets": len(assets), "spatial_epoch": epoch[0] if epoch and kind == "spatial" else None} + result[key] = { + "revision": revision, + "kind": kind, + "captured": bool(assets), + "unique_assets": len(assets), + "spatial_epoch": epoch[0] if epoch and kind == "spatial" else None, + } return result def snapshot(self): def read(): progress = self._progress() - epoch = self.db.execute("SELECT value FROM settings WHERE key='epoch'").fetchone() - return {"progress": progress, "epoch": epoch[0] if epoch else None, - "requests": self.db.execute("SELECT id,status,preview_frame,still_metadata,asset_hash FROM requests ORDER BY id").fetchall(), - "assets": self.db.execute("SELECT hash,length(payload),server_accepted FROM assets ORDER BY hash").fetchall()} + epoch = self.db.execute( + "SELECT value FROM settings WHERE key='epoch'" + ).fetchone() + return { + "progress": progress, + "epoch": epoch[0] if epoch else None, + "requests": self.db.execute( + "SELECT id,status,preview_frame,still_metadata,asset_hash FROM requests ORDER BY id" + ).fetchall(), + "assets": self.db.execute( + "SELECT hash,length(payload),server_accepted FROM assets ORDER BY hash" + ).fetchall(), + } + return self._read_view(read) diff --git a/experiments/research-handoff/capture-admission/test_admission.py b/experiments/research-handoff/capture-admission/test_admission.py index 43527e4d6..219a88811 100644 --- a/experiments/research-handoff/capture-admission/test_admission.py +++ b/experiments/research-handoff/capture-admission/test_admission.py @@ -1,4 +1,5 @@ """Fault controls for the protocol, not perception or installation tests.""" + from dataclasses import replace import hashlib import json @@ -16,8 +17,14 @@ def still(**changes): - base = ReturnedStill("actual-frame-107", 10.7, "epoch-a", "normal", True, - (("detail-a", 1), ("wall-a", 1))) + base = ReturnedStill( + "actual-frame-107", + 10.7, + "epoch-a", + "normal", + True, + (("detail-a", 1), ("wall-a", 1)), + ) return replace(base, **changes) @@ -43,7 +50,10 @@ def reopen(self): def test_good_preview_bad_returned_still(self): # The request itself represents an eligible preview, never an admission. self.assertFalse(self.db.progress()["detail-a"]["captured"]) - self.assertEqual(self.db.receive("capture-1", b"blurry source", still(usable=False)), "rejected") + self.assertEqual( + self.db.receive("capture-1", b"blurry source", still(usable=False)), + "rejected", + ) self.assertFalse(any(v["captured"] for v in self.db.progress().values())) self.assertEqual(len(self.db.snapshot()["assets"]), 1) @@ -59,24 +69,34 @@ def test_actual_still_metadata_and_local_progress_before_upload(self): def test_save_exception_rolls_back_and_retry_admits_once(self): def fail(): raise OSError("injected storage failure before commit") + with self.assertRaises(OSError): self.db.receive("capture-1", b"actual source", still(), fail) self.reopen() self.assertEqual(self.db.snapshot()["assets"], []) self.assertEqual(self.db.snapshot()["requests"][0][1], "pending") self.assertFalse(self.db.progress()["detail-a"]["captured"]) - self.assertEqual(self.db.receive("capture-1", b"actual source", still()), "admitted") + self.assertEqual( + self.db.receive("capture-1", b"actual source", still()), "admitted" + ) self.assertEqual(self.db.progress()["detail-a"]["unique_assets"], 1) def test_owned_process_killed_before_commit(self): self.db.close() marker = Path(self.tmp.name) / "transaction-entered" - proc = subprocess.Popen([sys.executable, __file__, "--crash-worker", str(self.path), str(marker)], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc = subprocess.Popen( + [sys.executable, __file__, "--crash-worker", str(self.path), str(marker)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) try: deadline = time.monotonic() + 10 - while not marker.exists() and proc.poll() is None and time.monotonic() < deadline: - time.sleep(.02) + while ( + not marker.exists() + and proc.poll() is None + and time.monotonic() < deadline + ): + time.sleep(0.02) self.assertTrue(marker.exists(), "Owned child never reached precommit seam") proc.kill() proc.communicate(timeout=3) @@ -94,15 +114,23 @@ def test_callback_idempotency_and_conflict_after_restart(self): self.db.receive("capture-1", b"actual source", still()) self.reopen() before = self.db.snapshot() - self.assertEqual(self.db.receive("capture-1", b"actual source", still()), "duplicate_callback") - for payload, frame in [(b"different source", still()), (b"actual source", still(timestamp=10.8))]: + self.assertEqual( + self.db.receive("capture-1", b"actual source", still()), + "duplicate_callback", + ) + for payload, frame in [ + (b"different source", still()), + (b"actual source", still(timestamp=10.8)), + ]: with self.assertRaises(Conflict): self.db.receive("capture-1", payload, frame) self.assertEqual(self.db.snapshot(), before) def test_other_current_requirement_does_not_inherit_requested_target(self): # Actual assessment supports B even though preview/request was for A. - self.db.receive("capture-1", b"actual B detail", still(supports=(("detail-b", 1),))) + self.db.receive( + "capture-1", b"actual B detail", still(supports=(("detail-b", 1),)) + ) self.assertFalse(self.db.progress()["detail-a"]["captured"]) self.assertTrue(self.db.progress()["detail-b"]["captured"]) @@ -121,11 +149,17 @@ def test_old_epoch_callback_and_limited_tracking_are_not_spatial_evidence(self): self.assertTrue(self.db.progress()["detail-a"]["captured"]) self.assertFalse(self.db.progress()["wall-a"]["captured"]) self.db.request("capture-2", "wall-a", "preview-b") - self.db.receive("capture-2", b"limited still", still(epoch="epoch-b", tracking="limited")) + self.db.receive( + "capture-2", b"limited still", still(epoch="epoch-b", tracking="limited") + ) self.assertFalse(self.db.progress()["wall-a"]["captured"]) def test_requirement_revision_and_unknown_target_do_not_reuse_support(self): - self.db.receive("capture-1", b"actual source", still(supports=(("detail-a", 1), ("future-detail", 1)))) + self.db.receive( + "capture-1", + b"actual source", + still(supports=(("detail-a", 1), ("future-detail", 1))), + ) self.db.require("detail-a", 2, "detail") self.db.require("future-detail", 1, "detail") self.assertFalse(self.db.progress()["detail-a"]["captured"]) @@ -141,7 +175,11 @@ def test_server_acceptance_matches_exact_committed_hash_and_length(self): data = b"actual source" self.db.receive("capture-1", data, still()) digest = hashlib.sha256(data).hexdigest() - for key, size in [("0" * 64, len(data)), (digest, len(data) + 1), (digest, True)]: + for key, size in [ + ("0" * 64, len(data)), + (digest, len(data) + 1), + (digest, True), + ]: with self.assertRaises(Conflict): self.db.acknowledge(key, size) self.assertEqual(self.db.snapshot()["assets"][0][2], 0) @@ -152,7 +190,11 @@ def test_server_acceptance_matches_exact_committed_hash_and_length(self): def test_identical_asset_under_two_requests_has_one_unique_asset(self): self.db.receive("capture-1", b"same bytes", still()) self.db.request("capture-2", "detail-a", "preview-frame-200") - self.db.receive("capture-2", b"same bytes", still(frame_id="actual-frame-207", timestamp=20.7)) + self.db.receive( + "capture-2", + b"same bytes", + still(frame_id="actual-frame-207", timestamp=20.7), + ) self.assertEqual(len(self.db.snapshot()["assets"]), 1) self.assertEqual(self.db.progress()["detail-a"]["unique_assets"], 1) self.assertEqual(len(self.db.snapshot()["requests"]), 2) @@ -176,7 +218,11 @@ def test_prior_epoch_cannot_resurrect_spatial_support(self): self.db.set_epoch("epoch-b") # Current-epoch assignment is idempotent. def test_unknown_future_epoch_does_not_gain_spatial_support_later(self): - self.db.receive("capture-1", b"unregistered future-epoch source", still(epoch="future-epoch")) + self.db.receive( + "capture-1", + b"unregistered future-epoch source", + still(epoch="future-epoch"), + ) self.assertFalse(self.db.progress()["wall-a"]["captured"]) self.db.set_epoch("future-epoch") self.assertFalse(self.db.progress()["wall-a"]["captured"]) @@ -184,8 +230,14 @@ def test_unknown_future_epoch_does_not_gain_spatial_support_later(self): def test_malformed_metadata_cannot_use_sqlite_affinity_to_match_task(self): self.db.require("1", 1, "detail") - bad = [still(frame_id=["frame"]), still(timestamp=True), still(timestamp=float("nan")), - still(epoch=1), still(supports=((True, 1),)), still(supports=(("1", True),))] + bad = [ + still(frame_id=["frame"]), + still(timestamp=True), + still(timestamp=float("nan")), + still(epoch=1), + still(supports=((True, 1),)), + still(supports=(("1", True),)), + ] before = self.db.snapshot() for frame in bad: with self.assertRaises(ValueError): @@ -198,9 +250,12 @@ def test_malformed_metadata_cannot_use_sqlite_affinity_to_match_task(self): def test_two_writers_cannot_regress_requirement_revision(self): self.db.require("detail-a", 2, "detail") - self.db.receive("capture-1", b"revision2 evidence", still(supports=(("detail-a", 2),))) + self.db.receive( + "capture-1", b"revision2 evidence", still(supports=(("detail-a", 2),)) + ) ready, go, done = threading.Event(), threading.Event(), threading.Event() failures = [] + def writer_b(): other = None try: @@ -215,16 +270,19 @@ def writer_b(): if other: other.close() done.set() + worker = threading.Thread(target=writer_b) worker.start() self.assertTrue(ready.wait(3)) + def before_insert(sql): if sql.startswith("INSERT OR REPLACE INTO requirements"): # A has already read rev2. Old separate autocommits allowed B to # install rev3 here, then A overwrote it with rev2. BEGIN IMMEDIATE # now holds the write reservation through the validated write. go.set() - done.wait(.3) + done.wait(0.3) + self.db.db.set_trace_callback(before_insert) try: self.db.require("detail-a", 2, "detail") @@ -243,13 +301,17 @@ def before_commit(): self.db.progress() with self.assertRaises(Conflict): self.db.snapshot() + self.db.receive("capture-1", b"actual source", still(), before_commit) self.assertTrue(self.db.progress()["detail-a"]["captured"]) - def test_snapshot_is_one_read_view_during_concurrent_epoch_and_revision_changes(self): + def test_snapshot_is_one_read_view_during_concurrent_epoch_and_revision_changes( + self, + ): self.db.receive("capture-1", b"actual source", still()) ready, go, done = threading.Event(), threading.Event(), threading.Event() errors = [] + def change_state(): other = None try: @@ -265,13 +327,16 @@ def change_state(): if other: other.close() done.set() + worker = threading.Thread(target=change_state) worker.start() self.assertTrue(ready.wait(3)) + def change_after_epoch_read(sql): if sql.startswith("SELECT id,revision,kind FROM requirements"): go.set() done.wait(2) + self.db.db.set_trace_callback(change_after_epoch_read) try: old_view = self.db.snapshot() @@ -293,15 +358,25 @@ def change_after_epoch_read(sql): if __name__ == "__main__": if len(sys.argv) == 4 and sys.argv[1] == "--crash-worker": ledger = Ledger(Path(sys.argv[2])) + def freeze_before_commit(): Path(sys.argv[3]).write_text("inside uncommitted admission transaction") signal.pause() + ledger.receive("capture-1", b"actual source", still(), freeze_before_commit) raise SystemExit("The owned worker should be killed before commit") suite = unittest.defaultTestLoader.loadTestsFromTestCase(AdmissionControls) result = unittest.TextTestRunner(verbosity=2).run(suite) - receipt = {"tests_run": result.testsRun, "failures": len(result.failures), "errors": len(result.errors), - "passed": result.wasSuccessful(), "scope": "synthetic admission protocol and SQLite process-crash rollback, no camera/perception/physical accuracy", - "python": sys.version, "sqlite": sqlite3.sqlite_version} - Path(__file__).with_name("test-receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + receipt = { + "tests_run": result.testsRun, + "failures": len(result.failures), + "errors": len(result.errors), + "passed": result.wasSuccessful(), + "scope": "synthetic admission protocol and SQLite process-crash rollback, no camera/perception/physical accuracy", + "python": sys.version, + "sqlite": sqlite3.sqlite_version, + } + Path(__file__).with_name("test-receipt.json").write_text( + json.dumps(receipt, indent=2) + "\n" + ) raise SystemExit(0 if result.wasSuccessful() else 1) diff --git a/experiments/research-handoff/capture-admission/test_regressions.py b/experiments/research-handoff/capture-admission/test_regressions.py index ac4053a19..16ca78ba7 100644 --- a/experiments/research-handoff/capture-admission/test_regressions.py +++ b/experiments/research-handoff/capture-admission/test_regressions.py @@ -1,4 +1,5 @@ """Existing independent synthetic regression controls, packaged without review tooling.""" + from dataclasses import replace import hashlib import sqlite3 @@ -15,8 +16,12 @@ def frame(**changes): - return replace(ReturnedStill("actual-1", 1.2, "epoch-a", "normal", True, - (("detail", 1), ("wall", 2))), **changes) + return replace( + ReturnedStill( + "actual-1", 1.2, "epoch-a", "normal", True, (("detail", 1), ("wall", 2)) + ), + **changes, + ) class FixControls(unittest.TestCase): @@ -50,9 +55,16 @@ def test_epoch_aba_reopen_and_late_old_epoch(self): self.assertTrue(after["progress"]["detail"]["captured"]) self.assertFalse(after["progress"]["wall"]["captured"]) self.assertEqual(after["progress"]["wall"]["spatial_epoch"], "epoch-b") - self.assertEqual(self.db.db.execute("SELECT requirement_id FROM support WHERE capture_id='capture-2'").fetchall(), [("detail",)]) - DETAILS["epoch_aba"] = {"after_reopen_and_old_epoch_attempt": before, - "late_old_epoch_callback": after} + self.assertEqual( + self.db.db.execute( + "SELECT requirement_id FROM support WHERE capture_id='capture-2'" + ).fetchall(), + [("detail",)], + ) + DETAILS["epoch_aba"] = { + "after_reopen_and_old_epoch_attempt": before, + "late_old_epoch_callback": after, + } def test_unknown_future_epoch_and_retry_stay_ineligible(self): future = frame(epoch="future-epoch") @@ -61,28 +73,39 @@ def test_unknown_future_epoch_and_retry_stay_ineligible(self): self.db.set_epoch("future-epoch") self.db.close() self.db = Ledger(self.path) - self.assertEqual(self.db.receive("capture-1", b"future", future), "duplicate_callback") + self.assertEqual( + self.db.receive("capture-1", b"future", future), "duplicate_callback" + ) after = self.db.snapshot() self.assertTrue(after["progress"]["detail"]["captured"]) self.assertFalse(after["progress"]["wall"]["captured"]) - self.assertEqual(self.db.db.execute("SELECT requirement_id FROM support").fetchall(), [("detail",)]) - DETAILS["future_epoch"] = {"before_epoch_introduction": first, - "after_introduction_reopen_identical_retry": after} + self.assertEqual( + self.db.db.execute("SELECT requirement_id FROM support").fetchall(), + [("detail",)], + ) + DETAILS["future_epoch"] = { + "before_epoch_introduction": first, + "after_introduction_reopen_identical_retry": after, + } def test_actual_two_connection_revision_race(self): self.db.receive("capture-1", b"revision 2", frame()) ready, go, attempted, done = [threading.Event() for _ in range(4)] failures, events, seam_checks = [], [], [] + def record(name): events.append({"event": name, "monotonic": time.monotonic()}) + def writer_b(): db = None try: db = Ledger(self.path) + def trace(sql): if sql == "BEGIN IMMEDIATE": record("B BEGIN IMMEDIATE attempted") attempted.set() + db.db.set_trace_callback(trace) ready.set() if not go.wait(3): @@ -95,18 +118,25 @@ def trace(sql): if db: db.close() done.set() + worker = threading.Thread(target=writer_b) worker.start() self.assertTrue(ready.wait(3)) + def at_a_write(sql): if sql.startswith("INSERT OR REPLACE INTO requirements"): record("A validated revision 2; write pending") go.set() saw_attempt = attempted.wait(2) - committed_while_a_held_write = done.wait(.15) - seam_checks.append({"b_attempted_begin": saw_attempt, - "b_completed_while_a_transaction_open": committed_while_a_held_write, - "a_in_transaction": self.db.db.in_transaction}) + committed_while_a_held_write = done.wait(0.15) + seam_checks.append( + { + "b_attempted_begin": saw_attempt, + "b_completed_while_a_transaction_open": committed_while_a_held_write, + "a_in_transaction": self.db.db.in_transaction, + } + ) + self.db.db.set_trace_callback(at_a_write) try: self.db.require("wall", 2, "spatial") @@ -117,30 +147,57 @@ def at_a_write(sql): worker.join(4) self.assertFalse(worker.is_alive()) self.assertEqual(failures, []) - self.assertEqual(seam_checks, [{"b_attempted_begin": True, - "b_completed_while_a_transaction_open": False, - "a_in_transaction": True}]) + self.assertEqual( + seam_checks, + [ + { + "b_attempted_begin": True, + "b_completed_while_a_transaction_open": False, + "a_in_transaction": True, + } + ], + ) after = self.db.progress() self.assertEqual(after["wall"]["revision"], 3) self.assertFalse(after["wall"]["captured"]) with self.assertRaises(Conflict): self.db.require("wall", 2, "spatial") - DETAILS["two_connection_revision_race"] = {"events": events, "seam_checks": seam_checks, "after": after} + DETAILS["two_connection_revision_race"] = { + "events": events, + "seam_checks": seam_checks, + "after": after, + } def test_runtime_types_reject_affinity_alias_and_malformed_shapes(self): self.db.require("1", 1, "detail") bad_changes = [ - {"frame_id": ["bad"]}, {"frame_id": True}, {"frame_id": " "}, - {"timestamp": True}, {"timestamp": False}, {"timestamp": "1.0"}, - {"timestamp": float("nan")}, {"timestamp": float("inf")}, {"timestamp": -1}, - {"epoch": 1}, {"epoch": {}}, {"epoch": "\t"}, - {"tracking": ["normal"]}, {"tracking": True}, {"tracking": "unknown"}, - {"usable": 1}, {"usable": "true"}, - {"supports": ((True, 1),)}, {"supports": (("1", True),)}, - {"supports": (("1", 1.0),)}, {"supports": (("1", 0),)}, - {"supports": ((" ", 1),)}, {"supports": ((["1"], 1),)}, - {"supports": [("1", 1)]}, {"supports": (["1", 1],)}, - {"supports": (("1",),)}, {"supports": (("1", 1, "extra"),)}, + {"frame_id": ["bad"]}, + {"frame_id": True}, + {"frame_id": " "}, + {"timestamp": True}, + {"timestamp": False}, + {"timestamp": "1.0"}, + {"timestamp": float("nan")}, + {"timestamp": float("inf")}, + {"timestamp": -1}, + {"epoch": 1}, + {"epoch": {}}, + {"epoch": "\t"}, + {"tracking": ["normal"]}, + {"tracking": True}, + {"tracking": "unknown"}, + {"usable": 1}, + {"usable": "true"}, + {"supports": ((True, 1),)}, + {"supports": (("1", True),)}, + {"supports": (("1", 1.0),)}, + {"supports": (("1", 0),)}, + {"supports": ((" ", 1),)}, + {"supports": ((["1"], 1),)}, + {"supports": [("1", 1)]}, + {"supports": (["1", 1],)}, + {"supports": (("1",),)}, + {"supports": (("1", 1, "extra"),)}, {"supports": None}, ] before = self.db.snapshot() @@ -168,12 +225,17 @@ def test_runtime_types_reject_affinity_alias_and_malformed_shapes(self): call() self.assertEqual(self.db.snapshot(), before) self.assertFalse(self.db.progress()["1"]["captured"]) - DETAILS["runtime_types"] = {"returned_still_rejections": cases, "public_api_rejections": len(bad_api), "no_snapshot_mutations": True} + DETAILS["runtime_types"] = { + "returned_still_rejections": cases, + "public_api_rejections": len(bad_api), + "no_snapshot_mutations": True, + } def test_read_view_does_not_mix_concurrent_admission_and_ack(self): ready, go, done = [threading.Event() for _ in range(3)] errors, seam_checks = [], [] payload = b"concurrent still" + def writer(): db = None try: @@ -189,14 +251,21 @@ def writer(): if db: db.close() done.set() + thread = threading.Thread(target=writer) thread.start() self.assertTrue(ready.wait(3)) + def after_first_epoch_read(sql): if sql.startswith("SELECT id,revision,kind FROM requirements"): go.set() - seam_checks.append({"writer_completed_during_reader_transaction": done.wait(3), - "reader_in_transaction": self.db.db.in_transaction}) + seam_checks.append( + { + "writer_completed_during_reader_transaction": done.wait(3), + "reader_in_transaction": self.db.db.in_transaction, + } + ) + self.db.db.set_trace_callback(after_first_epoch_read) try: old_view = self.db.snapshot() @@ -206,8 +275,15 @@ def after_first_epoch_read(sql): thread.join(4) self.assertFalse(thread.is_alive()) self.assertEqual(errors, []) - self.assertEqual(seam_checks, [{"writer_completed_during_reader_transaction": True, - "reader_in_transaction": True}]) + self.assertEqual( + seam_checks, + [ + { + "writer_completed_during_reader_transaction": True, + "reader_in_transaction": True, + } + ], + ) self.assertFalse(any(v["captured"] for v in old_view["progress"].values())) self.assertEqual(old_view["requests"][0][1], "pending") self.assertIsNone(old_view["requests"][0][3]) @@ -216,12 +292,17 @@ def after_first_epoch_read(sql): self.assertTrue(all(v["captured"] for v in new_view["progress"].values())) self.assertEqual(new_view["requests"][0][1], "admitted") self.assertEqual(new_view["assets"][0][2], 1) - DETAILS["read_view_admission_and_ack"] = {"seam_checks": seam_checks, "old_view": old_view, "new_view": new_view} + DETAILS["read_view_admission_and_ack"] = { + "seam_checks": seam_checks, + "old_view": old_view, + "new_view": new_view, + } def test_precommit_other_connection_view_and_rollback(self): other = Ledger(self.path) before = self.db.snapshot() observed = {} + def before_commit(): self.assertTrue(self.db.db.in_transaction) for method in (self.db.progress, self.db.snapshot): @@ -231,15 +312,22 @@ def before_commit(): observed["other_connection_during_write"] = other.snapshot() self.assertEqual(observed["other_connection_during_write"], before) raise OSError("independent rollback after failed publication attempts") + try: with self.assertRaises(OSError): self.db.receive("capture-1", b"rollback", frame(), before_commit) self.assertFalse(self.db.db.in_transaction) self.assertEqual(self.db.snapshot(), before) self.assertEqual(other.snapshot(), before) - self.assertEqual(self.db.receive("capture-1", b"rollback", frame()), "admitted") + self.assertEqual( + self.db.receive("capture-1", b"rollback", frame()), "admitted" + ) observed["other_connection_after_commit"] = other.snapshot() - self.assertTrue(observed["other_connection_after_commit"]["progress"]["wall"]["captured"]) + self.assertTrue( + observed["other_connection_after_commit"]["progress"]["wall"][ + "captured" + ] + ) finally: other.close() DETAILS["precommit_publication_and_rollback"] = observed @@ -248,12 +336,19 @@ def test_failed_read_rolls_back_read_transaction(self): self.db.receive("capture-1", b"saved", frame()) before = self.db.snapshot() denials = [] + def authorizer(action, table, column, database, source): if action == sqlite3.SQLITE_READ and table == "assets": - denials.append({"table": table, "column": column, - "inside_transaction": self.db.db.in_transaction}) + denials.append( + { + "table": table, + "column": column, + "inside_transaction": self.db.db.in_transaction, + } + ) return sqlite3.SQLITE_DENY return sqlite3.SQLITE_OK + self.db.db.set_authorizer(authorizer) try: with self.assertRaises(sqlite3.DatabaseError) as caught: @@ -267,7 +362,12 @@ def authorizer(action, table, column, database, source): self.assertEqual(self.db.snapshot(), before) self.db.require("detail", 2, "detail") self.assertFalse(self.db.progress()["detail"]["captured"]) - DETAILS["failed_read_cleanup"] = {"denials": denials, "error": error, "transaction_cleared": True, "subsequent_read_write_succeeded": True} + DETAILS["failed_read_cleanup"] = { + "denials": denials, + "error": error, + "transaction_cleared": True, + "subsequent_read_write_succeeded": True, + } if __name__ == "__main__": diff --git a/experiments/research-handoff/equipment-association/association.py b/experiments/research-handoff/equipment-association/association.py index 58eb220a0..4e94dfe15 100644 --- a/experiments/research-handoff/equipment-association/association.py +++ b/experiments/research-handoff/equipment-association/association.py @@ -1,4 +1,5 @@ """Prepared evidence-association reducer; software checks do not prove physical identity.""" + from __future__ import annotations import copy import hashlib @@ -30,181 +31,458 @@ def canonical(value): def scope_key(scope): - require(isinstance(scope, dict) and set(scope) == {"session_id", "epoch_id", "sequence_id"}, "scope fields") + require( + isinstance(scope, dict) + and set(scope) == {"session_id", "epoch_id", "sequence_id"}, + "scope fields", + ) require(all(text(v) for v in scope.values()), "scope identifiers") return canonical(scope) def photo(region): - require(isinstance(region, dict) and set(region) == { - "frame_id", "asset_ref", "asset_sha256", "width", "height", "xywh", "scope" - }, "photo region fields") - require(text(region["frame_id"]) and text(region["asset_ref"]) and digest(region["asset_sha256"]), "photo reference/hash") - require(all(type(region[k]) is int and region[k] > 0 for k in ("width", "height")), "photo dimensions") + require( + isinstance(region, dict) + and set(region) + == { + "frame_id", + "asset_ref", + "asset_sha256", + "width", + "height", + "xywh", + "scope", + }, + "photo region fields", + ) + require( + text(region["frame_id"]) + and text(region["asset_ref"]) + and digest(region["asset_sha256"]), + "photo reference/hash", + ) + require( + all(type(region[k]) is int and region[k] > 0 for k in ("width", "height")), + "photo dimensions", + ) box = region["xywh"] - require(isinstance(box, list) and len(box) == 4 and all( - type(v) in (int, float) and math.isfinite(v) for v in box), "finite photo region") + require( + isinstance(box, list) + and len(box) == 4 + and all(type(v) in (int, float) and math.isfinite(v) for v in box), + "finite photo region", + ) x, y, w, h = box - require(x >= 0 and y >= 0 and w > 0 and h > 0 and x+w <= region["width"] and y+h <= region["height"], "photo region bounds") + require( + x >= 0 + and y >= 0 + and w > 0 + and h > 0 + and x + w <= region["width"] + and y + h <= region["height"], + "photo region bounds", + ) scope_key(region["scope"]) return region class Associations: """One process-local actor; persist events/state atomically in a real app adapter.""" + def __init__(self, scope): key = scope_key(scope) self._lock = threading.RLock() - self._state = {"revision": 0, "scope": copy.deepcopy(scope), "seen_scopes": [key], - "candidates": {}, "labels": {}, "tracks": {}, "nominations": {}, - "tickets": {}, "reviews": {}, "attachments": {}, "events": {}} + self._state = { + "revision": 0, + "scope": copy.deepcopy(scope), + "seen_scopes": [key], + "candidates": {}, + "labels": {}, + "tracks": {}, + "nominations": {}, + "tickets": {}, + "reviews": {}, + "attachments": {}, + "events": {}, + } def apply(self, event): """Apply one event atomically; duplicate exact IDs return the original result.""" frozen = json.loads(canonical(event)) - require(isinstance(frozen, dict) and set(frozen) == {"id", "type", "data"}, "event fields") - require(text(frozen["id"]) and text(frozen["type"]) and isinstance(frozen["data"], dict), "event identity/data") + require( + isinstance(frozen, dict) and set(frozen) == {"id", "type", "data"}, + "event fields", + ) + require( + text(frozen["id"]) + and text(frozen["type"]) + and isinstance(frozen["data"], dict), + "event identity/data", + ) encoded = canonical(frozen) with self._lock: old = self._state["events"].get(frozen["id"]) if old: - require(old["event"] == frozen, "event ID reused with different content") + require( + old["event"] == frozen, "event ID reused with different content" + ) return copy.deepcopy(old["result"]) state = copy.deepcopy(self._state) result = self._reduce(state, frozen) state["revision"] += 1 result["snapshot_revision"] = state["revision"] - state["events"][frozen["id"]] = {"event": frozen, "sha256": hashlib.sha256(encoded.encode()).hexdigest(), "result": result} + state["events"][frozen["id"]] = { + "event": frozen, + "sha256": hashlib.sha256(encoded.encode()).hexdigest(), + "result": result, + } self._state = state return copy.deepcopy(result) @staticmethod def _reduce(s, e): kind, d, eid = e["type"], e["data"], e["id"] + def fields(*names): require(set(d) == set(names), "unexpected/missing event fields") + def known_photo(value): p = photo(value) - require(scope_key(p["scope"]) in s["seen_scopes"], "unregistered photo scope") + require( + scope_key(p["scope"]) in s["seen_scopes"], "unregistered photo scope" + ) return p + def current_photo(value): p = known_photo(value) require(p["scope"] == s["scope"], "context photo is not in current scope") return p + def candidate(cid, revision=None): require(cid in s["candidates"], "unknown equipment candidate") c = s["candidates"][cid] if revision is not None: - require(type(revision) is int and revision == c["revision"], "stale candidate revision") + require( + type(revision) is int and revision == c["revision"], + "stale candidate revision", + ) return c + def invalidate(cid, reason): for ticket in s["tickets"].values(): - if ticket["candidate_id"] == cid and ticket["status"] in {"needs_context", "needs_review", "unresolved"}: - ticket.update(status="stale", reason=reason, revision=ticket["revision"]+1) + if ticket["candidate_id"] == cid and ticket["status"] in { + "needs_context", + "needs_review", + "unresolved", + }: + ticket.update( + status="stale", reason=reason, revision=ticket["revision"] + 1 + ) if kind == "create_candidate": fields("candidate_id", "description") - require(text(d["candidate_id"]) and text(d["description"]), "candidate description") - require(d["candidate_id"] not in s["candidates"], "candidate ID already exists") - s["candidates"][d["candidate_id"]] = {"id": d["candidate_id"], "description": d["description"], "revision": 0, "status": "needs_context", "nomination_id": None, "contradictions": []} + require( + text(d["candidate_id"]) and text(d["description"]), + "candidate description", + ) + require( + d["candidate_id"] not in s["candidates"], "candidate ID already exists" + ) + s["candidates"][d["candidate_id"]] = { + "id": d["candidate_id"], + "description": d["description"], + "revision": 0, + "status": "needs_context", + "nomination_id": None, + "contradictions": [], + } return {"status": "needs_context", "reason": "context_photo_required"} if kind in {"record_label", "record_track"}: if kind == "record_label": - fields("observation_id", "text", "producer", "raw_result_ref", "raw_result_sha256", "photo") - require(isinstance(d["text"], str) and text(d["producer"]) and text(d["raw_result_ref"]) and digest(d["raw_result_sha256"]), "label source fields") + fields( + "observation_id", + "text", + "producer", + "raw_result_ref", + "raw_result_sha256", + "photo", + ) + require( + isinstance(d["text"], str) + and text(d["producer"]) + and text(d["raw_result_ref"]) + and digest(d["raw_result_sha256"]), + "label source fields", + ) table = s["labels"] else: fields("observation_id", "tracker_uuid", "confidence", "photo") - require(text(d["tracker_uuid"]) and type(d["confidence"]) in (int,float) and math.isfinite(d["confidence"]) and 0 <= d["confidence"] <= 1, "tracker diagnostics") + require( + text(d["tracker_uuid"]) + and type(d["confidence"]) in (int, float) + and math.isfinite(d["confidence"]) + and 0 <= d["confidence"] <= 1, + "tracker diagnostics", + ) table = s["tracks"] - require(text(d["observation_id"]) and d["observation_id"] not in table, "observation ID reused") + require( + text(d["observation_id"]) and d["observation_id"] not in table, + "observation ID reused", + ) known_photo(d["photo"]) table[d["observation_id"]] = copy.deepcopy(d) - return {"status": "source_saved", "reason": "no_equipment_attachment_inferred"} + return { + "status": "source_saved", + "reason": "no_equipment_attachment_inferred", + } if kind == "nominate_context": - fields("candidate_id", "expected_candidate_revision", "actor", "reason", "photo", "alternatives", "considered_contradictions") + fields( + "candidate_id", + "expected_candidate_revision", + "actor", + "reason", + "photo", + "alternatives", + "considered_contradictions", + ) c = candidate(d["candidate_id"], d["expected_candidate_revision"]) - require(text(d["actor"]) and text(d["reason"]), "explicit context reviewer required") + require( + text(d["actor"]) and text(d["reason"]), + "explicit context reviewer required", + ) current_photo(d["photo"]) - require(isinstance(d["alternatives"], list) and len(set(d["alternatives"])) == len(d["alternatives"]) and all(x in s["candidates"] and x != c["id"] for x in d["alternatives"]), "inspectable alternatives") - require(isinstance(d["considered_contradictions"], list) and sorted(d["considered_contradictions"]) == sorted(c["contradictions"]), "explicitly consider outstanding contradictions") + require( + isinstance(d["alternatives"], list) + and len(set(d["alternatives"])) == len(d["alternatives"]) + and all( + x in s["candidates"] and x != c["id"] for x in d["alternatives"] + ), + "inspectable alternatives", + ) + require( + isinstance(d["considered_contradictions"], list) + and sorted(d["considered_contradictions"]) + == sorted(c["contradictions"]), + "explicitly consider outstanding contradictions", + ) invalidate(c["id"], "context_changed") - c.update(revision=c["revision"]+1, status="context_nominated", nomination_id=eid, contradictions=[]) - s["nominations"][eid] = {**copy.deepcopy(d), "candidate_revision": c["revision"], "scope": copy.deepcopy(s["scope"])} - return {"status": "context_nominated", "candidate_revision": c["revision"], "nomination_id": eid, "reason": "label_specific_review_still_required"} + c.update( + revision=c["revision"] + 1, + status="context_nominated", + nomination_id=eid, + contradictions=[], + ) + s["nominations"][eid] = { + **copy.deepcopy(d), + "candidate_revision": c["revision"], + "scope": copy.deepcopy(s["scope"]), + } + return { + "status": "context_nominated", + "candidate_revision": c["revision"], + "nomination_id": eid, + "reason": "label_specific_review_still_required", + } if kind == "request_attachment": - fields("ticket_id", "candidate_id", "label_id", "expected_candidate_revision") + fields( + "ticket_id", "candidate_id", "label_id", "expected_candidate_revision" + ) c = candidate(d["candidate_id"], d["expected_candidate_revision"]) - require(text(d["ticket_id"]) and d["ticket_id"] not in s["tickets"] and d["label_id"] in s["labels"], "new ticket and saved label required") + require( + text(d["ticket_id"]) + and d["ticket_id"] not in s["tickets"] + and d["label_id"] in s["labels"], + "new ticket and saved label required", + ) # One authority per label/candidate/context; sibling tickets must not # re-admit an older decision after this pair was resolved elsewhere. - require(not any(t["candidate_id"] == c["id"] and t["label_id"] == d["label_id"] - and t["candidate_revision"] == c["revision"] - for t in s["tickets"].values()), - "ticket already exists for candidate/label/context revision") - status = "needs_review" if c["status"] == "context_nominated" else "needs_context" - ticket = {"id": d["ticket_id"], "candidate_id": c["id"], "label_id": d["label_id"], "candidate_revision": c["revision"], "nomination_id": c["nomination_id"], "scope": copy.deepcopy(s["scope"]), "revision": 1, "status": status, "reason": "review_label_and_context" if status == "needs_review" else "context_photo_required"} + require( + not any( + t["candidate_id"] == c["id"] + and t["label_id"] == d["label_id"] + and t["candidate_revision"] == c["revision"] + for t in s["tickets"].values() + ), + "ticket already exists for candidate/label/context revision", + ) + status = ( + "needs_review" + if c["status"] == "context_nominated" + else "needs_context" + ) + ticket = { + "id": d["ticket_id"], + "candidate_id": c["id"], + "label_id": d["label_id"], + "candidate_revision": c["revision"], + "nomination_id": c["nomination_id"], + "scope": copy.deepcopy(s["scope"]), + "revision": 1, + "status": status, + "reason": "review_label_and_context" + if status == "needs_review" + else "context_photo_required", + } s["tickets"][ticket["id"]] = ticket return copy.deepcopy(ticket) if kind == "review_attachment": - fields("ticket_id", "expected_ticket_revision", "expected_candidate_revision", "decision", "actor", "reason", "photo") - require(d["ticket_id"] in s["tickets"] and d["decision"] in {"attach", "unresolved", "reject"} and text(d["actor"]) and text(d["reason"]), "explicit label review required") + fields( + "ticket_id", + "expected_ticket_revision", + "expected_candidate_revision", + "decision", + "actor", + "reason", + "photo", + ) + require( + d["ticket_id"] in s["tickets"] + and d["decision"] in {"attach", "unresolved", "reject"} + and text(d["actor"]) + and text(d["reason"]), + "explicit label review required", + ) known_photo(d["photo"]) - ticket = s["tickets"][d["ticket_id"]];c = candidate(ticket["candidate_id"]) - fresh = (type(d["expected_ticket_revision"]) is int and type(d["expected_candidate_revision"]) is int - and d["expected_ticket_revision"] == ticket["revision"] and d["expected_candidate_revision"] == c["revision"] - and ticket["candidate_revision"] == c["revision"] and ticket["scope"] == s["scope"] - and d["photo"]["scope"] == s["scope"] and ticket["nomination_id"] == c["nomination_id"] - and c["status"] == "context_nominated" and ticket["status"] in {"needs_review", "unresolved"}) - outcome = {"status": "stale_review", "reason": "context_scope_or_review_revision_changed"} + ticket = s["tickets"][d["ticket_id"]] + c = candidate(ticket["candidate_id"]) + fresh = ( + type(d["expected_ticket_revision"]) is int + and type(d["expected_candidate_revision"]) is int + and d["expected_ticket_revision"] == ticket["revision"] + and d["expected_candidate_revision"] == c["revision"] + and ticket["candidate_revision"] == c["revision"] + and ticket["scope"] == s["scope"] + and d["photo"]["scope"] == s["scope"] + and ticket["nomination_id"] == c["nomination_id"] + and c["status"] == "context_nominated" + and ticket["status"] in {"needs_review", "unresolved"} + ) + outcome = { + "status": "stale_review", + "reason": "context_scope_or_review_revision_changed", + } conflicts = set() if fresh and d["decision"] == "attach": for link in s["attachments"].values(): other = candidate(link["candidate_id"]) - if (link["label_id"] == ticket["label_id"] and other["id"] != c["id"] - and other["status"] == "context_nominated" and other["revision"] == link["candidate_revision"] - and link["scope"] == s["scope"]): + if ( + link["label_id"] == ticket["label_id"] + and other["id"] != c["id"] + and other["status"] == "context_nominated" + and other["revision"] == link["candidate_revision"] + and link["scope"] == s["scope"] + ): conflicts.add(other["id"]) if conflicts: # A later incompatible explicit claim does not silently replace the earlier one. # Both require context review; the original observation and decisions survive. for cid in conflicts | {c["id"]}: disputed = candidate(cid) - disputed.update(revision=disputed["revision"]+1, status="contradicted") + disputed.update( + revision=disputed["revision"] + 1, status="contradicted" + ) disputed["contradictions"].append(eid) invalidate(cid, "conflicting_current_assignment") - outcome = {"status": "unresolved", "reason": "conflicting_current_assignment"} - s["reviews"][eid] = {**copy.deepcopy(d), "outcome": copy.deepcopy(outcome)} + outcome = { + "status": "unresolved", + "reason": "conflicting_current_assignment", + } + s["reviews"][eid] = { + **copy.deepcopy(d), + "outcome": copy.deepcopy(outcome), + } return outcome if fresh: - status = {"attach": "attached", "unresolved": "unresolved", "reject": "rejected"}[d["decision"]] - ticket.update(status=status, reason={"attach": "explicit_contextual_review", "unresolved": "association_unresolved", "reject": "review_rejected"}[d["decision"]], revision=ticket["revision"]+1) - outcome = {"status": status, "reason": ticket["reason"], "ticket_revision": ticket["revision"]} + status = { + "attach": "attached", + "unresolved": "unresolved", + "reject": "rejected", + }[d["decision"]] + ticket.update( + status=status, + reason={ + "attach": "explicit_contextual_review", + "unresolved": "association_unresolved", + "reject": "review_rejected", + }[d["decision"]], + revision=ticket["revision"] + 1, + ) + outcome = { + "status": status, + "reason": ticket["reason"], + "ticket_revision": ticket["revision"], + } if status == "attached": - s["attachments"][eid] = {"id": eid, "review_id": eid, "ticket_id": ticket["id"], "candidate_id": c["id"], "label_id": ticket["label_id"], "candidate_revision": c["revision"], "nomination_id": c["nomination_id"], "scope": copy.deepcopy(s["scope"])} + s["attachments"][eid] = { + "id": eid, + "review_id": eid, + "ticket_id": ticket["id"], + "candidate_id": c["id"], + "label_id": ticket["label_id"], + "candidate_revision": c["revision"], + "nomination_id": c["nomination_id"], + "scope": copy.deepcopy(s["scope"]), + } s["reviews"][eid] = {**copy.deepcopy(d), "outcome": copy.deepcopy(outcome)} return outcome if kind == "contradict": - fields("candidate_id", "expected_candidate_revision", "actor", "reason", "photo") + fields( + "candidate_id", + "expected_candidate_revision", + "actor", + "reason", + "photo", + ) c = candidate(d["candidate_id"], d["expected_candidate_revision"]) - require(text(d["actor"]) and text(d["reason"]), "contradiction source required") + require( + text(d["actor"]) and text(d["reason"]), "contradiction source required" + ) known_photo(d["photo"]) - c.update(revision=c["revision"]+1, status="contradicted") - c["contradictions"].append(eid);invalidate(c["id"], "association_contradicted") - return {"status": "contradicted", "reason": "context_review_required", "candidate_revision": c["revision"]} + c.update(revision=c["revision"] + 1, status="contradicted") + c["contradictions"].append(eid) + invalidate(c["id"], "association_contradicted") + return { + "status": "contradicted", + "reason": "context_review_required", + "candidate_revision": c["revision"], + } if kind == "reset_scope": fields("scope", "reason") - key = scope_key(d["scope"]);require(text(d["reason"]) and key not in s["seen_scopes"], "fresh reset scope and reason required") - new = d["scope"];old = s["scope"];seen = [json.loads(x) for x in s["seen_scopes"]] + key = scope_key(d["scope"]) + require( + text(d["reason"]) and key not in s["seen_scopes"], + "fresh reset scope and reason required", + ) + new = d["scope"] + old = s["scope"] + seen = [json.loads(x) for x in s["seen_scopes"]] if new["session_id"] != old["session_id"]: - require(not any(x["session_id"] == new["session_id"] for x in seen), "session ID cannot be resurrected") + require( + not any(x["session_id"] == new["session_id"] for x in seen), + "session ID cannot be resurrected", + ) elif new["epoch_id"] != old["epoch_id"]: - require(not any(x["session_id"] == new["session_id"] and x["epoch_id"] == new["epoch_id"] for x in seen), "coordinate epoch cannot be resurrected") - s["seen_scopes"].append(key);s["scope"] = copy.deepcopy(d["scope"]) + require( + not any( + x["session_id"] == new["session_id"] + and x["epoch_id"] == new["epoch_id"] + for x in seen + ), + "coordinate epoch cannot be resurrected", + ) + s["seen_scopes"].append(key) + s["scope"] = copy.deepcopy(d["scope"]) for c in s["candidates"].values(): - c.update(revision=c["revision"]+1, status="needs_context", nomination_id=None) + c.update( + revision=c["revision"] + 1, + status="needs_context", + nomination_id=None, + ) invalidate(c["id"], "scope_changed") - return {"status": "needs_context", "reason": "fresh_context_after_reset_required"} + return { + "status": "needs_context", + "reason": "fresh_context_after_reset_required", + } raise Conflict("unsupported event; tracker continuity never attaches labels") def snapshot(self): @@ -212,24 +490,56 @@ def snapshot(self): s = copy.deepcopy(self._state) for attachment in s["attachments"].values(): c = s["candidates"][attachment["candidate_id"]] - current = c["status"] == "context_nominated" and c["revision"] == attachment["candidate_revision"] and s["scope"] == attachment["scope"] - attachment.update(current=current, status="reviewed_current" if current else "historical_only") + current = ( + c["status"] == "context_nominated" + and c["revision"] == attachment["candidate_revision"] + and s["scope"] == attachment["scope"] + ) + attachment.update( + current=current, + status="reviewed_current" if current else "historical_only", + ) for ticket in s["tickets"].values(): c = s["candidates"][ticket["candidate_id"]] - if ticket["status"] == "attached" and (ticket["scope"] != s["scope"] or ticket["candidate_revision"] != c["revision"]): - ticket.update(status="historical_only", reason="current_context_requires_review") + if ticket["status"] == "attached" and ( + ticket["scope"] != s["scope"] + or ticket["candidate_revision"] != c["revision"] + ): + ticket.update( + status="historical_only", reason="current_context_requires_review" + ) return s def sidecar(self, scene_sha256, expected_snapshot_revision): """Proposed C1 adjunct only. No C1 object/admission/physical confidence emitted.""" require(digest(scene_sha256), "exact C1 scene byte hash required") with self._lock: - require(type(expected_snapshot_revision) is int and expected_snapshot_revision == self._state["revision"], "stale export snapshot") + require( + type(expected_snapshot_revision) is int + and expected_snapshot_revision == self._state["revision"], + "stale export snapshot", + ) s = self.snapshot() - return {"schema": "equipment-association-sidecar/1", "scene_sha256": scene_sha256, - "snapshot_revision": s["revision"], "scope": s["scope"], - "coordinate_contract": "Original encoded photo top-left pixel-cell xywh; caller must preserve camera/image transform provenance", - "candidates": s["candidates"], "label_observations": s["labels"], "track_observations": s["tracks"], - "nominations": s["nominations"], "review_tickets": s["tickets"], "reviews": s["reviews"], - "associations": s["attachments"], "events": s["events"], - "not_established": ["physical equipment identity", "photo/exposure provenance", "OCR correctness", "measurement uncertainty", "C1 ingestion compatibility", "capture-admission completion"]} + return { + "schema": "equipment-association-sidecar/1", + "scene_sha256": scene_sha256, + "snapshot_revision": s["revision"], + "scope": s["scope"], + "coordinate_contract": "Original encoded photo top-left pixel-cell xywh; caller must preserve camera/image transform provenance", + "candidates": s["candidates"], + "label_observations": s["labels"], + "track_observations": s["tracks"], + "nominations": s["nominations"], + "review_tickets": s["tickets"], + "reviews": s["reviews"], + "associations": s["attachments"], + "events": s["events"], + "not_established": [ + "physical equipment identity", + "photo/exposure provenance", + "OCR correctness", + "measurement uncertainty", + "C1 ingestion compatibility", + "capture-admission completion", + ], + } diff --git a/experiments/research-handoff/equipment-association/test_association.py b/experiments/research-handoff/equipment-association/test_association.py index 61127f8a6..4024d65d6 100644 --- a/experiments/research-handoff/equipment-association/test_association.py +++ b/experiments/research-handoff/equipment-association/test_association.py @@ -1,4 +1,5 @@ """Eleven synthetic v2 transition cases; no real media or model inputs.""" + import copy import importlib.util import json @@ -6,114 +7,282 @@ import sys import unittest -ROOT=Path(__file__).resolve().parent -spec=importlib.util.spec_from_file_location('equipment_association_prepared',ROOT/'association.py') -module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module) -Associations,Conflict=module.Associations,module.Conflict -F=json.loads((ROOT/'fixtures.json').read_text()) +ROOT = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location( + "equipment_association_prepared", ROOT / "association.py" +) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +Associations, Conflict = module.Associations, module.Conflict +F = json.loads((ROOT / "fixtures.json").read_text()) + class TransitionCases(unittest.TestCase): def setUp(self): - self.a=Associations(F['scope']);self.serial=0 - for cid in ('A','B'):self.send('create_candidate',candidate_id=cid,description='Synthetic enclosure '+cid) - def send(self,kind,**data): - self.serial+=1 - return self.a.apply({'id':f'event-{self.serial}','type':kind,'data':data}) - def photo(self,name):return copy.deepcopy(F['photos'][name]) - def revision(self):return self.a.snapshot()['candidates']['A']['revision'] - def label(self,oid='label-a',which='label_a'): - return self.send('record_label',observation_id=oid,text='Original label '+oid,producer='synthetic-extractor',raw_result_ref='synthetic/'+oid+'.json',raw_result_sha256='f'*64,photo=self.photo(which)) - def nominate(self,which='context_a',considered=()): - return self.send('nominate_context',candidate_id='A',expected_candidate_revision=self.revision(),actor='reviewer-1',reason='Inspect this contextual photo; no identity probability asserted',photo=self.photo(which),alternatives=['B'],considered_contradictions=list(considered)) - def ticket(self,tid='ticket-a',label='label-a'): - return self.send('request_attachment',ticket_id=tid,candidate_id='A',label_id=label,expected_candidate_revision=self.revision()) - def review(self,ticket,decision='attach',which='context_a'): - return self.send('review_attachment',ticket_id=ticket['id'],expected_ticket_revision=ticket['revision'],expected_candidate_revision=ticket['candidate_revision'],decision=decision,actor='reviewer-1',reason='Explicit inspection of original label and contextual enclosure evidence',photo=self.photo(which)) - def current(self):return [x for x in self.a.snapshot()['attachments'].values() if x['current']] + self.a = Associations(F["scope"]) + self.serial = 0 + for cid in ("A", "B"): + self.send( + "create_candidate", + candidate_id=cid, + description="Synthetic enclosure " + cid, + ) + + def send(self, kind, **data): + self.serial += 1 + return self.a.apply({"id": f"event-{self.serial}", "type": kind, "data": data}) + + def photo(self, name): + return copy.deepcopy(F["photos"][name]) + + def revision(self): + return self.a.snapshot()["candidates"]["A"]["revision"] + + def label(self, oid="label-a", which="label_a"): + return self.send( + "record_label", + observation_id=oid, + text="Original label " + oid, + producer="synthetic-extractor", + raw_result_ref="synthetic/" + oid + ".json", + raw_result_sha256="f" * 64, + photo=self.photo(which), + ) + + def nominate(self, which="context_a", considered=()): + return self.send( + "nominate_context", + candidate_id="A", + expected_candidate_revision=self.revision(), + actor="reviewer-1", + reason="Inspect this contextual photo; no identity probability asserted", + photo=self.photo(which), + alternatives=["B"], + considered_contradictions=list(considered), + ) + + def ticket(self, tid="ticket-a", label="label-a"): + return self.send( + "request_attachment", + ticket_id=tid, + candidate_id="A", + label_id=label, + expected_candidate_revision=self.revision(), + ) + + def review(self, ticket, decision="attach", which="context_a"): + return self.send( + "review_attachment", + ticket_id=ticket["id"], + expected_ticket_revision=ticket["revision"], + expected_candidate_revision=ticket["candidate_revision"], + decision=decision, + actor="reviewer-1", + reason="Explicit inspection of original label and contextual enclosure evidence", + photo=self.photo(which), + ) + + def current(self): + return [x for x in self.a.snapshot()["attachments"].values() if x["current"]] def test_same_uuid_wrong_target(self): - self.nominate();self.label('label-b','label_b') - for i,which in enumerate(('context_a','context_b')): - self.send('record_track',observation_id=f'track-{i}',tracker_uuid='unchanged-uuid',confidence=.95,photo=self.photo(which)) - ticket=self.ticket(label='label-b') - self.assertEqual(ticket['status'],'needs_review');self.assertEqual(self.current(),[]) - with self.assertRaises(Conflict):self.send('attach_from_tracker',ticket_id=ticket['id'],tracker_uuid='unchanged-uuid',confidence=.95) - self.assertEqual(self.review(ticket,'unresolved','context_b')['status'],'unresolved') - self.assertEqual(self.current(),[]);self.assertIn('label-b',self.a.snapshot()['labels']) + self.nominate() + self.label("label-b", "label_b") + for i, which in enumerate(("context_a", "context_b")): + self.send( + "record_track", + observation_id=f"track-{i}", + tracker_uuid="unchanged-uuid", + confidence=0.95, + photo=self.photo(which), + ) + ticket = self.ticket(label="label-b") + self.assertEqual(ticket["status"], "needs_review") + self.assertEqual(self.current(), []) + with self.assertRaises(Conflict): + self.send( + "attach_from_tracker", + ticket_id=ticket["id"], + tracker_uuid="unchanged-uuid", + confidence=0.95, + ) + self.assertEqual( + self.review(ticket, "unresolved", "context_b")["status"], "unresolved" + ) + self.assertEqual(self.current(), []) + self.assertIn("label-b", self.a.snapshot()["labels"]) def test_explicit_review_and_immutable_source(self): - event={'id':'original-label','type':'record_label','data':{'observation_id':'label-a','text':'SERIAL ORIGINAL','producer':'synthetic','raw_result_ref':'synthetic/original.json','raw_result_sha256':'f'*64,'photo':self.photo('label_a')}} - self.a.apply(event);event['data']['text']='MUTATED';event['data']['photo']['xywh'][0]=999 - self.nominate();self.assertEqual(self.review(self.ticket())['status'],'attached') - view=self.a.snapshot();self.assertEqual(view['labels']['label-a']['text'],'SERIAL ORIGINAL') - self.assertEqual(len(self.current()),1) - view['labels']['label-a']['text']='VIEW MUTATION' - self.assertEqual(self.a.snapshot()['labels']['label-a']['text'],'SERIAL ORIGINAL') - sidecar=self.a.sidecar('0'*64,self.a.snapshot()['revision']) - self.assertEqual(sidecar['schema'],'equipment-association-sidecar/1') - self.assertIn('physical equipment identity',sidecar['not_established']);self.assertNotIn('supports',sidecar) + event = { + "id": "original-label", + "type": "record_label", + "data": { + "observation_id": "label-a", + "text": "SERIAL ORIGINAL", + "producer": "synthetic", + "raw_result_ref": "synthetic/original.json", + "raw_result_sha256": "f" * 64, + "photo": self.photo("label_a"), + }, + } + self.a.apply(event) + event["data"]["text"] = "MUTATED" + event["data"]["photo"]["xywh"][0] = 999 + self.nominate() + self.assertEqual(self.review(self.ticket())["status"], "attached") + view = self.a.snapshot() + self.assertEqual(view["labels"]["label-a"]["text"], "SERIAL ORIGINAL") + self.assertEqual(len(self.current()), 1) + view["labels"]["label-a"]["text"] = "VIEW MUTATION" + self.assertEqual( + self.a.snapshot()["labels"]["label-a"]["text"], "SERIAL ORIGINAL" + ) + sidecar = self.a.sidecar("0" * 64, self.a.snapshot()["revision"]) + self.assertEqual(sidecar["schema"], "equipment-association-sidecar/1") + self.assertIn("physical equipment identity", sidecar["not_established"]) + self.assertNotIn("supports", sidecar) def test_reset_and_late_label(self): - self.label();self.nominate();old=self.ticket() - self.send('reset_scope',scope=copy.deepcopy(F['reset_scope']),reason='Tracking reset') - self.label('late-label','label_b') - self.assertEqual(self.review(old)['status'],'stale_review') - self.assertIn('late-label',self.a.snapshot()['labels']);self.assertEqual(self.current(),[]) - self.nominate('new_context_a') - new=self.ticket('new-ticket','late-label') - self.assertEqual(self.review(new,which='new_context_a')['status'],'attached') - self.assertEqual(len(self.current()),1) + self.label() + self.nominate() + old = self.ticket() + self.send( + "reset_scope", + scope=copy.deepcopy(F["reset_scope"]), + reason="Tracking reset", + ) + self.label("late-label", "label_b") + self.assertEqual(self.review(old)["status"], "stale_review") + self.assertIn("late-label", self.a.snapshot()["labels"]) + self.assertEqual(self.current(), []) + self.nominate("new_context_a") + new = self.ticket("new-ticket", "late-label") + self.assertEqual(self.review(new, which="new_context_a")["status"], "attached") + self.assertEqual(len(self.current()), 1) def test_contradiction_preserves_history(self): - self.label();self.label('label-b','label_b');self.nominate();self.review(self.ticket()) - self.ticket('pending-b','label-b') - original=copy.deepcopy(self.a.snapshot()['labels']) - self.send('contradict',candidate_id='A',expected_candidate_revision=self.revision(),actor='reviewer-2',reason='Context suggests a different enclosure',photo=self.photo('context_b')) - snapshot=self.a.snapshot();cid=snapshot['candidates']['A']['contradictions'][0] - self.assertEqual(self.current(),[]);self.assertEqual(snapshot['labels'],original) - self.assertEqual(next(iter(snapshot['attachments'].values()))['status'],'historical_only') - self.assertEqual(snapshot['tickets']['pending-b']['status'],'stale') - with self.assertRaises(Conflict):self.nominate() + self.label() + self.label("label-b", "label_b") + self.nominate() + self.review(self.ticket()) + self.ticket("pending-b", "label-b") + original = copy.deepcopy(self.a.snapshot()["labels"]) + self.send( + "contradict", + candidate_id="A", + expected_candidate_revision=self.revision(), + actor="reviewer-2", + reason="Context suggests a different enclosure", + photo=self.photo("context_b"), + ) + snapshot = self.a.snapshot() + cid = snapshot["candidates"]["A"]["contradictions"][0] + self.assertEqual(self.current(), []) + self.assertEqual(snapshot["labels"], original) + self.assertEqual( + next(iter(snapshot["attachments"].values()))["status"], "historical_only" + ) + self.assertEqual(snapshot["tickets"]["pending-b"]["status"], "stale") + with self.assertRaises(Conflict): + self.nominate() self.nominate(considered=[cid]) - self.assertEqual(self.current(),[],'New nomination does not automatically restore historical links') - self.review(self.ticket('reattach-a')) - self.send('nominate_context',candidate_id='B',expected_candidate_revision=0,actor='reviewer-2',reason='Competing explicit context claim',photo=self.photo('context_b'),alternatives=['A'],considered_contradictions=[]) - competing=self.send('request_attachment',ticket_id='competing-b',candidate_id='B',label_id='label-a',expected_candidate_revision=1) - result=self.review(competing,which='context_b') - self.assertEqual(result['reason'],'conflicting_current_assignment') - self.assertEqual(self.current(),[]) - self.assertTrue(all(c['status']=='contradicted' for c in self.a.snapshot()['candidates'].values())) - self.assertEqual(self.a.snapshot()['labels'],original) + self.assertEqual( + self.current(), + [], + "New nomination does not automatically restore historical links", + ) + self.review(self.ticket("reattach-a")) + self.send( + "nominate_context", + candidate_id="B", + expected_candidate_revision=0, + actor="reviewer-2", + reason="Competing explicit context claim", + photo=self.photo("context_b"), + alternatives=["A"], + considered_contradictions=[], + ) + competing = self.send( + "request_attachment", + ticket_id="competing-b", + candidate_id="B", + label_id="label-a", + expected_candidate_revision=1, + ) + result = self.review(competing, which="context_b") + self.assertEqual(result["reason"], "conflicting_current_assignment") + self.assertEqual(self.current(), []) + self.assertTrue( + all( + c["status"] == "contradicted" + for c in self.a.snapshot()["candidates"].values() + ) + ) + self.assertEqual(self.a.snapshot()["labels"], original) def test_candidate_revision_race(self): - self.label();self.nominate();old=self.ticket() - self.nominate('context_b') - self.assertEqual(self.review(old)['status'],'stale_review') - self.assertEqual(self.current(),[]) - self.assertEqual(self.a.snapshot()['candidates']['A']['status'],'context_nominated') + self.label() + self.nominate() + old = self.ticket() + self.nominate("context_b") + self.assertEqual(self.review(old)["status"], "stale_review") + self.assertEqual(self.current(), []) + self.assertEqual( + self.a.snapshot()["candidates"]["A"]["status"], "context_nominated" + ) def test_review_ticket_revision_race(self): - self.label();self.nominate();old=self.ticket() - self.assertEqual(self.review(old,'unresolved')['status'],'unresolved') - self.assertEqual(self.review(old,'attach')['status'],'stale_review');self.assertEqual(self.current(),[]) - current=self.a.snapshot()['tickets']['ticket-a'] - self.assertEqual(self.review(current,'attach')['status'],'attached') - self.assertEqual(self.review(old,'unresolved')['status'],'stale_review') - self.assertEqual(self.a.snapshot()['tickets']['ticket-a']['status'],'attached') - self.assertEqual(len(self.current()),1) + self.label() + self.nominate() + old = self.ticket() + self.assertEqual(self.review(old, "unresolved")["status"], "unresolved") + self.assertEqual(self.review(old, "attach")["status"], "stale_review") + self.assertEqual(self.current(), []) + current = self.a.snapshot()["tickets"]["ticket-a"] + self.assertEqual(self.review(current, "attach")["status"], "attached") + self.assertEqual(self.review(old, "unresolved")["status"], "stale_review") + self.assertEqual(self.a.snapshot()["tickets"]["ticket-a"]["status"], "attached") + self.assertEqual(len(self.current()), 1) def test_idempotency_and_export_revision(self): - event={'id':'sample','type':'record_track','data':{'observation_id':'sample','tracker_uuid':'uuid','confidence':.99,'photo':self.photo('context_a')}} - old_revision=self.a.snapshot()['revision'];first=self.a.apply(event) - self.assertEqual(self.a.apply(copy.deepcopy(event)),first) - event['data']['confidence']=.8 - with self.assertRaises(Conflict):self.a.apply(event) - with self.assertRaises(Conflict):self.a.sidecar('0'*64,old_revision) - self.send('reset_scope',scope=copy.deepcopy(F['reset_scope']),reason='Reset') - with self.assertRaises(Conflict):self.send('reset_scope',scope=copy.deepcopy(F['scope']),reason='Old sequence must not return') - unknown=self.photo('label_a');unknown['scope']['epoch_id']='future' - with self.assertRaises(Conflict):self.send('record_label',observation_id='future',text='saved?',producer='synthetic',raw_result_ref='synthetic/future.json',raw_result_sha256='f'*64,photo=unknown) - self.assertNotIn('future',self.a.snapshot()['labels']) + event = { + "id": "sample", + "type": "record_track", + "data": { + "observation_id": "sample", + "tracker_uuid": "uuid", + "confidence": 0.99, + "photo": self.photo("context_a"), + }, + } + old_revision = self.a.snapshot()["revision"] + first = self.a.apply(event) + self.assertEqual(self.a.apply(copy.deepcopy(event)), first) + event["data"]["confidence"] = 0.8 + with self.assertRaises(Conflict): + self.a.apply(event) + with self.assertRaises(Conflict): + self.a.sidecar("0" * 64, old_revision) + self.send("reset_scope", scope=copy.deepcopy(F["reset_scope"]), reason="Reset") + with self.assertRaises(Conflict): + self.send( + "reset_scope", + scope=copy.deepcopy(F["scope"]), + reason="Old sequence must not return", + ) + unknown = self.photo("label_a") + unknown["scope"]["epoch_id"] = "future" + with self.assertRaises(Conflict): + self.send( + "record_label", + observation_id="future", + text="saved?", + producer="synthetic", + raw_result_ref="synthetic/future.json", + raw_result_sha256="f" * 64, + photo=unknown, + ) + self.assertNotIn("future", self.a.snapshot()["labels"]) def assert_sibling_refused_atomically(self, ticket_id): before = self.a.snapshot() @@ -122,61 +291,87 @@ def assert_sibling_refused_atomically(self, ticket_id): self.assertEqual(self.a.snapshot(), before) def test_sibling_ticket_rejection_order(self): - self.label(); self.nominate(); first = self.ticket() + self.label() + self.nominate() + first = self.ticket() # A queued second form cannot create independent authority before a decision. - self.assert_sibling_refused_atomically('sibling-before-reject') - self.assertEqual(self.review(first, 'reject')['status'], 'rejected') + self.assert_sibling_refused_atomically("sibling-before-reject") + self.assertEqual(self.review(first, "reject")["status"], "rejected") # Nor can a later request bypass this revision's rejection. - self.assert_sibling_refused_atomically('sibling-after-reject') + self.assert_sibling_refused_atomically("sibling-after-reject") self.assertEqual(self.current(), []) - self.nominate('context_b') - fresh = self.ticket('fresh-after-rejected-nomination') - self.assertEqual(self.review(fresh, which='context_b')['status'], 'attached') + self.nominate("context_b") + fresh = self.ticket("fresh-after-rejected-nomination") + self.assertEqual(self.review(fresh, which="context_b")["status"], "attached") self.assertEqual(len(self.current()), 1) - self.assertEqual(self.a.snapshot()['tickets']['ticket-a']['status'], 'rejected') + self.assertEqual(self.a.snapshot()["tickets"]["ticket-a"]["status"], "rejected") def test_sibling_ticket_unresolved_order(self): - self.label(); self.nominate(); first = self.ticket() - self.assert_sibling_refused_atomically('sibling-before-unresolved') - self.assertEqual(self.review(first, 'unresolved')['status'], 'unresolved') - self.assert_sibling_refused_atomically('sibling-after-unresolved') + self.label() + self.nominate() + first = self.ticket() + self.assert_sibling_refused_atomically("sibling-before-unresolved") + self.assertEqual(self.review(first, "unresolved")["status"], "unresolved") + self.assert_sibling_refused_atomically("sibling-after-unresolved") self.assertEqual(self.current(), []) - current = self.a.snapshot()['tickets']['ticket-a'] - self.assertEqual(self.review(current)['status'], 'attached') + current = self.a.snapshot()["tickets"]["ticket-a"] + self.assertEqual(self.review(current)["status"], "attached") self.assertEqual(len(self.current()), 1) def test_attached_ticket_replay_and_new_nomination(self): - self.label(); self.nominate() - event = {'id': 'exact-ticket-request', 'type': 'request_attachment', - 'data': {'ticket_id': 'ticket-a', 'candidate_id': 'A', - 'label_id': 'label-a', 'expected_candidate_revision': self.revision()}} + self.label() + self.nominate() + event = { + "id": "exact-ticket-request", + "type": "request_attachment", + "data": { + "ticket_id": "ticket-a", + "candidate_id": "A", + "label_id": "label-a", + "expected_candidate_revision": self.revision(), + }, + } original_result = self.a.apply(event) - self.assertEqual(self.review(original_result)['status'], 'attached') + self.assertEqual(self.review(original_result)["status"], "attached") before = self.a.snapshot() self.assertEqual(self.a.apply(copy.deepcopy(event)), original_result) - self.assertEqual(self.a.snapshot(), before, 'Exact replay does not reapply or advance state') - self.assert_sibling_refused_atomically('sibling-after-attached') - self.nominate('context_b') + self.assertEqual( + self.a.snapshot(), before, "Exact replay does not reapply or advance state" + ) + self.assert_sibling_refused_atomically("sibling-after-attached") + self.nominate("context_b") self.assertEqual(self.current(), []) - fresh = self.ticket('fresh-after-attached-nomination') - self.assertEqual(self.review(fresh, which='context_b')['status'], 'attached') + fresh = self.ticket("fresh-after-attached-nomination") + self.assertEqual(self.review(fresh, which="context_b")["status"], "attached") self.assertEqual(len(self.current()), 1) - links = list(self.a.snapshot()['attachments'].values()) - self.assertEqual(sum(link['status'] == 'historical_only' for link in links), 1) + links = list(self.a.snapshot()["attachments"].values()) + self.assertEqual(sum(link["status"] == "historical_only" for link in links), 1) def test_same_pair_after_scope_reset(self): - self.label(); self.nominate(); self.review(self.ticket()) - source = copy.deepcopy(self.a.snapshot()['labels']['label-a']) - self.send('reset_scope', scope=copy.deepcopy(F['reset_scope']), reason='New capture scope') - provisional = self.ticket('reset-needs-context') - self.assertEqual(provisional['status'], 'needs_context') - self.nominate('new_context_a') - self.assertEqual(self.a.snapshot()['tickets']['reset-needs-context']['status'], 'stale') - fresh = self.ticket('reset-fresh-context') - self.assertEqual(self.review(fresh, which='new_context_a')['status'], 'attached') + self.label() + self.nominate() + self.review(self.ticket()) + source = copy.deepcopy(self.a.snapshot()["labels"]["label-a"]) + self.send( + "reset_scope", + scope=copy.deepcopy(F["reset_scope"]), + reason="New capture scope", + ) + provisional = self.ticket("reset-needs-context") + self.assertEqual(provisional["status"], "needs_context") + self.nominate("new_context_a") + self.assertEqual( + self.a.snapshot()["tickets"]["reset-needs-context"]["status"], "stale" + ) + fresh = self.ticket("reset-fresh-context") + self.assertEqual( + self.review(fresh, which="new_context_a")["status"], "attached" + ) self.assertEqual(len(self.current()), 1) - self.assertEqual(self.a.snapshot()['labels']['label-a'], source) - links = list(self.a.snapshot()['attachments'].values()) - self.assertEqual(sum(link['status'] == 'historical_only' for link in links), 1) + self.assertEqual(self.a.snapshot()["labels"]["label-a"], source) + links = list(self.a.snapshot()["attachments"].values()) + self.assertEqual(sum(link["status"] == "historical_only" for link in links), 1) + -if __name__=='__main__':unittest.main(verbosity=2) +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/experiments/research-handoff/native-proposals/README.md b/experiments/research-handoff/native-proposals/README.md index 9f1589daa..0b358d004 100644 --- a/experiments/research-handoff/native-proposals/README.md +++ b/experiments/research-handoff/native-proposals/README.md @@ -13,7 +13,7 @@ One Swift 6 host build and one run exercised nine paired cases: **18 observation The ground helper proposes synchronous recomputation from a caller-qualified camera ledger in original acceptance order, including observations whose original delta was zero. It retains latent skip flags and marked-end coordinates, discards photographic row state, replays existing geometry methods, and advances revision from the prior map. Unchanged samples and meter-y-only updates preserve current evidence/revision. Changed horizontal geometry, nonfinite heights or a nonzero residual cell shift are refused without mutation. -**The ground helper is not a complete geometry fix.** Ordinary anchor refinement can leave a residual shift and prevent its use. Carrying that residual into freshly projected evidence, or dropping it while retaining rounded skip cells, would mix coordinate histories. Caller ledger admission, world/session provenance, refusal handling, coherent UI publication and export snapshot integration remain unresolved. This helper does not implement them. Nine source-prepared controls cover fresh-map equivalence, useful zero-delta observations, order dependence, latent skips/bounds and revision collisions; none was compiled or run. It makes no physical-accuracy, occlusion or phone-performance claim. +**The ground helper is not a complete geometry fix.** Ordinary anchor refinement can leave a residual shift and prevent its use. Carrying that residual into freshly projected evidence, or dropping it while retaining rounded skip cells, would mix coordinate histories. Caller ledger admission, world/session provenance, refusal handling, coherent UI publication and export snapshot integration remain unresolved. This helper does not implement them. It also loses evidence across a marked end. `observe` skips cells past a marked end, so replaying cameras recorded before the end was marked drops what they saw beyond it, and `clearEnd` replays nothing to bring it back. A fix needs each observation's admission-time ends in the caller's ledger, or a replay when an end is cleared; the patch does neither ([review](https://github.com/SamGu-NRX/BaseScanning/pull/18#discussion_r4111600380)). Alpha's `CoverageMap.replayObservedCameras` also loses regular wall and ground sightings past a marked end. When clearing a limit end, Alpha's `clearEnd` calls `replayPastLimits()` to rebuild past-limit ground state, but it does not replay those regular sightings to restore them. Nine source-prepared controls cover fresh-map equivalence, useful zero-delta observations, order dependence, latent skips/bounds and revision collisions; none was compiled or run. It makes no physical-accuracy, occlusion or phone-performance claim. To check applicability later, use a separate checkout already at the exact baseline and inspect the output of `git rev-parse HEAD` first. These commands only check whether the patches apply; they do not apply them: diff --git a/packet/README.md b/packet/README.md index 541ce1982..eed07d10e 100644 --- a/packet/README.md +++ b/packet/README.md @@ -38,9 +38,14 @@ performance. adding once a device study measures it. 2. **The ground's height at the meter.** 0.4 notes that world y = 0 is the phone's height at session start, not the ground. The app measures the ground where it can: the horizontal plane - below the meter. Proposal: `meterAnchor.groundY` (world meters) with `groundSource` + below the meter. Proposal: `meterAnchor.groundOffset`, the ground's height in the meter + anchor's frame (meters along its +y, negative below the meter), with `groundSource` (`plane`, `mesh`, `assumed`), so headroom and "ground under the battery" start from a - measurement when there is one. + measurement when there is one. A raw world `groundY` would stay put when ARKit corrects the + anchor (proposal 10), and the server would derive a wrong meter-to-ground offset; held in + the anchor's frame, the ground moves with the meter. The app's local packet 1.1 already + writes it this way: `meter_anchor.ground_y_m` is the ground's world y minus the anchor's, and + with `.gravity` alignment the anchor's +y is world up. 3. **What a marked end or feature is.** 0.4's tap labels name the point, not the answers the homeowner gave about it: - whether a wall end is a real end (something blocks it) or unexplored (it turns a corner, @@ -85,18 +90,20 @@ performance. 6. **Estimated depth with its uncertainty** (S6). A phone without LiDAR can infer metric depth from the image, but used without its uncertainty it creates surfaces that are not there. Proposal: a depth `source` of `estimated`, which requires a `sigma` file (float32 meters, one - standard deviation per depth pixel, the depth's layout). + standard deviation per depth pixel, the depth's layout). To keep malformed uncertainty from + reaching geometry calculations, the proposed validator must require exactly `w * h` samples + (`4 * w * h` bytes), all finite and non-negative. A missing, truncated or unreadable map is a + validation problem, not an exception. 7. **The frame of a plane's boundary.** 0.4 lists `planes[].boundary` without saying its frame. ARKit's `boundaryVertices` are relative to the anchor transform, not to `planeExtent`'s centre and `rotationOnYAxis`, and packet 1.1 found that mixing the two is an easy mistake. Proposal: carry the anchor-frame boundary in a new field whose frame is stated, rather than giving the existing `boundary` a meaning an existing reader might not share, and have the validator check it lies within `planeExtent` once the centre and rotation are applied. -8. **Photos that frame one span.** With learned depth on ETH3D photos, an along-wall edge came to - 0.9 in p90 when seen within 15° of face-on and 24 in beyond 45° (#59, - `experiments/edge-geometry`), so a distance measured inside one face-on photo is likely far - better than one chained across the walk. Proposal: a still or - keyframe purpose `span` that names the marks the photo frames. +8. **Photos that frame one span.** With learned depth on ETH3D electro photos scaled using the + laser reference, in-plane edges had along-wall error of 0.9 in p90 within 15° of face-on and + 24 in beyond 45° (#59, `experiments/edge-geometry`). Proposal: a still or keyframe purpose + `span` that names the marks the photo frames. 9. **Scale references in view.** One reference near the meter may not be enough: on MARViN's walks ARKit's scale varied 1.8% (robust SD) within a walk (#59, `experiments/drift-anatomy`), so a reference helps where it appears. Proposal: `references[]` with `kind` (door, brick course, and hand-held ones such diff --git a/packet/validator-checks.md b/packet/validator-checks.md index 37a8cf041..f5be618be 100644 --- a/packet/validator-checks.md +++ b/packet/validator-checks.md @@ -52,7 +52,8 @@ case. `geometry/mesh.ply` is a binary little-endian PLY. 0.4 does not fix its index and count types, so this check proposes one layout as well: - `float x, y, z` per vertex, and per face a `uchar` count followed by that many `int` indices, - then an optional `uchar classification`; + then `uchar classification` when `mesh.classified` is true. The property may be absent only + when the flag is false, so a mesh cannot claim face classes without carrying them; - the body length equals exactly what the vertex and face counts imply; - vertex and face counts are non-negative, and every vertex coordinate is finite; - every face is a triangle, and every index satisfies `0 <= index < vertex_count`; diff --git a/recon/HANDOFF.md b/recon/HANDOFF.md index 16b84e388..bd8c458d7 100644 --- a/recon/HANDOFF.md +++ b/recon/HANDOFF.md @@ -126,17 +126,16 @@ In `recon/` on this branch: ## 6. Known defects -Reviewers confirmed these in the worker at this branch's head, and none is fixed. Each can let the server settle a check on evidence the worker doesn't have. No replacement below has been built or validated. +These defects remain after the review-debt repairs. They can let the server settle a check on evidence the worker does not have. The saved ETH3D results above predate those repairs; dataset acceptance has not been rerun. -- **One view can count as two for free space.** `recon/coverage.py:183-187, 205-243`: `_slab` treats a voxel as observed when its fused weight is above zero, and any single frame gives it weight. Repro: a capture in which one frame alone sees the space in front of a wall cell reports a facing clearance there. Wall and ground cells need two positions 0.25 m apart for the same claim. The server at 903d86f can settle a check on that clearance. A fix needs visibility kept per frame for free space, with the same two-position test as the wall and ground. -- **The overhead strip is narrower than a battery.** `recon/coverage.py:42, 231-242`: overhead space is sampled only 0.10 to 0.56 m out from the wall front. Repro: an eave 0.05 m out from the wall, or 0.7 m out over a 3 ft deep battery, is not found, and the cell reports clearance above it. A fix needs the footprint of the battery the chosen rule places, passed from the server's rules or request. -- **Old facing and overhead entries survive a small meter move.** `recon/scene.py:50-57, 87-89`: when the meter moves 5 cm or less, the bundle's existing `facing` and `overheads` entries are kept and the new ones appended, although the reconstructed wall's line and orientation replaced the phone's. Repro: a bundle with a phone-measured facing gap, and a reconstructed wall rotated a few degrees about the meter, sends both the old gap and the new one. A fix needs the target wall's earlier entries dropped whenever its line is replaced, as they already are when the meter moves more than 5 cm. +- **Overhead sampling assumes one battery depth.** `recon/coverage.py`: the worker samples both edges of the existing 0.56 m footprint. A 0.7 m eave over a 3 ft deep battery still lies beyond that assumed depth. Other battery depths need the footprint passed from the server's rules or request. - **The photos-only clearance error is not calibrated.** `recon/pipeline.py:29-35`: facing gaps and overhead clearances from photos carry ±0.56 ft. That comes from the 95% interval of the evals' surface pair-error p90 over 1 to 3 m spans, not from any measurement of where the fused volume puts a boundary. The evals put surface p90 at 5.64 in [4.63, 6.66] and edges at 9.65 in [4.93, 15.01]. Repro: a photos-only overhead clearance measured at 7.10 ft passes the 6.5 ft headroom rule, since 7.10 − 0.56 = 6.54 > 6.5. A fix needs the clearance estimator's own measured uncertainty, or a gate that returns the check unresolved until that exists. -- **Partly occupied space counts as clear.** `recon/coverage.py:223, 235` (threshold at 179): a step out from the wall, or up over the footprint, counts as blocked only when at least 20% of its sampled voxels are occupied, on two steps in a row. That 20% has no source. Otherwise `free_space` reports the space clear through voxels the reconstruction marks occupied. Repro: at 5 cm voxels, a 5 cm rail crossing the facing band (0.3 to 1.8 m high) fills 1 of its 31 height rows, about 3%, and the facing clearance runs straight through it. An obstruction thinner than two steps passes the same way. A fix needs any occupied voxel in the band to end the clear distance or leave the check unresolved. -- **LiDAR without a confidence map is fused unfiltered.** `recon/depth.py:90-92` (accepted at `recon/capture.py:182-189`): a keyframe whose `depth` names a `file` but no `confidenceFile` is accepted, and `lidar()` then keeps every positive sample, including ARKit's low-confidence ones that section 4 says to drop. Repro: a packet keyframe with `"depth": {"file": "k.f32", "w": 256, "h": 192}` has all its samples fused, with no confidence filter. A fix needs a missing confidence map either refused, or treated as that frame having no usable LiDAR. +### Fixed in the worker -Two more are filed as issues for the server team: coverage samples ground on a flat plane at the meter's height rather than the fitted slope (#95), and coverage cells can reach past the fitted wall's ends (#96). +Free-space coverage now uses `SeenFromTwo` in `compute()`: facing and overhead samples need visibility from two camera positions at least 0.25 m apart. One-view and nearby-camera regressions are in `tests/test_coverage_views.py`. + +The worker now replaces the target wall's prior measurements, stops clearance at any occupied sampled voxel, and refuses LiDAR without a confidence map. Ground coverage follows the fitted slope, and only whole cells within the fitted wall extent are exported. The regressions are in `tests/test_review_debt.py`; they do not validate the remaining clearance policies above. ## 7. Data diff --git a/recon/models/moge_depth.py b/recon/models/moge_depth.py index 0c049b79b..f26c881dd 100644 --- a/recon/models/moge_depth.py +++ b/recon/models/moge_depth.py @@ -16,6 +16,7 @@ import math import os import sys +import tempfile from pathlib import Path DATA = Path(os.environ.get("HOUSE_SCANNING_DATA", Path.home() / "house-scanning-data")) @@ -28,6 +29,21 @@ MPS_CAP_GB = 3.6 +def write_depth(target: Path, depth) -> None: + import numpy as np + + temporary = None + try: + with tempfile.NamedTemporaryFile(dir=target.parent, suffix=".npz", delete=False) as f: + temporary = Path(f.name) + np.savez(f, depth=depth.astype(np.float32)) + # A killed writer cannot leave a partially written archive at the reusable cache path. + os.replace(temporary, target) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + def main(manifest: Path) -> None: import cv2 import numpy as np @@ -58,7 +74,7 @@ def main(manifest: Path) -> None: out = model.infer(rgb, resolution_level=RESOLUTION_LEVEL, fov_x=fov_x, use_fp16=True) depth = out["depth"].float().cpu().numpy() depth[~out["mask"].cpu().numpy().astype(bool)] = np.nan - np.savez(item["out"], depth=depth.astype(np.float32)) + write_depth(Path(item["out"]), depth) print(f"{Path(item['image']).name}: median {np.nanmedian(depth):.2f} m", file=sys.stderr) del out, rgb if device == "mps": diff --git a/recon/recon/capture.py b/recon/recon/capture.py index 3c573ff87..e0733cd7c 100644 --- a/recon/recon/capture.py +++ b/recon/recon/capture.py @@ -23,10 +23,15 @@ from __future__ import annotations +import fcntl import hashlib import json import re +import shutil +import struct +import tempfile import zipfile +import zlib from dataclasses import dataclass, field from pathlib import Path @@ -94,6 +99,10 @@ def __post_init__(self) -> None: ROTATION_TOL = 1e-2 # scene.json rounds poses to 4 decimals; anything further off is not a pose +# Existing bundle admission policy in verification/hsverify/e2e.py, not measured capture maxima. +MAX_BUNDLE_ENTRIES = 2000 +MAX_BUNDLE_BYTES = 512 * 2**20 + def column_major(pose: list[float]) -> np.ndarray: """A 4x4 from 16 column-major numbers, its rotation snapped to the nearest proper rotation. @@ -115,22 +124,160 @@ def outward_of(along: np.ndarray) -> np.ndarray: return np.array([-along[2], 0.0, along[0]]) -def unpack(path: Path, work: Path) -> Path: - """A folder as is; a zip extracted once under `work`, keyed by its sha256.""" +def bundle_root(path: Path, work: Path) -> Path: + """The capture folder, including a ZIP's cache path before extraction or cleanup.""" if path.is_dir(): return path - digest = hashlib.sha256(path.read_bytes()).hexdigest()[:16] - out = work / "bundles" / digest - if not out.exists(): - with zipfile.ZipFile(path) as z: - names = [n for n in z.namelist() if not n.startswith("__MACOSX/")] - for n in names: - if n.startswith("/") or ".." in Path(n).parts: - raise ValueError(f"{path}: unsafe entry {n!r}") - z.extractall(out, members=names) + if path.stat().st_size > MAX_BUNDLE_BYTES: + raise ValueError(f"{path}: archive exceeds {MAX_BUNDLE_BYTES} bytes") + with path.open("rb") as f: + digest = hashlib.file_digest(f, "sha256").hexdigest()[:16] + return work / "bundles" / digest + + +def _bundle_intact(root: Path, members: list[zipfile.ZipInfo]) -> bool: + """Check the extracted file set and each file's size and CRC against the archive.""" + if root.is_symlink() or not root.is_dir(): + return False + files = {Path(i.filename): i for i in members if not i.is_dir()} + directories = {Path(i.filename) for i in members if i.is_dir()} + for name in files.keys() | directories: + directories.update(p for p in name.parents if p != Path(".")) + expected = files.keys() | directories + try: + for entry in root.rglob("*"): + name = entry.relative_to(root) + if entry.is_symlink() or name not in expected: + return False + if any(not (root / name).is_dir() for name in directories): + return False + for name, info in files.items(): + entry = root / name + if not entry.is_file() or entry.stat().st_size != info.file_size: + return False + crc, size = 0, 0 + with entry.open("rb") as source: + while chunk := source.read(2**20): + size += len(chunk) + if size > info.file_size: + return False + crc = zlib.crc32(chunk, crc) + if size != info.file_size or crc != info.CRC: + return False + except OSError: + return False + return True + + +def unpack(path: Path, work: Path) -> Path: + """A folder as is; a ZIP cache reused only while its extracted members remain intact.""" + out = bundle_root(path, work) + if path.is_dir(): + return out + _zip_directory_limits(path) + with zipfile.ZipFile(path) as z: + infos = z.infolist() + if len(infos) > MAX_BUNDLE_ENTRIES: + raise ValueError(f"{path}: {len(infos)} entries, limit {MAX_BUNDLE_ENTRIES}") + if sum(i.file_size for i in infos) > MAX_BUNDLE_BYTES: + raise ValueError(f"{path}: expanded archive exceeds {MAX_BUNDLE_BYTES} bytes") + members = [i for i in infos if not i.filename.startswith("__MACOSX/")] + for info in members: + n = info.filename + if n.startswith("/") or ".." in Path(n).parts: + raise ValueError(f"{path}: unsafe entry {n!r}") + out.parent.mkdir(parents=True, exist_ok=True) + # Cache verification and replacement share a lock so one repair cannot delete another + # run's newly published extraction. The lock stays outside the replaceable directory. + with (work / f".bundle-{out.name}.lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + if _bundle_intact(out, members): + return out + # Extraction stays private until every CRC-checked member finishes. Failed extraction + # cannot leave a partial cache directory that a later run mistakes for a complete one. + stage = Path(tempfile.mkdtemp(dir=out.parent, prefix=f".{out.name}-")) + try: + written = 0 + for info in members: + dest = stage / info.filename + if info.is_dir(): + dest.mkdir(parents=True, exist_ok=True) + continue + dest.parent.mkdir(parents=True, exist_ok=True) + with z.open(info) as src, dest.open("wb") as dst: + while chunk := src.read(2**20): + written += len(chunk) + if written > MAX_BUNDLE_BYTES: + raise ValueError(f"{path}: expanded archive exceeds byte limit") + dst.write(chunk) + if out.is_symlink() or out.is_file(): + out.unlink() + elif out.exists(): + shutil.rmtree(out) + stage.rename(out) + finally: + if stage.exists(): + shutil.rmtree(stage) return out +def _zip_directory_limits(path: Path) -> None: + """Check central-directory records in bounded reads before ZipFile allocates their metadata. + ZIP64 and split archives exceed this worker's ordinary ZIP admission format and are refused. + The record layouts are PKWARE APPNOTE sections 4.3.12 and 4.3.16.""" + with path.open("rb") as f: + size = path.stat().st_size + f.seek(max(0, size - 22 - 65535)) + tail = f.read(22 + 65535) + # Match ZipFile's footer selection: a no-comment footer at EOF, otherwise the last + # signature. Backtracking past malformed signatures can validate a different directory + # from the one ZipFile later allocates. Refuse that ambiguity before constructing it. + if len(tail) >= 22 and tail[-22:-18] == b"PK\x05\x06" and tail[-2:] == b"\0\0": + pos = len(tail) - 22 + else: + pos = tail.rfind(b"PK\x05\x06") + if pos < 0: + raise ValueError(f"{path}: missing ZIP end record") + if pos + 22 > len(tail): + raise ValueError(f"{path}: truncated ZIP end record") + record = struct.unpack("<4s4H2IH", tail[pos : pos + 22]) + if pos + 22 + record[-1] != len(tail): + raise ValueError(f"{path}: ambiguous ZIP end record or invalid comment length") + _, disk, central_disk, disk_entries, total_entries, length, offset, _ = record + if disk or central_disk or disk_entries != total_entries: + raise ValueError(f"{path}: split ZIP archives are not supported") + if total_entries > MAX_BUNDLE_ENTRIES: + raise ValueError(f"{path}: {total_entries} entries, limit {MAX_BUNDLE_ENTRIES}") + directory_end = offset + length + end_offset = size - len(tail) + pos + # ZipFile checks this locator even when the ordinary footer has no ZIP64 sentinels. + # It must not replace the directory fields after the bounded scan below. + if end_offset >= 20: + f.seek(end_offset - 20) + if f.read(4) == b"PK\x06\x07": + raise ValueError(f"{path}: ZIP64 archive metadata is not supported") + if length == 0xFFFFFFFF or offset == 0xFFFFFFFF or directory_end != end_offset: + raise ValueError(f"{path}: ZIP64 or inconsistent ZIP directory") + f.seek(offset) + count, expanded = 0, 0 + while f.tell() < directory_end: + header = f.read(46) + if len(header) != 46 or header[:4] != b"PK\x01\x02": + raise ValueError(f"{path}: invalid ZIP central-directory record") + fields = struct.unpack("<4s6H3I5H2I", header) + count += 1 + expanded += fields[9] + if count > MAX_BUNDLE_ENTRIES: + raise ValueError(f"{path}: {count} entries, limit {MAX_BUNDLE_ENTRIES}") + if expanded > MAX_BUNDLE_BYTES: + raise ValueError(f"{path}: expanded archive exceeds {MAX_BUNDLE_BYTES} bytes") + f.seek(sum(fields[10:13]), 1) + if f.tell() > directory_end: + raise ValueError(f"{path}: ZIP central-directory record exceeds directory") + if count != total_entries: + raise ValueError(f"{path}: inconsistent ZIP entry count") + + def _find(root: Path, name: str) -> Path | None: hits = [p for p in root.rglob(name) if "__MACOSX" not in p.parts] if len(hits) > 1: diff --git a/recon/recon/coverage.py b/recon/recon/coverage.py index a04d50d01..ffff18b72 100644 --- a/recon/recon/coverage.py +++ b/recon/recon/coverage.py @@ -18,8 +18,9 @@ otherwise, so a bush in front of the wall hides the wall instead of standing in for it. Facing and overhead space needs no surface: it is observed where the fused volume shows it seen -empty (every voxel observed, none occupied), up to the first occupied voxel (a measured gap or -clearance) or the first unobserved one (seen clear only that far). +empty (every voxel observed, none occupied) and two camera positions at least 0.25 m apart each +saw through it (`SeenFromTwo`). An occupied slab always stops the march, but supports a measured +gap or clearance only when that slab also passes the observation test. """ from __future__ import annotations @@ -31,7 +32,7 @@ from recon.capture import FEET, UP, Frame from recon.depth import Depth from recon.fusion import MAX_RANGE_M, Mesh, Volume -from recon.geometry import WallFrame +from recon.geometry import Ground, WallFrame CELL_M = 0.1524 # half a foot, the app's cell WALL_TOP_M = 1.9812 # 6.5 ft, the headroom the export's wall band claims @@ -39,7 +40,7 @@ GROUND_MAX_M = 3.048 # 10 ft out FACING_MAX_M = 6.0 FACING_BAND_M = (0.3, 1.8) -OVERHEAD_OUT_M = (0.1, 0.56) # above the battery's footprint (1.83 ft deep) +OVERHEAD_OUT_M = (0.0, 0.56) # the whole existing 1.83 ft footprint, from the first free wall step OVERHEAD_MAX_M = 3.0 MAX_ANGLE = np.radians(65) # the app's limit on how oblique a view may be IMAGE_MARGIN = 0.03 @@ -98,20 +99,134 @@ def seen_by( return out -def two_positions(saw: np.ndarray, centres: np.ndarray) -> np.ndarray: - far = (np.linalg.norm(centres[:, None] - centres[None], axis=-1) >= BASELINE_M).astype( +def far_pairs(centres: np.ndarray) -> np.ndarray: + """(frames, frames) float32: 1 where two camera positions are at least BASELINE_M apart.""" + return (np.linalg.norm(centres[:, None] - centres[None], axis=-1) >= BASELINE_M).astype( np.float32 ) + + +def two_positions( + saw: np.ndarray, centres: np.ndarray, far: np.ndarray | None = None +) -> np.ndarray: + """Per row of `saw` (N, frames), whether two frames at least BASELINE_M apart both saw it. + `far` is `far_pairs(centres)`, for a caller that asks many times about the same frames.""" + if far is None: + far = far_pairs(centres) s = saw.astype(np.float32) return ((s @ far) * s).sum(axis=-1) > 0 -def face_offsets(wall: WallFrame, mesh: Mesh, cells: np.ndarray) -> np.ndarray: +class SeenFromTwo: + """The fused volume for free space, with `observed` narrowed to samples that two camera + positions at least BASELINE_M apart each saw: the frame's own depth at the sample's pixel + reaches the sample (lies no nearer than it, less the tolerance), so its ray passed through. + + The volume's weight alone says only that some frame's ray reached a voxel, so one view could + vouch for a clearance; wall and ground already need two positions, and so does free space. + Queries project one frame at a time in bounded point batches, without copying the depth maps.""" + + def __init__(self, vol: Volume, frames: list[Frame], depths: dict[str, Depth]): + self.vol, self.voxel = vol, vol.voxel + self.centres = np.array([f.center for f in frames]).reshape(-1, 3) + # Camera separation and partner masks stay the same across every slab batch. + self.far = far_pairs(self.centres) + T = np.array([f.cam_to_world for f in frames]).reshape(-1, 4, 4) + self.R, self.C = T[:, :3, :3], T[:, :3, 3] + self.K = np.array([f.intrinsics for f in frames], float).reshape(-1, 4) + self.size = np.array([(f.width, f.height) for f in frames], float).reshape(-1, 2) + self.depths = [depths[f.id].depth for f in frames] + self.partner_bits = np.packbits(self.far.astype(bool), axis=1) + self.partner_frames = np.flatnonzero(self.partner_bits.any(axis=1)) + + def occupied(self, points: np.ndarray) -> np.ndarray: + return self.vol.occupied(points) + + def seen_through(self, points: np.ndarray) -> np.ndarray: + """(N, frames) bool: frame j's ray reached point i.""" + out = np.zeros((len(points), len(self.C)), bool) + for j, depth_map in enumerate(self.depths): + d = points - self.C[j] + dist = np.linalg.norm(d, axis=1) + p = d @ self.R[j] # camera axes, as in seen_by + z = -p[:, 2] + fx, fy, cx, cy = self.K[j] + w, h = self.size[j] + with np.errstate(divide="ignore", invalid="ignore"): + u = cx + fx * p[:, 0] / z + v = cy - fy * p[:, 1] / z + ok = (dist > 0.1) & (dist <= MAX_RANGE_M) & (z > 0.05) + ok &= (u >= w * IMAGE_MARGIN) & (u <= w * (1 - IMAGE_MARGIN)) + ok &= (v >= h * IMAGE_MARGIN) & (v <= h * (1 - IMAGE_MARGIN)) + n_i = np.flatnonzero(ok) + dh, dw = depth_map.shape + scale = dw / w + du = np.clip((u[n_i] * scale).astype(np.int64), 0, dw - 1) + dv = np.clip((v[n_i] * scale).astype(np.int64), 0, dh - 1) + measured = depth_map[dv, du] + tol = tolerance(dist[n_i]) + reached = np.isfinite(measured) & (measured >= z[n_i] - tol) + keep = n_i[reached] + if len(keep): + # A depth map may miss an obstruction retained by fusion. Ignore only the + # sample's own surface, using the same stop-short distance as seen_by. + blocked = self.vol.blocked(self.C[j], points[keep], tol[reached] + self.voxel) + out[keep[~blocked], j] = True + + return out + + def observed(self, points: np.ndarray) -> np.ndarray: + seen = self.vol.observed(points) + if not seen.any() or not len(self.partner_frames): + return np.zeros_like(seen) + result = np.zeros_like(seen) + indices = np.flatnonzero(seen) + chunk = min(VIEW_POINTS_PER_BATCH, max(1, VIEW_SAMPLES_PER_BATCH // len(self.C))) + for start in range(0, len(indices), chunk): + idx = indices[start : start + chunk] + result[idx] = _two_positions_packed( + self.seen_through(points[idx]), self.partner_bits, self.partner_frames + ) + return result + + +# Bound each visibility matrix to 1 MiB and each per-frame projection to 4096 points, even when +# a caller passes all points along a long wall. These are allocation limits, not measured tuning. +VIEW_SAMPLES_PER_BATCH = 2**20 +VIEW_POINTS_PER_BATCH = 4096 + + +def _two_positions_packed( + saw: np.ndarray, partners: np.ndarray, partner_frames: np.ndarray | None = None +) -> np.ndarray: + """Exact two-view test using packed partner masks instead of N x F by F x F multiplication. + A row stops once one visible frame has another visible, sufficiently separated partner.""" + packed = np.packbits(saw, axis=1) + result = np.zeros(len(saw), bool) + eligible = np.count_nonzero(saw, axis=1) >= 2 + if partner_frames is None: + partner_frames = np.flatnonzero(partners.any(axis=1)) + for j in partner_frames: + idx = np.flatnonzero(eligible & ~result & saw[:, j]) + if len(idx): + result[idx] = (packed[idx] & partners[j]).any(axis=1) + if np.all(result | ~eligible): + break + return result + + +def face_offsets( + wall: WallFrame, mesh: Mesh, cells: np.ndarray, ground: Ground | None = None +) -> np.ndarray: """Per cell, how far a flat, full-height face stands in front of the fitted plane (0 if none). Flat: in at least 75% of the 10 cm height bins from 0.3 to 1.9 m, the front-most surface within 0.6 m lies within 3 cm of their median. A bush or bin is neither, so it stays an occluder.""" + if not len(cells): + return np.zeros(0) loc = wall.local(mesh.vertices.astype(np.float64)) s, h, out = loc[:, 0], loc[:, 1], loc[:, 2] + if ground is not None: + h = mesh.vertices[:, 1] - ground.height_at(mesh.vertices[:, 0], mesh.vertices[:, 2]) keep = (out > -0.15) & (out < 0.6) & (h > 0.3) & (h < 1.9) col = np.floor((s[keep] - cells[0]) / CELL_M).astype(np.int64) row = np.floor((h[keep] - 0.3) / 0.1).astype(np.int64) @@ -146,15 +261,18 @@ def wall_and_ground( vol: Volume, mesh: Mesh, cells: np.ndarray, + ground: Ground | None = None, ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: centres = np.array([f.center for f in frames]) - faces = face_offsets(wall, mesh, cells) + faces = face_offsets(wall, mesh, cells, ground) alongs = np.stack([cells + 0.25 * CELL_M, cells + 0.75 * CELL_M], axis=1) # (C, 2) heights = np.linspace(0.05, WALL_TOP_M, WALL_ROWS) S = np.broadcast_to(alongs[:, :, None], (len(cells), 2, WALL_ROWS)) Hh = np.broadcast_to(heights[None, None, :], S.shape) offs = np.broadcast_to(faces[:, None, None], S.shape) wpts = wall.world(S, Hh, offs).reshape(-1, 3) + if ground is not None: + wpts[:, 1] = Hh.ravel() + ground.height_at(wpts[:, 0], wpts[:, 2]) wseen = two_positions(seen_by(wpts, wall.outward, frames, depths, vol), centres) wall_ok = wseen.reshape(len(cells), 2, WALL_ROWS).all(axis=(1, 2)) @@ -164,7 +282,10 @@ def wall_and_ground( S = np.broadcast_to(alongs[:, :, None], (len(cells), 2, len(outs))) Oo = np.maximum(np.broadcast_to(outs[None, None, :], S.shape), faces[:, None, None] + 0.05) gpts = wall.world(S, np.zeros_like(S), Oo).reshape(-1, 3) - gseen = two_positions(seen_by(gpts, UP, frames, depths, vol), centres).reshape( + if ground is not None: + gpts[:, 1] = ground.height_at(gpts[:, 0], gpts[:, 2]) + gnormal = UP if ground is None else ground.normal + gseen = two_positions(seen_by(gpts, gnormal, frames, depths, vol), centres).reshape( len(cells), 2, len(outs) ) row_ok = gseen.all(axis=1) # (C, outs) @@ -180,21 +301,27 @@ def wall_and_ground( WALL_SEARCH_M = (-0.3, 1.0) -def _slab(wall: WallFrame, vol: Volume, across, heights, outs) -> tuple[float, bool]: +def _slab( + wall: WallFrame, vol: Volume, across, heights, outs, ground: Ground | None = None +) -> tuple[float, bool]: """Occupied share and whether every voxel was observed, over a grid of wall coordinates.""" S, Hh, Oo = np.meshgrid(across, heights, outs, indexing="ij") pts = wall.world(S, Hh, Oo).reshape(-1, 3) + if ground is not None: + pts[:, 1] = Hh.ravel() + ground.height_at(pts[:, 0], pts[:, 2]) return float(vol.occupied(pts).mean()), bool(vol.observed(pts).all()) -def wall_front(wall: WallFrame, vol: Volume, across: np.ndarray) -> float: +def wall_front( + wall: WallFrame, vol: Volume, across: np.ndarray, ground: Ground | None = None +) -> float: """Where the reconstructed wall surface ends toward the homeowner: marching out from behind the fitted plane, the first step past the wall's occupied voxels. A noisy reconstruction smears a wall several inches thick; space inside that smear is neither free nor an obstacle.""" heights = np.arange(0.5, FACING_BAND_M[1] + 1e-9, vol.voxel) front, inside = 0.0, False for out in np.arange(WALL_SEARCH_M[0], WALL_SEARCH_M[1], vol.voxel): - share, _ = _slab(wall, vol, across, heights, [out]) + share, _ = _slab(wall, vol, across, heights, [out], ground) if share >= OCCUPIED_SHARE: inside, front = True, out elif inside: @@ -202,54 +329,104 @@ def wall_front(wall: WallFrame, vol: Volume, across: np.ndarray) -> float: return front + vol.voxel -def free_space(wall: WallFrame, vol: Volume, cells: np.ndarray): +# At 5 cm voxels a facing batch has 3 x 31 x 16 = 1488 points, below the projection limit. +# Keep short marches bounded too: a failed slab evaluates at most the rest of its 16-step batch. +SLABS_PER_BATCH = 16 + + +def _slabs(wall: WallFrame, vol, across, heights, outs, *, axis: int, ground: Ground | None = None): + """Occupied and observed flags per facing step (axis=2) or overhead step (axis=1).""" + if axis not in (1, 2): + raise ValueError("slab axis must be height (1) or outward distance (2)") + S, Hh, Oo = np.meshgrid(across, heights, outs, indexing="ij") + points = wall.world(S, Hh, Oo).reshape(-1, 3) + if ground is not None: + # Clearance is above each sample's ground, including slope away from the wall. + points[:, 1] = Hh.ravel() + ground.height_at(points[:, 0], points[:, 2]) + reduce = (0, 1) if axis == 2 else (0, 2) + occupied = vol.occupied(points).reshape(S.shape).any(axis=reduce) + seen = vol.observed(points).reshape(S.shape).all(axis=reduce) + return occupied, seen + + +def free_space( + wall: WallFrame, vol: Volume | SeenFromTwo, cells: np.ndarray, ground: Ground | None = None +): """Per cell: (facing gap or NaN, facing clear, overhead clearance or NaN, overhead clear). The facing gap and clear distance are measured out from the wall's reconstructed front, not from the fitted plane: in front of a pilaster 0.3 m proud, something 1.3 m out of the plane is 1.0 m from the face the battery would stand against. The front is the first free step - past the wall's occupied voxels, so both read up to a voxel short, never long. Overhead + past the wall's occupied voxels. The gap also backs off one voxel from the first occupied + sample, so voxel sampling does not extend it into the obstacle. Overhead clearance is a height above the ground over the footprint in front of that face.""" step = vol.voxel n = len(cells) facing_gap, facing_clear = np.full(n, np.nan), np.zeros(n) over_gap, over_clear = np.full(n, np.nan), np.zeros(n) fh = np.arange(FACING_BAND_M[0], FACING_BAND_M[1] + 1e-9, step) + raw = vol.vol if isinstance(vol, SeenFromTwo) else vol for c in range(n): across = cells[c] + CELL_M * np.array([0.25, 0.5, 0.75]) - front = wall_front(wall, vol, across) - hits = 0 - for out in np.arange(front + step, front + FACING_MAX_M, step): - share, seen = _slab(wall, vol, across, fh, [out]) - hits = hits + 1 if share >= OCCUPIED_SHARE else 0 - if hits == 2: - facing_gap[c] = out - step - front - break - if not seen: - break - if hits == 0: + front = wall_front(wall, raw, across, ground) + outs = np.arange(front + step, front + FACING_MAX_M, step) + for start in range(0, len(outs), SLABS_PER_BATCH): + chunk = outs[start : start + SLABS_PER_BATCH] + occupied, seen = _slabs(wall, vol, across, fh, chunk, axis=2, ground=ground) + stopped = False + for out, hit, observed in zip(chunk, occupied, seen, strict=True): + # Any occupied sampled voxel stops clearance, including one-step thin rails. + if hit: + # A fused hit alone does not establish a two-view measurement. + if observed: + facing_gap[c] = max(0.0, out - step - front) + stopped = True + break + if not observed: + stopped = True + break facing_clear[c] = out - front - oo = front + np.arange(OVERHEAD_OUT_M[0], OVERHEAD_OUT_M[1] + 1e-9, step) - hits = 0 - for h in np.arange(0.3, OVERHEAD_MAX_M, step): - share, seen = _slab(wall, vol, across, [h], oo) - hits = hits + 1 if share >= OCCUPIED_SHARE else 0 - if hits == 2: - over_gap[c] = h - step + if stopped: break - if not seen: - break - if hits == 0: + # Include both footprint edges. Skipping the first 10 cm missed a 1.90 m eave in the + # full-depth regression, while a voxel step alone can end short of the outer edge. + oo = front + np.append( + np.arange(OVERHEAD_OUT_M[0], OVERHEAD_OUT_M[1], step), OVERHEAD_OUT_M[1] + ) + heights = np.arange(0.3, OVERHEAD_MAX_M, step) + for start in range(0, len(heights), SLABS_PER_BATCH): + chunk = heights[start : start + SLABS_PER_BATCH] + occupied, seen = _slabs(wall, vol, across, chunk, oo, axis=1, ground=ground) + stopped = False + for h, hit, observed in zip(chunk, occupied, seen, strict=True): + if hit: + if observed: + over_gap[c] = max(0.0, h - step) + stopped = True + break + if not observed: + stopped = True + break over_clear[c] = h + if stopped: + break + return facing_gap, facing_clear, over_gap, over_clear def compute( - wall: WallFrame, frames: list[Frame], depths: dict[str, Depth], vol: Volume, mesh: Mesh + wall: WallFrame, + frames: list[Frame], + depths: dict[str, Depth], + vol: Volume, + mesh: Mesh, + ground: Ground | None = None, ) -> CellCoverage: lo, hi = wall.s_range - cells = np.arange(np.floor(lo / CELL_M) * CELL_M, hi - 1e-9, CELL_M) - wall_ok, ground_out, _ = wall_and_ground(wall, frames, depths, vol, mesh, cells) - fg, fc, og, oc = free_space(wall, vol, cells) + # Only complete cells inside the fitted extent can support the full exported half-foot span. + start = np.ceil(lo / CELL_M - 1e-9) * CELL_M + cells = np.arange(start, hi - CELL_M + 1e-9, CELL_M) + wall_ok, ground_out, _ = wall_and_ground(wall, frames, depths, vol, mesh, cells, ground) + fg, fc, og, oc = free_space(wall, SeenFromTwo(vol, frames, depths), cells, ground) return CellCoverage(cells, wall_ok, ground_out, fg, fc, og, oc) diff --git a/recon/recon/depth.py b/recon/recon/depth.py index 14889b09e..7635a6afe 100644 --- a/recon/recon/depth.py +++ b/recon/recon/depth.py @@ -16,11 +16,14 @@ from __future__ import annotations +import fcntl import hashlib import json import os import subprocess import sys +import zipfile +import zlib from dataclasses import dataclass from pathlib import Path @@ -83,20 +86,22 @@ def rotated_intrinsics(k: np.ndarray, w: float, h: float, turns: int) -> np.ndar def lidar(frame: Frame) -> Depth: d = frame.lidar + if d.confidence is None: + raise ValueError(f"{d.file}: LiDAR depth requires its confidence map") depth = np.fromfile(d.file, dtype=" 0)] = np.nan color = cv2.resize(_image(frame), (d.width, d.height), interpolation=cv2.INTER_AREA) return Depth(depth, scaled(frame.intrinsics, d.width / frame.width), color, "lidar") def _image(frame: Frame) -> np.ndarray: - img = cv2.imread(str(frame.image), cv2.IMREAD_COLOR) + # Poses, intrinsics and LiDAR describe sensor pixels, regardless of JPEG EXIF orientation. + img = cv2.imread(str(frame.image), cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) if img is None: raise FileNotFoundError(frame.image) if img.shape[:2] != (frame.height, frame.width): @@ -169,6 +174,23 @@ def cache_file(work: Path, fid: str, suffix: str) -> Path: def moge(capture: Capture, work: Path) -> dict[str, Depth]: """MoGe-2 depth per frame at up to MAX_SIDE px, in the frame's own (unrotated) orientation, cached under `work` by the capture's content (`moge_cache`).""" + work.mkdir(parents=True, exist_ok=True) + # The lock covers key repair, inputs, inference and reads, so concurrent runs cannot read a + # partially written archive or launch duplicate inference for the same cache root. + with (work / ".moge.lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + return _moge_locked(capture, work) + + +def _cached_depth(path: Path, shape: tuple[int, int]) -> np.ndarray: + with np.load(path, allow_pickle=False) as archive: + d = archive["depth"] + if d.shape != shape or d.dtype.kind != "f": + raise ValueError(f"{path}: expected floating depth with shape {shape}, got {d.shape}") + return d.astype(np.float32) + + +def _moge_locked(capture: Capture, work: Path) -> dict[str, Depth]: work = moge_cache(capture, work) manifest, meta = [], {} for f in capture.frames: @@ -184,7 +206,26 @@ def moge(capture: Capture, work: Path) -> dict[str, Depth]: k_up = rotated_intrinsics(k_small, small.shape[1], small.shape[0], turns) out = cache_file(work, f.id, ".moge2.npz") meta[f.id] = (out, turns, k_small, small) - if not out.exists(): + valid = False + if out.exists(): + try: + _cached_depth(out, np.rot90(small, turns).shape[:2]) + valid = True + except ( + ValueError, + OSError, + KeyError, + EOFError, + TypeError, + # A damaged ZIP encryption flag makes NumPy request a password. + RuntimeError, + # ZIP corruption can name an unsupported compression method. + NotImplementedError, + zipfile.BadZipFile, + zlib.error, + ): + out.unlink() + if not valid: manifest.append({"image": str(up_path), "fx": float(k_up[0]), "out": str(out)}) if manifest: (work / "moge2.json").write_text(json.dumps(manifest, indent=1)) @@ -203,7 +244,7 @@ def moge(capture: Capture, work: Path) -> dict[str, Depth]: ) depths = {} for fid, (out, turns, k_small, small) in meta.items(): - d = np.rot90(np.load(out)["depth"], -turns).astype(np.float32) + d = np.rot90(_cached_depth(out, np.rot90(small, turns).shape[:2]), -turns) if d.shape != small.shape[:2]: raise ValueError(f"{out}: depth {d.shape} does not match the image {small.shape[:2]}") depths[fid] = Depth(np.ascontiguousarray(d), k_small, small, "moge2") diff --git a/recon/recon/pipeline.py b/recon/recon/pipeline.py index 15794a1cc..c27829686 100644 --- a/recon/recon/pipeline.py +++ b/recon/recon/pipeline.py @@ -57,7 +57,7 @@ def reconstruct( # Only frames with a depth map can see anything (depth.depth_maps: LiDAR frames saved # without depth get none). seeing = [f for f in capture.frames if f.id in depths] - cov = coverage.compute(wall, seeing, depths, vol, surface) + cov = coverage.compute(wall, seeing, depths, vol, surface, ground) _log(f"coverage over {len(cov.cells)} cells ({time.perf_counter() - t0:.0f} s)") return { "depths": depths, @@ -124,7 +124,31 @@ def write_model(r: dict, path: Path) -> None: def run( bundle: Path, out: Path, work: Path, depth_mode: str, server_url: str | None, move_meter: bool ) -> dict: + owned = ( + "model.glb", + "geometry.json", + "scene.json", + "coverage.json", + "report.md", + "result.json", + "site-plan.svg", + ) + bundle_path, out_path = bundle.resolve(), out.resolve() + if bundle.is_dir(): + overlap = bundle_path.is_relative_to(out_path) or out_path.is_relative_to(bundle_path) + else: + overlap = bundle_path in {(out_path / name).resolve() for name in owned} + if not overlap and bundle.is_file(): + capture_root = cap.bundle_root(bundle, work).resolve() + overlap = capture_root.is_relative_to(out_path) or out_path.is_relative_to(capture_root) + if overlap: + raise ValueError( + f"output {out} overlaps capture input {bundle}; use a separate output folder" + ) out.mkdir(parents=True, exist_ok=True) + # A failed or --no-server rerun must not leave an earlier run's answer beside new outputs. + for name in owned: + (out / name).unlink(missing_ok=True) capture = cap.load(bundle, work) _log(f"{capture.source}: {len(capture.frames)} frames from {bundle}") t0 = time.perf_counter() diff --git a/recon/recon/scene.py b/recon/recon/scene.py index 6f0ce4bcc..c0ee6db48 100644 --- a/recon/recon/scene.py +++ b/recon/recon/scene.py @@ -4,8 +4,8 @@ A scan bundle's scene is updated: the meter's wall takes the fitted line (over the union of its fitted extent and the phone's baseline), `coverage.observed` is replaced by the occlusion-aware -coverage, and the measured facing gaps and overhead clearances are added to whatever the phone -marked. A Measure Lab session gets a new scene with the same parts. +coverage, and the target wall's measured facing gaps and overhead clearances are replaced. +A Measure Lab session gets a new scene with the same parts. The fitted wall takes the source of the depth that built it (`pipeline.WALL_SOURCE`: "mesh" from LiDAR, "plane" from photos alone) and no `plus_minus_ft`; a bundle's tapped bound is removed. The @@ -83,10 +83,17 @@ def build( target["baseline"] = [_plan_ft(wall.world(s, 0.0)) for s in (lo, hi)] target["source"] = wall_source target.pop("plus_minus_ft", None) - doc.setdefault("coverage", {})["observed"] = observed(cov)[:500] + entries = observed(cov) + # scene.schema.json caps observed entries at 500; a partial export hides valid observations. + if len(entries) > 500: + raise ValueError( + f"coverage.observed has {len(entries)} entries; scene contract limit is 500" + ) + doc.setdefault("coverage", {})["observed"] = entries facing, overheads = measurements(cov, wid, plus_minus_ft) - doc["facing"] = [*doc.get("facing", []), *facing] - doc["overheads"] = [*doc.get("overheads", []), *overheads] + # Replacing the wall invalidates its old spans even when the meter stays in place. + doc["facing"] = [*[m for m in doc.get("facing", []) if m["wall_id"] != wid], *facing] + doc["overheads"] = [*[m for m in doc.get("overheads", []) if m["wall_id"] != wid], *overheads] return doc diff --git a/recon/tests/test_coverage_batches.py b/recon/tests/test_coverage_batches.py new file mode 100644 index 000000000..fde5a2bd9 --- /dev/null +++ b/recon/tests/test_coverage_batches.py @@ -0,0 +1,145 @@ +"""Compare the batched march with individual slab queries and count repeated work.""" + +from dataclasses import replace + +import numpy as np +import pytest +from test_coverage_views import views +from test_worker import _slab_volume + +from recon import coverage +from recon.geometry import Ground, WallFrame + +WALL = WallFrame( + np.array([0.0, 1.0, 0.0]), np.array([1.0, 0, 0]), np.array([0, 0, 1.0]), 0.0, (0, 0.3) +) + + +def scalar_march(wall, volume, cells): + step = volume.voxel + raw = volume.vol if isinstance(volume, coverage.SeenFromTwo) else volume + arrays = [ + np.full(len(cells), np.nan), + np.zeros(len(cells)), + np.full(len(cells), np.nan), + np.zeros(len(cells)), + ] + fg, fc, og, oc = arrays + heights = np.arange(coverage.FACING_BAND_M[0], coverage.FACING_BAND_M[1] + 1e-9, step) + for c, cell in enumerate(cells): + across = cell + coverage.CELL_M * np.array([0.25, 0.5, 0.75]) + front = coverage.wall_front(wall, raw, across) + for out in np.arange(front + step, front + coverage.FACING_MAX_M, step): + share, seen = coverage._slab(wall, volume, across, heights, [out]) + if share > 0: + if seen: + fg[c] = max(0, out - step - front) + break + if not seen: + break + fc[c] = out - front + outs = front + np.append( + np.arange(coverage.OVERHEAD_OUT_M[0], coverage.OVERHEAD_OUT_M[1], step), + coverage.OVERHEAD_OUT_M[1], + ) + for height in np.arange(0.3, coverage.OVERHEAD_MAX_M, step): + share, seen = coverage._slab(wall, volume, across, [height], outs) + if share > 0: + if seen: + og[c] = max(0, height - step) + break + if not seen: + break + oc[c] = height + return arrays + + +@pytest.mark.parametrize("use_views", [False, True]) +@pytest.mark.parametrize("kind", ["clear", "proud", "thin", "unknown"]) +def test_batched_march_matches_scalar_stop_order(kind, use_views): + occupied = [(-0.3, 0.3), (1.3, 1.7)] if kind == "proud" else [(-0.3, 0.0)] + raw = _slab_volume(occupied, 1.1 if kind == "unknown" else 7.0) + if kind == "thin": + raw.tsdf[:, 20, 30] = -1.0 + raw.tsdf[:, 30, 16] = -1.0 + if use_views: + frames, depths = views(7) + volume = coverage.SeenFromTwo(raw, frames, depths) + else: + volume = raw + cells = np.array([0.0, coverage.CELL_M]) + actual = coverage.free_space(WALL, volume, cells) + expected = scalar_march(WALL, volume, cells) + for a, b in zip(actual, expected, strict=True): + np.testing.assert_array_equal(a, b) + + +def test_wall_front_gets_raw_volume_and_uses_no_visibility(monkeypatch): + raw = _slab_volume([(-0.3, 0.0)], 7.0) + frames, depths = views(2) + view = coverage.SeenFromTwo(raw, frames, depths) + calls = [] + real_front = coverage.wall_front + + def wall_front(wall, volume, across, ground=None): + assert volume is raw + calls.append(len(across)) + return real_front(wall, volume, across, ground) + + monkeypatch.setattr(coverage, "wall_front", wall_front) + coverage.free_space(WALL, view, np.array([0.0, coverage.CELL_M])) + assert calls == [3, 3] + + +def test_clear_march_batches_queries_and_builds_pairs_once(monkeypatch): + raw = _slab_volume([(-0.3, 0.0)], 7.0) + frames, depths = views(9) + real_far = coverage.far_pairs + pair_calls, query_sizes = [], [] + + def far(centres): + pair_calls.append(len(centres)) + return real_far(centres) + + monkeypatch.setattr(coverage, "far_pairs", far) + view = coverage.SeenFromTwo(raw, frames, depths) + + def seen(points): + query_sizes.append(len(points)) + return np.ones((len(points), len(frames)), bool) + + monkeypatch.setattr(view, "seen_through", seen) + monkeypatch.setattr(coverage, "two_positions", lambda *_: pytest.fail("dense pair multiply")) + cells = np.array([0.0, coverage.CELL_M]) + coverage.free_space(WALL, view, cells) + facing_steps = len(np.arange(0.10, 0.05 + coverage.FACING_MAX_M, raw.voxel)) + overhead_steps = len(np.arange(0.3, coverage.OVERHEAD_MAX_M, raw.voxel)) + expected_calls = len(cells) * ( + int(np.ceil(facing_steps / coverage.SLABS_PER_BATCH)) + + int(np.ceil(overhead_steps / coverage.SLABS_PER_BATCH)) + ) + assert pair_calls == [len(frames)] + assert len(query_sizes) == expected_calls == 24 + assert len(query_sizes) < (facing_steps + overhead_steps) * len(cells) // 10 + assert max(query_sizes) <= coverage.VIEW_POINTS_PER_BATCH + + +def test_compute_retains_ground_and_complete_cells_after_combining_prs(monkeypatch): + raw = _slab_volume([], 7.0) + frames, depths = views(2) + wall = replace(WALL, s_range=(0.9 * coverage.CELL_M, 2.1 * coverage.CELL_M)) + ground = Ground(np.zeros(3), np.array([0.0, 1.0, 0.0]), 0.0, 100) + + def wall_and_ground(wall, frames, depths, volume, mesh, cells, fitted_ground): + assert fitted_ground is ground + np.testing.assert_allclose(cells, [coverage.CELL_M]) + return np.zeros(len(cells), bool), np.zeros(len(cells)), np.zeros(len(cells)) + + monkeypatch.setattr(coverage, "wall_and_ground", wall_and_ground) + result = coverage.compute(wall, frames, depths, raw, None, ground) + np.testing.assert_allclose(result.cells, [coverage.CELL_M]) + + +def test_slab_batch_refuses_an_unsupported_march_axis(): + with pytest.raises(ValueError, match="slab axis must be height"): + coverage._slabs(WALL, None, [0], [0], [0], axis=0) diff --git a/recon/tests/test_coverage_local_ground.py b/recon/tests/test_coverage_local_ground.py new file mode 100644 index 000000000..c1ee4b47b --- /dev/null +++ b/recon/tests/test_coverage_local_ground.py @@ -0,0 +1,85 @@ +"""Vertical samples follow the fitted ground across and away from the wall.""" + +import numpy as np +import pytest + +from recon import coverage +from recon.geometry import Ground, WallFrame + + +@pytest.mark.parametrize("axis", [1, 2]) +@pytest.mark.parametrize("slope", [0.25, -0.25]) +def test_vertical_slabs_are_relative_to_local_ground(axis, slope): + wall = WallFrame( + np.array([0.0, 1.0, 0.0]), + np.array([1.0, 0.0, 0.0]), + np.array([0.0, 0.0, 1.0]), + 0.0, + (0.0, 3.0), + ) + ground = Ground(np.zeros(3), np.array([-slope, 1.0, -0.1]), 0.0, 100) + queried = [] + + class Volume: + def occupied(self, points): + queried.append(points.copy()) + return np.zeros(len(points), bool) + + def observed(self, points): + return np.ones(len(points), bool) + + coverage._slabs(wall, Volume(), [2.0, 3.0], [1.0, 2.0], [0.1, 0.5], axis=axis, ground=ground) + s, h, out = np.meshgrid([2.0, 3.0], [1.0, 2.0], [0.1, 0.5], indexing="ij") + points = queried[0] + np.testing.assert_allclose(points[:, 1], h.ravel() + ground.height_at(s.ravel(), out.ravel())) + + +def test_uphill_eave_reports_clearance_above_cell_ground(): + wall = WallFrame( + np.array([0.0, 1.0, 0.0]), + np.array([1.0, 0.0, 0.0]), + np.array([0.0, 0.0, 1.0]), + 0.0, + (0.0, 3.0), + ) + ground = Ground(np.zeros(3), np.array([-0.5, 1.0, 0.0]), 0.0, 100) + + class Eave: + voxel = 0.05 + + def occupied(self, points): + # Horizontal eave at world y=2.5 m; the cell ground is about 1 m higher. + return points[:, 1] >= 2.5 + + def observed(self, points): + return np.ones(len(points), bool) + + _, _, gap, clear = coverage.free_space(wall, Eave(), np.array([2.0]), ground=ground) + assert gap[0] == pytest.approx(1.40) + assert clear[0] == pytest.approx(1.40) + + +def test_wall_band_heights_are_relative_to_local_ground(monkeypatch): + wall = WallFrame( + np.array([0.0, 1.0, 0.0]), + np.array([1.0, 0.0, 0.0]), + np.array([0.0, 0.0, 1.0]), + 0.0, + (0.0, 3.0), + ) + ground = Ground(np.zeros(3), np.array([-0.5, 1.0, -0.1]), 0.0, 100) + queried = [] + + def seen(points, *args): + queried.append(points.copy()) + return np.ones((len(points), 2), bool) + + monkeypatch.setattr(coverage, "seen_by", seen) + monkeypatch.setattr( + coverage, "two_positions", lambda visibility, centres: visibility.all(axis=1) + ) + monkeypatch.setattr(coverage, "face_offsets", lambda *args: np.zeros(1)) + coverage.wall_and_ground(wall, [], {}, None, None, np.array([2.0]), ground) + points = queried[0] + height = np.tile(np.linspace(0.05, coverage.WALL_TOP_M, coverage.WALL_ROWS), 2) + np.testing.assert_allclose(points[:, 1], height + ground.height_at(points[:, 0], points[:, 2])) diff --git a/recon/tests/test_coverage_views.py b/recon/tests/test_coverage_views.py new file mode 100644 index 000000000..a1a764321 --- /dev/null +++ b/recon/tests/test_coverage_views.py @@ -0,0 +1,224 @@ +"""Exact visibility and camera-pair checks across batching limits.""" + +from pathlib import Path + +import numpy as np +import pytest + +from recon import coverage +from recon.capture import Frame +from recon.depth import Depth +from recon.fusion import Volume +from recon.geometry import WallFrame + + +class ClearVolume: + voxel = 0.05 + blocked = Volume.blocked + + def observed(self, points): + return np.ones(points.shape[:-1], bool) + + def occupied(self, points): + return np.zeros(points.shape[:-1], bool) + + +def views(count): + frames, depths = [], {} + for j in range(count): + pose = np.eye(4) + pose[:3, 3] = [j * 0.13 - 0.3, 1.0, 4.0] + angle = (j % 3 - 1) * 0.08 + c, s = np.cos(angle), np.sin(angle) + pose[:3, :3] = [[c, 0, s], [0, 1, 0], [-s, 0, c]] + frame = Frame(str(j), Path("unused"), 64, 48, np.array([24, 24, 32, 24]), pose) + frames.append(frame) + shape = (24, 32) if j % 2 else (48, 64) + values = np.full(shape, 4.0, np.float32) + values[: shape[0] // 3, : shape[1] // 3] = np.nan + depths[frame.id] = Depth(values, frame.intrinsics, np.zeros((*shape, 3)), "lidar") + return frames, depths + + +def scalar_visible(points, frames, depths): + result = np.zeros((len(points), len(frames)), bool) + for i, point in enumerate(points): + for j, frame in enumerate(frames): + delta = point - frame.center + distance = np.linalg.norm(delta) + if not 0.1 < distance <= coverage.MAX_RANGE_M: + continue + p = delta @ frame.cam_to_world[:3, :3] + z = -p[2] + if z <= 0.05: + continue + fx, fy, cx, cy = frame.intrinsics + u, v = cx + fx * p[0] / z, cy - fy * p[1] / z + margin = coverage.IMAGE_MARGIN + if not frame.width * margin <= u <= frame.width * (1 - margin): + continue + if not frame.height * margin <= v <= frame.height * (1 - margin): + continue + values = depths[frame.id].depth + h, w = values.shape + scale = w / frame.width + value = values[min(int(v * scale), h - 1), min(int(u * scale), w - 1)] + result[i, j] = np.isfinite(value) and value >= z - coverage.tolerance(distance) + return result + + +@pytest.mark.parametrize("frames", [0, 1, 2, 7, 9, 33, 65]) +def test_packed_pair_test_matches_dense_relation(frames): + rng = np.random.default_rng(40 + frames) + centres = rng.normal(0, 0.2, (frames, 3)) + saw = rng.random((113, frames)) > 0.45 + far = coverage.far_pairs(centres) + expected = coverage.two_positions(saw, centres, far) + actual = coverage._two_positions_packed(saw, np.packbits(far.astype(bool), axis=1)) + np.testing.assert_array_equal(actual, expected) + + +def test_pair_test_does_not_require_first_view_to_have_a_far_partner(): + centres = np.array([[0, 0, 0], [-0.2, 0, 0], [0.2, 0, 0]]) + saw = np.array([[True, True, True], [True, True, False]]) + partners = np.packbits(coverage.far_pairs(centres).astype(bool), axis=1) + np.testing.assert_array_equal(coverage._two_positions_packed(saw, partners), [True, False]) + + +def test_batched_projection_matches_independent_scalar_visibility(): + frames, depths = views(9) + view = coverage.SeenFromTwo(ClearVolume(), frames, depths) + points = np.random.default_rng(5).uniform([-2, -1, -1], [3, 3, 5], (137, 3)) + expected = scalar_visible(points, frames, depths) + np.testing.assert_array_equal(view.seen_through(points), expected) + np.testing.assert_array_equal(view.observed(points), coverage.two_positions(expected, view.C)) + assert all(a is depths[f.id].depth for a, f in zip(view.depths, frames, strict=True)) + + +def test_projection_queries_and_matrix_memory_are_bounded(monkeypatch): + frames, depths = views(129) + view = coverage.SeenFromTwo(ClearVolume(), frames, depths) + monkeypatch.setattr(coverage, "VIEW_POINTS_PER_BATCH", 64) + monkeypatch.setattr(coverage, "VIEW_SAMPLES_PER_BATCH", 1032) + sizes = [] + + def seen(points): + sizes.append(len(points)) + return np.ones((len(points), len(frames)), bool) + + monkeypatch.setattr(view, "seen_through", seen) + points = np.zeros((257, 3)) + assert view.observed(points).all() + assert len(sizes) == 33 and max(sizes) == 8 + assert max(sizes) * len(frames) <= coverage.VIEW_SAMPLES_PER_BATCH + + +def test_one_view_or_one_position_never_claims_free_space(): + for count in (0, 1, 3): + frames, depths = views(count) + for frame in frames: + frame.cam_to_world[:3, 3] = [0, 1, 4] + view = coverage.SeenFromTwo(ClearVolume(), frames, depths) + assert not view.observed(np.array([[0, 1, 1]])).any() + + +class FirstSlabObstacle(ClearVolume): + def __init__(self, band): + self.band = band + + def occupied(self, points): + if self.band == "facing": + # A thin hit in the first outward slab, too small to become the wall front. + return np.isclose(points[..., 2], 0.1) & np.isclose(points[..., 1], 1.0) + return np.isclose(points[..., 1], 0.3) & np.isclose(points[..., 2], 0.3) + + +def free_space_from_positions(raw, xs): + frames, depths = views(len(xs)) + for frame, x in zip(frames, xs, strict=True): + frame.cam_to_world[:3, :3] = np.eye(3) + frame.cam_to_world[:3, 3] = [x, 1.0, 4.0] + depths[frame.id].depth[:] = 4.0 + wall = WallFrame( + np.array([0.0, 1.0, 0.0]), np.array([1.0, 0, 0]), np.array([0, 0, 1.0]), 0.0, (0, 0.3) + ) + cells = np.array([0.0]) + arrays = coverage.free_space(wall, coverage.SeenFromTwo(raw, frames, depths), cells) + return coverage.CellCoverage(cells, np.zeros(1, bool), np.zeros(1), *arrays) + + +@pytest.mark.parametrize("band", ["facing", "overhead"]) +@pytest.mark.parametrize( + ("xs", "supported"), + [([0.075], False), ([0.025, 0.125], False), ([-0.125, 0.275], True)], + ids=["one_camera", "nearby_cameras", "separated_cameras"], +) +def test_first_occupied_slab_needs_two_views_before_exporting_measurement(band, xs, supported): + cov = free_space_from_positions(FirstSlabObstacle(band), xs) + gap = cov.facing_gap if band == "facing" else cov.overhead_clearance + clear = cov.facing_clear if band == "facing" else cov.overhead_clear + entries = [entry for entry in coverage.observed(cov) if entry["band"] == band] + measurements = coverage.measurements(cov, "wall", 0.5)[0 if band == "facing" else 1] + # An occupied first slab never counts as empty, even when its measurement is unsupported. + assert clear[0] == 0.0 + if supported: + assert gap[0] == pytest.approx(0.0 if band == "facing" else 0.25) + assert entries == [{"band": band, "span_ft": [0.0, 0.5]}] + assert len(measurements) == 1 + else: + assert np.isnan(gap[0]) + assert entries == [] + assert measurements == [] + + +@pytest.mark.parametrize( + ("xs", "supported"), + [([0.075], False), ([0.025, 0.125], False), ([-0.125, 0.275], True)], + ids=["one_camera", "nearby_cameras", "separated_cameras"], +) +def test_empty_volume_needs_two_views_before_exporting_clear_space(xs, supported): + cov = free_space_from_positions(ClearVolume(), xs) + assert np.isnan(cov.facing_gap[0]) and np.isnan(cov.overhead_clearance[0]) + assert coverage.measurements(cov, "wall", 0.5) == ([], []) + entries = coverage.observed(cov) + if supported: + assert cov.facing_clear[0] > 0 and cov.overhead_clear[0] > 0 + assert {entry["band"] for entry in entries} == {"facing", "overhead"} + assert all(entry["out_ft"] > 0 for entry in entries) + else: + assert cov.facing_clear[0] == cov.overhead_clear[0] == 0.0 + assert entries == [] + + +@pytest.mark.parametrize( + ("obstruction_z", "visible"), + [(2.0, False), (None, True), (0.0, True)], + ids=["fused_obstruction", "clear_ray", "supported_surface"], +) +def test_free_space_rays_respect_fused_obstructions_but_not_the_sample_surface( + obstruction_z, visible +): + frames, depths = views(2) + for frame, x in zip(frames, [-0.2, 0.2], strict=True): + frame.cam_to_world[:3, :3] = np.eye(3) + frame.cam_to_world[:3, 3] = [x, 1.0, 4.0] + depths[frame.id].depth[:] = 4.0 + shape = (21, 31, 51) + tsdf = np.ones(shape, np.float32) + if obstruction_z is not None: + tsdf[:, :, round(obstruction_z / 0.1)] = -1.0 + raw = Volume( + origin=np.array([-1.0, 0.0, 0.0]), + voxel=0.1, + shape=shape, + tsdf=tsdf, + weight=np.ones(shape, np.float32), + color=np.zeros((*shape, 3), np.float32), + color_n=np.zeros(shape, np.float32), + ) + view = coverage.SeenFromTwo(raw, frames, depths) + points = np.array([[0.0, 1.0, 0.0]]) + # Both depth maps put the sample on a surface. Fusion independently retains a + # nearer obstacle in the failing case; a hit on the sample itself remains supported. + np.testing.assert_array_equal(view.seen_through(points), [[visible, visible]]) + np.testing.assert_array_equal(view.observed(points), [visible]) diff --git a/recon/tests/test_review_debt.py b/recon/tests/test_review_debt.py new file mode 100644 index 000000000..f175dc90a --- /dev/null +++ b/recon/tests/test_review_debt.py @@ -0,0 +1,434 @@ +"""Regressions for review findings that survived the reconstruction handoff.""" + +import fcntl +import importlib.util +import io +import json +import struct +import zipfile +from dataclasses import replace +from pathlib import Path + +import cv2 +import numpy as np +import pytest +from test_worker import ( + NO_COVERAGE, + WALL, + _all_visible_setup, + _capture_named_scan, + _fake_moge, + _lidar_frames, + _slab_volume, +) + +from recon import capture as cap +from recon import coverage, depth, geometry, pipeline, scene +from recon.coverage import CELL_M + + +def test_replacing_wall_drops_its_old_measurements_and_keeps_other_walls(): + old_facing = {"wall_id": "w", "span_ft": [-1, 1], "depth_ft": 20} + old_overhead = {"wall_id": "w", "span_ft": [-1, 1], "clearance_ft": 10} + other = {"wall_id": "v", "span_ft": [1, 2], "depth_ft": 3} + prior = { + "meter": {"wall_id": "w", "pos": [0, 5, 0]}, + "walls": [{"id": "w", "baseline": [[-1, 0], [1, 0]]}], + "facing": [old_facing, other], + "overheads": [old_overhead], + } + capture = cap.Capture("scan-bundle", Path("b"), [], 0.0, WALL.meter, None, prior) + rotated = replace(WALL, along=np.array([np.cos(0.1), 0, np.sin(0.1)])) + doc = scene.build(capture, rotated, NO_COVERAGE, "mesh", 0.5) + assert doc["facing"] == [other] and doc["overheads"] == [] + assert prior["facing"] == [old_facing, other] + + +@pytest.mark.parametrize("n", [500, 501]) +def test_export_observations_refuses_contract_overflow_without_slicing(monkeypatch, n): + entries = [{"band": "wall", "span_ft": [i, i + 0.5]} for i in range(n)] + monkeypatch.setattr(scene, "observed", lambda _: entries) + capture = cap.Capture("measure-lab", Path("b"), [], 0.0, None, None) + if n == 501: + with pytest.raises(ValueError, match=r"501 entries.*limit is 500"): + scene.build(capture, WALL, NO_COVERAGE, "mesh", 0.5) + else: + assert ( + scene.build(capture, WALL, NO_COVERAGE, "mesh", 0.5)["coverage"]["observed"] == entries + ) + + +def test_coverage_exports_only_cells_inside_fitted_extent(monkeypatch): + wall, frames, depths, vol, surface, _ = _all_visible_setup() + monkeypatch.setattr(coverage, "seen_by", lambda points, *_: np.ones((len(points), 2), bool)) + # This test isolates cell bounds; the two-view wrapper has separate real-depth regressions. + monkeypatch.setattr(coverage, "SeenFromTwo", lambda vol, frames, depths: vol) + monkeypatch.setattr( + coverage, "free_space", lambda wall, vol, cells, ground=None: (np.zeros(len(cells)),) * 4 + ) + wall = replace(wall, s_range=(0.9 * CELL_M, 2.1 * CELL_M)) + cov = coverage.compute(wall, frames, depths, vol, surface) + np.testing.assert_allclose(cov.cells, [CELL_M]) + entries = coverage.observed(cov) + assert all(0.5 <= e["span_ft"][0] < e["span_ft"][1] <= 1.0 for e in entries) + cov = coverage.compute(replace(wall, s_range=(0.01, 0.02)), frames, depths, vol, surface) + assert cov.cells.size == 0 and coverage.observed(cov) == [] + + +def test_ground_coverage_uses_local_slope_height_and_normal(monkeypatch): + setup = _all_visible_setup() + ground = geometry.Ground(np.zeros(3), np.array([0.0, 1, -0.2]), 0.0, 10) + sampled = [] + + def see(points, normal, *_): + if normal is ground.normal: + sampled.append(points.copy()) + seen = np.abs(points[:, 1] - 0.2 * points[:, 2]) < 1e-9 + else: + seen = np.ones(len(points), bool) + return np.repeat(seen[:, None], 2, axis=1) + + monkeypatch.setattr(coverage, "seen_by", see) + _, reach, _ = coverage.wall_and_ground(*setup, ground) + assert sampled and np.all(sampled[0][:, 1] > 0) + np.testing.assert_allclose(reach, coverage.GROUND_MAX_M) + + +def test_depth_test_sees_sloped_ground_and_rejects_horizontal_datum(): + # Two cameras look vertically down at y = 0.2*z. Pixel-ray intersections supply metric depth. + from test_worker import DOWN, GROUND_FRAMES, H, K, W + + frames = GROUND_FRAMES + v, _u = np.mgrid[:H, :W] + dz_per_depth = (v + 0.5 - K[3]) / K[1] + d = 1.5 / (1 + 0.2 * dz_per_depth) + depths = { + f.id: depth.Depth(d.astype(np.float32), K, np.zeros((H, W, 3), np.uint8), "lidar") + for f in frames + } + assert np.array_equal(frames[0].cam_to_world[:3, :3], DOWN) + + class EmptyRays: + voxel = 0.05 + + def blocked(self, start, ends, stop_short): + return np.zeros(len(ends), bool) + + points = np.array([[0.0, 0.12, 0.6], [0.0, -0.12, -0.6]]) + normal = np.array([0.0, 1, -0.2]) / np.sqrt(1.04) + assert coverage.seen_by(points, normal, frames, depths, EmptyRays()).all() + horizontal = points.copy() + horizontal[:, 1] = 0.0 + assert not coverage.seen_by(horizontal, normal, frames, depths, EmptyRays()).any() + + +def test_reconstruction_passes_fitted_ground_to_coverage(monkeypatch): + wall, frames, depths, _, surface, _ = _all_visible_setup() + capture = cap.Capture("measure-lab", Path("b"), frames, 0.0, wall.meter, None) + ground = geometry.Ground(np.zeros(3), np.array([0.0, 1, -0.2]), 0.0, 10) + volume = _slab_volume([], 7) + monkeypatch.setattr(pipeline.fusion, "integrate", lambda *_: volume) + monkeypatch.setattr(pipeline.fusion, "mesh", lambda _: surface) + monkeypatch.setattr(pipeline.geometry, "fit_ground", lambda *_: ground) + monkeypatch.setattr(pipeline.geometry, "wall_lines", lambda *_: ["line"]) + monkeypatch.setattr(pipeline.geometry, "choose_wall", lambda *_: "line") + monkeypatch.setattr(pipeline.geometry, "wall_frame", lambda *_: wall) + + def compute(*args): + assert args[-1] is ground + return NO_COVERAGE + + monkeypatch.setattr(pipeline.coverage, "compute", compute) + assert pipeline.reconstruct(capture, depths, {"source": "lidar"})["coverage"] is NO_COVERAGE + + +@pytest.mark.parametrize("band", ["facing", "overhead"]) +def test_one_voxel_obstruction_ends_clearance(band): + wall, *_ = _all_visible_setup() + vol = _slab_volume([(-0.3, 0.0)], 7.0) + if band == "facing": + # One 5 cm row at y=1 m, one step at z=1 m: under 20% of the facing slab. + vol.tsdf[:, 20, 30] = -1 + else: + # One outward voxel at z=0.3 m, one height step at y=1.5 m. + vol.tsdf[:, 30, 16] = -1 + fg, fc, og, oc = coverage.free_space(wall, vol, np.array([0.0])) + if band == "facing": + assert np.isfinite(fg[0]) and 0.8 < fg[0] < 1.0 and fc[0] < 1.0 + else: + assert np.isfinite(og[0]) and 1.3 < og[0] < 1.5 and oc[0] < 1.5 + + +def test_lidar_without_confidence_is_refused_before_depth_read(tmp_path): + frames = _lidar_frames(tmp_path, {"a": True}) + f = replace(frames[0], lidar=replace(frames[0].lidar, confidence=None)) + f.lidar.file.unlink() + with pytest.raises(ValueError, match="requires its confidence map"): + depth.lidar(f) + + +@pytest.mark.parametrize("orientation", [3, 6]) +def test_keyframe_jpeg_exif_does_not_rotate_sensor_pixels(tmp_path, orientation): + image = np.zeros((16, 24, 3), np.uint8) + image[:8, :12] = 255 + ok, jpeg = cv2.imencode(".jpg", image) + assert ok + tiff = struct.pack("<2sHIH", b"II", 42, 8, 1) + tiff += struct.pack("H", len(exif) + 2) + exif + raw[2:] + path = tmp_path / "k.jpg" + path.write_bytes(tagged) + frame = cap.Frame("k", path, 24, 16, np.array([24.0, 24, 12, 8]), np.eye(4)) + sensor = cv2.imdecode(jpeg, cv2.IMREAD_COLOR) + np.testing.assert_array_equal(depth._image(frame), sensor) + default = cv2.imread(str(path), cv2.IMREAD_COLOR) + assert default.shape != sensor.shape or not np.array_equal(default, sensor) + + +def corrupt_compressed_npz() -> bytes: + archive = io.BytesIO() + np.savez_compressed(archive, depth=np.ones((2, 2), np.float32)) + raw = bytearray(archive.getvalue()) + name_bytes, extra_bytes = struct.unpack_from(" bytes: + archive = io.BytesIO() + np.savez(archive, depth=np.ones((2, 2), np.float32)) + raw = bytearray(archive.getvalue()) + central = raw.index(b"PK\x01\x02") + # Both ZIP headers agree on an unsupported method, so the failure occurs on member read. + struct.pack_into(" bytes: + archive = io.BytesIO() + np.savez(archive, depth=np.ones((2, 2), np.float32)) + raw = bytearray(archive.getvalue()) + central = raw.index(b"PK\x01\x02") + # Corrupt only the directory's encryption bit. zipfile uses that bit when + # NumPy reads depth.npy, so a normal cache read now asks for a password. + flags = struct.unpack_from("= low) + & (points[:, 2] <= high) + & (points[:, 1] >= 1.9 - 1e-9) + ) + + def observed(self, points): + low, high = self.band + missing = (points[:, 2] >= low) & (points[:, 2] <= high) & (points[:, 1] >= 1.9 - 1e-9) + return np.ones(len(points), bool) if self.obstruction else ~missing + + +@pytest.mark.parametrize("band", [(0.05, 0.10), (0.609, 0.611)], ids=["near_wall", "far_edge"]) +@pytest.mark.parametrize("obstruction", [True, False], ids=["eave", "unobserved"]) +def test_overhead_checks_full_footprint_depth(band, obstruction): + # The first free step is 0.05 m out, so the 0.56 m footprint ends at z=0.61 m. + _, _, gap, clear = coverage.free_space( + WALL, _FootprintVolume(band, obstruction), np.array([0.0]) + ) + assert clear[0] < 1.9 + if obstruction: + assert gap[0] == pytest.approx(1.85) + else: + assert np.isnan(gap[0]) diff --git a/recon/tests/test_worker.py b/recon/tests/test_worker.py index e4b639cdf..b71183a64 100644 --- a/recon/tests/test_worker.py +++ b/recon/tests/test_worker.py @@ -410,7 +410,8 @@ def _lidar_frames(root: Path, with_depth: dict[str, bool]) -> list[cap.Frame]: lid = None if has: np.full((2, 2), 2.0, " float: + """Facing clear distance in front of a wall at z = 0 (outward +z), its space seen empty in + the volume to z = 7, when only cameras at x = xs, 4 m out and 1 m up, looking at it see it.""" + wall = WallFrame( + np.array([0.0, 1.0, 0.0]), np.array([1.0, 0, 0]), np.array([0, 0, 1.0]), 0.0, (0, 0.3) + ) + frames = [ + cap.Frame(f.id, f.image, f.width, f.height, f.intrinsics, f.cam_to_world + _AT_Z4) + for f in _frames_facing(xs, 1.0, np.eye(3)) + ] + view = coverage.SeenFromTwo(_slab_volume([(-0.3, 0.0)], 7.0), frames, _flat_depths(frames, 4.0)) + _, clear, _, _ = coverage.free_space(wall, view, np.array([0.0])) + return float(clear[0]) + + +_AT_Z4 = np.zeros((4, 4)) +_AT_Z4[2, 3] = 4.0 + + +def test_free_space_seen_from_one_position_is_not_clearance_evidence(): + # The volume alone calls all of it observed; one camera, or two closer than 0.25 m, is not + # the two positions wall and ground need, so no clear distance may be claimed. + assert _facing_clear_seen_from({"a": 0.075}) == 0.0 + assert _facing_clear_seen_from({"a": 0.025, "b": 0.125}) == 0.0 + # Two positions 0.4 m apart see it clear until the facing band's foot leaves their view, + # 2.5 m out with this lens. + assert _facing_clear_seen_from({"a": -0.125, "b": 0.275}) == pytest.approx(2.5, abs=0.06) + + # --- The server call ------------------------------------------------------------------------ diff --git a/tests/makefile.sh b/tests/makefile.sh index c87f78501..133a160a0 100755 --- a/tests/makefile.sh +++ b/tests/makefile.sh @@ -31,10 +31,10 @@ EOF done export PATH="$work/bin:$PATH" -suites="scoring measure-lab evals recon meter-closeup integration-viewer" +suites="scoring measure-lab evals recon meter-closeup verification integration-viewer" folder() { case $1 in - recon) echo recon ;; + recon | verification) echo "$1" ;; integration-viewer) echo tools/integration-viewer ;; *) echo "experiments/$1" ;; esac @@ -80,8 +80,8 @@ fail() { } # 1. `make check` runs server, web and ios, then exactly the experiment suites present, in order, -# for all 64 combinations. "." is the ios build or project drift check at the root. -for mask in $(seq 0 63); do +# for all 128 combinations. "." is the ios build or project drift check at the root. +for mask in $(seq 0 127); do present=() expected="server web ." i=0 @@ -125,6 +125,8 @@ experiments/evals make test" expect_commands recon "recon make test" expect_commands meter-closeup "experiments/meter-closeup uv sync --locked experiments/meter-closeup make check" +expect_commands verification "verification uv sync --locked +verification make test" expect_commands integration-viewer "tools/integration-viewer node --test test/*.test.js" # 3. Naming a missing suite fails loudly and runs nothing. @@ -145,6 +147,7 @@ for FAIL_AT in "experiments/scoring uv" "experiments/measure-lab make" \ ". git ls-files --others --exclude-standard -- experiments/measure-lab/MeasureLab.xcodeproj" \ "experiments/evals uv" "experiments/evals make" "recon make" \ "experiments/meter-closeup uv" "experiments/meter-closeup make" \ + "verification uv" "verification make" \ "tools/integration-viewer node"; do make_tree $suites if run_make check; then diff --git a/tools/integration-viewer/README.md b/tools/integration-viewer/README.md index e833493a6..8617073ab 100644 --- a/tools/integration-viewer/README.md +++ b/tools/integration-viewer/README.md @@ -76,7 +76,7 @@ Sources are fixed at startup. The relay refuses other methods and paths, and any | `public/live.js` | Long-poll loop: catch-up reads until history is drained, then long polls that end by the next 5 s status refresh, with a 1 s floor, backoff up to 5 s that honors `Retry-After`, result and preview reads when events call for them | | `public/replay.js`, `public/scenario.js` | Illustrative replay and the synthetic scenarios both sources share | | `public/ply.js`, `public/cloud.js` | PLY reader (ascii and binary) and the canvas renderer | -| `public/conduit.js`, `public/app.js` | The file line and the page wiring. Counts update on timers, never on an animation finishing, and a probe turns motion and transitions off when the browser's animation clock is stopped. | +| `public/conduit.js`, `public/count-timers.js`, `public/app.js` | The file line, the timers that update the counts, and the page wiring. Counts update on timers, never on an animation finishing, and a probe turns motion and transitions off when the browser's animation clock is stopped. | ## Tests diff --git a/tools/integration-viewer/public/app.js b/tools/integration-viewer/public/app.js index cbca79a95..05f84419b 100644 --- a/tools/integration-viewer/public/app.js +++ b/tools/integration-viewer/public/app.js @@ -3,6 +3,7 @@ import { CloudView } from "./cloud.js"; import { Conduit } from "./conduit.js"; +import { createCountTimers } from "./count-timers.js"; import { followCapture } from "./live.js"; import { connectionView, hasPreview, initialState, reduce, stageRows, stageTimingUninterpretable, viewsToShow } from "./model.js"; import { REPLAY_CAPTURE_ID, ReplayPlayer } from "./replay.js"; @@ -50,7 +51,7 @@ let follower = null; const reducedMotion = matchMedia("(prefers-reduced-motion: reduce)"); let motionPaused = reducedMotion.matches; let shownArrival = 0; -const countTimers = new Set(); +const countTimers = createCountTimers({ write: writeCounts }); let shownCounts = { session: -1, value: 0 }; let clockStalled = false; let renderQueued = false; @@ -74,7 +75,6 @@ function dispatch(action) { // ---- modes ---- function stopAll() { - for (const timer of countTimers) clearTimeout(timer); countTimers.clear(); player?.stop(); player = null; @@ -223,22 +223,13 @@ function renderPhone() { function renderArrivals() { const { counts } = state; const fresh = state.arrivals.filter((a) => a.id > shownArrival); + let landsIn = 0; if (fresh.length > 0) { shownArrival = fresh.at(-1).id; - const landsIn = Math.max(...fresh.map((a) => conduit.send(a, { motion: motionOn() }))); - if (landsIn > 0) { - // The counts change as the marks land. A timer does the update, so the counts stay current - // even where the marks' animations cannot run. - // Each timer shows the totals as of its own batch, so a counter never runs ahead of its marks. - const snapshot = state; - const timer = setTimeout(() => { - countTimers.delete(timer); - writeCounts(snapshot); - }, landsIn); - countTimers.add(timer); - } + landsIn = Math.max(...fresh.map((a) => conduit.send(a, { motion: motionOn() }))); } - if (countTimers.size === 0) writeCounts(); + // The counts change as the marks land. + countTimers.rendered(state, landsIn); const backlog = counts.backlog; const parts = []; if (counts.registered != null) parts.push(`of ${counts.registered} registered`); @@ -248,7 +239,7 @@ function renderArrivals() { } /** Shows the acknowledged-file totals from `from`; never goes back to a smaller total in a session. */ -function writeCounts(from = state) { +function writeCounts(from) { if (from.session !== state.session) return; if (shownCounts.session === from.session && from.counts.acknowledged < shownCounts.value) return; shownCounts = { session: from.session, value: from.counts.acknowledged }; diff --git a/tools/integration-viewer/public/conduit.js b/tools/integration-viewer/public/conduit.js index 721307359..69af1f3ff 100644 --- a/tools/integration-viewer/public/conduit.js +++ b/tools/integration-viewer/public/conduit.js @@ -50,14 +50,14 @@ export class Conduit { * stopped, such as a hidden embedded preview. */ send(arrival, { motion }) { - const groups = Object.entries(arrival.groups).flatMap(([group, n]) => Array(n).fill(group)); - const marks = Math.min(groups.length, MAX_MARKS_PER_ARRIVAL, Math.max(0, MAX_IN_FLIGHT - this.inFlight)); - if (!motion || arrival.backlog || marks === 0 || this.samples.length === 0) return 0; - const step = groups.length / marks; + if (!motion || arrival.backlog || this.samples.length === 0) return 0; + const groups = markGroups(arrival.groups, Math.min(MAX_MARKS_PER_ARRIVAL, MAX_IN_FLIGHT - this.inFlight)); + const marks = groups.length; + if (marks === 0) return 0; for (let i = 0; i < marks; i += 1) { const chip = document.createElement("span"); chip.className = "chip"; - chip.dataset.group = groups[Math.floor(i * step)]; + chip.dataset.group = groups[i]; this.layer.append(chip); this.inFlight += 1; const frames = this.samples.map((pt, k) => ({ @@ -83,3 +83,26 @@ export class Conduit { } } } + +/** + * The group of each mark for a batch with `counts` files per group, at most `max` marks. The + * marks sample the batch evenly: list every file in group order, then take every + * (files / marks)-th. This walks the group totals instead of building that list, so a batch of + * any size costs a handful of steps. + * @param {Record} counts + * @param {number} max + * @returns {string[]} + */ +export function markGroups(counts, max) { + let files = 0; + for (const group in counts) files += counts[group]; + const marks = Math.max(0, Math.min(files, max)); + const picked = []; + const step = files / marks; + let end = 0; + for (const group in counts) { + end += counts[group]; + while (picked.length < marks && Math.floor(picked.length * step) < end) picked.push(group); + } + return picked; +} diff --git a/tools/integration-viewer/public/count-timers.js b/tools/integration-viewer/public/count-timers.js new file mode 100644 index 000000000..f58efcc7b --- /dev/null +++ b/tools/integration-viewer/public/count-timers.js @@ -0,0 +1,52 @@ +// Decides when the acknowledged-file counters change. Each render hands over the state it drew +// and when that render's marks land. A render's totals show once its own marks and the marks of +// every earlier render have landed, so the counters never run ahead of a mark still on the line, +// even when a later, shorter batch lands first. Totals reach the counters only from a render: the +// reducer's state can already hold an arrival whose marks have not started yet. +// Timers do the work, never an animation finishing, because an animation never finishes where +// the animation clock is stopped. + +/** + * @template S + * @param {{ write: (from: S) => void, setTimer?: typeof setTimeout, clearTimer?: typeof clearTimeout }} options + * `write` shows the totals in a state. + */ +export function createCountTimers({ write, setTimer = setTimeout, clearTimer = clearTimeout }) { + /** Renders not shown yet, oldest first: { from, landed, timer }. */ + const queue = []; + + function release() { + let latest = null; + while (queue.length > 0 && queue[0].landed) latest = queue.shift(); + if (latest) write(latest.from); + } + + return { + /** + * Records the state a render drew. `landsIn` is the milliseconds until its last mark lands, + * or 0 when it sent no marks (motion paused, clock stopped, mark cap reached, nothing new). + */ + rendered(from, landsIn) { + const tail = queue.at(-1); + if (landsIn > 0) { + const entry = { from, landed: false, timer: null }; + entry.timer = setTimer(() => { + entry.landed = true; + release(); + }, landsIn); + queue.push(entry); + } else if (tail?.landed) { + // Waiting behind the same marks as the previous render, so only the newest totals matter. + tail.from = from; + } else { + queue.push({ from, landed: true, timer: null }); + } + release(); + }, + /** Drops pending updates, for a capture switch. */ + clear() { + for (const entry of queue) if (!entry.landed) clearTimer(entry.timer); + queue.length = 0; + }, + }; +} diff --git a/tools/integration-viewer/test/conduit.test.js b/tools/integration-viewer/test/conduit.test.js new file mode 100644 index 000000000..220ebf5b9 --- /dev/null +++ b/tools/integration-viewer/test/conduit.test.js @@ -0,0 +1,65 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { Conduit, markGroups } from "../public/conduit.js"; + +// The rule markGroups must keep, written the way send() used to: list every file, take every step-th. +function expanded(counts, max) { + const files = Object.entries(counts).flatMap(([group, n]) => Array(n).fill(group)); + const marks = Math.max(0, Math.min(files.length, max)); + const step = files.length / marks; + return Array.from({ length: marks }, (_, i) => files[Math.floor(i * step)]); +} + +test("marks match the evenly spaced picks from the full file list", () => { + for (let photo = 0; photo <= 9; photo += 1) + for (let depth = 0; depth <= 7; depth += 1) + for (const other of [0, 1, 13]) + for (let max = 0; max <= 6; max += 1) { + const counts = { photo, depth, mesh: 0, other }; + assert.deepEqual(markGroups(counts, max), expanded(counts, max), JSON.stringify({ counts, max })); + } +}); + +test("a batch of billions picks its marks without listing the files", () => { + // Array(5e9) throws at once (the limit is 2 ** 32 - 1), so expanding the batch fails fast here + // instead of filling memory. + assert.deepEqual(markGroups({ photo: 5e9, depth: 5e9, mesh: 0, other: 0 }, 6), ["photo", "photo", "photo", "depth", "depth", "depth"]); + assert.deepEqual(markGroups({ photo: 5, depth: 1 }, -3), []); +}); + +function conduit() { + globalThis.ResizeObserver ??= class { + observe() {} + }; + const chips = []; + const c = new Conduit({}, [], { append: (chip) => chips.push(chip) }); + c.samples = Array.from({ length: 25 }, (_, i) => ({ x: i, y: 0 })); + return { c, chips }; +} + +test("with motion off, send returns at once without reading the batch", () => { + const { c, chips } = conduit(); + const arrival = { + backlog: false, + get groups() { + throw new Error("send read the batch with motion off"); + }, + }; + assert.equal(c.send(arrival, { motion: false }), 0); + assert.equal(chips.length, 0); +}); + +test("a huge batch sends at most six marks", (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }); + globalThis.document ??= { createElement: () => ({ dataset: {}, animate() {}, remove() {} }) }; + const { c, chips } = conduit(); + const landsIn = c.send({ backlog: false, groups: { photo: 5e9, depth: 1, mesh: 0, other: 0 } }, { motion: true }); + assert.equal(chips.length, 6); + assert.equal(landsIn, 1100 + 5 * 70); + assert.deepEqual( + chips.map((chip) => chip.dataset.group), + Array(6).fill("photo"), + ); + t.mock.timers.tick(landsIn + 50); + assert.equal(c.inFlight, 0); +}); diff --git a/tools/integration-viewer/test/count-timers.test.js b/tools/integration-viewer/test/count-timers.test.js new file mode 100644 index 000000000..6c63a61ac --- /dev/null +++ b/tools/integration-viewer/test/count-timers.test.js @@ -0,0 +1,111 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { createCountTimers } from "../public/count-timers.js"; + +// The page's side of the counters, on a virtual clock. `dispatch` changes the reducer's state +// at once, as app.js does; `render` is the queued frame that draws it and starts its marks. +// Timers fire at their due time when the clock passes it. +function page() { + let now = 0; + let reducer = { acknowledged: 0 }; + let shown = 0; + const timers = []; + const counts = createCountTimers({ + write: (from) => { + shown = from.acknowledged; + }, + setTimer: (fn, ms) => { + const t = { due: now + ms, fn }; + timers.push(t); + return t; + }, + clearTimer: (t) => timers.splice(timers.indexOf(t), 1), + }); + return { + counts, + dispatch(acknowledged) { + reducer = { acknowledged }; + }, + /** Draws the reducer's current state; `landsIn` is 0 when the render sends no marks. */ + render(landsIn = 0) { + counts.rendered(reducer, landsIn); + }, + at(t) { + for (;;) { + const next = timers.filter((x) => x.due <= t).sort((a, b) => a.due - b.due)[0]; + if (!next) break; + timers.splice(timers.indexOf(next), 1); + now = next.due; + next.fn(); + } + now = t; + }, + get shown() { + return shown; + }, + get pending() { + return timers.length; + }, + }; +} + +test("an arrival the reducer holds but no render has drawn stays off the counters", () => { + const p = page(); + p.dispatch(6); + p.render(1450); // six marks, the last landing at 1450 ms + p.at(1449); + p.dispatch(7); // arrives; its frame is queued, so its mark has not started + p.at(1450); + assert.equal(p.shown, 6, "the counters counted a file whose mark had not started"); + p.at(1456); + p.render(1100); + p.at(2555); + assert.equal(p.shown, 6); + p.at(2556); + assert.equal(p.shown, 7); +}); + +test("a later batch that lands first waits for the earlier batch's marks", () => { + const p = page(); + p.dispatch(6); + p.render(1450); // six marks from 0 ms, landing by 1450 ms + p.at(70); + p.dispatch(7); + p.render(1100); // one mark from 70 ms, landing at 1170 ms + p.at(1170); + assert.equal(p.shown, 0, "the counters ran ahead of marks still travelling"); + p.at(1450); + assert.equal(p.shown, 7); +}); + +test("a batch that drew no marks while older marks travelled shows when they land", () => { + const p = page(); + p.dispatch(5); + p.render(1100); + p.at(300); + p.dispatch(8); + p.render(0); // motion paused, clock stopped or mark cap reached + assert.equal(p.shown, 0); + p.at(1100); + assert.equal(p.shown, 8); +}); + +test("with nothing travelling, a render shows its totals at once", () => { + const p = page(); + p.dispatch(3); + p.render(0); + assert.equal(p.shown, 3); +}); + +test("clear drops pending updates", () => { + const p = page(); + p.dispatch(5); + p.render(1100); + p.counts.clear(); + assert.equal(p.pending, 0); + p.at(2000); + assert.equal(p.shown, 0); + p.dispatch(2); + p.render(0); + assert.equal(p.shown, 2); +}); diff --git a/verification/Makefile b/verification/Makefile index de79d452a..b19381ff9 100644 --- a/verification/Makefile +++ b/verification/Makefile @@ -22,12 +22,17 @@ sim: uv run python -m hsverify.simrun --ref $(REF) $(ARGS) # The app at REF on the real ADVIO replay (C3), driven by its autopilot, uploading to the server -# started from SERVER_REF (or at SERVER_URL): a screenshot per STATE in ~/house-scanning-data/reports/sim/. Fails -# unless the run reaches the result state; the scan bundle is copied into the report as scan.zip. +# started from SERVER_REF (or at SERVER_URL): a screenshot per STATE in ~/house-scanning-data/reports/sim/. +# Fails unless the run walks the flow in order (meter, close-up, wall walk, feature review, +# upload, result) and uploads after every gap request it shows, so each recapture round ends in +# an upload. Reaching `result` alone would also pass an app that showed a cached answer. A round +# the app never opens leaves no STATE to check. The scan bundle is copied into the report as +# scan.zip. State names are ScanPhase in ios/HouseScan/Contract/ScanContract.swift. sim-app: uv run python -m hsverify.simrun --ref $(REF) --replay $(REPLAY) --autopilot \ - $(SERVER) --extra-arg=-autopilotHold --extra-arg=3 \ - --idle 60 --timeout 900 --require result --require-export --label replay $(ARGS) + $(SERVER) --extra-arg=-autopilotHold --extra-arg=3 --idle 60 --timeout 900 \ + --require-order findMeter,meterCloseUp,wallWalk,markFeatures,uploading,result \ + --require-then gapRequest:uploading --require-export --label replay $(ARGS) # Every case in e2e/cases/ plus the ETH3D facade scene through the server's API at SERVER_REF; # each result is validated against server/schemas/result.schema.json at the same ref. diff --git a/verification/README.md b/verification/README.md index b20d4031c..3862d5a14 100644 --- a/verification/README.md +++ b/verification/README.md @@ -19,6 +19,14 @@ Reports go to `~/house-scanning-data/reports/`, outside git, because replay fram non-commercial datasets. What each check asserts is in the docstrings of `hsverify/e2e.py`, `resultcheck.py` and `simrun.py`. +When a case's battery width or corner allowance differs from public rules, e2e uses the +value reported in the result's spot or rule-keyed check threshold. If the result omits it, +the report says "assumed public rules; server may override" and marks the case +`assumptions unverified`. It skips the rule-dependent checks and case expectations, while +retaining rule-independent invariants. These records cannot establish acceptance. Exit +code 2 means the run is inconclusive for this reason; exit code 1 still means another +failure or incomplete check. The verifier does not read private backend files. + **Result.** At `t3/server` `3baa338`, with the app's upload from `c820cf1` (#10's freeze, synthetic replay, hosted server), 49 of 50 scenes pass (45 answered and checked; slowest real scene 0.48 s). The ETH3D scene fails: after its one requested capture the server leaves pool unsure with no diff --git a/verification/e2e/cases/c5-no-coverage.json b/verification/e2e/cases/c5-no-coverage.json index 29e9225b6..4777661c2 100644 --- a/verification/e2e/cases/c5-no-coverage.json +++ b/verification/e2e/cases/c5-no-coverage.json @@ -67,7 +67,9 @@ } ] }, - "rules_assumed": {}, + "rules_assumed": { + "battery_width": 2.5833333333333335 + }, "expect": { "decision_in": [ "manual_review" diff --git a/verification/e2e/cases/d-gas-drift.json b/verification/e2e/cases/d-gas-drift.json index e94419104..3beac4277 100644 --- a/verification/e2e/cases/d-gas-drift.json +++ b/verification/e2e/cases/d-gas-drift.json @@ -119,6 +119,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0, "route_length": 20.0 }, diff --git a/verification/e2e/cases/d-reach-drift-left.json b/verification/e2e/cases/d-reach-drift-left.json index 1f015c85b..2b03a1f4e 100644 --- a/verification/e2e/cases/d-reach-drift-left.json +++ b/verification/e2e/cases/d-reach-drift-left.json @@ -107,6 +107,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/d-reach-drift-right.json b/verification/e2e/cases/d-reach-drift-right.json index 826109a2e..e8dd80374 100644 --- a/verification/e2e/cases/d-reach-drift-right.json +++ b/verification/e2e/cases/d-reach-drift-right.json @@ -107,6 +107,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/g01-unseen-beyond-garage.json b/verification/e2e/cases/g01-unseen-beyond-garage.json index d382da71b..b11340fe9 100644 --- a/verification/e2e/cases/g01-unseen-beyond-garage.json +++ b/verification/e2e/cases/g01-unseen-beyond-garage.json @@ -124,7 +124,9 @@ ] } }, - "rules_assumed": {}, + "rules_assumed": { + "battery_width": 2.5833333333333335 + }, "expect": { "missing_evidence_empty": true, "sweep_runs": [ diff --git a/verification/e2e/cases/g03-negative-s-near-edge.json b/verification/e2e/cases/g03-negative-s-near-edge.json index c7d60170b..9b66fdccf 100644 --- a/verification/e2e/cases/g03-negative-s-near-edge.json +++ b/verification/e2e/cases/g03-negative-s-near-edge.json @@ -204,7 +204,10 @@ ] } }, - "rules_assumed": {}, + "rules_assumed": { + "battery_width": 2.5833333333333335, + "corner_allowance": 0.5 + }, "expect": { "spot": { "wall_id": "w1", diff --git a/verification/e2e/cases/g05-inside-corner-gas.json b/verification/e2e/cases/g05-inside-corner-gas.json index 78e61e185..d9542b1ac 100644 --- a/verification/e2e/cases/g05-inside-corner-gas.json +++ b/verification/e2e/cases/g05-inside-corner-gas.json @@ -149,7 +149,9 @@ } }, "rules_assumed": { - "gas": 3.0 + "battery_width": 2.5833333333333335, + "gas": 3.0, + "corner_allowance": 0.5 }, "expect": { "decision_not": [ diff --git a/verification/e2e/cases/g06-regulator-footprint.json b/verification/e2e/cases/g06-regulator-footprint.json index 2d12e8c84..6d941c54e 100644 --- a/verification/e2e/cases/g06-regulator-footprint.json +++ b/verification/e2e/cases/g06-regulator-footprint.json @@ -150,6 +150,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g07-headroom-fail.json b/verification/e2e/cases/g07-headroom-fail.json index 4cbbcc947..406c90015 100644 --- a/verification/e2e/cases/g07-headroom-fail.json +++ b/verification/e2e/cases/g07-headroom-fail.json @@ -120,6 +120,7 @@ ] }, "rules_assumed": { + "battery_width": 2.5833333333333335, "headroom": 6.5 }, "expect": { diff --git a/verification/e2e/cases/g07-headroom-margin.json b/verification/e2e/cases/g07-headroom-margin.json index 385c3d930..54126f030 100644 --- a/verification/e2e/cases/g07-headroom-margin.json +++ b/verification/e2e/cases/g07-headroom-margin.json @@ -120,6 +120,7 @@ ] }, "rules_assumed": { + "battery_width": 2.5833333333333335, "headroom": 6.5 }, "expect": { diff --git a/verification/e2e/cases/g07-headroom-partial-span.json b/verification/e2e/cases/g07-headroom-partial-span.json index 7da273f41..40166dffa 100644 --- a/verification/e2e/cases/g07-headroom-partial-span.json +++ b/verification/e2e/cases/g07-headroom-partial-span.json @@ -120,6 +120,7 @@ ] }, "rules_assumed": { + "battery_width": 2.5833333333333335, "headroom": 6.5 }, "expect": { diff --git a/verification/e2e/cases/g07-headroom-pass.json b/verification/e2e/cases/g07-headroom-pass.json index 5184547de..b817e2d94 100644 --- a/verification/e2e/cases/g07-headroom-pass.json +++ b/verification/e2e/cases/g07-headroom-pass.json @@ -120,6 +120,7 @@ ] }, "rules_assumed": { + "battery_width": 2.5833333333333335, "headroom": 6.5 }, "expect": { diff --git a/verification/e2e/cases/g07-headroom-unobserved.json b/verification/e2e/cases/g07-headroom-unobserved.json index 4f224cdd9..302eaaf85 100644 --- a/verification/e2e/cases/g07-headroom-unobserved.json +++ b/verification/e2e/cases/g07-headroom-unobserved.json @@ -102,6 +102,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "headroom": 6.5 }, "expect": { diff --git a/verification/e2e/cases/g08a-garage-blocks-route.json b/verification/e2e/cases/g08a-garage-blocks-route.json index 8d44257c0..f405b0dbf 100644 --- a/verification/e2e/cases/g08a-garage-blocks-route.json +++ b/verification/e2e/cases/g08a-garage-blocks-route.json @@ -124,7 +124,9 @@ ] } }, - "rules_assumed": {}, + "rules_assumed": { + "battery_width": 2.5833333333333335 + }, "expect": { "decision_not": [ "pass" diff --git a/verification/e2e/cases/g08b-missing-wall-blocks-route.json b/verification/e2e/cases/g08b-missing-wall-blocks-route.json index 6b20e34a5..d72f7fff0 100644 --- a/verification/e2e/cases/g08b-missing-wall-blocks-route.json +++ b/verification/e2e/cases/g08b-missing-wall-blocks-route.json @@ -131,7 +131,9 @@ ] } }, - "rules_assumed": {}, + "rules_assumed": { + "battery_width": 2.5833333333333335 + }, "expect": { "decision_not": [ "pass" diff --git a/verification/e2e/cases/g09-reach-14.json b/verification/e2e/cases/g09-reach-14.json index c38cb8b27..3b02b7295 100644 --- a/verification/e2e/cases/g09-reach-14.json +++ b/verification/e2e/cases/g09-reach-14.json @@ -131,6 +131,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/g09-reach-15.json b/verification/e2e/cases/g09-reach-15.json index 2a3beb2ff..515b99fd9 100644 --- a/verification/e2e/cases/g09-reach-15.json +++ b/verification/e2e/cases/g09-reach-15.json @@ -131,6 +131,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/g09-reach-16.json b/verification/e2e/cases/g09-reach-16.json index 3d44710e6..b044445e2 100644 --- a/verification/e2e/cases/g09-reach-16.json +++ b/verification/e2e/cases/g09-reach-16.json @@ -131,6 +131,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/g09-reach-20.json b/verification/e2e/cases/g09-reach-20.json index 74e3350cf..861f163ce 100644 --- a/verification/e2e/cases/g09-reach-20.json +++ b/verification/e2e/cases/g09-reach-20.json @@ -131,6 +131,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/g09-reach-20p2-pm03.json b/verification/e2e/cases/g09-reach-20p2-pm03.json index b47ac4c51..22cefea9b 100644 --- a/verification/e2e/cases/g09-reach-20p2-pm03.json +++ b/verification/e2e/cases/g09-reach-20p2-pm03.json @@ -131,6 +131,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/g09-reach-20p4-pm03.json b/verification/e2e/cases/g09-reach-20p4-pm03.json index 7ff4ecd72..2b8cfe85e 100644 --- a/verification/e2e/cases/g09-reach-20p4-pm03.json +++ b/verification/e2e/cases/g09-reach-20p4-pm03.json @@ -131,6 +131,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/g09-reach-21.json b/verification/e2e/cases/g09-reach-21.json index 2ea2706f7..6cbbe2d8a 100644 --- a/verification/e2e/cases/g09-reach-21.json +++ b/verification/e2e/cases/g09-reach-21.json @@ -131,6 +131,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/g09-vertical-run.json b/verification/e2e/cases/g09-vertical-run.json index ea0be5f32..b13ec63a0 100644 --- a/verification/e2e/cases/g09-vertical-run.json +++ b/verification/e2e/cases/g09-vertical-run.json @@ -144,6 +144,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "route_length": 20.0 }, "expect": { diff --git a/verification/e2e/cases/g10-derived-compound-error.json b/verification/e2e/cases/g10-derived-compound-error.json index d17aeb721..164a591d7 100644 --- a/verification/e2e/cases/g10-derived-compound-error.json +++ b/verification/e2e/cases/g10-derived-compound-error.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e03-fail.json b/verification/e2e/cases/g10-e03-fail.json index 9fdef2f3d..99acb6e5d 100644 --- a/verification/e2e/cases/g10-e03-fail.json +++ b/verification/e2e/cases/g10-e03-fail.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e03-lower-edge.json b/verification/e2e/cases/g10-e03-lower-edge.json index 2596a533b..48452bea5 100644 --- a/verification/e2e/cases/g10-e03-lower-edge.json +++ b/verification/e2e/cases/g10-e03-lower-edge.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e03-pass.json b/verification/e2e/cases/g10-e03-pass.json index ecc3a6cd2..ad77a1311 100644 --- a/verification/e2e/cases/g10-e03-pass.json +++ b/verification/e2e/cases/g10-e03-pass.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e03-upper-edge.json b/verification/e2e/cases/g10-e03-upper-edge.json index b2014bcc3..5568ed123 100644 --- a/verification/e2e/cases/g10-e03-upper-edge.json +++ b/verification/e2e/cases/g10-e03-upper-edge.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e05-fail.json b/verification/e2e/cases/g10-e05-fail.json index b9e3050c5..39732668e 100644 --- a/verification/e2e/cases/g10-e05-fail.json +++ b/verification/e2e/cases/g10-e05-fail.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e05-lower-edge.json b/verification/e2e/cases/g10-e05-lower-edge.json index 95736f521..bfa44a8ef 100644 --- a/verification/e2e/cases/g10-e05-lower-edge.json +++ b/verification/e2e/cases/g10-e05-lower-edge.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e05-pass.json b/verification/e2e/cases/g10-e05-pass.json index 69f63c6be..9270ff2d4 100644 --- a/verification/e2e/cases/g10-e05-pass.json +++ b/verification/e2e/cases/g10-e05-pass.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e05-upper-edge.json b/verification/e2e/cases/g10-e05-upper-edge.json index f59d16188..b110ad62c 100644 --- a/verification/e2e/cases/g10-e05-upper-edge.json +++ b/verification/e2e/cases/g10-e05-upper-edge.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e15-fail.json b/verification/e2e/cases/g10-e15-fail.json index 1f3d62be4..c09577b41 100644 --- a/verification/e2e/cases/g10-e15-fail.json +++ b/verification/e2e/cases/g10-e15-fail.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e15-lower-edge.json b/verification/e2e/cases/g10-e15-lower-edge.json index a11fae0c5..920148f7c 100644 --- a/verification/e2e/cases/g10-e15-lower-edge.json +++ b/verification/e2e/cases/g10-e15-lower-edge.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e15-pass.json b/verification/e2e/cases/g10-e15-pass.json index b5ea1c602..6afe24d62 100644 --- a/verification/e2e/cases/g10-e15-pass.json +++ b/verification/e2e/cases/g10-e15-pass.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g10-e15-upper-edge.json b/verification/e2e/cases/g10-e15-upper-edge.json index 5b4204ba3..36c014efa 100644 --- a/verification/e2e/cases/g10-e15-upper-edge.json +++ b/verification/e2e/cases/g10-e15-upper-edge.json @@ -139,6 +139,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g11a-unexplored-corner.json b/verification/e2e/cases/g11a-unexplored-corner.json index d837ed903..d878a851e 100644 --- a/verification/e2e/cases/g11a-unexplored-corner.json +++ b/verification/e2e/cases/g11a-unexplored-corner.json @@ -138,6 +138,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g11b-both-ends-real-all-fail.json b/verification/e2e/cases/g11b-both-ends-real-all-fail.json index 60fc39539..17a8379c2 100644 --- a/verification/e2e/cases/g11b-both-ends-real-all-fail.json +++ b/verification/e2e/cases/g11b-both-ends-real-all-fail.json @@ -197,7 +197,9 @@ } }, "rules_assumed": { - "gas": 3.0 + "battery_width": 2.5833333333333335, + "gas": 3.0, + "corner_allowance": 0.5 }, "expect": { "decision_in": [ diff --git a/verification/e2e/cases/g11c-unobserved-ground.json b/verification/e2e/cases/g11c-unobserved-ground.json index 6ddd3c15f..8018fa307 100644 --- a/verification/e2e/cases/g11c-unobserved-ground.json +++ b/verification/e2e/cases/g11c-unobserved-ground.json @@ -138,6 +138,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g12a-subgrid-start.json b/verification/e2e/cases/g12a-subgrid-start.json index c05e3a148..bd75a484f 100644 --- a/verification/e2e/cases/g12a-subgrid-start.json +++ b/verification/e2e/cases/g12a-subgrid-start.json @@ -140,6 +140,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "opening": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g12b-exact-fit-last-start.json b/verification/e2e/cases/g12b-exact-fit-last-start.json index 6ad22a994..33e40ed14 100644 --- a/verification/e2e/cases/g12b-exact-fit-last-start.json +++ b/verification/e2e/cases/g12b-exact-fit-last-start.json @@ -168,7 +168,10 @@ ] } }, - "rules_assumed": {}, + "rules_assumed": { + "battery_width": 2.5833333333333335, + "corner_allowance": 0.5 + }, "expect": { "spot": { "wall_id": "w1", diff --git a/verification/e2e/cases/g13a-unsure-never-outranks-pass.json b/verification/e2e/cases/g13a-unsure-never-outranks-pass.json index 90a65f0a7..9ed6d39e7 100644 --- a/verification/e2e/cases/g13a-unsure-never-outranks-pass.json +++ b/verification/e2e/cases/g13a-unsure-never-outranks-pass.json @@ -138,6 +138,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/g13b-unsure-alone.json b/verification/e2e/cases/g13b-unsure-alone.json index a92e83c01..b262a3323 100644 --- a/verification/e2e/cases/g13b-unsure-alone.json +++ b/verification/e2e/cases/g13b-unsure-alone.json @@ -138,6 +138,7 @@ } }, "rules_assumed": { + "battery_width": 2.5833333333333335, "gas": 3.0 }, "expect": { diff --git a/verification/e2e/cases/generate.py b/verification/e2e/cases/generate.py index e30079fec..1377f0924 100644 --- a/verification/e2e/cases/generate.py +++ b/verification/e2e/cases/generate.py @@ -17,6 +17,8 @@ # Battery footprint from the lane C review's parameter table: 31 in wide, 22 in deep. W = 31 / 12 D = 11 / 6 +# Public server/rules.yaml route.corner_allowance_ft, a day-1 placeholder, not measured. +CORNER_ALLOWANCE = 0.5 METER_Y = 4.0 # Walls placed at z = -D put a flush battery's front edge at z = 0 exactly in floating point, # so a gas strip at z = d is measured as exactly d. The margin cases need that: 3.3 - 0.3 and @@ -159,7 +161,8 @@ def case(cid, source, description, scene, expect, rules_assumed=None): "source": source, "description": description, "scene": scene, - "rules_assumed": rules_assumed or {}, + # The fixed fixture spans and sweep ranges were derived with this battery width. + "rules_assumed": {"battery_width": W} | (rules_assumed or {}), "expect": expect, } ) @@ -230,6 +233,7 @@ def straight_scene(x0, x1, z=0.0, meter_x=0.0, objects=(), ground=None, cov=None "sweep_runs": [{"wall_id": "w1", "start_ft": [-8.4, 6.9], "outcome": "fail"}], "missing_evidence_empty": True, }, + {"corner_allowance": CORNER_ALLOWANCE}, ) # g05: inside corner, gas meter on the return wall at [4, 4]. @@ -258,7 +262,7 @@ def straight_scene(x0, x1, z=0.0, meter_x=0.0, objects=(), ground=None, cov=None {"wall_id": "w1", "start_ft": [6, 7.4], "outcome": "fail", "failing_match": "gas"} ], }, - {"gas": 3.0}, + {"gas": 3.0, "corner_allowance": CORNER_ALLOWANCE}, ) # g06: gas meter body 10 ft out, regulator reaching [7, 23/6]. @@ -448,7 +452,7 @@ def g10_scene(d, e, wall_pm=0.0): {"wall_id": "w2", "start_ft": [8, 11.4], "outcome": "fail", "failing_match": "gas"}, ], }, - {"gas": 3.0}, + {"gas": 3.0, "corner_allowance": CORNER_ALLOWANCE}, ) # g11c: observed starts fail gas; the ground right of s = 1 was never seen (closed gate). @@ -531,6 +535,7 @@ def g10_scene(d, e, wall_pm=0.0): "coverage": full_coverage((-3, 6 + W)), }, {"spot": {"wall_id": "w1", "span_within": [6, 6 + W]}, "missing_evidence_empty": True}, + {"corner_allowance": CORNER_ALLOWANCE}, ) diff --git a/verification/fixtures/a11y-audit/AccessibilityAudit/AccessibilityAuditTests.swift b/verification/fixtures/a11y-audit/AccessibilityAudit/AccessibilityAuditTests.swift index b2639d7e7..4fb333452 100644 --- a/verification/fixtures/a11y-audit/AccessibilityAudit/AccessibilityAuditTests.swift +++ b/verification/fixtures/a11y-audit/AccessibilityAudit/AccessibilityAuditTests.swift @@ -3,9 +3,10 @@ import XCTest /// Launches the app under test by bundle id with the launch arguments it is given, then runs /// `performAccessibilityAudit` once per distinct screen until the app stops changing. /// -/// A screen is identified by the labels of its static texts and buttons, so a new instruction -/// counts as a new screen. Each issue is printed as one `A11Y_ISSUE=` line and each -/// screen as `A11Y_SCREEN=`, which hsverify.a11yaudit reads from the test output. +/// A screen is identified by every element in its accessibility tree (see `screenIdentity`), so +/// a new instruction or a new control counts as a new screen. Each issue is printed as one +/// `A11Y_ISSUE=` line and each screen as `A11Y_SCREEN=`, which hsverify.a11yaudit +/// reads from the test output. /// Screenshots go to AUDIT_OUT on the host. /// /// Inputs arrive as environment variables (set on the xcodebuild command line with the @@ -34,39 +35,58 @@ final class AccessibilityAuditTests: XCTestCase { emit("A11Y_SCREEN", ["index": seen.count + 1, "error": "app not running"]) break } - guard let labels = screenLabels(app) else { + guard let screen = screenIdentity(app) else { RunLoop.current.run(until: Date().addingTimeInterval(0.5)) continue } - let key = labels.joined(separator: "\n") - if !seen.contains(key) { - seen.append(key) + if !seen.contains(screen.key) { + seen.append(screen.key) lastChange = Date() - audit(app, index: seen.count, labels: labels, out: out, + audit(app, index: seen.count, labels: screen.labels, out: out, at: Date().timeIntervalSince(start)) } RunLoop.current.run(until: Date().addingTimeInterval(0.5)) } } - /// Labels from one snapshot of the accessibility tree. Querying elements and then reading - /// each label is two round trips, and a button that disappears in between (the screen - /// changing) fails the whole test. + /// Containers that only group other elements. Without a label or identifier they show + /// nothing a person or the audit would notice, so they stay out of a screen's identity, + /// which then does not depend on how the views happen to be nested. + private static let grouping: Set = [ + .application, .window, .other, .group, .layoutArea, .layoutItem, .scrollView, + ] + + /// One snapshot of the accessibility tree: the screen's identity, and the static-text and + /// button labels the report shows. Querying elements and then reading each label is two + /// round trips, and a button that disappears in between (the screen changing) fails the + /// whole test. + /// + /// The identity lists every element in tree order by type, identifier and label, so two + /// screens with the same texts and buttons still differ when one adds a text field, toggle, + /// image or link, labelled or not. It leaves out what changes while the screen stays the + /// same: values (typed text, a toggle's state, a slider's position) and frames (animations, + /// the keyboard). Each change of those would otherwise start another audit of one screen + /// and keep resetting the idle timer. @MainActor - private func screenLabels(_ app: XCUIApplication) -> [String]? { + private func screenIdentity(_ app: XCUIApplication) -> (key: String, labels: [String])? { guard let root = try? app.snapshot() else { return nil } + var elements: [String] = [] var texts: [String] = [] var buttons: [String] = [] func walk(_ node: any XCUIElementSnapshot) { - switch node.elementType { + let type = node.elementType + switch type { case .staticText: texts.append("text:" + node.label) case .button: buttons.append("button:" + node.label) default: break } + if !Self.grouping.contains(type) || !node.label.isEmpty || !node.identifier.isEmpty { + elements.append("\(type.rawValue)|\(node.identifier)|\(node.label)") + } node.children.forEach(walk) } walk(root) - return texts + buttons + return (elements.joined(separator: "\n"), texts + buttons) } @MainActor diff --git a/verification/hsverify/a11yaudit.py b/verification/hsverify/a11yaudit.py index f02b604bb..31c87bd04 100644 --- a/verification/hsverify/a11yaudit.py +++ b/verification/hsverify/a11yaudit.py @@ -6,8 +6,9 @@ The app is built and installed exactly as `hsverify.simrun` does. Then the UI test bundle in `fixtures/a11y-audit` launches it by bundle id with the same launch arguments, and runs `performAccessibilityAudit(for: .all)` on each distinct screen: contrast, Dynamic Type, hit -areas, clipped text, element descriptions and traits. A screen is new when the labels of its -texts and buttons change. +areas, clipped text, element descriptions and traits. A screen is new when any element in its +accessibility tree is added, removed or relabelled; values and positions don't count. Each +screen's `labels` in the report are still only its texts and buttons. Apple's audit accepts a button whose label SwiftUI took from an SF Symbol name, which VoiceOver then reads aloud ("gearshape"). Those labels are flagged here as well. @@ -90,7 +91,7 @@ def audit_problems(screens: list[dict], harness_ran: bool, min_screens: int) -> """Why this run is not evidence of an audit; empty when it is.""" problems = [] if not harness_ran: - problems.append("the audit harness did not run (no TEST SUCCEEDED or TEST FAILED line)") + problems.append("the audit harness did not finish successfully") problems += [f"screen {s.get('index')}: {s['error']}" for s in screens if "error" in s] audited = audited_screens(screens) if not audited: @@ -102,7 +103,7 @@ def audit_problems(screens: list[dict], harness_ran: bool, min_screens: int) -> def run_harness( udid: str, bundle_id: str, args: argparse.Namespace, launch: list[str], out: Path -) -> str: +) -> tuple[str, int]: env = os.environ | { "TEST_RUNNER_AUDIT_BUNDLE": bundle_id, "TEST_RUNNER_AUDIT_ARGS": ARG_SEPARATOR.join(launch), @@ -126,7 +127,7 @@ def run_harness( simrun.wait_for_other_builds(args.build_wait) result = subprocess.run(cmd, env=env, capture_output=True, text=True) (out / "xcodebuild-test.log").write_text(result.stdout + result.stderr) - return result.stdout + return result.stdout, result.returncode def write_report(out: Path, data: dict) -> None: @@ -171,6 +172,7 @@ def write_report(out: Path, data: dict) -> None: (out / "index.html").write_text(page) +@simrun.device_lock() def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--ref", default="origin/t3/ios-mvf") @@ -222,7 +224,11 @@ def main(argv: list[str] | None = None) -> int: if app is None: print(f"Build failed; see {out / 'build.log'}") return 1 - device = simrun.ensure_device() + try: + device = simrun.ensure_device() + except BaseException: + simrun.discard_app(app) + raise udid = device["udid"] try: bundle_id = simrun.bundle_id_of(app) @@ -234,13 +240,18 @@ def main(argv: list[str] | None = None) -> int: ["xcrun", "simctl", "privacy", udid, "grant", "camera", bundle_id], capture_output=True ) print(f"Auditing {bundle_id} at {sha[:12]}...", flush=True) - output = run_harness(udid, bundle_id, args, launch, out) + output, test_returncode = run_harness(udid, bundle_id, args, launch, out) finally: + simrun.discard_app(app) subprocess.run(["xcrun", "simctl", "shutdown", udid], capture_output=True) screens, issues = parse_output(output) symbols = symbol_name_labels(screens) - harness_ran = "** TEST SUCCEEDED **" in output or "** TEST FAILED **" in output + harness_ran = ( + test_returncode == 0 + and "** TEST SUCCEEDED **" in output + and "** TEST FAILED **" not in output + ) problems = audit_problems(screens, harness_ran, args.min_screens) data = { # ok: the run is evidence of an audit. Issues found are counted separately below. diff --git a/verification/hsverify/bundle_limits.py b/verification/hsverify/bundle_limits.py new file mode 100644 index 000000000..6f4eb3885 --- /dev/null +++ b/verification/hsverify/bundle_limits.py @@ -0,0 +1,61 @@ +"""Bound ZIP metadata before the verification harness constructs a ZipFile.""" + +import struct +from pathlib import Path + + +def zip_directory_limits(path: Path, max_entries: int, max_bytes: int) -> None: + """Check central-directory records in bounded reads before ZipFile allocates their metadata. + ZIP64 and split archives exceed this worker's ordinary ZIP admission format and are refused. + The record layouts are PKWARE APPNOTE sections 4.3.12 and 4.3.16.""" + with path.open("rb") as f: + size = path.stat().st_size + f.seek(max(0, size - 22 - 65535)) + tail = f.read(22 + 65535) + # Match ZipFile's footer selection: a no-comment footer at EOF, otherwise the last + # signature. Backtracking past malformed signatures can validate a different directory + # from the one ZipFile later allocates. Refuse that ambiguity before constructing it. + if len(tail) >= 22 and tail[-22:-18] == b"PK\x05\x06" and tail[-2:] == b"\0\0": + pos = len(tail) - 22 + else: + pos = tail.rfind(b"PK\x05\x06") + if pos < 0: + raise ValueError(f"{path}: missing ZIP end record") + if pos + 22 > len(tail): + raise ValueError(f"{path}: truncated ZIP end record") + record = struct.unpack("<4s4H2IH", tail[pos : pos + 22]) + if pos + 22 + record[-1] != len(tail): + raise ValueError(f"{path}: ambiguous ZIP end record or invalid comment length") + _, disk, central_disk, disk_entries, total_entries, length, offset, _ = record + if disk or central_disk or disk_entries != total_entries: + raise ValueError(f"{path}: split ZIP archives are not supported") + if total_entries > max_entries: + raise ValueError(f"{path}: {total_entries} entries, limit {max_entries}") + directory_end = offset + length + end_offset = size - len(tail) + pos + # ZipFile checks this locator even when the ordinary footer has no ZIP64 sentinels. + # It must not replace the directory fields after the bounded scan below. + if end_offset >= 20: + f.seek(end_offset - 20) + if f.read(4) == b"PK\x06\x07": + raise ValueError(f"{path}: ZIP64 archive metadata is not supported") + if length == 0xFFFFFFFF or offset == 0xFFFFFFFF or directory_end != end_offset: + raise ValueError(f"{path}: ZIP64 or inconsistent ZIP directory") + f.seek(offset) + count, expanded = 0, 0 + while f.tell() < directory_end: + header = f.read(46) + if len(header) != 46 or header[:4] != b"PK\x01\x02": + raise ValueError(f"{path}: invalid ZIP central-directory record") + fields = struct.unpack("<4s6H3I5H2I", header) + count += 1 + expanded += fields[9] + if count > max_entries: + raise ValueError(f"{path}: {count} entries, limit {max_entries}") + if expanded > max_bytes: + raise ValueError(f"{path}: expanded archive exceeds {max_bytes} bytes") + f.seek(sum(fields[10:13]), 1) + if f.tell() > directory_end: + raise ValueError(f"{path}: ZIP central-directory record exceeds directory") + if count != total_entries: + raise ValueError(f"{path}: inconsistent ZIP entry count") diff --git a/verification/hsverify/diagnostics.py b/verification/hsverify/diagnostics.py new file mode 100644 index 000000000..3c81ed8ee --- /dev/null +++ b/verification/hsverify/diagnostics.py @@ -0,0 +1,86 @@ +"""Bound diagnostic output independently of geometry work and input size.""" + +import json + +# Operational output limits, not measured scan maxima: 1,000 messages and 256 KiB +# leave a readable report without multiplying input-sized strings across cases. +# Reserve one message and 128 characters for the omitted-message count. +MAX_DIAGNOSTIC_COUNT = 1_000 +MAX_DIAGNOSTIC_CHARACTERS = 256 * 1024 +SUMMARY_RESERVE = 128 +MAX_EXPECTATION_EXAMPLES = 3 + + +class DiagnosticBudget: + def __init__(self) -> None: + self.count = 0 + self.characters = 0 + self.omitted = 0 + + def accept(self, message: str) -> bool: + prefix = "further problems omitted (" + if message.startswith(prefix) and message.endswith(")"): + count = message[len(prefix) : -1] + if len(count) <= 20 and count.isdecimal(): + self.omitted += int(count) + return False + if ( + self.omitted + or self.count >= MAX_DIAGNOSTIC_COUNT - 1 + or self.characters + len(message) > MAX_DIAGNOSTIC_CHARACTERS - SUMMARY_RESERVE + ): + self.omitted += 1 + return False + # JSON escapes control and non-ASCII characters; budget the saved text, + # not only the shorter Python string. The raw-length check above bounds + # this temporary encoding too. + characters = len(json.dumps(message)) + if self.characters + characters > MAX_DIAGNOSTIC_CHARACTERS - SUMMARY_RESERVE: + self.omitted += 1 + return False + self.count += 1 + self.characters += characters + return True + + def summary(self) -> list[str]: + return [f"further problems omitted ({self.omitted})"] if self.omitted else [] + + +class DiagnosticMessages: + def __init__(self) -> None: + self.budget = DiagnosticBudget() + self.messages: list[str] = [] + + def append(self, message: str, *values: object) -> None: + if self.budget.omitted: + self.budget.omitted += 1 + return + if values: + message = message.format(*values) + if self.budget.accept(message): + self.messages.append(message) + + def finish(self) -> list[str]: + return self.messages + self.budget.summary() + + +def bounded_report_records(records: list[dict]) -> tuple[list[dict], list[str]]: + """Use one budget across cases and duplicate diagnostic fields in the JSON report.""" + budget = DiagnosticBudget() + bounded = [] + for record in records: + copy = dict(record) + # Markdown renders problems; retain its explanations before their JSON duplicates. + for field in ( + "problems", + "contract_problems", + "expectation_problems", + "assumption_mismatches", + "assumption_warnings", + ): + if field in copy: + copy[field] = [message for message in copy[field] if budget.accept(message)] + if copy.get("skip_reason") and not budget.accept(copy["skip_reason"]): + copy["skip_reason"] = "" + bounded.append(copy) + return bounded, budget.summary() diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index 87e8bde35..6fd48bbde 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -21,7 +21,9 @@ decision and outcome lengths), without or with less coverage, with ground trimmed just short of the largest clearance, with more error or tape re-measured by tap (nothing improves), and with every requested view added, up to three rounds (no check left unsure for coverage). Five -hostile inputs must be refused or answered within 10 s and 500 MB of server growth. +hostile inputs must be refused or answered within 10 s and 500 MB of server growth. Every +answer, a result or an error, must arrive whole within its deadline and within 4 MB, or it +counts as no answer. A run claims the contract (`contract_ok`) only when at least one scene was answered and checked and none broke it, and real-scene latency only from an HTTP 200 result. @@ -36,37 +38,50 @@ import contextlib import datetime as dt import hashlib +import http.client import io import json import os import re import signal import socket +import ssl import subprocess import sys +import threading import time import urllib.error import urllib.parse import urllib.request import uuid import zipfile -from collections.abc import Iterator +from collections.abc import Callable, Iterator from dataclasses import dataclass, field from pathlib import Path from hsverify import gitref, memory +from hsverify.bundle_limits import zip_directory_limits +from hsverify.diagnostics import bounded_report_records from hsverify.memory import peak_rss_mb from hsverify.resultcheck import ( RuleSet, + WorkLimitExceeded, + WorkMeter, assumption_mismatches, + case_metadata_problems, comparable, coverage_blocks, expectation_problems, + geometry_admission_problems, + ingestion_problems, invariant_problems, less_coverage_problems, mirror_scene, more_error_problems, outcome_lengths, + public_assumption_warnings, + public_dimension_problems, + reported_rules, schema_errors, tape_to_tap, unobserved_checks, @@ -144,16 +159,40 @@ def read_bounded(bundle: zipfile.ZipFile, info: zipfile.ZipInfo, limit: int) -> def read_bundle(path: Path) -> tuple[bytes, dict[str, bytes]]: """scene.json and the JPEGs of a C1 zip bundle, within the limits above.""" + if path.stat().st_size > MAX_BUNDLE_BYTES: + raise SystemExit(f"{path}: archive exceeds {MAX_BUNDLE_BYTES} bytes") + try: + zip_directory_limits(path, MAX_BUNDLE_ENTRIES, MAX_BUNDLE_BYTES) + except ValueError as exc: + raise SystemExit(str(exc)) from exc with zipfile.ZipFile(path) as bundle: - infos = [i for i in bundle.infolist() if not i.is_dir()] + infos = bundle.infolist() if len(infos) > MAX_BUNDLE_ENTRIES: raise SystemExit(f"{path}: {len(infos)} entries, limit {MAX_BUNDLE_ENTRIES}") declared = sum(i.file_size for i in infos) if declared > MAX_BUNDLE_BYTES: raise SystemExit(f"{path}: declares {declared} bytes, limit {MAX_BUNDLE_BYTES}") - scene = next((i for i in infos if i.filename.endswith("scene.json")), None) - if scene is None: - raise SystemExit(f"{path}: no scene.json in the bundle") + capture_infos = [i for i in infos if not i.filename.startswith("__MACOSX/")] + infos = [i for i in capture_infos if not i.is_dir()] + # Mirror the public API layouts, but refuse competing scenes instead of + # preferring the root: the verifier must know which input it is testing. + candidates = [i for i in infos if i.filename.split("/")[-1] == "scene.json"] + if len({i.filename for i in candidates}) > 1: + raise SystemExit(f"{path}: ambiguous scene.json paths") + scene_name = "scene.json" + if not any(i.filename == scene_name for i in infos): + # The public API also accepts a capture wrapped in one top-level folder. + tops = {i.filename.split("/", 1)[0] for i in capture_infos} + if len(tops) == 1: + (top,) = tops + if all(i.filename.startswith(f"{top}/") for i in capture_infos): + scene_name = f"{top}/scene.json" + scenes = [i for i in infos if i.filename == scene_name] + if len(scenes) != 1: + raise SystemExit( + f"{path}: expected exactly one canonical scene.json, found {len(scenes)}" + ) + scene = scenes[0] raw = read_bounded(bundle, scene, MAX_SCENE_BYTES) images = { Path(i.filename).name: read_bounded(bundle, i, MAX_IMAGE_BYTES) @@ -354,10 +393,23 @@ def bundle_zip(item: SceneInput) -> bytes: return buf.getvalue() +# The most of any response body the harness reads, a result or an error. Not derived from any +# requirement: the largest result in the 20260927-102101 run against t3/server 1d1e7e1 was 16 KB +# (eth3d-facade), so this leaves room for 250 times that. +MAX_RESPONSE_BYTES = 4 * 2**20 +READ_CHUNK_BYTES = 2**16 + + def post( url: str, endpoint: Endpoint, item: SceneInput, timeout_s: float = 60 ) -> tuple[int | None, bytes, float]: - """Send one scene. A timeout or a dropped connection comes back as status None.""" + """Send one scene and read the whole answer within `timeout_s`, headers and body together. + + Status None means no complete answer, and the payload then says why: the deadline passed, + the body ran past MAX_RESPONSE_BYTES, it ended short of its Content-Length or chunk framing, + or the connection failed. A socket timeout alone bounds + each read, not the response. A bounded worker wait covers DNS as well as TCP, TLS, headers + and body; deadline expiry shuts down sockets already owned by the worker.""" if item.bundle is not None and endpoint.accepts_zip: body, ctype = item.bundle, "application/zip" elif endpoint.content_type == "application/json": @@ -373,18 +425,112 @@ def post( + f"\r\n--{boundary}--\r\n".encode() ) ctype = f"multipart/form-data; boundary={boundary}" - request = urllib.request.Request( - url + endpoint.path, data=body, headers={"Content-Type": ctype}, method="POST" + target = urllib.parse.urlsplit(url + endpoint.path) + https = target.scheme == "https" + conn = http.client.HTTPConnection( + target.hostname, port=target.port or (443 if https else 80), timeout=timeout_s ) start = time.perf_counter() - try: - with urllib.request.urlopen(request, timeout=timeout_s) as response: - status, payload = response.status, response.read() - except urllib.error.HTTPError as exc: - status, payload = exc.code, exc.read() - except (urllib.error.URLError, TimeoutError, ConnectionError) as exc: - status, payload = None, str(exc).encode() - return status, payload, (time.perf_counter() - start) * 1000 + deadline = start + timeout_s + expired = threading.Event() + done = threading.Event() + lock = threading.Lock() + sockets: list[socket.socket] = [] + answer: list[tuple[int | None, bytes]] = [] + + def remaining() -> float: + left = deadline - time.perf_counter() + if expired.is_set() or left <= 0: + raise TimeoutError + return left + + def expire() -> None: + with lock: + expired.set() + for sock in sockets: + with contextlib.suppress(OSError): + sock.shutdown(socket.SHUT_RDWR) + sock.close() + + def connect() -> None: + # getaddrinfo has no cancellable stdlib deadline. The daemon worker bounds the caller's + # wait; a late DNS result must check the deadline before allocating or sending anything. + addresses = socket.getaddrinfo(conn.host, conn.port, type=socket.SOCK_STREAM) + error = None + for family, kind, proto, _, address in addresses: + with lock: + remaining() + sock = socket.socket(family, kind, proto) + sockets.append(sock) # own it before a blocking TCP connect or TLS handshake + try: + sock.settimeout(remaining()) + sock.connect(address) + if https: + context = ssl.create_default_context() + with lock: + remaining() + sock = context.wrap_socket( + sock, server_hostname=conn.host, do_handshake_on_connect=False + ) + sockets.append(sock) + sock.settimeout(remaining()) + sock.do_handshake() + sock.settimeout(remaining()) + conn.sock = sock + return + except OSError as exc: + sock.close() + error = exc + remaining() + if error is not None: + raise error + raise OSError("DNS returned no connection addresses") + + def exchange() -> None: + status: int | None = None + payload = bytearray() + try: + connect() + path = target.path + (f"?{target.query}" if target.query else "") + conn.sock.settimeout(remaining()) + conn.request("POST", path, body=body, headers={"Content-Type": ctype}) + conn.sock.settimeout(remaining()) + response = conn.getresponse() + while len(payload) <= MAX_RESPONSE_BYTES and ( + chunk := response.read(min(READ_CHUNK_BYTES, MAX_RESPONSE_BYTES + 1 - len(payload))) + ): + remaining() + payload += chunk + if len(payload) <= MAX_RESPONSE_BYTES and response.length: + raise http.client.IncompleteRead(bytes(payload), response.length) + remaining() + status = response.status + except TimeoutError: + expired.set() + except (OSError, http.client.HTTPException) as exc: + status, payload = None, bytearray(f"{type(exc).__name__}: {exc}".encode()) + finally: + conn.close() + with lock: + for sock in sockets: + sock.close() + answer.append((status, bytes(payload))) + done.set() + + # DNS may remain blocked after this returns. The worker is a daemon and owns no socket + # during DNS; once lookup returns, remaining() prevents a late request from being sent. + worker = threading.Thread(target=exchange, daemon=True) + worker.start() + if not done.wait(max(0.0, deadline - time.perf_counter())): + expire() + ms = (time.perf_counter() - start) * 1000 + if expired.is_set() or time.perf_counter() >= deadline: + expire() + return None, f"no answer within {timeout_s:g} s".encode(), ms + status, payload = answer[0] + if len(payload) > MAX_RESPONSE_BYTES: + return None, f"HTTP {status} body over {MAX_RESPONSE_BYTES} bytes".encode(), ms + return status, payload, ms # --- Judging --------------------------------------------------------------------------------- @@ -412,7 +558,7 @@ def answer(url: str, endpoint: Endpoint, item: SceneInput) -> tuple[dict | None, """The server's result for a scene, or why there is none.""" status, payload, _ = post(url, endpoint, item) if status is None: - return None, f"no answer: {payload.decode(errors='replace')[:200]}" + return None, payload.decode(errors="replace")[:200] if status != 200: return None, f"HTTP {status}: {payload[:300]!r}" result = parse_result(payload) @@ -428,7 +574,10 @@ def judge( schemas: dict, rules: RuleSet, limit: memory.TreeLimit | None = None, + *, + meter: WorkMeter | None = None, ) -> dict: + meter = meter if meter is not None else WorkMeter() record: dict = { "name": item.name, "source": item.source, @@ -440,7 +589,11 @@ def judge( return record | {"status": "skipped", "problems": [], "skip_reason": item.skip_reason} if item.hostile: return record | {"hostile": True} | judge_hostile(url, endpoint, item, schemas, limit) - input_errors = schema_errors(item.scene, schemas["scene"]) + input_errors = ( + case_metadata_problems(item.rules_assumed, item.expect) + or ingestion_problems(item.scene, "scene") + or schema_errors(item.scene, schemas["scene"]) + ) if input_errors: return record | {"status": "bad input", "problems": input_errors[:20]} @@ -450,7 +603,7 @@ def judge( failure = [ f"HTTP {status} for a scene that validates: {payload[:500]!r}" if status is not None - else f"no answer within 60 s: {payload[:200]!r}" + else payload.decode(errors="replace")[:200] ] return record | {"status": "fail", "problems": failure, "contract_problems": failure} result = parse_result(payload) @@ -458,54 +611,178 @@ def judge( failure = [f"HTTP 200 with a body that is not a JSON object: {payload[:200]!r}"] return record | {"status": "fail", "problems": failure, "contract_problems": failure} record["result"] = result - problems = [f"result schema: {e}" for e in schema_errors(result, schemas["result"])] + + def check(name: str, function: Callable[[], list[str]]) -> list[str]: + try: + return function() + except WorkLimitExceeded: + return [f"verification work limit exceeded ({meter.samples} samples)"] + except Exception as exc: + return [f"{name}: {type(exc).__name__}: {exc}"] + + problems = check( + "result schema", + lambda: [ + f"result schema: {e}" + for e in ( + ingestion_problems(result, "result") or schema_errors(result, schemas["result"]) + ) + ], + ) if problems: return record | {"status": "fail", "problems": problems[:20], "contract_problems": problems} - # Contract problems hold for any scene; expectation problems depend on one case's geometry. - contract = invariant_problems(item.scene, result, sent=item.raw, rules=rules) - contract += property_problems(url, endpoint, item, result, schemas, rules) + # Admission errors stop geometry, including report summaries that subtract spans. + contract = check( + "geometry admission", + lambda: geometry_admission_problems(item.scene, result, rules, meter=meter), + ) + admitted = not contract + mismatches: list[str] = [] + warnings: list[str] = [] + active_rules = rules + expected: list[str] = [] + if admitted: + + def check_reported_rules() -> list[str]: + nonlocal active_rules + mismatches.extend(assumption_mismatches(item.rules_assumed, result)) + warnings.extend(public_assumption_warnings(item.rules_assumed, result, rules)) + active_rules = reported_rules(result, rules) + return [] + + contract += check("reported rules", check_reported_rules) + admitted = not contract + if admitted: + contract += check("public dimensions", lambda: public_dimension_problems(result, rules)) + if warnings: + record["rule_checks_unverified"] = True + if admitted: + contract += check( + "invariants", + lambda: invariant_problems( + item.scene, + result, + sent=item.raw, + rules=None if warnings else active_rules, + meter=meter, + ), + ) + if not contract: + contract += check( + "properties", + lambda: property_problems( + url, + endpoint, + item, + result, + schemas, + active_rules, + meter=meter, + rule_checks=not warnings, + public_rules=rules, + ), + ) if item.real and ms > LATENCY_BUDGET_MS: contract.append(f"real-derived scene took {ms:.0f} ms, budget {LATENCY_BUDGET_MS:.0f}") - mismatches = assumption_mismatches(item.rules_assumed, result) - expected = expectation_problems(item.expect, result) if item.expect and not mismatches else [] - problems = contract + expected + exhausted = any(p.startswith("verification work limit exceeded (") for p in contract) + if admitted and not exhausted: + if item.expect and not mismatches and not warnings: + expected = check( + "expectations", lambda: expectation_problems(item.expect, result, meter=meter) + ) - record["decision"] = result["decision"] - record["outcome_lengths_ft"] = outcome_lengths(result) + def summarize() -> list[str]: + record["outcome_lengths_ft"] = outcome_lengths(result) + return [] + + contract += check("outcome lengths", summarize) + if "decision" in result: + record["decision"] = result["decision"] if mismatches: record["assumption_mismatches"] = mismatches + if warnings: + record["assumption_warnings"] = warnings + record["verification_samples"] = meter.samples record["contract_problems"] = contract record["expectation_problems"] = expected - record["problems"] = problems - record["status"] = "fail" if problems else "assumption mismatch" if mismatches else "pass" + record["problems"] = contract + expected + record["status"] = ( + "fail" + if record["problems"] + else "assumption mismatch" + if mismatches + else "assumptions unverified" + if warnings + else "pass" + ) return record def property_problems( - url: str, endpoint: Endpoint, item: SceneInput, result: dict, schemas: dict, rules: RuleSet + url: str, + endpoint: Endpoint, + item: SceneInput, + result: dict, + schemas: dict, + rules: RuleSet, + *, + meter: WorkMeter | None = None, + rule_checks: bool = True, + public_rules: RuleSet | None = None, ) -> list[str]: """The same scene resent unchanged and transformed. Every answer to a transformed scene must also satisfy the invariants, and the transforms that only lose information must not make the server surer.""" + meter = meter if meter is not None else WorkMeter() + public_rules = public_rules if public_rules is not None else rules + checked_rules = rules if rule_checks else None + if admission := geometry_admission_problems(item.scene, result, checked_rules, meter=meter): + return admission problems: list[str] = [] def resend(name: str, scene: dict) -> dict | None: - other, failure = answer(url, endpoint, variant(item, name, scene)) + if admission := ingestion_problems(scene, "scene"): + problems.extend(f"{name}: scene ingestion: {p}" for p in admission) + return None + transformed = variant(item, name, scene) + other, failure = answer(url, endpoint, transformed) if failure: problems.append(f"{name}: {failure}") return None - errors = schema_errors(other, schemas["result"]) - if errors: - problems.append(f"{name}: result schema: {errors[0]}") + check_name = "result schema" + try: + errors = ingestion_problems(other, "result") or schema_errors(other, schemas["result"]) + if errors: + problems.append(f"{name}: result schema: {errors[0]}") + return None + check_name = "geometry admission" + admission = geometry_admission_problems(scene, other, checked_rules, meter=meter) + if admission: + problems.extend(f"{name}: {p}" for p in admission) + return None + check_name = "invariants" + variant_rules = reported_rules(other, public_rules) if rule_checks else None + dimensions = public_dimension_problems(other, public_rules) if rule_checks else [] + invariants = dimensions + invariant_problems( + scene, other, sent=transformed.raw, rules=variant_rules, meter=meter + ) + if invariants: + problems.extend(f"{name}: {p}" for p in invariants) + return None + except WorkLimitExceeded: + raise + except Exception as exc: + problems.append(f"{name} {check_name}: {type(exc).__name__}: {exc}") return None - problems.extend(f"{name}: {p}" for p in invariant_problems(scene, other, rules=rules)) return other # 1. Same input, same answer. status, payload, _ = post(url, endpoint, item) again = parse_result(payload) if status == 200 else None - if again is None or comparable(again) != comparable(result): + if again is not None and (admission := ingestion_problems(again, "result")): + problems.extend(f"identical resend: {p}" for p in admission) + elif again is None or comparable(again) != comparable(result): problems.append("sending the same scene twice gave different results") # 2. Mirrored left to right: same decision, same amount of each outcome along the wall. @@ -527,19 +804,22 @@ def resend(name: str, scene: dict) -> dict | None: if "coverage" in item.scene: bare = {k: v for k, v in item.scene.items() if k != "coverage"} if (other := resend("no coverage", bare)) is not None: - problems += less_coverage_problems(result, other, "no coverage") - for name, scene in ( - ("less coverage", with_less_coverage(item.scene)), - ("ground short of the largest clearance", with_ground_short_of(item.scene, rules)), - ): - if scene is not None and (other := resend(name, scene)) is not None: - problems += less_coverage_problems(result, other, name) - for name, scene in ( - ("more error", with_more_error(item.scene, rules)), - ("tape re-measured by tap", tape_to_tap(item.scene)), - ): + problems += less_coverage_problems(result, other, "no coverage", meter=meter) + coverage_variants = [("less coverage", with_less_coverage(item.scene))] + if rule_checks: + coverage_variants.append( + ("ground short of the largest clearance", with_ground_short_of(item.scene, rules)) + ) + for name, scene in coverage_variants: if scene is not None and (other := resend(name, scene)) is not None: - problems += more_error_problems(result, other, name) + problems += less_coverage_problems(result, other, name, meter=meter) + if rule_checks: + for name, scene in ( + ("more error", with_more_error(item.scene, rules)), + ("tape re-measured by tap", tape_to_tap(item.scene)), + ): + if scene is not None and (other := resend(name, scene)) is not None: + problems += more_error_problems(result, other, name, meter=meter) # 4. Showing everything the result asked for settles every check that was unsure only # because an area was unobserved (the chosen spot may move; three rounds at most). @@ -570,6 +850,15 @@ def resend(name: str, scene: dict) -> dict | None: HOSTILE_BUDGET_MS = 10_000.0 +HOSTILE_PROBE_NAMES = ( + "hostile: 150 objects, each its own error", + "hostile: coordinates of 1e300 ft", + "hostile: a 5000 ft wall", + "hostile: nested 5000 deep", + "hostile: 400 MB scene.json in a small zip", +) + + def hostile_inputs() -> list[SceneInput]: """Small requests that cost a careless server a lot. Built here, not stored, so the repository carries no blobs; none takes the harness more than a few MB to build.""" @@ -613,13 +902,11 @@ def hostile_inputs() -> list[SceneInput]: } nested = ('{"meter": ' + "[" * 5000 + "]" * 5000 + "}").encode() items = [ - SceneInput( - "hostile: 150 objects, each its own error", crowded, json.dumps(crowded).encode() - ), - SceneInput("hostile: coordinates of 1e300 ft", huge, json.dumps(huge).encode()), - SceneInput("hostile: a 5000 ft wall", long_wall, json.dumps(long_wall).encode()), - SceneInput("hostile: nested 5000 deep", {}, nested), - SceneInput("hostile: 400 MB scene.json in a small zip", {}, b"", bundle=zip_bomb(400)), + SceneInput(HOSTILE_PROBE_NAMES[0], crowded, json.dumps(crowded).encode()), + SceneInput(HOSTILE_PROBE_NAMES[1], huge, json.dumps(huge).encode()), + SceneInput(HOSTILE_PROBE_NAMES[2], long_wall, json.dumps(long_wall).encode()), + SceneInput(HOSTILE_PROBE_NAMES[3], {}, nested), + SceneInput(HOSTILE_PROBE_NAMES[4], {}, b"", bundle=zip_bomb(400)), ] for item in items: item.hostile = True @@ -661,19 +948,22 @@ def judge_hostile( and not (endpoint.accepts_zip) ): return { - "status": "skipped", - "skip_reason": "the endpoint takes neither a zip nor an upload", + "status": "fail", + "problems": ["large-input probe was not exercised: endpoint accepts no archive"], + "contract_problems": [ + "large-input probe was not exercised: endpoint accepts no archive" + ], } before = limit.reset() if limit else 0.0 status, payload, ms = post(url, endpoint, item, timeout_s=HOSTILE_BUDGET_MS / 1000 + 5) - record = {"http_status": status, "latency_ms": round(ms, 1)} + record = {"http_status": status, "latency_ms": round(ms, 1), "hostile_probe_executed": True} grew = limit.peak_mb - before if limit else 0.0 if limit: record["server_growth_mb"] = round(grew, 1) if grew > HOSTILE_MEMORY_MB: failure = f"the server grew by {grew:.0f} MB, budget {HOSTILE_MEMORY_MB} MB" elif status is None: - failure = f"no answer within {HOSTILE_BUDGET_MS / 1000 + 5:.0f} s" + failure = payload.decode(errors="replace")[:200] elif ms > HOSTILE_BUDGET_MS: failure = f"answered HTTP {status} after {ms:.0f} ms, budget {HOSTILE_BUDGET_MS:.0f}" elif status in (400, 413, 422): @@ -710,11 +1000,21 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: exported = [r for r in records if r.get("app_export")] contract_problems = sum(len(r.get("contract_problems", [])) for r in records) killed = bool(meta["peak_memory"].get("server_killed")) + all_passed = all(r["status"] == "pass" for r in records) and bool(checked) and not killed + contract_ok = ( + bool(checked) + and contract_problems == 0 + and not killed + and not any(r.get("rule_checks_unverified") for r in checked) + ) + executed = sorted( + {r["name"] for r in records if r.get("hostile") and r.get("hostile_probe_executed") is True} + ) + hostile_complete = set(HOSTILE_PROBE_NAMES).issubset(executed) report = meta | { "counts": counts, - "all_passed": all(r["status"] in ("pass", "skipped") for r in records) - and bool(checked) - and not killed, + # A missing listed scene leaves the run incomplete, even when other scenes passed. + "all_passed": all_passed, "latency_ms": max(real) if real else None, "scenes_answered": sum(1 for r in records if "decision" in r), "scenes_checked": len(checked), @@ -724,7 +1024,12 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: # expectations are counted separately because a case can be wrong itself. "contract_problem_count": contract_problems, # The contract holds only on evidence: at least one scene checked, none broken. - "contract_ok": bool(checked) and contract_problems == 0 and not killed, + "contract_ok": contract_ok, + "hostile_probes_required": list(HOSTILE_PROBE_NAMES), + "hostile_probes_executed": executed, + "hostile_probes_complete": hostile_complete, + # Local --no-hostile success proves only that shortened run, not S2-8's full guarantee. + "full_contract_ok": all_passed and contract_ok and hostile_complete, "expectation_problem_count": sum(len(r.get("expectation_problems", [])) for r in records), "real_scene_passed": any(r.get("real") and r["status"] == "pass" for r in records), # Null when the run had no app export, so the scoreboard looks for an older run. @@ -740,6 +1045,10 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: "app_export_source": [r["source"] for r in exported], "records": records, } + records, omitted = bounded_report_records(records) + report["records"] = records + if omitted: + report["diagnostic_summary"] = omitted (out / "report.json").write_text(json.dumps(report, indent=2) + "\n") lines = [ f"# End-to-end run against `{meta['server_ref']}` at `{meta['server_sha'][:12]}`", @@ -760,12 +1069,17 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: "| --- | --- | --- | --- | --- |", ] for r in records: - issues = "; ".join(r.get("problems", []) + r.get("assumption_mismatches", [])) + issues = "; ".join( + r.get("problems", []) + + r.get("assumption_mismatches", []) + + r.get("assumption_warnings", []) + ) issues = issues or r.get("skip_reason", "") lines.append( f"| {r['name']}{' (real)' if r.get('real') else ''} | {r['status']} | " f"{r.get('decision', '')} | {r.get('latency_ms', '')} | {issues.replace('|', '/')} |" ) + lines.extend(omitted) (out / "report.md").write_text("\n".join(lines) + "\n") return report @@ -859,7 +1173,7 @@ def main(argv: list[str] | None = None) -> int: else "" ) ) - for problem in record.get("problems", [])[:6]: + for problem in (record.get("problems", []) + record.get("assumption_warnings", []))[:6]: print(f" - {problem}") meta = { @@ -879,7 +1193,16 @@ def main(argv: list[str] | None = None) -> int: } report = write_report(out, meta, records) print(f"Report: {out / 'report.md'}") - return 0 if report["all_passed"] else 1 + if report["all_passed"]: + return 0 + # An unreported override is incomplete evidence, not a proven server failure. + if ( + not meta["peak_memory"].get("server_killed") + and all(r["status"] in ("pass", "assumptions unverified") for r in records) + and any(r["status"] == "assumptions unverified" for r in records) + ): + return 2 + return 1 if __name__ == "__main__": diff --git a/verification/hsverify/gitref.py b/verification/hsverify/gitref.py index e711e295d..59f1431d6 100644 --- a/verification/hsverify/gitref.py +++ b/verification/hsverify/gitref.py @@ -24,7 +24,7 @@ def git(*args: str, cwd: Path = REPO) -> str: def fetch() -> None: - git("fetch", "--quiet", "origin") + git("fetch", "--quiet", "--prune", "origin") def resolve(ref: str) -> str: diff --git a/verification/hsverify/intervalindex.py b/verification/hsverify/intervalindex.py new file mode 100644 index 000000000..ced4e9ddd --- /dev/null +++ b/verification/hsverify/intervalindex.py @@ -0,0 +1,189 @@ +"""Compare sweep outcomes without scanning every run for each sampled start.""" + +from __future__ import annotations + +import heapq +import itertools +import math +from dataclasses import dataclass +from typing import Protocol + + +def numeric_key(value: int | float) -> str: + """Hash numbers as exact strings so adversarial JSON integers cannot collide cheaply. + + Python hashes integers modulo a public prime. Rational form also keeps equal integer + and float inputs in one cache entry without rounding distinct large integers together. + """ + if isinstance(value, float) and not math.isfinite(value): + return repr(value) + numerator, denominator = value.as_integer_ratio() + return f"{numerator}/{denominator}" + + +class Meter(Protocol): + def charge(self, n: int = 1) -> None: ... + + +def _charge(meter: Meter | None, n: int = 1) -> None: + if meter is not None: + meter.charge(n) + + +def _bisect(values: list[float], value: float, meter: Meter | None, *, right=False) -> int: + lo, hi = 0, len(values) + while lo < hi: + _charge(meter) + mid = (lo + hi) // 2 + if values[mid] < value or (right and values[mid] == value): + lo = mid + 1 + else: + hi = mid + return lo + + +@dataclass +class _WallIndex: + endpoints: list[float] + at_endpoint: list[str | None] + after_endpoint: list[str | None] + starts: list[float] + start_outcomes: list[str] + ends: list[float] + end_outcomes: list[str] + + +class SweepIndex: + """Index closed intervals, choosing the first original run when intervals overlap. + + Endpoint outcomes and the outcomes between endpoints are stored separately. This keeps + an interval's inclusive endpoint, including EPS expansion, without changing overlap order. + """ + + def __init__(self, result: dict, meter: Meter | None = None, eps: float = 1e-6): + self.meter = meter + groups: dict[str, list[dict]] = {} + for run in result.get("sweep", []): + _charge(meter) + groups.setdefault(run["wall_id"], []).append(run) + self.walls = {wall: self._build(runs, eps) for wall, runs in groups.items()} + + def _build(self, runs: list[dict], eps: float) -> _WallIndex: + events = [] + starts, ends = {}, {} + for rank, run in enumerate(runs): + _charge(self.meter) + a, b = run["start_ft"] + events.extend(((a - eps, 0, rank), (b + eps, 1, rank))) + starts.setdefault(numeric_key(a), (a, run["outcome"])) + ends.setdefault(numeric_key(b), (b, run["outcome"])) + events.sort() + active = [False] * len(runs) + heap: list[int] = [] + endpoints, at_endpoint, after_endpoint = [], [], [] + + def first() -> str | None: + while heap and not active[heap[0]]: + _charge(self.meter) + heapq.heappop(heap) + return runs[heap[0]]["outcome"] if heap else None + + pos = 0 + while pos < len(events): + value = events[pos][0] + ending = [] + while pos < len(events) and events[pos][0] == value: + _charge(self.meter) + _, kind, rank = events[pos] + if kind == 0: + active[rank] = True + heapq.heappush(heap, rank) + else: + ending.append(rank) + pos += 1 + endpoints.append(value) + at_endpoint.append(first()) + for rank in ending: + _charge(self.meter) + active[rank] = False + after_endpoint.append(first()) + start_entries, end_entries = sorted(starts.values()), sorted(ends.values()) + return _WallIndex( + endpoints, + at_endpoint, + after_endpoint, + [value for value, _ in start_entries], + [outcome for _, outcome in start_entries], + [value for value, _ in end_entries], + [outcome for _, outcome in end_entries], + ) + + def outcome_at(self, wall_id: str, start_ft: float) -> str | None: + _charge(self.meter) + wall = self.walls.get(wall_id) + if wall is None: + return None + i = _bisect(wall.endpoints, start_ft, self.meter) + if i < len(wall.endpoints) and wall.endpoints[i] == start_ft: + return wall.at_endpoint[i] + return wall.after_endpoint[i - 1] if i else None + + def outcomes_near(self, wall_id: str, start_ft: float) -> tuple[set[str], str]: + if (inside := self.outcome_at(wall_id, start_ft)) is not None: + return {inside}, inside + wall = self.walls.get(wall_id) + if wall is None: + return set(), "no run on its wall" + left = _bisect(wall.ends, start_ft, self.meter) - 1 + right = _bisect(wall.starts, start_ft, self.meter, right=True) + if left < 0 or right == len(wall.starts): + return {"fail"}, "past the original sweep" + a, b = wall.end_outcomes[left], wall.start_outcomes[right] + return {a, b}, f"unswept beside {a} and {b}" + + +def probe_starts(*results: dict, meter: Meter | None = None) -> list[tuple[str, float]]: + points = [] + for result in results: + for run in result.get("sweep", []): + _charge(meter, 3) + a, b = run["start_ft"] + # Halving first avoids overflowing the sum of two admitted finite endpoints. + points.extend( + ((run["wall_id"], a), (run["wall_id"], b), (run["wall_id"], a / 2 + b / 2)) + ) + return [point for point, _ in itertools.groupby(sorted(points))] + + +def more_error_problems( + before: dict, after: dict, change: str, meter: Meter | None = None +) -> list[str]: + original, changed = SweepIndex(before, meter), SweepIndex(after, meter) + problems = [] + for wall_id, s in probe_starts(before, after, meter=meter): + _charge(meter) + b = changed.outcome_at(wall_id, s) + allowed, was = original.outcomes_near(wall_id, s) + if b is None or b == "unsure" or b in allowed or (not allowed and b == "fail"): + continue + problems.append(f"{change}: start {wall_id} {s:.3f} went from {was} to {b}") + if before["decision"] != "pass" and after["decision"] == "pass": + problems.append(f"{change}: the decision became pass") + return problems + + +def less_coverage_problems( + before: dict, after: dict, change: str, meter: Meter | None = None +) -> list[str]: + original, changed = SweepIndex(before, meter), SweepIndex(after, meter) + problems = [] + for wall_id, s in probe_starts(before, after, meter=meter): + _charge(meter) + if changed.outcome_at(wall_id, s) != "pass": + continue + allowed, was = original.outcomes_near(wall_id, s) + if "pass" not in allowed: + problems.append(f"{change}: start {wall_id} {s:.3f} went from {was} to pass") + if before["decision"] != "pass" and after["decision"] == "pass": + problems.append(f"{change}: the decision became pass") + return problems diff --git a/verification/hsverify/replaycheck.py b/verification/hsverify/replaycheck.py index eaf415414..1f6fab57a 100644 --- a/verification/hsverify/replaycheck.py +++ b/verification/hsverify/replaycheck.py @@ -46,7 +46,11 @@ def numbers(value: Any, count: int) -> list[float] | None: return None if not all(isinstance(v, (int, float)) and not isinstance(v, bool) for v in value): return None - return [float(v) for v in value] if all(math.isfinite(v) for v in value) else None + try: + converted = [float(v) for v in value] + except OverflowError: + return None + return converted if all(math.isfinite(v) for v in converted) else None def rigid_problems(m: np.ndarray, tol: float = 1e-3) -> list[str]: @@ -246,7 +250,7 @@ def main(argv: list[str] | None = None) -> int: errors, summary = check_format(folder, session) report: dict = {"session": str(folder), "format_errors": errors, "summary": summary} truth_path = folder / "ground_truth.json" - if truth_path.exists(): + if truth_path.exists() and not errors: report["pose_accuracy"] = accuracy(session, json.loads(truth_path.read_text())) if args.json: print(json.dumps(report, indent=2)) diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index bb276281a..3d366b4da 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -28,16 +28,133 @@ import copy import hashlib import itertools +import json import math -from collections.abc import Callable -from dataclasses import dataclass +from collections.abc import Callable, Iterator +from dataclasses import dataclass, replace from typing import Any import yaml from jsonschema import Draft202012Validator +from .diagnostics import MAX_EXPECTATION_EXAMPLES, DiagnosticMessages +from .intervalindex import ( + SweepIndex, + numeric_key, +) +from .intervalindex import ( + less_coverage_problems as indexed_less_coverage_problems, +) +from .intervalindex import ( + more_error_problems as indexed_more_error_problems, +) +from .intervalindex import ( + probe_starts as indexed_probe_starts, +) + EPS = 1e-6 NUMBER_TOL_FT = 1e-5 +# Geometry loops share WorkMeter. Metadata traversal, hashing, copying and interval +# sorting are bounded separately before schema validation: 50 walls, 500 baseline +# vertices per wall (C1's existing limits), 30,000 sweep runs, 16,000 checks and +# 16,000 requests, 1,000,000 numeric leaves, 2,000,000 JSON nodes, depth 32, and +# 4 MiB of string/key characters per document, and 64 case rule queries. +# Diagnostic output has a separate shared report budget in diagnostics.py. +# Case queries may scan the bounded response; their count is fixed. Other metadata +# passes are linear or +# O(n log n); geometry comparisons and samples charge the meter. These operational +# limits retain the 30,000-run and 12,000-distinct-check regressions. They are not +# measured scan maxima; the node/numeric limits also bound otherwise uncapped C2 +# arrays and C1's combined polygons/keyframes before recursive library traversal. +MAX_RESULT_RUNS = 30_000 +MAX_RESULT_CHECKS = 16_000 +MAX_EVIDENCE_REQUESTS = 16_000 +MAX_NUMERIC_LEAVES = 1_000_000 +MAX_METADATA_NODES = 2_000_000 +MAX_METADATA_DEPTH = 32 +MAX_METADATA_CHARACTERS = 4 * 1024 * 1024 +MAX_SCHEMA_PROBLEMS = 100 +MAX_CASE_RULE_QUERIES = 64 +# Bound verifier work even when a server span or rules step requests billions of samples. +MAX_SWEEP_SAMPLES = 10_000 +# On 2026-10-14, after caching scene derivatives and deferring ground fallback, +# 35 valid public fixture judgments including property variants peaked at 31,649 +# units. Twenty measurements of 20 four-foot walls at 1/4/12/13/20 subdivisions, +# original and mirrored, including full-judge resends, peaked at 88,999 units. +# Another 468 distinct saved responses peaked at 391,700 invariant units on ETH3D +# facade. Allow 20 times that largest measured valid workload for other scans and +# their variants. This is an operational margin, not physical-error calibration. +MAX_WORK_SAMPLES = 20 * 391_700 + + +class WorkLimitExceeded(Exception): + def __init__(self, samples: int) -> None: + self.samples = samples + super().__init__(f"verification work limit exceeded ({samples} samples)") + + +class WorkMeter: + """Count actual geometry work across one response and its property comparisons.""" + + def __init__(self, limit: int | None = None) -> None: + self.limit = MAX_WORK_SAMPLES if limit is None else limit + self.samples = 0 + self.cache: dict[tuple, Any] = {} + self.inputs: dict[int, tuple[Any, int]] = {} + self.input_values: dict[Any, int] = {} + + def charge(self, n: int = 1) -> None: + if n < 0: + raise ValueError("verification work charge must be nonnegative") + self.samples += n + if self.samples > self.limit: + raise WorkLimitExceeded(self.samples) + + def derived( + self, name: str, scene: dict, rules: RuleSet | None, build: Callable[[], Any] + ) -> Any: + key = (name, self.input_key(scene), self.input_key(rules)) + if key not in self.cache: + self.cache[key] = build() + return self.cache[key] + + def input_key(self, value: Any) -> Any: + # judge's scenes and rule sets are immutable during a response's checks. Keep the + # source alive so a later transformed scene cannot reuse its object id. + identity = id(value) + if identity not in self.inputs: + frozen = _input_key(value, self) + token = self.input_values.setdefault(frozen, len(self.input_values)) + self.inputs[identity] = (value, token) + return self.inputs[identity][1] + + +def finite(value: Any) -> bool: + """JSON integers may exceed the float range accepted by math.isfinite.""" + try: + return ( + isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + ) + except (OverflowError, TypeError, ValueError): + return False + + +def _input_key(value: Any, meter: WorkMeter | None = None) -> Any: + if meter is not None: + meter.charge() + if isinstance(value, dict): + return tuple((key, _input_key(item, meter)) for key, item in sorted(value.items())) + if isinstance(value, (tuple, list)): + return tuple(_input_key(item, meter) for item in value) + if isinstance(value, RuleSet): + return _input_key({k: v for k, v in vars(value).items() if k != "rules_sha256"}, meter) + if isinstance(value, Need): + return _input_key(vars(value), meter) + if isinstance(value, (int, float)) and not isinstance(value, bool): + return numeric_key(value).encode("ascii") + return value + + # Spot centres and offsets are rounded by the server; a hundredth of a foot is 1/8 inch. OFFSET_TOL_FT = 0.01 @@ -91,6 +208,8 @@ class RuleSet: battery_height_ft: float | None = None # the rules' headroom.min_ft: how high a wall view without out_ft reached headroom_ft: float | None = None + corner_allowance_ft: float | None = None + rules_sha256: str | None = None @classmethod def from_yaml(cls, text: str) -> RuleSet: @@ -150,12 +269,118 @@ def reach(key: str) -> Need: value("sweep", "wall_join_ft"), value("battery", "height_ft"), headroom, + value("route", "corner_allowance_ft"), + public_rules_fingerprint(data), ) +def public_rules_fingerprint(data: dict) -> str: + """Mirror public rules_from_dict's normalized JSON hash without importing the server.""" + + def normalize(value: Any) -> Any: + if isinstance(value, dict): + normalized = {key: normalize(item) for key, item in value.items()} + if "value" in normalized and "source" in normalized: + normalized["value"] = float(normalized["value"]) + normalized.setdefault("placeholder", False) + return normalized + if isinstance(value, list): + return [normalize(item) for item in value] + return value + + canonical = normalize(data) + canonical.get("policy", {}).setdefault("notice", None) + openings = canonical.get("openings", {}) + if openings.get("exempt_bottom_above_ft") is not None: + openings["exempt_bottom_above_ft"] = float(openings["exempt_bottom_above_ft"]) + payload = json.dumps(canonical, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode()).hexdigest() + + +def ingestion_problems(instance: Any, kind: str | None = None) -> list[str]: + """Bound metadata before validation or copies, using a depth-sized iterator stack.""" + if kind is None and isinstance(instance, dict): + kind = "scene" if "walls" in instance else "result" + label = kind or "document" + if isinstance(instance, dict): + limits = ( + {"walls": 50} + if kind == "scene" + else { + "sweep": MAX_RESULT_RUNS, + "checks": MAX_RESULT_CHECKS, + "missing_evidence": MAX_EVIDENCE_REQUESTS, + } + ) + for field, limit in limits.items(): + items = instance.get(field) + if isinstance(items, (list, tuple)) and len(items) > limit: + return [f"{label} ingestion limit exceeded: {field} exceeds {limit} entries"] + if kind == "scene": + for wall in ( + instance.get("walls", []) if isinstance(instance.get("walls"), list) else [] + ): + points = wall.get("baseline") if isinstance(wall, dict) else None + if isinstance(points, (list, tuple)) and len(points) > 500: + return [f"{label} ingestion limit exceeded: baseline exceeds 500 vertices"] + + nodes = numeric = characters = 0 + stack = [iter((instance,))] + while stack: + try: + value = next(stack[-1]) + except StopIteration: + stack.pop() + continue + nodes += 1 + if nodes > MAX_METADATA_NODES: + return [f"{label} ingestion limit exceeded: more than {MAX_METADATA_NODES} JSON nodes"] + if isinstance(value, (int, float)) and not isinstance(value, bool): + numeric += 1 + if numeric > MAX_NUMERIC_LEAVES: + return [ + f"{label} ingestion limit exceeded: more than " + f"{MAX_NUMERIC_LEAVES} numeric leaves" + ] + elif isinstance(value, str): + characters += len(value) + elif isinstance(value, dict): + if len(value) > MAX_METADATA_NODES - nodes: + return [ + f"{label} ingestion limit exceeded: more than {MAX_METADATA_NODES} JSON nodes" + ] + characters += sum(len(key) for key in value if isinstance(key, str)) + if len(stack) > MAX_METADATA_DEPTH: + return [f"{label} ingestion limit exceeded: nesting exceeds {MAX_METADATA_DEPTH}"] + stack.append(iter(value.values())) + elif isinstance(value, (list, tuple)): + if len(value) > MAX_METADATA_NODES - nodes: + return [ + f"{label} ingestion limit exceeded: more than {MAX_METADATA_NODES} JSON nodes" + ] + if len(stack) > MAX_METADATA_DEPTH: + return [f"{label} ingestion limit exceeded: nesting exceeds {MAX_METADATA_DEPTH}"] + stack.append(iter(value)) + if characters > MAX_METADATA_CHARACTERS: + return [ + f"{label} ingestion limit exceeded: strings/keys exceed " + f"{MAX_METADATA_CHARACTERS} characters" + ] + return [] + + def schema_errors(instance: Any, schema: dict) -> list[str]: + properties = schema.get("properties", {}) + kind = "scene" if "walls" in properties else "result" if "sweep" in properties else None + if problems := ingestion_problems(instance, kind): + return problems validator = Draft202012Validator(schema) - errors = sorted(validator.iter_errors(instance), key=lambda e: list(e.absolute_path)) + # A malformed document may produce a diagnostic per node. Stop after enough + # diagnostics to fail it; valid documents still complete schema validation. + errors = sorted( + itertools.islice(validator.iter_errors(instance), MAX_SCHEMA_PROBLEMS), + key=lambda e: tuple(str(part) for part in e.absolute_path), + ) return [f"{'/'.join(map(str, e.absolute_path)) or '(root)'}: {e.message}" for e in errors] @@ -181,6 +406,7 @@ def observed( min_out_ft: float = 0.0, beyond: bool = False, wall_default_ft: float | None = None, + meter: WorkMeter | None = None, ) -> list[tuple[float, float]]: """Merged observed s-intervals for a band, counting only entries whose view reached `min_out_ft` (out from the wall for ground and facing, up for wall and overhead). With @@ -191,8 +417,20 @@ def observed( headroom.min_ft) and settles any height up to it inclusively, `beyond` or not; S2 chose this category default at 6200098 (server/scene.py `observed_intervals`). None keeps the old reading, all the way up.""" + meter = meter if meter is not None else WorkMeter() + key = ( + "observed", + meter.input_key(scene), + band, + numeric_key(min_out_ft), + beyond, + None if wall_default_ft is None else numeric_key(wall_default_ft), + ) + if key in meter.cache: + return meter.cache[key] spans = [] for entry in scene.get("coverage", {}).get("observed", []): + meter.charge() if entry["band"] != band: continue if band == "wall" and "out_ft" not in entry and wall_default_ft is not None: @@ -210,15 +448,28 @@ def observed( spans.sort() merged: list[list[float]] = [] for a, b in spans: + meter.charge() if merged and a <= merged[-1][1] + EPS: merged[-1][1] = max(merged[-1][1], b) else: merged.append([a, b]) - return [(a, b) for a, b in merged] + meter.cache[key] = [(a, b) for a, b in merged] + return meter.cache[key] -def covers(intervals: list[tuple[float, float]], a: float, b: float, slack: float = 0.0) -> bool: - return any(lo - slack <= a + EPS and b - EPS <= hi + slack for lo, hi in intervals) +def covers( + intervals: list[tuple[float, float]], + a: float, + b: float, + slack: float = 0.0, + meter: WorkMeter | None = None, +) -> bool: + meter = meter if meter is not None else WorkMeter() + for lo, hi in intervals: + meter.charge() + if lo - slack <= a + EPS and b - EPS <= hi + slack: + return True + return False # A gap in a 1D band narrower than this is rounding between the capture's spans and the unrolled @@ -227,12 +478,14 @@ def covers(intervals: list[tuple[float, float]], a: float, b: float, slack: floa def gaps_over( - intervals: list[tuple[float, float]], a: float, b: float + intervals: list[tuple[float, float]], a: float, b: float, meter: WorkMeter | None = None ) -> list[tuple[float, float]]: """The parts of [a, b] no interval covers, as the server's missing() finds them: gaps narrower than COVERAGE_TOLERANCE_FT don't count.""" + meter = meter if meter is not None else WorkMeter() gaps, at = [], a for lo, hi in sorted(intervals): + meter.charge() if hi <= at: continue if lo > at: @@ -302,8 +555,16 @@ def margin_problem(check: dict) -> str | None: def invariant_problems( - scene: dict, result: dict, sent: bytes | None = None, rules: RuleSet | None = None + scene: dict, + result: dict, + sent: bytes | None = None, + rules: RuleSet | None = None, + meter: WorkMeter | None = None, ) -> list[str]: + meter = meter if meter is not None else WorkMeter() + if sampling := geometry_admission_problems(scene, result, rules, meter=meter): + return sampling + result = deduplicate_result_runs(result) problems: list[str] = [] decision = result["decision"] spot = result.get("spot") @@ -371,15 +632,27 @@ def invariant_problems( if max(abs(want[0] - got[0]), abs(want[1] - got[1])) > OFFSET_TOL_FT: problems.append(f"spot.meter_offset_ft {got} != centre - meter {want}") - problems += spot_sweep_problems(result) + def checked(name: str, run: Callable[[], list[str]]) -> None: + try: + problems.extend(run()) + except WorkLimitExceeded: + raise + except Exception as exc: + problems.append(f"{name}: {type(exc).__name__}: {exc}") + + checked("spot_sweep", lambda: spot_sweep_problems(result, meter=meter)) if rules is not None: - problems += footprint_problems(scene, result, rules) - problems += spot_position_problems(scene, result, rules) - problems += missing_evidence_problems(scene, result, rules) + checked("footprint", lambda: footprint_problems(scene, result, rules, meter=meter)) + checked("spot_position", lambda: spot_position_problems(scene, result, rules, meter=meter)) + checked( + "missing_evidence", lambda: missing_evidence_problems(scene, result, rules, meter=meter) + ) if rules is not None: - problems += coverage_problems(scene, result, rules) - problems += declared_height_problems(scene, result, rules) - problems += dropped_check_problems(scene, result, rules) + checked("coverage", lambda: coverage_problems(scene, result, rules, meter=meter)) + checked( + "declared_height", lambda: declared_height_problems(scene, result, rules, meter=meter) + ) + checked("dropped_checks", lambda: dropped_check_problems(scene, result, rules)) return problems @@ -393,12 +666,19 @@ def invariant_problems( COLLINEAR_FT = 0.05 -def straight_pieces(scene: dict, rules: RuleSet) -> list[tuple[float, float]]: +def straight_pieces( + scene: dict, rules: RuleSet, meter: WorkMeter | None = None +) -> list[tuple[float, float]]: """The s-extent of each straight stretch of scanned wall a battery can back onto: baseline segments joined where the next one starts at the previous one's end and continues its line, within a wall or across two walls that meet.""" + meter = meter if meter is not None else WorkMeter() + key = ("straight_pieces", meter.input_key(scene), meter.input_key(rules)) + if key in meter.cache: + return meter.cache[key] pieces: list[list] = [] # [s0, s1, start point, end point] - for _, s0, s1, a, b in segments_s(scene, rules) or []: + for _, s0, s1, a, b in segments_s(scene, rules, meter=meter) or []: + meter.charge() if pieces: prev = pieces[-1] length = math.dist(prev[2], prev[3]) @@ -410,23 +690,40 @@ def straight_pieces(scene: dict, rules: RuleSet) -> list[tuple[float, float]]: prev[1], prev[3] = s1, b continue pieces.append([s0, s1, a, b]) - return [(p[0], p[1]) for p in pieces] + meter.cache[key] = [(p[0], p[1]) for p in pieces] + return meter.cache[key] -def footprint_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: +def footprint_problems( + scene: dict, result: dict, rules: RuleSet, meter: WorkMeter | None = None +) -> list[str]: """A battery that passes or is left to a person backs onto one straight stretch of scanned wall along its whole width; a footprint past the stretch's end (a corner, the end of the wall, a gap) can't sit flush, so the server fails it (server/solver.py check_backing).""" - pieces = straight_pieces(scene, rules) + meter = meter if meter is not None else WorkMeter() + if sampling := geometry_admission_problems(scene, result, rules, meter=meter): + return sampling + result = deduplicate_result_runs(result) + pieces = straight_pieces(scene, rules, meter=meter) if not pieces: return [] def off_wall(lo: float, hi: float, outcome: str, wall_id: str) -> str | None: + meter.charge() # The stretch it starts on, or else ends on: the one it would back onto. A start on a # boundary belongs to the stretch it runs onto, an end to the one it runs off. - piece = next((p for p in pieces if p[0] - EPS <= lo < p[1] - EPS), None) or next( - (p for p in pieces if p[0] + EPS < hi <= p[1] + EPS), None - ) + piece = None + for candidate in pieces: + meter.charge() + if candidate[0] - EPS <= lo < candidate[1] - EPS: + piece = candidate + break + if piece is None: + for candidate in pieces: + meter.charge() + if candidate[0] + EPS < hi <= candidate[1] + EPS: + piece = candidate + break if piece is None: return "is not on any scanned wall" if lo < piece[0] - COLLINEAR_FT or hi > piece[1] + COLLINEAR_FT: @@ -435,8 +732,9 @@ def off_wall(lo: float, hi: float, outcome: str, wall_id: str) -> str | None: return None # A pass also keeps its position error clear of the stretch's ends: within it the # battery may not sit flush, and the server leaves it unsure (check_backing). - e = battery_error(scene, rules, wall_id, lo, hi) + e = battery_error(scene, rules, wall_id, lo, hi, meter=meter) for end, clear in ((piece[0], lo - e - piece[0]), (piece[1], piece[1] - hi - e)): + meter.charge() if clear < -COLLINEAR_FT: return f"ends within its error ({e:.2f} ft) of its straight wall's end at {end:.2f}" return None @@ -450,18 +748,23 @@ def off_wall(lo: float, hi: float, outcome: str, wall_id: str) -> str | None: for run in result.get("sweep", []): if run["outcome"] not in ("pass", "unsure"): continue - for start in dict.fromkeys(run["start_ft"]): # a run's two ends bound all its starts + for start in run_starts(run["start_ft"], rules.step_ft): + meter.charge() battery = (start, start + rules.width_ft) if (why := off_wall(*battery, run["outcome"], run["wall_id"])) is not None: problems.append(f"sweep {run['outcome']} start {start:.2f} {why}") + break return problems -def spot_position_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: +def spot_position_problems( + scene: dict, result: dict, rules: RuleSet, meter: WorkMeter | None = None +) -> list[str]: """Where the app places the battery must be where the result says it stands: spot.center is its footprint's centre, and that is the middle of span_ft on the wall, half the battery's depth out. meter_offset_ft is checked against center elsewhere, so a centre moved together with its offset is caught here.""" + meter = meter if meter is not None else WorkMeter() spot = result.get("spot") if spot is None: return [] @@ -477,7 +780,7 @@ def spot_position_problems(scene: dict, result: dict, rules: RuleSet) -> list[st f"[{fx:.2f}, {fz:.2f}]" ) lo, hi = spot["span_ft"] - plan = GroundPlan(scene, rules) + plan = ground_plan(scene, rules, meter) want = plan.point((lo + hi) / 2, (lo + hi) / 2, rules.depth_ft / 2) if want is not None and math.dist((cx, cz), want) > COLLINEAR_FT: problems.append( @@ -487,7 +790,7 @@ def spot_position_problems(scene: dict, result: dict, rules: RuleSet) -> list[st return problems -def spot_sweep_problems(result: dict) -> list[str]: +def spot_sweep_problems(result: dict, meter: WorkMeter | None = None) -> list[str]: """The chosen spot is one of the evaluated starts, so the sweep must give its start the spot's own outcome, and a passing spot must lie in a passing run. A result could otherwise report a pass at a start its own sweep says fails, with stats that agree with the sweep.""" @@ -495,7 +798,7 @@ def spot_sweep_problems(result: dict) -> list[str]: if spot is None: return [] start = spot["span_ft"][0] - got = outcome_at(result, spot["wall_id"], start) + got = outcome_at(result, spot["wall_id"], start, meter=meter) if got == spot["outcome"]: return [] verb = {"pass": "passes", "fail": "fails"}.get(spot["outcome"], "is unsure") @@ -515,23 +818,31 @@ def required_span(lo: float, hi: float, radius: float) -> tuple[float, float]: return lo - radius, hi + radius -def battery_error(scene: dict, rules: RuleSet, wall_id: str, lo: float, hi: float) -> float: +def battery_error( + scene: dict, rules: RuleSet, wall_id: str, lo: float, hi: float, meter: WorkMeter | None = None +) -> float: """Position error of a battery over [lo, hi] on `wall_id`, as server 3baa338 takes it for its checks (solver.py _errors): its wall's error at its far edge from the meter, plus the larger of the meter's error in plan and the battery's slide along the walls (slide).""" - wall = next((w for w in scene["walls"] if w["id"] == wall_id), None) + meter = meter if meter is not None else WorkMeter() + wall = wall_lookup(scene, meter).get(wall_id) if wall is None: raise ValueError(f"the result names wall {wall_id!r}, which the scene does not have") return wall_error_at(wall, rules, max(abs(lo), abs(hi))) + max( - meter_error(scene, rules), slide(scene, rules, lo, hi) + meter_error(scene, rules), slide(scene, rules, lo, hi, meter=meter) ) -def along_error(scene: dict, rules: RuleSet, wall_id: str, lo: float, hi: float) -> float: +def along_error( + scene: dict, rules: RuleSet, wall_id: str, lo: float, hi: float, meter: WorkMeter | None = None +) -> float: """The battery's error against spans measured along the walls: its wall's at its far edge plus its slide, without the meter's plan error (server 3baa338 _errors, along_err).""" - wall = next(w for w in scene["walls"] if w["id"] == wall_id) - return wall_error_at(wall, rules, max(abs(lo), abs(hi))) + slide(scene, rules, lo, hi) + meter = meter if meter is not None else WorkMeter() + wall = wall_lookup(scene, meter)[wall_id] + return wall_error_at(wall, rules, max(abs(lo), abs(hi))) + slide( + scene, rules, lo, hi, meter=meter + ) def wall_error_at(wall: dict, rules: RuleSet, s: float) -> float: @@ -547,16 +858,28 @@ def meter_error(scene: dict, rules: RuleSet) -> float: return scene.get("meter", {}).get("plus_minus_ft", rules.errors["meter"]) -def slide(scene: dict, rules: RuleSet, lo: float, hi: float) -> float: +def slide( + scene: dict, rules: RuleSet, lo: float, hi: float, meter: WorkMeter | None = None +) -> float: """How far a battery over [lo, hi] can move along the walls against spans measured in s when the meter or a corner between them moves within its error (server 3baa338 _slide): the meter's error times the difference between its wall's direction and the battery's, plus each other wall between the meter and the battery's near edge, at its far end, times the same difference for that wall. 0 on the meter's own wall.""" - segments = segments_s(scene, rules) + meter = meter if meter is not None else WorkMeter() + key = ( + "slide", + meter.input_key(scene), + meter.input_key(rules), + numeric_key(lo), + numeric_key(hi), + ) + if key in meter.cache: + return meter.cache[key] + segments = segments_s(scene, rules, meter=meter) if not segments: return 0.0 - walls = {w["id"]: w for w in scene["walls"]} + walls = wall_lookup(scene, meter) def along(seg) -> Point: _, _, _, p, q = seg @@ -564,7 +887,11 @@ def along(seg) -> Point: return ((q[0] - p[0]) / n, (q[1] - p[1]) / n) def under(s: float): - return min(segments, key=lambda g: max(g[1] - s, s - g[2], 0.0)) + def distance(g): + meter.charge() + return max(g[1] - s, s - g[2], 0.0) + + return min(segments, key=distance) battery = along(under((lo + hi) / 2)) @@ -573,6 +900,7 @@ def turn(seg) -> float: return math.hypot(battery[0] - a[0], battery[1] - a[1]) meter_wall = scene["meter"]["wall_id"] + meter.charge(len(segments)) meter_seg = min( (g for g in segments if g[0] == meter_wall), key=lambda g: max(g[1], -g[2], 0.0) ) @@ -580,10 +908,12 @@ def turn(seg) -> float: a0, b0 = min(0.0, near), max(0.0, near) total = meter_error(scene, rules) * turn(meter_seg) for seg in segments: + meter.charge() a, b = max(seg[1], a0), min(seg[2], b0) if b - a <= EPS: continue total += wall_error_at(walls[seg[0]], rules, max(abs(a), abs(b))) * turn(seg) + meter.cache[key] = total return total @@ -593,7 +923,13 @@ def wall_error(wall: dict, rules: RuleSet) -> float: def reach_gaps( - scene: dict, rules: RuleSet, check: str, wall_id: str, lo: float, hi: float + scene: dict, + rules: RuleSet, + check: str, + wall_id: str, + lo: float, + hi: float, + meter: WorkMeter | None = None, ) -> list[tuple[str, str]]: """(band, what is missing) for each Need of `check` a battery over [lo, hi] does not have. @@ -610,77 +946,285 @@ def reach_gaps( - facing and overhead bands: over [lo - e, hi + e], every place the battery may sit, beyond the check's distance unless measured. """ + meter = meter if meter is not None else WorkMeter() + key = ( + "reach_gaps", + meter.input_key(scene), + meter.input_key(rules), + check, + wall_id, + numeric_key(lo), + numeric_key(hi), + ) + if key in meter.cache: + return meter.cache[key] gaps = [] for need in rules.needs[check]: + meter.charge() if not need.widen: error = 0.0 elif need.along: - error = along_error(scene, rules, wall_id, lo, hi) + error = along_error(scene, rules, wall_id, lo, hi, meter=meter) else: - error = battery_error(scene, rules, wall_id, lo, hi) + error = battery_error(scene, rules, wall_id, lo, hi, meter=meter) reach = need.radius + error if need.band == "ground": - plan = GroundPlan(scene, rules) if rules.wall_join_ft is not None else None - gap = ground_gap(scene, lo, hi, rules.depth_ft, reach, plan) + plan = ( + (lambda: ground_plan(scene, rules, meter)) + if rules.wall_join_ft is not None + else None + ) + gap = ground_gap(scene, lo, hi, rules.depth_ft, reach, plan, meter=meter) elif need.band == "wall": a, b = required_span(lo, hi, reach) wall = observed( - scene, "wall", need.height, beyond=True, wall_default_ft=rules.headroom_ft + scene, + "wall", + need.height, + beyond=True, + wall_default_ft=rules.headroom_ft, + meter=meter, ) - seen = not gaps_over(wall, a, b) + seen = not gaps_over(wall, a, b, meter=meter) gap = None if seen else f"wall [{a:.2f}, {b:.2f}] observed higher than {need.height} ft" else: a, b = required_span(lo, hi, reach) - gap = measured_band_gap(scene, need.band, wall_id, a, b, need.height) + gap = measured_band_gap(scene, need.band, wall_id, a, b, need.height, meter=meter) if gap: gaps.append((need.band, gap)) + meter.cache[key] = gaps return gaps def measured_band_gap( - scene: dict, band: str, wall_id: str, lo: float, hi: float, need: float + scene: dict, + band: str, + wall_id: str, + lo: float, + hi: float, + need: float, + meter: WorkMeter | None = None, ) -> str | None: """The first stretch of [lo, hi] the facing or overhead band leaves unsettled: not observed, or observed short of `need` where no measurement covers it.""" - entries = [ - (*sorted(e["span_ft"]), reached(e)) - for e in scene.get("coverage", {}).get("observed", []) - if e["band"] == band - ] - measured = [ - tuple(sorted(m["span_ft"])) - for m in scene.get(MEASUREMENTS[band], []) - if m.get("wall_id", wall_id) == wall_id + meter = meter if meter is not None else WorkMeter() + entries = band_entries(scene, band, meter) + measured = meter.derived( + f"measurements:{band}:{wall_id}", + scene, + None, + lambda: [ + tuple(sorted(m["span_ft"])) + for m in scene.get(MEASUREMENTS[band], []) + if m.get("wall_id", wall_id) == wall_id + ], + ) + edges = itertools.chain((x for e in entries for x in e[:2]), (x for m in measured for x in m)) + cuts = [ + value + for value, _ in itertools.groupby(sorted([lo, hi, *(x for x in edges if lo < x < hi)])) ] - edges = {x for e in entries for x in e[:2]} | {x for m in measured for x in m} - cuts = sorted({lo, hi} | {x for x in edges if lo < x < hi}) for p, q in itertools.pairwise(cuts): + meter.charge() if q - p <= EPS: continue + meter.charge(len(entries)) seen = [out for x, y, out in entries if x <= p + EPS and q - EPS <= y] where = f"{band} [{p:.2f}, {q:.2f}] observed" if not seen: if q - p < COVERAGE_TOLERANCE_FT: continue return f"{where}, none seen" - settled = any(x <= p + EPS and q - EPS <= y for x, y in measured) + settled = False + for x, y in measured: + meter.charge() + if x <= p + EPS and q - EPS <= y: + settled = True + break # Strictly beyond, as the server requires (server/solver.py at 6c7ca23). if not settled and max(seen) <= need: return f"{where} out to {need:.2f} ft or measured, seen {max(seen):.2f} ft" return None -def run_starts(span: list[float], step_ft: float | None) -> list[float]: - """Starts sampled from a sweep run: its first, every step after it, and its last. +def run_start_count(span: list[float], step_ft: float | None) -> int: + """Count pre-end starts; run_starts appends the final endpoint separately. A sample, not the server's own list: the server also evaluates boundaries and midpoints - that a run reports only as its two ends.""" - if step_ft is None or step_ft <= 0: - raise ValueError("checking a sweep run needs the rules' sweep.step_ft") + that a run reports only as its two ends. Oversized runs fail rather than lose coverage + through truncation.""" + if step_ft is None or not finite(step_ft) or step_ft <= 0: + raise ValueError("checking a sweep run needs finite positive rules' sweep.step_ft") a, b = span + if not finite(a) or not finite(b) or a > b: + raise ValueError("sweep start_ft must be finite and ordered") + steps = (b - a) / step_ft + if not finite(steps) or steps > MAX_SWEEP_SAMPLES: + raise ValueError(f"sweep sampling exceeds the {MAX_SWEEP_SAMPLES} sample limit") # Every step strictly before the last start, the first included however short the run. - count = max(0, math.ceil((b - a) / step_ft - EPS)) - return [a + k * step_ft for k in range(count)] + [b] + count = max(int(a < b), math.ceil(steps - EPS)) + if count + 1 > MAX_SWEEP_SAMPLES: + raise ValueError(f"sweep sampling exceeds the {MAX_SWEEP_SAMPLES} sample limit") + return count + + +def run_starts(span: list[float], step_ft: float | None) -> Iterator[float]: + count = run_start_count(span, step_ft) + a, b = span + return itertools.chain((a + k * step_ft for k in range(count)), (b,)) + + +def deduplicate_result_runs(result: dict) -> dict: + """Identical records cannot add geometry evidence or multiply sampling work.""" + unique = {} + for run in result.get("sweep", []): + unique.setdefault(json.dumps(run, sort_keys=True), run) + return result | {"sweep": list(unique.values())} + + +def reported_dimension_problems(result: dict, rules: RuleSet | None) -> list[str]: + """Reject inconsistent C2 geometry before its dimensions can override public rules.""" + problems = [] + + spots = [(name, result.get(name)) for name in ("spot", "nearest_considered")] + for name, spot in spots: + if spot is None: + continue + for field in ("width_ft", "depth_ft", "height_ft"): + if field in spot and (not finite(spot[field]) or spot[field] <= 0): + problems.append(f"{name}.{field} must be finite and positive") + span = spot.get("span_ft") + if span is not None: + if len(span) != 2 or not all(finite(v) for v in span) or span[0] >= span[1]: + problems.append(f"{name}.span_ft must be finite and increasing") + elif ( + finite(spot.get("width_ft")) + and abs(span[1] - span[0] - spot["width_ft"]) > OFFSET_TOL_FT + ): + problems.append(f"{name}.width_ft does not match span_ft") + corners = spot.get("footprint") + if corners is None: + continue + if len(corners) != 4 or any( + len(point) != 2 or not all(finite(v) for v in point) for point in corners + ): + problems.append(f"{name}.footprint must have four finite plan corners") + continue + # C2 orders back-left, back-right, front-right, front-left. Use the verifier's existing + # hundredth-foot position tolerance when comparing its rounded plan edges and spans. + for field, edges in (("width_ft", ((0, 1), (2, 3))), ("depth_ft", ((1, 2), (3, 0)))): + if finite(spot.get(field)) and any( + abs(math.dist(corners[a], corners[b]) - spot[field]) > OFFSET_TOL_FT + for a, b in edges + ): + problems.append(f"{name}.{field} does not match footprint") + + widths = [ + (f"{name}.width_ft", spot["width_ft"]) + for name, spot in spots + if spot is not None and "width_ft" in spot + ] + widths += [ + (c["id"], c["threshold_ft"]) + for c in result.get("checks", []) + if c.get("rule", {}).get("key") == "battery.width_ft" and c.get("threshold_ft") is not None + ] + for source, width in widths: + if not finite(width) or width <= 0: + message = f"{source} must be finite and positive" + problems.append(message) + for check in result.get("checks", []): + if check.get("rule", {}).get("key") == "route.corner_allowance_ft": + allowance = check.get("threshold_ft") + if allowance is not None and (not finite(allowance) or allowance < 0): + problems.append(f"{check['id']} corner allowance must be finite and nonnegative") + valid_widths = [(source, width) for source, width in widths if finite(width) and width > 0] + if valid_widths and any( + abs(width - valid_widths[0][1]) > NUMBER_TOL_FT for _, width in valid_widths + ): + problems.append("reported battery widths disagree") + first, second = (spot for _, spot in spots) + if first is not None and second is not None: + for field in ("depth_ft", "height_ft"): + if ( + finite(first.get(field)) + and finite(second.get(field)) + and abs(first[field] - second[field]) > NUMBER_TOL_FT + ): + problems.append(f"spot and nearest_considered {field} disagree") + return _unique_messages(problems) + + +def _unique_messages(messages: list[str]) -> list[str]: + """Preserve diagnostic order without comparing each message to every predecessor.""" + return list(dict.fromkeys(messages)) + + +def geometry_admission_problems( + scene: dict, result: dict, rules: RuleSet | None, meter: WorkMeter | None = None +) -> list[str]: + """Validate response numbers and references before geometry. Work is counted as it runs.""" + meter = meter if meter is not None else WorkMeter() + if problems := ingestion_problems(scene, "scene") + ingestion_problems(result, "result"): + return problems + problems = reported_dimension_problems(result, rules) + + def numbers(value: Any, path: str) -> None: + if isinstance(value, dict): + for key, item in value.items(): + numbers(item, f"{path}.{key}" if path else key) + elif isinstance(value, (tuple, list)): + for i, item in enumerate(value): + numbers(item, f"{path}[{i}]") + elif isinstance(value, (int, float)) and not isinstance(value, bool) and not finite(value): + problems.append(f"{path} must be finite") + + numbers(result, "result") + numbers(scene, "scene") + wall_ids = {wall["id"] for wall in scene.get("walls", [])} + bodies = [(name, result.get(name)) for name in ("spot", "nearest_considered")] + bodies += [(f"sweep[{i}]", run) for i, run in enumerate(result.get("sweep", []))] + for name, body in bodies: + if body is not None and body.get("wall_id") not in wall_ids: + problems.append(f"{name} names unknown wall_id {body.get('wall_id')!r}") + for run in result.get("sweep", []): + a, b = run["start_ft"] + if not finite(a) or not finite(b) or a > b: + problems.append( + f"sweep {run['outcome']} starts {run['start_ft']}: " + "start_ft must be finite and ordered" + ) + elif not finite(b - a): + problems.append(f"sweep {run['outcome']} start_ft span length must be finite") + if problems: + return list(dict.fromkeys(problems)) + ends = ( + chain_ends_s(scene, rules, meter=meter) + if rules is not None and rules.wall_join_ft is not None + else None + ) + for name in ("spot", "nearest_considered"): + if (spot := result.get(name)) is not None: + lo, hi = spot["span_ft"] + if ends is not None and (lo < ends[0] - COLLINEAR_FT or hi > ends[1] + COLLINEAR_FT): + problems.append(f"{name}: span_ft exceeds scanned range [{ends[0]}, {ends[1]}]") + for run in deduplicate_result_runs(result)["sweep"]: + a, b = run["start_ft"] + prefix = f"sweep {run['outcome']} starts {run['start_ft']}: " + if a > b: + problems.append(prefix + "start_ft must be finite and ordered") + continue + if run["outcome"] in ("pass", "unsure") and rules is not None: + if ends is not None and (a < ends[0] - COLLINEAR_FT or b > ends[1] + COLLINEAR_FT): + problems.append( + prefix + f"start_ft span exceeds scanned range [{ends[0]}, {ends[1]}]" + ) + continue + try: + run_start_count(run["start_ft"], rules.step_ft) + except ValueError as exc: + problems.append(prefix + str(exc)) + return problems # Checks every result lists besides those with coverage needs (server README, "What settles @@ -708,15 +1252,26 @@ def dropped_check_problems(scene: dict, result: dict, rules: RuleSet) -> list[st ] -def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: +def coverage_problems( + scene: dict, result: dict, rules: RuleSet, meter: WorkMeter | None = None +) -> list[str]: """Missing coverage is never a pass (C5), to the reach each check's rule looks out to (see reach_gaps). A sweep run that passes needs, at each start sampled from it (run_starts), every check's area around that start's battery with that battery's own position error, and the wall and cable route back to the meter over the whole run. Checking the whole run as one battery at its farthest start's error would ask for more than any single start needs.""" + meter = meter if meter is not None else WorkMeter() + if sampling := geometry_admission_problems(scene, result, rules, meter=meter): + return sampling + result = deduplicate_result_runs(result) problems: list[str] = [] route_wall = observed( - scene, "wall", rules.route_height_ft, beyond=True, wall_default_ft=rules.headroom_ft + scene, + "wall", + rules.route_height_ft, + beyond=True, + wall_default_ft=rules.headroom_ft, + meter=meter, ) # Coverage is needed for every check the rules define for this scene, whether or not the # result lists it: a result that drops a check must not skip its coverage. @@ -731,10 +1286,10 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: # The route starts at the meter, which may stand its error either side of s = 0 # (server README, route_path), within the scanned chain. m = meter_error(scene, rules) - ends = chain_ends_s(scene, rules) or (-math.inf, math.inf) + ends = chain_ends_s(scene, rules, meter=meter) or (-math.inf, math.inf) route_lo = max(min(lo, -m), ends[0]) route_hi = min(max(hi, m), ends[1]) - if gaps_over(route_wall, route_lo, route_hi): + if gaps_over(route_wall, route_lo, route_hi, meter=meter): problems.append( f"sweep pass for starts {run['start_ft']} but the wall and cable route " f"[{route_lo:.2f}, {route_hi:.2f}] were not all observed higher than " @@ -743,8 +1298,11 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: for check in sorted(set(rules.needs) & evaluated): gaps = {} # each gap once, at the first start that needs it for start in run_starts(run["start_ft"], rules.step_ft): + meter.charge() battery = (start, start + rules.width_ft) - for _, gap in reach_gaps(scene, rules, check, run["wall_id"], *battery): + for _, gap in reach_gaps( + scene, rules, check, run["wall_id"], *battery, meter=meter + ): gaps.setdefault(gap, start) problems += [ f"sweep pass for starts {run['start_ft']} but at start {start:.2f} {check} " @@ -755,11 +1313,10 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: spot = result.get("spot") if spot is not None: lo, hi = spot["span_ft"] - for check in result.get("checks", []): - if check["outcome"] != "pass" or check["id"] not in rules.needs: - continue - for _, gap in reach_gaps(scene, rules, check["id"], spot["wall_id"], lo, hi): - problems.append(f"check {check['id']} passes but needs {gap}") + passing = {check["id"] for check in result.get("checks", []) if check["outcome"] == "pass"} + for check_id in sorted(passing & set(rules.needs)): + for _, gap in reach_gaps(scene, rules, check_id, spot["wall_id"], lo, hi, meter=meter): + problems.append(f"check {check_id} passes but needs {gap}") for request in result.get("missing_evidence", []): if request["kind"] != "band" or "span_ft" not in request or "band" not in request: @@ -771,8 +1328,8 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: # only where the band was seen all the way (no `out_ft`). default = GROUND_FAR_FT if request["band"] == "ground" else math.inf far = request.get("out_ft", default) - seen = observed(scene, request["band"], far, wall_default_ft=rules.headroom_ft) - if b - a > EPS and covers(seen, a, b): + seen = observed(scene, request["band"], far, wall_default_ft=rules.headroom_ft, meter=meter) + if b - a > EPS and covers(seen, a, b, meter=meter): problems.append( f"missing_evidence asks for {request['band']} {request['span_ft']}, " "which the scene lists as observed" @@ -786,19 +1343,23 @@ def ground_gap( hi: float, depth: float, radius: float, - plan: GroundPlan | None = None, + plan: GroundPlan | Callable[[], GroundPlan] | None = None, + meter: WorkMeter | None = None, ) -> str | None: """The first stretch of ground seen less far out than a battery over [lo, hi] needs (see coverage_problems), or None. With `plan`, ground not seen in the battery's own wall's band still counts where it lies in another wall's observed band in plan (GroundPlan).""" - entries = [ - (*sorted(e["span_ft"]), e.get("out_ft", 0.0)) - for e in scene.get("coverage", {}).get("observed", []) - if e["band"] == "ground" - ] + meter = meter if meter is not None else WorkMeter() + entries = band_entries(scene, "ground", meter) a, b = lo - radius, hi + radius - cuts = sorted({a, lo, hi, b} | {x for e in entries for x in e[:2] if a < x < b}) + cuts = [ + value + for value, _ in itertools.groupby( + sorted([a, lo, hi, b, *(x for e in entries for x in e[:2] if a < x < b)]) + ) + ] for p, q in itertools.pairwise(cuts): + meter.charge() if q - p <= EPS: continue @@ -809,8 +1370,11 @@ def need_at(s: float) -> float: return depth + math.sqrt(max(0.0, radius**2 - d**2)) need = need_at(q if q <= lo else p if p >= hi else (lo + hi) / 2) + meter.charge(len(entries)) seen = max((out for x, y, out in entries if x <= p + EPS and q - EPS <= y), default=None) if seen is None or seen + SEEN_GROWTH_FT < need: + if callable(plan): + plan = plan() if plan is not None and plan.covers((lo + hi) / 2, p, q, seen or 0.0, need_at): continue where = f"ground [{p:.2f}, {q:.2f}] observed" @@ -820,7 +1384,9 @@ def need_at(s: float) -> float: return None -def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = None) -> list[str]: +def missing_evidence_problems( + scene: dict, result: dict, rules: RuleSet | None = None, meter: WorkMeter | None = None +) -> list[str]: """When coverage is what stands in the way (reason unobserved_area), every check left unsure for an unobserved area is named by a request, and a clearance check by a request of each band it lacks around the chosen spot: a check that forgets one band leaves the next capture @@ -832,9 +1398,13 @@ def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = the reason and its requests; leaving both out would never ask the homeowner for the view that settles the check. Without a spot and without the reason, every position fails anyway, no photo would change the answer, and C2 lets missing_evidence stay empty.""" + meter = meter if meter is not None else WorkMeter() + if sampling := geometry_admission_problems(scene, result, rules, meter=meter): + return sampling + result = deduplicate_result_runs(result) codes = {r["code"] for r in result.get("reasons", [])} spot = result.get("spot") - unseen = unobserved_checks(result) + unseen = list(dict.fromkeys(unobserved_checks(result))) requests = result.get("missing_evidence", []) if "unobserved_area" not in codes: if spot is None or not unseen: @@ -847,31 +1417,32 @@ def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = "but no reason unobserved_area and no missing_evidence" ] lo, hi = spot["span_ft"] - asked = with_past_ends_asked(scene, requests, rules) + asked = with_past_ends_asked(scene, requests, rules, meter=meter) return [ f"check {c} is unsure and needs {gap}, not past an end the result asks to walk " f"past, but there is no reason unobserved_area and no {band} request" for c in unseen if c in rules.needs - for band, gap in reach_gaps(asked, rules, c, spot["wall_id"], lo, hi) + for band, gap in reach_gaps(asked, rules, c, spot["wall_id"], lo, hi, meter=meter) ] if not requests: return ["reason unobserved_area but missing_evidence is empty"] problems: list[str] = [] named = {(r.get("band"), c) for r in requests for c in r.get("checks", [])} + requested_ids = {check for _, check in named} problems += [ f"check {c} is unsure (unobserved) but no missing_evidence entry names it" for c in unseen - if not any(check == c for _, check in named) + if c not in requested_ids ] if rules is None or spot is None: return problems lo, hi = spot["span_ft"] - asked = with_past_ends_asked(scene, requests, rules) - for c in unobserved_checks(result): + asked = with_past_ends_asked(scene, requests, rules, meter=meter) + for c in unseen: if c not in rules.needs: continue - for band, gap in reach_gaps(asked, rules, c, spot["wall_id"], lo, hi): + for band, gap in reach_gaps(asked, rules, c, spot["wall_id"], lo, hi, meter=meter): if (band, c) not in named: problems.append( f"check {c} is unsure and needs {gap}, but no {band} request names it" @@ -886,7 +1457,18 @@ def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = Point = tuple[float, float] -def segments_s(scene: dict, rules: RuleSet) -> list[tuple[str, float, float, Point, Point]] | None: +def band_entries(scene: dict, band: str, meter: WorkMeter) -> list[tuple[float, float, float]]: + def build() -> list[tuple[float, float, float]]: + entries = scene.get("coverage", {}).get("observed", []) + meter.charge(len(entries)) + return [(*sorted(e["span_ft"]), reached(e)) for e in entries if e["band"] == band] + + return meter.derived(f"coverage_entries:{band}", scene, None, build) + + +def segments_s( + scene: dict, rules: RuleSet, meter: WorkMeter | None = None +) -> list[tuple[str, float, float, Point, Point]] | None: """Each straight baseline segment as (wall id, s at its start, s at its end, start point, end point), with s measured from the meter's projection, or None without a meter wall. @@ -894,6 +1476,10 @@ def segments_s(scene: dict, rules: RuleSet) -> list[tuple[str, float, float, Poi when it is a gap: wider than both walls' own errors together (no drift), capped at sweep.wall_join_ft. Narrower, the walls meet and s skips the space (server/scene.py at 7133283, `meets`).""" + meter = meter if meter is not None else WorkMeter() + key = ("segments_s", meter.input_key(scene), meter.input_key(rules)) + if key in meter.cache: + return meter.cache[key] if rules.wall_join_ft is None: raise ValueError("laying out the walls needs the rules' sweep.wall_join_ft") walls = scene.get("walls", []) @@ -904,12 +1490,14 @@ def segments_s(scene: dict, rules: RuleSet) -> list[tuple[str, float, float, Poi mx, _, mz = scene["meter"]["pos"] s, prev = 0.0, None for wall in walls: + meter.charge(len(wall["baseline"])) pts = [(float(pt[0]), float(pt[1])) for pt in wall["baseline"]] if prev is not None: space = math.dist(prev[0], pts[0]) meets = min(rules.wall_join_ft, max(prev[1] + wall_error(wall, rules), SERVER_EPS)) s += space if space > meets else 0.0 for p, q in itertools.pairwise(pts): + meter.charge() seg = math.dist(p, q) if wall["id"] == scene["meter"]["wall_id"] and seg > 0: t = ((mx - p[0]) * (q[0] - p[0]) + (mz - p[1]) * (q[1] - p[1])) / seg**2 @@ -922,21 +1510,56 @@ def segments_s(scene: dict, rules: RuleSet) -> list[tuple[str, float, float, Poi prev = (pts[-1], wall_error(wall, rules)) if meter_s is None: return None - return [(wid, a - meter_s[1], b - meter_s[1], p, q) for wid, a, b, p, q in raw] + merged: list[tuple[str, float, float, Point, Point]] = [] + # Only redundant vertices on the same straight wall are removed. Keep wall IDs, + # s distances and real turns, since wall errors and corner slides depend on them. + for wid, a, b, p, q in raw: + if b - a <= SERVER_EPS: + continue + if merged and merged[-1][0] == wid: + prev_wid, prev_a, _, start, end = merged[-1] + ux, uz = end[0] - start[0], end[1] - start[1] + vx, vz = q[0] - p[0], q[1] - p[1] + cross = abs(ux * vz - uz * vx) + if ( + cross <= SERVER_EPS * math.hypot(ux, uz) * math.hypot(vx, vz) + and ux * vx + uz * vz > 0 + ): + merged[-1] = (prev_wid, prev_a, b, start, q) + continue + merged.append((wid, a, b, p, q)) + meter.cache[key] = [(wid, a - meter_s[1], b - meter_s[1], p, q) for wid, a, b, p, q in merged] + return meter.cache[key] -def wall_spans_s(scene: dict, rules: RuleSet) -> dict[str, tuple[float, float]] | None: +def wall_spans_s( + scene: dict, rules: RuleSet, meter: WorkMeter | None = None +) -> dict[str, tuple[float, float]] | None: """Each wall's stretch of s (see segments_s), or None without a meter wall.""" - segments = segments_s(scene, rules) + meter = meter if meter is not None else WorkMeter() + key = ("wall_spans_s", meter.input_key(scene), meter.input_key(rules)) + if key in meter.cache: + return meter.cache[key] + segments = segments_s(scene, rules, meter=meter) if segments is None: return None spans: dict[str, tuple[float, float]] = {} for wid, a, b, _, _ in segments: + meter.charge() lo, hi = spans.get(wid, (a, b)) spans[wid] = (min(lo, a), max(hi, b)) + meter.cache[key] = spans return spans +def wall_lookup(scene: dict, meter: WorkMeter) -> dict[str, dict]: + return meter.derived("wall_lookup", scene, None, lambda: {w["id"]: w for w in scene["walls"]}) + + +def ground_plan(scene: dict, rules: RuleSet, meter: WorkMeter) -> GroundPlan: + return meter.derived("ground_plan", scene, rules, lambda: GroundPlan(scene, rules, meter=meter)) + + class GroundPlan: """The observed ground in plan: each entry's stretch of every segment it runs along, out to its `out_ft` on the segment's outward side (the baseline turned 90 degrees clockwise seen @@ -947,18 +1570,32 @@ class GroundPlan: # point of a region was seen: a thin gap shaped to avoid them can still be missed. STEP_FT = 0.05 - def __init__(self, scene: dict, rules: RuleSet) -> None: - self.segments = segments_s(scene, rules) or [] - self.strips = [] # (origin, along, outward, from s, to s, out) - for e in scene.get("coverage", {}).get("observed", []): - if e["band"] != "ground": - continue - a, b = sorted(e["span_ft"]) - for _, s0, s1, p, q in self.segments: - if min(b, s1) - max(a, s0) > EPS and s1 - s0 > EPS: - along, outward = self._frame(p, q) - strip = (p, along, outward, max(a, s0) - s0, min(b, s1) - s0) - self.strips.append((*strip, e.get("out_ft", 0.0))) + def __init__(self, scene: dict, rules: RuleSet, meter: WorkMeter | None = None) -> None: + self.meter = meter if meter is not None else WorkMeter() + self.segments = segments_s(scene, rules, meter=self.meter) or [] + self.scene = scene + self.rules = rules + + @property + def strips(self) -> list: + # Direct s-coverage and spot positioning need no ground polygons. Construct + # these strips only when an uncovered reach actually needs the plan fallback. + def build() -> list: + strips = [] # (origin, along, outward, from s, to s, out) + for e in self.scene.get("coverage", {}).get("observed", []): + self.meter.charge() + if e["band"] != "ground": + continue + a, b = sorted(e["span_ft"]) + for _, s0, s1, p, q in self.segments: + self.meter.charge() + if min(b, s1) - max(a, s0) > EPS and s1 - s0 > EPS: + along, outward = self._frame(p, q) + strip = (p, along, outward, max(a, s0) - s0, min(b, s1) - s0) + strips.append((*strip, e.get("out_ft", 0.0))) + return strips + + return self.meter.derived("ground_strips", self.scene, self.rules, build) @staticmethod def _frame(p: Point, q: Point) -> tuple[Point, Point]: @@ -968,7 +1605,12 @@ def _frame(p: Point, q: Point) -> tuple[Point, Point]: def point(self, s_mid: float, s: float, out: float) -> Point | None: """The plan point at s along the line of the segment under s_mid, `out` in front.""" - seg = next((g for g in self.segments if g[1] - EPS <= s_mid <= g[2] + EPS), None) + seg = None + for candidate in self.segments: + self.meter.charge() + if candidate[1] - EPS <= s_mid <= candidate[2] + EPS: + seg = candidate + break if seg is None: return None _, s0, _, p, q = seg @@ -978,6 +1620,7 @@ def point(self, s_mid: float, s: float, out: float) -> Point | None: def seen(self, x: Point) -> bool: for origin, along, outward, t0, t1, out in self.strips: + self.meter.charge() dx, dz = x[0] - origin[0], x[1] - origin[1] t = dx * along[0] + dz * along[1] n = dx * outward[0] + dz * outward[1] @@ -991,6 +1634,7 @@ def required(self, x: Point) -> bool: front of some wall segment, or past either end of the chain. Past a corner, behind the next wall, is neither: the house.""" for i, (_, _, _, p, q) in enumerate(self.segments): + self.meter.charge() along, outward = self._frame(p, q) dx, dz = x[0] - p[0], x[1] - p[1] t = dx * along[0] + dz * along[1] @@ -1015,45 +1659,73 @@ def covers( if not self.strips: return False for s in self._samples(p, q): + self.meter.charge() # Up to out_lo the stretch's own band already saw the ground. - deeper = [o for o in self._samples(out_lo, need(s)) if o > out_lo + SERVER_EPS] - for out in deeper: + for out in self._samples(out_lo, need(s)): + self.meter.charge() + if out <= out_lo + SERVER_EPS: + continue x = self.point(s_mid, s, out) if x is None or (self.required(x) and not self.seen(x)): return False return True @classmethod - def _samples(cls, a: float, b: float) -> list[float]: - n = max(1, math.ceil((b - a) / cls.STEP_FT)) - edges = [a + (b - a) * i / n for i in range(n + 1)] - centres = [a + (b - a) * (i + 0.5) / n for i in range(n)] - return sorted(edges + centres) - - -def chain_ends_s(scene: dict, rules: RuleSet) -> tuple[float, float] | None: + def _samples(cls, a: float, b: float) -> Iterator[float]: + if not finite(a) or not finite(b): + raise ValueError("ground sampling bounds must be finite") + cells = (b - a) / cls.STEP_FT + if not finite(cells): + raise ValueError("ground sampling span must be finite") + n = max(1, math.ceil(max(0, cells))) + + def points(): + for i in range(n): + yield a + (b - a) * (i / n) + yield a + (b - a) * ((i + 0.5) / n) + yield b + + return points() + + +def chain_ends_s( + scene: dict, rules: RuleSet, meter: WorkMeter | None = None +) -> tuple[float, float] | None: """s of the wall chain's left and right ends, or None without a meter wall.""" - spans = wall_spans_s(scene, rules) + meter = meter if meter is not None else WorkMeter() + key = ("chain_ends_s", meter.input_key(scene), meter.input_key(rules)) + if key in meter.cache: + return meter.cache[key] + spans = wall_spans_s(scene, rules, meter=meter) if spans is None: return None - return min(a for a, _ in spans.values()), max(b for _, b in spans.values()) + meter.cache[key] = min(a for a, _ in spans.values()), max(b for _, b in spans.values()) + return meter.cache[key] -def declared_height_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: +def declared_height_problems( + scene: dict, result: dict, rules: RuleSet, meter: WorkMeter | None = None +) -> list[str]: """A battery backs onto a wall that declares `height_ft` only if the wall is taller than the battery. The server fails wall_backing for a lower wall surely behind the battery and leaves it unsure for one within the battery's error of it (server/solver.py at 7133283, check_backing), so wall_backing passes only if every declared wall overlapping [s0 - e, s1 + e] is taller than battery.height_ft.""" + meter = meter if meter is not None else WorkMeter() + if sampling := geometry_admission_problems(scene, result, rules, meter=meter): + return sampling + result = deduplicate_result_runs(result) declared = {w["id"]: w["height_ft"] for w in scene.get("walls", []) if "height_ft" in w} if not declared or rules.battery_height_ft is None: return [] - spans = wall_spans_s(scene, rules) or {} + spans = wall_spans_s(scene, rules, meter=meter) or {} height = rules.battery_height_ft def short_near(wall_id: str, lo: float, hi: float) -> str | None: - e = battery_error(scene, rules, wall_id, lo, hi) + meter.charge() + e = battery_error(scene, rules, wall_id, lo, hi, meter=meter) for wid, h in declared.items(): + meter.charge() a, b = spans.get(wid, (math.inf, -math.inf)) near = a < hi + e - SERVER_EPS and b > lo - e + SERVER_EPS if near and h - height <= SERVER_EPS: @@ -1071,6 +1743,7 @@ def short_near(wall_id: str, lo: float, hi: float) -> str | None: if run["outcome"] != "pass": continue for start in run_starts(run["start_ft"], rules.step_ft): + meter.charge() short = short_near(run["wall_id"], start, start + rules.width_ft) if short: problems.append( @@ -1080,21 +1753,25 @@ def short_near(wall_id: str, lo: float, hi: float) -> str | None: return problems -def with_past_ends_asked(scene: dict, requests: list[dict], rules: RuleSet) -> dict: +def with_past_ends_asked( + scene: dict, requests: list[dict], rules: RuleSet, meter: WorkMeter | None = None +) -> dict: """The scene as if everything past each end the result asks to walk past were observed. Past an unexplored end the wall may turn, so a view along the same line settles nothing; the server asks to keep walking (`past_end`) instead of for a band there, and that request covers every band beyond the end. Only a request at the chain's actual end, on a side not marked `limit`, earns that credit.""" + meter = meter if meter is not None else WorkMeter() kinds = scene.get("coverage", {}).get("ends", {}) past = [r for r in requests if r["kind"] == "past_end" and "span_ft" in r and "side" in r] - ends = chain_ends_s(scene, rules) if past else None + ends = chain_ends_s(scene, rules, meter=meter) if past else None if not past or ends is None: return scene m = copy.deepcopy(scene) observed_ = m.setdefault("coverage", {}).setdefault("observed", []) for r in past: + meter.charge() side = r["side"] end = ends[0] if side == "left" else ends[1] if kinds.get(side, {}).get("kind") == "limit" or abs(r["span_ft"][0] - end) > END_TOL_FT: @@ -1109,31 +1786,162 @@ def with_past_ends_asked(scene: dict, requests: list[dict], rules: RuleSet) -> d # --- Case expectations ----------------------------------------------------------------------- +def case_metadata_problems(assumed: dict, expect: dict | None = None) -> list[str]: + """Bound local case queries too: substring expectations scan the response once each.""" + if not isinstance(assumed, dict): + return ["case rules_assumed must be an object"] + if expect is None: + expect = {} + if not isinstance(expect, dict): + return ["case expect must be an object"] + for field in ("sweep_runs", "checks", "start_outcomes"): + if field in expect and not isinstance(expect[field], list): + return [f"case expect.{field} must be an array"] + queries = len(assumed) + sum( + len(expect.get(field, [])) for field in ("sweep_runs", "checks", "start_outcomes") + ) + if queries > MAX_CASE_RULE_QUERIES: + return [f"case ingestion limit exceeded: more than {MAX_CASE_RULE_QUERIES} rule queries"] + return ingestion_problems({"assumed": assumed, "expect": expect}, "case") + + +def reported_assumption_values(key: str, result: dict) -> list[tuple[str, float]]: + """Rule values carried by C2 spots or check thresholds, without reading server files.""" + values = [] + if key == "battery_width": + for name in ("spot", "nearest_considered"): + spot = result.get(name) + if spot is not None and "width_ft" in spot: + values.append((f"{name}.width_ft", spot["width_ft"])) + rule_key = { + "battery_width": "battery.width_ft", + "corner_allowance": "route.corner_allowance_ft", + }.get(key) + for check in result.get("checks", []): + threshold = check.get("threshold_ft") + matches = check.get("rule", {}).get("key") == rule_key if rule_key else key in check["id"] + if matches and threshold is not None: + values.append((check["id"], threshold)) + return values + + +def reported_policy_overrides(result: dict, rules: RuleSet) -> bool: + """A reported fingerprint must identify a policy other than the loaded public rules.""" + policy = result.get("policy", {}) + fingerprint = policy.get("rules_sha256") + if not isinstance(fingerprint, str): + return False + if rules.rules_sha256 is not None: + return fingerprint != rules.rules_sha256 + return "private" in policy.get("sources", []) + + +def public_dimension_problems(result: dict, rules: RuleSet) -> list[str]: + if reported_policy_overrides(result, rules): + return [] + values = [] + for name in ("spot", "nearest_considered"): + spot = result.get(name) + if spot is not None: + for field, expected in ( + ("width_ft", rules.width_ft), + ("depth_ft", rules.depth_ft), + ("height_ft", rules.battery_height_ft), + ): + if field in spot and expected is not None: + values.append((f"{name}.{field}", spot[field], expected)) + for key, expected in ( + ("battery_width", rules.width_ft), + ("corner_allowance", rules.corner_allowance_ft), + ): + if expected is not None: + values.extend( + (source, actual, expected) + for source, actual in reported_assumption_values(key, result) + ) + if result.get("route") is not None and "height_ft" in result["route"]: + values.append(("route.height_ft", result["route"]["height_ft"], rules.route_height_ft)) + configured = { + "battery.depth_ft": rules.depth_ft, + "battery.height_ft": rules.battery_height_ft, + "route.height_ft": rules.route_height_ft, + } + for check in result.get("checks", []): + expected = configured.get(check.get("rule", {}).get("key")) + actual = check.get("threshold_ft") + if expected is not None and actual is not None: + values.append((check["id"], actual, expected)) + # C2 rounds dimensions to six decimals; preserve that representation tolerance. + return _unique_messages( + [ + f"{source} {actual} does not match loaded public rules {expected}" + for source, actual, expected in values + if abs(actual - expected) > NUMBER_TOL_FT + ] + ) + + +def reported_rules(result: dict, rules: RuleSet) -> RuleSet: + """Use validated reported geometry only for an explicitly different policy.""" + if problems := reported_dimension_problems(result, rules): + raise ValueError("; ".join(problems)) + if not reported_policy_overrides(result, rules): + return rules + changes = {} + for key, field in (("battery_width", "width_ft"), ("corner_allowance", "corner_allowance_ft")): + if values := reported_assumption_values(key, result): + changes[field] = values[0][1] + return replace(rules, **changes) + + def assumption_mismatches(assumed: dict[str, float], result: dict) -> list[str]: - out = [] - for key, value in assumed.items(): - for check in result.get("checks", []): - threshold = check.get("threshold_ft") - if key in check["id"] and threshold is not None and abs(threshold - value) > EPS: - out.append( - f"case assumes {key} threshold {value}, server's {check['id']} uses " - f"{check['threshold_ft']}" - ) - return out + if problems := case_metadata_problems(assumed): + return problems + return [ + f"case assumes {key} threshold {value}, server's {source} uses {actual}" + for key, value in assumed.items() + for source, actual in reported_assumption_values(key, result) + if abs(actual - value) > EPS + ] -def expectation_problems(expect: dict, result: dict) -> list[str]: - problems: list[str] = [] +def public_assumption_warnings( + assumed: dict[str, float], result: dict, rules: RuleSet +) -> list[str]: + if problems := case_metadata_problems(assumed): + return problems + configured = {"battery_width": rules.width_ft, "corner_allowance": rules.corner_allowance_ft} + policy = result.get("policy", {}) + nonpublic = "private" in policy.get("sources", []) or ( + rules.rules_sha256 is not None + and policy.get("rules_sha256") is not None + and policy["rules_sha256"] != rules.rules_sha256 + ) + return [ + f"case assumes {key} {value}, public rules use {configured[key]}; " + "assumed public rules; server may override; result does not report this value, " + "so case expectations are unverified" + for key, value in assumed.items() + if configured.get(key) is not None + and not reported_assumption_values(key, result) + and (nonpublic or abs(configured[key] - value) > EPS) + ] + + +def expectation_problems(expect: dict, result: dict, meter: WorkMeter | None = None) -> list[str]: + if admission := case_metadata_problems({}, expect): + return admission + problems = DiagnosticMessages() decision = result["decision"] if "decision_in" in expect and decision not in expect["decision_in"]: - problems.append(f"decision {decision}, expected one of {expect['decision_in']}") + problems.append("decision {}, expected one of {}", decision, expect["decision_in"]) if decision in expect.get("decision_not", []): - problems.append(f"decision {decision} is ruled out for this case") + problems.append("decision {} is ruled out for this case", decision) if "spot" in expect: want, spot = expect["spot"], result.get("spot") if want is None and spot is not None: - problems.append(f"expected no spot, got {spot['wall_id']} {spot['span_ft']}") + problems.append("expected no spot, got {} {}", spot["wall_id"], spot["span_ft"]) if want is not None: if spot is None: problems.append("expected a spot, got none") @@ -1142,8 +1950,11 @@ def expectation_problems(expect: dict, result: dict) -> list[str]: s0, s1 = spot["span_ft"] if spot["wall_id"] != want["wall_id"] or s0 < a - 1 / 12 or s1 > b + 1 / 12: problems.append( - f"spot {spot['wall_id']} {spot['span_ft']} outside expected " - f"{want['wall_id']} {want['span_within']}" + "spot {} {} outside expected {} {}", + spot["wall_id"], + spot["span_ft"], + want["wall_id"], + want["span_within"], ) for rule in expect.get("sweep_runs", []): @@ -1154,62 +1965,109 @@ def expectation_problems(expect: dict, result: dict) -> list[str]: if r["wall_id"] == rule["wall_id"] and r["start_ft"][0] < b and r["start_ft"][1] > a ] if not runs and "outcome" in rule: - problems.append(f"no sweep run overlaps {rule['wall_id']} starts ({a}, {b})") + problems.append("no sweep run overlaps {} starts ({}, {})", rule["wall_id"], a, b) + failed = 0 + examples = [] for run in runs: - if "outcome" in rule and run["outcome"] != rule["outcome"]: - problems.append( - f"sweep {run['wall_id']} starts {run['start_ft']} is {run['outcome']}, " - f"expected {rule['outcome']}" - ) - if "outcome_not" in rule and run["outcome"] == rule["outcome_not"]: - why = f" ({rule['reason']})" if "reason" in rule else "" - problems.append( - f"sweep {run['wall_id']} starts {run['start_ft']} is {run['outcome']}, " - f"which this case rules out{why}" + wrong = ( + ("outcome" in rule and run["outcome"] != rule["outcome"]) + or ("outcome_not" in rule and run["outcome"] == rule["outcome_not"]) + or any( + key in rule and not any(rule[key] in check for check in run[field]) + for field, key in (("failing", "failing_match"), ("unsure", "unsure_match")) ) - for field, key in (("failing", "failing_match"), ("unsure", "unsure_match")): - if key in rule and not any(rule[key] in c for c in run[field]): - problems.append( - f"sweep {run['wall_id']} starts {run['start_ft']}: no {field} check " - f"matching {rule[key]!r} (has {run[field]})" + ) + if wrong: + failed += 1 + if len(examples) < MAX_EXPECTATION_EXAMPLES: + examples.append( + f"{run['wall_id']} starts {run['start_ft']} is {run['outcome']}" ) + if failed: + required = { + key: rule[key] + for key in ("outcome", "outcome_not", "failing_match", "unsure_match") + if key in rule + } + why = f"; reason: {rule['reason']}" if "reason" in rule else "" + problems.append( + "sweep expectation {} starts ({}, {}) failed for {} runs; " + "expected {}; examples: {}{}", + rule["wall_id"], + a, + b, + failed, + required, + "; ".join(examples), + why, + ) for rule in expect.get("checks", []): matching = [c for c in result.get("checks", []) if rule["match"] in c["id"]] if not matching: - problems.append(f"no check matching {rule['match']!r}") + problems.append("no check matching {!r}", rule["match"]) + failed = 0 + examples = [] for check in matching: - for field in ("outcome", "unsure_cause"): - if field in rule and check.get(field) != rule[field]: - problems.append( - f"check {check['id']}: {field} {check.get(field)}, expected {rule[field]}" + wrong = [ + field + for field in ("outcome", "unsure_cause") + if field in rule and check.get(field) != rule[field] + ] + # The public solver rounds measurements to six decimals. + wrong += [ + field + for field in ("measured_ft", "plus_minus_ft") + if field in rule + and (check.get(field) is None or abs(check[field] - rule[field]) > NUMBER_TOL_FT) + ] + if wrong: + failed += 1 + if len(examples) < MAX_EXPECTATION_EXAMPLES: + examples.append( + "; ".join( + f"check {check['id']}: {field} {check.get(field)}, " + f"expected {rule[field]}" + for field in wrong + ) ) - # Pinned numbers catch a server that measures from somewhere else (for example - # another battery depth) and lands on the right outcome by accident. The server - # rounds to 6 decimals. - for field in ("measured_ft", "plus_minus_ft"): - got = check.get(field) - if field in rule and (got is None or abs(got - rule[field]) > NUMBER_TOL_FT): - problems.append(f"check {check['id']}: {field} {got}, expected {rule[field]}") - - for rule in expect.get("start_outcomes", []): - got = outcome_at(result, rule["wall_id"], rule["start_ft"]) + if failed: + why = f"; reason: {rule['reason']}" if "reason" in rule else "" + if failed == 1: + problems.append("{}{}", examples[0], why) + else: + problems.append( + "check expectation {!r} failed for {} checks; examples: {}{}", + rule["match"], + failed, + "; ".join(examples), + why, + ) + + starts = expect.get("start_outcomes", []) + index = SweepIndex(result, meter=meter) if starts else None + for rule in starts: + got = index.outcome_at(rule["wall_id"], rule["start_ft"]) if got != rule["outcome"]: why = f" ({rule['why']})" if "why" in rule else "" problems.append( - f"start {rule['wall_id']} {rule['start_ft']} is {got or 'not evaluated'}, " - f"expected {rule['outcome']}{why}" + "start {} {} is {}, expected {}{}", + rule["wall_id"], + rule["start_ft"], + got or "not evaluated", + rule["outcome"], + why, ) if "missing_evidence_empty" in expect: empty = not result.get("missing_evidence") if empty != expect["missing_evidence_empty"]: problems.append( - "missing_evidence " - + ("is empty" if empty else "is not empty") - + f", expected {'empty' if expect['missing_evidence_empty'] else 'requests'}" + "missing_evidence {}, expected {}", + "is empty" if empty else "is not empty", + "empty" if expect["missing_evidence_empty"] else "requests", ) - return problems + return problems.finish() # --- Transforms for black-box properties ----------------------------------------------------- @@ -1279,50 +2137,30 @@ def comparable(result: dict) -> dict: # numbers: more measurement error or less coverage can only make the server less sure. -def outcome_at(result: dict, wall_id: str, start_ft: float) -> str | None: - """Outcome of the sweep run containing this battery start, or None if none contains it.""" - for run in result.get("sweep", []): - a, b = run["start_ft"] - if run["wall_id"] == wall_id and a - EPS <= start_ft <= b + EPS: - return run["outcome"] - return None +def outcome_at( + result: dict, wall_id: str, start_ft: float, meter: WorkMeter | None = None +) -> str | None: + meter = meter if meter is not None else WorkMeter() + return SweepIndex(result, meter=meter).outcome_at(wall_id, start_ft) -def probe_starts(*results: dict) -> list[tuple[str, float]]: - """Every run's ends and middle, from all results: where the outcomes can be compared.""" - points = set() - for result in results: - for run in result.get("sweep", []): - a, b = run["start_ft"] - points |= {(run["wall_id"], a), (run["wall_id"], b), (run["wall_id"], (a + b) / 2)} - return sorted(points) +def probe_starts(*results: dict, meter: WorkMeter | None = None) -> list[tuple[str, float]]: + meter = meter if meter is not None else WorkMeter() + return indexed_probe_starts(*results, meter=meter) -def more_error_problems(before: dict, after: dict, change: str) -> list[str]: - """With more error, a start may only move toward UNSURE: UNSURE stays, PASS and FAIL stay - or become UNSURE.""" - problems = [] - for wall_id, s in probe_starts(before, after): - a, b = outcome_at(before, wall_id, s), outcome_at(after, wall_id, s) - if a is None or b is None or a == b or b == "unsure": - continue - problems.append(f"{change}: start {wall_id} {s:.3f} went from {a} to {b}") - if before["decision"] != "pass" and after["decision"] == "pass": - problems.append(f"{change}: the decision became pass") - return problems +def more_error_problems( + before: dict, after: dict, change: str, meter: WorkMeter | None = None +) -> list[str]: + meter = meter if meter is not None else WorkMeter() + return indexed_more_error_problems(before, after, change, meter=meter) -def less_coverage_problems(before: dict, after: dict, change: str) -> list[str]: - """With less observed, no start may newly PASS.""" - problems = [ - f"{change}: start {wall_id} {s:.3f} went from {outcome_at(before, wall_id, s)} to pass" - for wall_id, s in probe_starts(before, after) - if outcome_at(after, wall_id, s) == "pass" - and outcome_at(before, wall_id, s) not in (None, "pass") - ] - if before["decision"] != "pass" and after["decision"] == "pass": - problems.append(f"{change}: the decision became pass") - return problems +def less_coverage_problems( + before: dict, after: dict, change: str, meter: WorkMeter | None = None +) -> list[str]: + meter = meter if meter is not None else WorkMeter() + return indexed_less_coverage_problems(before, after, change, meter=meter) def chain_length_ft(scene: dict) -> float: @@ -1394,7 +2232,7 @@ def with_ground_short_of(scene: dict, rules: RuleSet, margin_ft: float = 0.1) -> m = copy.deepcopy(scene) for entry in m["coverage"]["observed"]: if entry["band"] == "ground": - entry["out_ft"] = min(entry["out_ft"], reach) + entry["out_ft"] = min(entry.get("out_ft", 0.0), reach) return m diff --git a/verification/hsverify/scoreboard.py b/verification/hsverify/scoreboard.py index 3cca83fe9..725557158 100644 --- a/verification/hsverify/scoreboard.py +++ b/verification/hsverify/scoreboard.py @@ -750,7 +750,19 @@ def render_markdown(board: Board) -> str: def list_prs() -> list[dict]: out = subprocess.run( - ["gh", "pr", "list", "--repo", GITHUB_REPO, "--state", "open", "--json", PR_FIELDS], + [ + "gh", + "pr", + "list", + "--repo", + GITHUB_REPO, + "--state", + "open", + "--limit", + "10000", + "--json", + PR_FIELDS, + ], check=True, capture_output=True, text=True, diff --git a/verification/hsverify/simrun.py b/verification/hsverify/simrun.py index 235d96d8e..01efff5be 100644 --- a/verification/hsverify/simrun.py +++ b/verification/hsverify/simrun.py @@ -8,7 +8,10 @@ a screenshot once it has been on screen for `--settle` seconds; a state replaced sooner is captured immediately and flagged `transient`, because its image may already show the next state. The run ends at `--until`, after `--idle` seconds without a new state, or at -`--timeout`. A state named with `--require` that never appeared makes the run fail. If the app +`--timeout`. States named with `--require` must appear in the order given, as do +states missing from a `--require-order` sequence or out of its order, and a FIRST state of a +`--require-then FIRST:THEN` pair that no THEN state answered before the next FIRST or the +last state of a `--require-order` sequence. If the app logged a scan bundle, it is copied into the report as scan.zip (required with `--require-export`) and named in report.json's `app_export`, so the report folder can be replayed as an app export. Reports go outside git @@ -30,6 +33,7 @@ import shutil import subprocess import sys +import tempfile import threading import time from collections.abc import Iterable @@ -52,6 +56,7 @@ DEFAULT_REPORTS = Path.home() / "house-scanning-data" / "reports" / "sim" DERIVED_DATA = Path("/tmp/hs-verify-derived-data") BUILD_LOCK = Path("/tmp/hs-verify-xcodebuild.lock") +DEVICE_LOCK = Path("/tmp/hs-verify-simulator.lock") BUILT_APP_DIR = "built-app" # in the report folder, until the Simulator has installed it @@ -219,11 +224,14 @@ def build_app( built = DERIVED_DATA / "Build" / "Products" / "Debug-iphonesimulator" / f"{scheme}.app" app = None if code == 0 and built.exists(): - app = out / BUILT_APP_DIR / built.name - shutil.rmtree(app.parent, ignore_errors=True) - shutil.copytree(built, app, symlinks=True) + app = Path(tempfile.mkdtemp(prefix=f"{BUILT_APP_DIR}-", dir=out)) / built.name + try: + shutil.copytree(built, app, symlinks=True) + except BaseException: + discard_app(app) + raise text = log_path.read_text(errors="replace") - warnings = sorted(set(re.findall(r"^.*: warning: .*$", text, re.M))) + warnings = sorted(set(re.findall(r"^(?:.*: )?warning: .*$", text, re.M))) errors = sorted(set(re.findall(r"^.*: error: .*$", text, re.M))) result = { "ok": code == 0, @@ -239,7 +247,7 @@ def build_app( def discard_app(app: Path) -> None: """Remove this run's copy of the app once the Simulator has installed its own.""" - if app.parent.name == BUILT_APP_DIR: + if app.parent.name.startswith(f"{BUILT_APP_DIR}-"): shutil.rmtree(app.parent, ignore_errors=True) @@ -326,7 +334,68 @@ def wait_for_log_stream(events: queue.Queue, timeout_s: float) -> str | None: def missing_required(required: list[str], states: list[ShotRecord]) -> list[str]: seen = {s.state for s in states} - return [f"required state {name} never appeared" for name in required if name not in seen] + missing = [f"required state {name} never appeared" for name in required if name not in seen] + if missing: + return missing + remaining = iter(required) + next_required = next(remaining, None) + for state in states: + if state.state == next_required: + next_required = next(remaining, None) + if next_required is not None: + return [f"required states did not appear in order: {' -> '.join(required)}"] + return [] + + +def out_of_order(order: list[str], states: list[ShotRecord]) -> list[str]: + """`order` must appear among the run's states in that order; other states may come between. + Reaching the last one is not enough: a run that skips the steps before it did not walk them.""" + next_required = 0 + closing = order[-1] if order else None + for record in states: + if next_required == len(order): + return [] + if record.state == order[next_required]: + next_required += 1 + elif record.state == closing: + return [f"required order {' > '.join(order)}: {closing} before {order[next_required]}"] + if next_required < len(order): + want = order[next_required] + after = f"after {order[next_required - 1]}" if next_required else "at all" + return [f"required order {' > '.join(order)}: no {want} {after}"] + return [] + + +def unanswered( + pairs: list[tuple[str, str]], states: list[ShotRecord], closing: frozenset[str] = frozenset() +) -> list[str]: + """For each (first, then): every `first` state must be followed by a `then` state within its + own round, which ends at the next `first` or at a `closing` state. A later upload therefore + cannot answer a request after the app showed a result, nor answer two requests at once.""" + names = [s.state for s in states] + problems = [] + for first, then in pairs: + for i, name in enumerate(names): + if name != first: + continue + for later in names[i + 1 :]: + if later == then: + break + if later == first or later in closing: + problems.append( + f"state {i + 1} {first} was not followed by {then} before {later}" + ) + break + else: + problems.append(f"state {i + 1} {first} was never followed by {then}") + return problems + + +def state_pair(text: str) -> tuple[str, str]: + first, sep, then = text.partition(":") + if not (sep and first and then): + raise argparse.ArgumentTypeError(f"expected FIRST:THEN, got {text!r}") + return first, then def logged_message(line: str) -> str: @@ -529,6 +598,16 @@ def collect_crashes(since: float, out: Path) -> list[str]: return found +@contextlib.contextmanager +def device_lock(): + """Hold the shared Simulator until install, capture and shutdown have finished.""" + DEVICE_LOCK.touch() + with DEVICE_LOCK.open() as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + yield + + +@device_lock() def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--ref", default="origin/main", help="git ref to build") @@ -548,7 +627,24 @@ def main(argv: list[str] | None = None) -> int: action="append", default=[], metavar="STATE", - help="a state the run must reach; one that never appears is a problem (exit 1)", + help="a state the run must reach; repeat in the required journey order (exit 1 on failure)", + ) + parser.add_argument( + "--require-order", + action="append", + default=[], + type=lambda text: text.split(","), + metavar="STATE,STATE,...", + help="states the run must pass through in this order (others may come between)", + ) + parser.add_argument( + "--require-then", + action="append", + default=[], + type=state_pair, + metavar="FIRST:THEN", + help="every FIRST state must be followed by a THEN state before the next FIRST or the last " + "state of a --require-order sequence", ) parser.add_argument( "--require-export", @@ -608,7 +704,12 @@ def main(argv: list[str] | None = None) -> int: flush=True, ) - device = ensure_device(args.udid) + try: + device = ensure_device(args.udid) + except BaseException: + if app is not None: + discard_app(app) + raise rep = RunReport( ref=args.ref, sha=sha, @@ -648,12 +749,19 @@ def main(argv: list[str] | None = None) -> int: print("Running...", flush=True) follow(args, udid, bundle_id, out, rep) rep.problems += missing_required(args.require, rep.states) + for order in args.require_order: + rep.problems += out_of_order(order, rep.states) + # Each --require-order sequence ends at the state that closes a round (the result). + closing = frozenset(order[-1] for order in args.require_order) + rep.problems += unanswered(args.require_then, rep.states, closing) rep.app_export, export_problems = app_export(out, args.require_export) rep.problems += export_problems rep.crash_reports = collect_crashes(started, out) if rep.crash_reports: rep.problems.append(f"Crash reports: {', '.join(rep.crash_reports)}") finally: + if app is not None: + discard_app(app) services.close() rep.peak_memory = peak_rss_mb() if not args.keep_booted: diff --git a/verification/scenes/eth3d-facade/README.md b/verification/scenes/eth3d-facade/README.md index 26fe14937..586cef340 100644 --- a/verification/scenes/eth3d-facade/README.md +++ b/verification/scenes/eth3d-facade/README.md @@ -17,12 +17,18 @@ Data: `facade_dslr_undistorted.7z` and `facade_dslr_scan_eval.7z` from eth3d.net `~/house-scanning-data/verify/eth3d/` (it creates `facade/`). Then, from `verification/`: ``` -uv run python scenes/eth3d-facade/build_scene.py [--schema path/to/scene.schema.json] [--overlays] +uv run python scenes/eth3d-facade/build_scene.py [--schema path/to/scene.schema.json] [--overlays] [--rules-ref origin/t3/server] ``` This writes `~/house-scanning-data/verify/scenes/eth3d-facade/`: `scene.json`, `scene.zip` (scene.json plus 7 keyframes at the top level), `bundle/`, and `build_report.json` with every fit -statistic. `--overlays` also draws the wall chain and the annotations on each keyframe. The run +statistic and the rules ref, commit and headroom the scene was built with. The overhead probe +height is `headroom.min_ft` from `server/rules.yaml` at `--rules-ref` (default +`origin/t3/server`, the e2e harness's default server). `case.json` pins that value in +`rules_assumed.headroom`. When the server under test applies another headroom, because its +rules changed or a private override sets one, its `headroom` check reports that threshold and +the e2e run marks the case `assumption mismatch` instead of judging it. Rebuild the scene +against that ref, then update the pin. `--overlays` also draws the wall chain and the annotations on each keyframe. The run takes about 10 s and is deterministic (fixed RANSAC seed). The output validates against `server/schemas/scene.schema.json` from `origin/t3/server` (44607e0). @@ -64,7 +70,8 @@ takes about 10 s and is deterministic (fixed RANSAC seed). The output validates probe points. A point counts as seen when it is inside the image, on the camera side of its wall, not behind another wall of the chain, and within 75 degrees of the wall's normal. The 75-degree limit is my judgement, not a measured value. The probes are: wall, ground line and - battery height (3.29 ft); ground, out to 6 ft; overhead, 6.5 ft up at 0 and 3 ft out; facing, + battery height (3.29 ft); ground, out to 6 ft; overhead, at the rules' headroom (6.5 ft at + `origin/t3/server` 1d1e7e1) at 0 and 3 ft out; facing, 3 ft up, straight out. No camera sees w2 at better than grazing, so w2 is in no band. - **Overheads and facing** come from the scan. An overhead is anything within 3 ft of the wall, lower than 12 ft, whose underside is above 4 ft. With a 2 ft floor, the tops of the two bins by diff --git a/verification/scenes/eth3d-facade/build_scene.py b/verification/scenes/eth3d-facade/build_scene.py index a5cb9f841..42134121f 100644 --- a/verification/scenes/eth3d-facade/build_scene.py +++ b/verification/scenes/eth3d-facade/build_scene.py @@ -20,6 +20,7 @@ import math import re import shutil +import subprocess import sys import zipfile from dataclasses import dataclass @@ -27,6 +28,7 @@ import jsonschema import numpy as np +import yaml from PIL import Image, ImageDraw HERE = Path(__file__).resolve().parent @@ -45,8 +47,9 @@ COVERAGE_STEP_FT = 0.25 # Base Core height, docs/04 (What Base does today); the wall band's top probe BATTERY_HEIGHT_FT = 39.5 / 12 -# NEC 110.26 headroom, docs/04 (Public rule values); the overhead band's top probe -HEADROOM_FT = 6.5 +# The overhead band's top probe is the rules' headroom.min_ft, read from server/rules.yaml at +# --rules-ref: a rule change then moves the probe instead of leaving it at an old height. +DEFAULT_RULES_REF = "origin/t3/server" # the e2e harness's default server ref GROUND_OUT_FT = 6.0 # how far out the ground band claims to have been seen OVERHEAD_OUT_FT = 3.0 # overhead search depth in front of the wall OVERHEAD_MAX_FT = 12.0 # anything higher than this is not reported as an overhead @@ -445,7 +448,40 @@ def visible(kf: Keyframe, wall: Wall, X, walls) -> bool: FACING_PROBE_MAX_FT = 20.0 # the facing probe looks no further out than this -def band_reach(band: str, facing_depth: float | None) -> float | None: +def rules_headroom_ft(text: str, where: str) -> float: + """headroom.min_ft from the text of a rules.yaml.""" + node = yaml.safe_load(text) + for key in ("headroom", "min_ft", "value"): + if not isinstance(node, dict) or key not in node: + raise SystemExit(f"{where} has no headroom.min_ft.value") + node = node[key] + return float(node) + + +def read_rules_headroom(ref: str) -> tuple[str, float]: + """The commit `ref` names and the headroom.min_ft of its server/rules.yaml.""" + + def git(*args: str) -> str: + done = subprocess.run(["git", *args], cwd=HERE, capture_output=True, text=True) + if done.returncode != 0: + raise SystemExit(f"git {' '.join(args)}: {done.stderr.strip()}") + return done.stdout + + sha = git("rev-parse", "--verify", f"{ref}^{{commit}}").strip() + headroom_ft = rules_headroom_ft(git("show", f"{sha}:server/rules.yaml"), f"rules.yaml at {ref}") + # scan_profiles drops returns at or above OVERHEAD_MAX_FT, so an overhead probe that high + # would read an obstruction above that height as clear space. + if headroom_ft >= OVERHEAD_MAX_FT: + raise SystemExit( + f"rules.yaml at {ref} sets headroom.min_ft to {headroom_ft:g} ft, but the scene only " + f"reports overheads below OVERHEAD_MAX_FT = {OVERHEAD_MAX_FT:g} ft" + ) + return sha, headroom_ft + + +def band_reach( + band: str, facing_depth: float | None, headroom_ft: float | None = None +) -> float | None: """How far a band's probes reach, and so the most a view that sees them all demonstrates: up the wall and up above the ground (wall, overhead), out from the wall (ground, facing). It becomes the entry's `out_ft`: without one, the server reads a wall view as reaching @@ -455,16 +491,20 @@ def band_reach(band: str, facing_depth: float | None) -> float | None: if band == "ground": return GROUND_OUT_FT if band == "overhead": - return HEADROOM_FT + if headroom_ft is None: + raise ValueError("the overhead band is probed at the rules' headroom.min_ft") + return headroom_ft if band == "facing": return None if facing_depth is None else min(facing_depth, FACING_PROBE_MAX_FT) raise ValueError(band) -def band_probes(band: str, wall: Wall, s: float, facing_depth: float | None): +def band_probes( + band: str, wall: Wall, s: float, facing_depth: float | None, headroom_ft: float | None = None +): q = wall.plan_at(s) o = wall.outward - reach = band_reach(band, facing_depth) + reach = band_reach(band, facing_depth, headroom_ft) def at(out, y): p = q + out * o @@ -481,18 +521,20 @@ def at(out, y): return [at(0.0, 3.0), at(reach, 3.0)] -def band_coverage(band, walls, facing, kfs, seen=visible): +def band_coverage(band, walls, facing, kfs, seen=visible, headroom_ft=None): """Observed entries for one band: stretches where some keyframe sees every probe, merged across corners, each with the smallest reach demonstrated inside it as `out_ft`, rounded - down so it never claims more. `seen(kf, wall, X, walls)` is `visible`; tests replace it.""" + down so it never claims more. `seen(kf, wall, X, walls)` is `visible`; tests replace it. + The overhead band needs `headroom_ft`, the rules' headroom.min_ft.""" samples = [] # (s0, s1, reach) per sample seen for w in walls: - for s in np.arange(w.s0 + COVERAGE_STEP_FT / 2, w.s0 + w.length, COVERAGE_STEP_FT): + for a in np.arange(w.s0, w.s0 + w.length, COVERAGE_STEP_FT): + b = min(a + COVERAGE_STEP_FT, w.s0 + w.length) + s = (a + b) / 2 fd = next((v for (wid, a, b, v) in facing if wid == w.id and a <= s < b), None) - probes = band_probes(band, w, s, fd) + probes = band_probes(band, w, s, fd, headroom_ft) if probes is not None and any(all(seen(k, w, X, walls) for X in probes) for k in kfs): - half = COVERAGE_STEP_FT / 2 - samples.append([s - half, s + half, band_reach(band, fd)]) + samples.append([a, b, band_reach(band, fd, headroom_ft)]) merged = [] # consecutive samples, also across a corner, keep the smaller reach for a, b, reach in samples: if merged and abs(merged[-1][1] - a) < 1e-6: @@ -672,6 +714,11 @@ def main(): ap.add_argument("--out", type=Path, default=DEFAULT_OUT) ap.add_argument("--annotations", type=Path, default=HERE / "annotations.json") ap.add_argument("--schema", type=Path, help="validate scene.json against this JSON Schema") + ap.add_argument( + "--rules-ref", + default=DEFAULT_RULES_REF, + help="git ref whose server/rules.yaml sets the overhead probe height (headroom.min_ft)", + ) ap.add_argument( "--overlays", action="store_true", @@ -681,7 +728,8 @@ def main(): ann = json.loads(args.annotations.read_text()) rng = np.random.default_rng(RNG_SEED) - report = {} + rules_sha, headroom_ft = read_rules_headroom(args.rules_ref) + report = {"rules": {"ref": args.rules_ref, "sha": rules_sha, "headroom_min_ft": headroom_ft}} # 1. Laser scan, ground plane, scene frame --------------------------------------------- P = load_scan_crop(args.data / "dslr_scan_eval", ann["roi_scan_xy_m"]) @@ -803,7 +851,7 @@ def main(): observed = [] per_band = {} for band in ("wall", "ground", "overhead", "facing"): - entries, per_band[band] = band_coverage(band, walls, facing, kfs) + entries, per_band[band] = band_coverage(band, walls, facing, kfs, headroom_ft=headroom_ft) observed += entries # 7. Assemble ---------------------------------------------------------------------------- @@ -914,7 +962,10 @@ def main(): print( json.dumps( - {k: report[k] for k in ("ground", "walls", "arkit_projection_check", "reprojection")}, + { + k: report[k] + for k in ("rules", "ground", "walls", "arkit_projection_check", "reprojection") + }, indent=1, default=float, ) diff --git a/verification/scenes/eth3d-facade/case.json b/verification/scenes/eth3d-facade/case.json index 9cffd67cd..e472c939b 100644 --- a/verification/scenes/eth3d-facade/case.json +++ b/verification/scenes/eth3d-facade/case.json @@ -4,7 +4,7 @@ "scene_path": "~/house-scanning-data/verify/scenes/eth3d-facade/scene.zip", "source": "ETH3D 'facade' (dslr_undistorted + dslr_scan_eval, research-only license), south courtyard wing; built by verification/scenes/eth3d-facade/build_scene.py from annotations.json. Walls, windows and door are real; the electric meter is invented.", "description": "Three-wall chain: w1 (52.0 ft, five ground-floor windows), w2 (11.6 ft tower side, never observed), w3 (16.2 ft tower front with the entrance door). Meter invented on w1 between windows 4 and 5. No start position can pass: every w1 window gap (5.11-5.71 ft) is at least 2.87 ft short of the 8.58 ft a battery plus two 3 ft clearances needs, w2 is unobserved, and all of w3 is past the 20 ft route. Reject is also wrong because w2 is within reach and unseen. See README.md for the arithmetic.", - "rules_assumed": {"opening": 3.0, "route_length": 20.0, "corner_allowance": 0.5, "battery_width": 2.583333333333}, + "rules_assumed": {"opening": 3.0, "route_length": 20.0, "corner_allowance": 0.5, "battery_width": 2.583333333333, "headroom": 6.5}, "expect": { "decision_in": ["manual_review"], "sweep_runs": [ diff --git a/verification/scoreboard/metrics.yaml b/verification/scoreboard/metrics.yaml index c73e9cc0c..9d93cd12f 100644 --- a/verification/scoreboard/metrics.yaml +++ b/verification/scoreboard/metrics.yaml @@ -161,7 +161,10 @@ metrics: report: glob: ~/house-scanning-data/reports/e2e/*/report.json # True only when at least one scene was answered and checked and none broke the contract. - key: contract_ok + # Requires every hostile probe; shortened --no-hostile runs cannot satisfy this field. + key: full_contract_ok + # A failed or incomplete run cannot establish the advertised black-box acceptance. + valid_key: all_passed equals: true require_current_sha: true diff --git a/verification/tests/fixtures/segmented-public-pass.json b/verification/tests/fixtures/segmented-public-pass.json new file mode 100644 index 000000000..4190d2822 --- /dev/null +++ b/verification/tests/fixtures/segmented-public-pass.json @@ -0,0 +1,1383 @@ +{ + "scene": { + "meter": { + "pos": [ + 0, + 4, + 0 + ], + "wall_id": "w11", + "plus_minus_ft": 0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -40.0, + 0 + ], + [ + -36.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w2", + "baseline": [ + [ + -36.0, + 0 + ], + [ + -32.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w3", + "baseline": [ + [ + -32.0, + 0 + ], + [ + -28.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w4", + "baseline": [ + [ + -28.0, + 0 + ], + [ + -24.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w5", + "baseline": [ + [ + -24.0, + 0 + ], + [ + -20.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w6", + "baseline": [ + [ + -20.0, + 0 + ], + [ + -16.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w7", + "baseline": [ + [ + -16.0, + 0 + ], + [ + -12.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w8", + "baseline": [ + [ + -12.0, + 0 + ], + [ + -8.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w9", + "baseline": [ + [ + -8.0, + 0 + ], + [ + -4.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w10", + "baseline": [ + [ + -4.0, + 0 + ], + [ + 0.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w11", + "baseline": [ + [ + 0.0, + 0 + ], + [ + 4.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w12", + "baseline": [ + [ + 4.0, + 0 + ], + [ + 8.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w13", + "baseline": [ + [ + 8.0, + 0 + ], + [ + 12.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w14", + "baseline": [ + [ + 12.0, + 0 + ], + [ + 16.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w15", + "baseline": [ + [ + 16.0, + 0 + ], + [ + 20.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w16", + "baseline": [ + [ + 20.0, + 0 + ], + [ + 24.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w17", + "baseline": [ + [ + 24.0, + 0 + ], + [ + 28.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w18", + "baseline": [ + [ + 28.0, + 0 + ], + [ + 32.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w19", + "baseline": [ + [ + 32.0, + 0 + ], + [ + 36.0, + 0 + ] + ], + "height_ft": 9 + }, + { + "id": "w20", + "baseline": [ + [ + 36.0, + 0 + ], + [ + 40.0, + 0 + ] + ], + "height_ft": 9 + } + ], + "objects": [], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -40, + 0 + ], + [ + 40, + 0 + ], + [ + 40, + 30 + ], + [ + -40, + 30 + ] + ], + "plus_minus_ft": 0 + } + ], + "overheads": [ + { + "wall_id": "w1", + "span_ft": [ + -40, + -36 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w2", + "span_ft": [ + -36, + -32 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w3", + "span_ft": [ + -32, + -28 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w4", + "span_ft": [ + -28, + -24 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w5", + "span_ft": [ + -24, + -20 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w6", + "span_ft": [ + -20, + -16 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w7", + "span_ft": [ + -16, + -12 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w8", + "span_ft": [ + -12, + -8 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w9", + "span_ft": [ + -8, + -4 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w10", + "span_ft": [ + -4, + 0 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w11", + "span_ft": [ + 0, + 4 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w12", + "span_ft": [ + 4, + 8 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w13", + "span_ft": [ + 8, + 12 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w14", + "span_ft": [ + 12, + 16 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w15", + "span_ft": [ + 16, + 20 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w16", + "span_ft": [ + 20, + 24 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w17", + "span_ft": [ + 24, + 28 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w18", + "span_ft": [ + 28, + 32 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w19", + "span_ft": [ + 32, + 36 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w20", + "span_ft": [ + 36, + 40 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -40, + -36 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w2", + "span_ft": [ + -36, + -32 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w3", + "span_ft": [ + -32, + -28 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w4", + "span_ft": [ + -28, + -24 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w5", + "span_ft": [ + -24, + -20 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w6", + "span_ft": [ + -20, + -16 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w7", + "span_ft": [ + -16, + -12 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w8", + "span_ft": [ + -12, + -8 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w9", + "span_ft": [ + -8, + -4 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w10", + "span_ft": [ + -4, + 0 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w11", + "span_ft": [ + 0, + 4 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w12", + "span_ft": [ + 4, + 8 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w13", + "span_ft": [ + 8, + 12 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w14", + "span_ft": [ + 12, + 16 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w15", + "span_ft": [ + 16, + 20 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w16", + "span_ft": [ + 20, + 24 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w17", + "span_ft": [ + 24, + 28 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w18", + "span_ft": [ + 28, + 32 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w19", + "span_ft": [ + 32, + 36 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + }, + { + "wall_id": "w20", + "span_ft": [ + 36, + 40 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -40, + 40 + ] + }, + { + "band": "ground", + "span_ft": [ + -40, + 40 + ], + "out_ft": 30 + }, + { + "band": "overhead", + "span_ft": [ + -40, + 40 + ] + }, + { + "band": "facing", + "span_ft": [ + -40, + 40 + ] + } + ] + } + }, + "result": { + "schema_version": "1.0", + "decision": "pass", + "summary": "The battery fits 3 ft 8 in left of the meter with a 2 ft 4 in cable run: every check passes. Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's.", + "reasons": [ + { + "code": "all_checks_pass", + "message": "A fully observed spot passes every check." + } + ], + "policy": { + "id": "demo", + "version": "2026-09-26", + "auto_approve": true, + "sources": [ + "public" + ], + "rules_sha256": "0df93f73b7c3176d33411888638189544a0848ffb51803219ab662a7ca51db66", + "notice": "Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's." + }, + "spot": { + "outcome": "pass", + "wall_id": "w10", + "segment": 0, + "span_ft": [ + -4.958333, + -2.375 + ], + "width_ft": 2.583333, + "depth_ft": 1.833333, + "height_ft": 3.291667, + "footprint": [ + [ + -4.958333, + 0.0 + ], + [ + -2.375, + 0.0 + ], + [ + -2.375, + 1.833333 + ], + [ + -4.958333, + 1.833333 + ] + ], + "center": [ + -3.666667, + 0.916667 + ], + "along": [ + 1.0, + 0.0 + ], + "outward": [ + -0.0, + 1.0 + ], + "meter_offset_ft": [ + -3.666667, + 0.916667 + ], + "route_length_ft": 2.375 + }, + "route": { + "outcome": "pass", + "length_ft": 2.375, + "plus_minus_ft": 1.093333, + "height_ft": 1.0, + "polyline": [ + [ + 0.0, + 0.0 + ], + [ + -2.375, + 0.0 + ] + ], + "detours": [], + "crossings": [] + }, + "checks": [ + { + "id": "wall_backing", + "label": "Flush against one straight wall", + "outcome": "pass", + "reason": "The whole footprint backs onto one straight, observed wall segment.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "battery.width_ft", + "source": "Base Core spec: 30.68 in wide, rounded up to 31 in (docs/04, What Base does today)", + "placeholder": false + } + }, + { + "id": "ground_surface", + "label": "Ground under the battery", + "outcome": "pass", + "reason": "The whole footprint stands on an allowed surface.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "ground.allowed", + "source": "Ground types from schemas/scene.schema.json; which surfaces are allowed is a demo choice", + "placeholder": true + } + }, + { + "id": "meter_working_space", + "label": "Clear of the meter's working space", + "outcome": "pass", + "reason": "The battery is 1 ft 1 in clear of the meter's 2 ft 6 in wide by 3 ft 0 in deep working space.", + "measured_ft": 1.125, + "plus_minus_ft": 1.093333, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": "meter", + "rule": { + "key": "meter_working_space.width_ft", + "source": "NEC 110.26(A)(2): 30 in wide working space", + "placeholder": false + } + }, + { + "id": "gas_clearance", + "label": "Distance from gas equipment", + "outcome": "pass", + "reason": "No gas meter or pipe within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.gas_ft", + "source": "Base help page: 3 ft from the gas meter; Austin Energy \u00a71.9; Texas Gas Service", + "placeholder": false + } + }, + { + "id": "ac_clearance", + "label": "Distance from AC units", + "outcome": "pass", + "reason": "No AC unit within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.ac_ft", + "source": "Base help page: 3 ft from AC units, fences and other batteries", + "placeholder": false + } + }, + { + "id": "drive_clearance", + "label": "Distance from the driveway", + "outcome": "pass", + "reason": "No drivable surface within 5 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 5.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.drive_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "pool_clearance", + "label": "Distance from a pool", + "outcome": "pass", + "reason": "No pool within 10 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 10.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.pool_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "opening_clearance", + "label": "Distance from doors and windows", + "outcome": "pass", + "reason": "No door or window within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.opening_ft", + "source": "IRC 2021 R328.4: 3 ft from doors and windows directly entering the dwelling. Applying it to garage doors (openings.types) is a demo choice the code does not require", + "placeholder": false + } + }, + { + "id": "wall_equipment_above", + "label": "No box or vent above the battery", + "outcome": "pass", + "reason": "Nothing is mounted on the wall above the battery.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.wall_equipment_ft", + "source": "No public value; 0 means only overlap fails", + "placeholder": true + } + }, + { + "id": "facing_gap", + "label": "Open space in front", + "outcome": "pass", + "reason": "The gap in front of the battery is 7 ft 2 in (\u00b1 0 ft 0 in), clear of the 3 ft 0 in rule.", + "measured_ft": 7.166667, + "plus_minus_ft": 0.0, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": "facing[8]", + "rule": { + "key": "facing.min_ft", + "source": "Base help page: 3 ft from fences", + "placeholder": false + } + }, + { + "id": "headroom", + "label": "Headroom above", + "outcome": "pass", + "reason": "The headroom is 9 ft 0 in (\u00b1 0 ft 0 in), clear of the 6 ft 6 in rule.", + "measured_ft": 9.0, + "plus_minus_ft": 0.0, + "threshold_ft": 6.5, + "comparison": "at_least", + "subject": "overheads[8]", + "rule": { + "key": "headroom.min_ft", + "source": "NEC 110.26(A)(3) working-space headroom, applied to the battery as a demo choice", + "placeholder": true + } + }, + { + "id": "route_path", + "label": "Cable route along the wall", + "outcome": "pass", + "reason": "The cable runs along continuous, observed wall with nothing blocking it.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "route.crossing", + "source": "Demo rule: the cable can't cross a door, a garage or a stretch with no wall", + "placeholder": false + } + }, + { + "id": "route_length", + "label": "Cable run length", + "outcome": "pass", + "reason": "The cable run is 2 ft 4 in (\u00b1 1 ft 1 in), within the 15 ft 0 in confident reach.", + "measured_ft": 2.375, + "plus_minus_ft": 1.093333, + "threshold_ft": 20.0, + "comparison": "at_most", + "subject": null, + "rule": { + "key": "route.max_ft", + "source": "Base help page: within 20 ft of the meter. Review past 15 ft 0 in: No public value", + "placeholder": true + }, + "review_threshold_ft": 15.0 + } + ], + "nearest_considered": null, + "missing_evidence": [], + "objects_not_used": [], + "ends": { + "left": { + "kind": "limit", + "s_ft": -40.0, + "point": [ + -40.0, + 0.0 + ], + "beyond_reach": true + }, + "right": { + "kind": "limit", + "s_ft": 40.0, + "point": [ + 40.0, + 0.0 + ], + "beyond_reach": true + } + }, + "sweep": [ + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -40.0, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.242064, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.183532, + -25.625 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -25.541667, + -25.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + -25.208333, + -21.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w6", + "segment": 0, + "start_ft": [ + -21.208333, + -17.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -17.208333, + -14.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -14.875, + -13.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w8", + "segment": 0, + "start_ft": [ + -13.208333, + -9.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w9", + "segment": 0, + "start_ft": [ + -9.208333, + -5.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -5.208333, + -4.958333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -4.875, + -3.125 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -3.041667, + -1.291667 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + -1.270833, + 0.458333 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 0.541667, + 2.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 2.375, + 2.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w12", + "segment": 0, + "start_ft": [ + 2.791667, + 6.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w13", + "segment": 0, + "start_ft": [ + 6.791667, + 10.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 10.791667, + 12.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 12.375, + 14.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w15", + "segment": 0, + "start_ft": [ + 14.791667, + 18.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + 18.791667, + 22.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 22.791667, + 22.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 23.041667, + 24.600198 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 24.65873, + 24.658731 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + 24.658731, + 37.416667 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + } + ], + "stats": { + "candidates": 815, + "pass": 312, + "unsure": 444, + "fail": 59, + "elapsed_ms": 0, + "input_sha256": "429ef7a1eeb84485c6d19036bfb71a589501b08206fb9a5a34c1458f5a8791cf" + } + } +} diff --git a/verification/tests/fixtures/segmented-public-workload.json b/verification/tests/fixtures/segmented-public-workload.json new file mode 100644 index 000000000..467ae56ab --- /dev/null +++ b/verification/tests/fixtures/segmented-public-workload.json @@ -0,0 +1,4565 @@ +{ + "surveyed wall": { + "schema_version": "1.0", + "decision": "pass", + "summary": "The battery fits 3 ft 8 in left of the meter with a 2 ft 4 in cable run: every check passes. Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's.", + "reasons": [ + { + "code": "all_checks_pass", + "message": "A fully observed spot passes every check." + } + ], + "policy": { + "id": "demo", + "version": "2026-09-26", + "auto_approve": true, + "sources": [ + "public" + ], + "rules_sha256": "0df93f73b7c3176d33411888638189544a0848ffb51803219ab662a7ca51db66", + "notice": "Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's." + }, + "spot": { + "outcome": "pass", + "wall_id": "w10", + "segment": 0, + "span_ft": [ + -4.958333, + -2.375 + ], + "width_ft": 2.583333, + "depth_ft": 1.833333, + "height_ft": 3.291667, + "footprint": [ + [ + -4.958333, + 0.0 + ], + [ + -2.375, + 0.0 + ], + [ + -2.375, + 1.833333 + ], + [ + -4.958333, + 1.833333 + ] + ], + "center": [ + -3.666667, + 0.916667 + ], + "along": [ + 1.0, + 0.0 + ], + "outward": [ + -0.0, + 1.0 + ], + "meter_offset_ft": [ + -3.666667, + 0.916667 + ], + "route_length_ft": 2.375 + }, + "route": { + "outcome": "pass", + "length_ft": 2.375, + "plus_minus_ft": 1.093333, + "height_ft": 1.0, + "polyline": [ + [ + 0.0, + 0.0 + ], + [ + -2.375, + 0.0 + ] + ], + "detours": [], + "crossings": [] + }, + "checks": [ + { + "id": "wall_backing", + "label": "Flush against one straight wall", + "outcome": "pass", + "reason": "The whole footprint backs onto one straight, observed wall segment.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "battery.width_ft", + "source": "Base Core spec: 30.68 in wide, rounded up to 31 in (docs/04, What Base does today)", + "placeholder": false + } + }, + { + "id": "ground_surface", + "label": "Ground under the battery", + "outcome": "pass", + "reason": "The whole footprint stands on an allowed surface.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "ground.allowed", + "source": "Ground types from schemas/scene.schema.json; which surfaces are allowed is a demo choice", + "placeholder": true + } + }, + { + "id": "meter_working_space", + "label": "Clear of the meter's working space", + "outcome": "pass", + "reason": "The battery is 1 ft 1 in clear of the meter's 2 ft 6 in wide by 3 ft 0 in deep working space.", + "measured_ft": 1.125, + "plus_minus_ft": 1.093333, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": "meter", + "rule": { + "key": "meter_working_space.width_ft", + "source": "NEC 110.26(A)(2): 30 in wide working space", + "placeholder": false + } + }, + { + "id": "gas_clearance", + "label": "Distance from gas equipment", + "outcome": "pass", + "reason": "No gas meter or pipe within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.gas_ft", + "source": "Base help page: 3 ft from the gas meter; Austin Energy \u00a71.9; Texas Gas Service", + "placeholder": false + } + }, + { + "id": "ac_clearance", + "label": "Distance from AC units", + "outcome": "pass", + "reason": "No AC unit within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.ac_ft", + "source": "Base help page: 3 ft from AC units, fences and other batteries", + "placeholder": false + } + }, + { + "id": "drive_clearance", + "label": "Distance from the driveway", + "outcome": "pass", + "reason": "No drivable surface within 5 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 5.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.drive_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "pool_clearance", + "label": "Distance from a pool", + "outcome": "pass", + "reason": "No pool within 10 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 10.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.pool_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "opening_clearance", + "label": "Distance from doors and windows", + "outcome": "pass", + "reason": "No door or window within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.opening_ft", + "source": "IRC 2021 R328.4: 3 ft from doors and windows directly entering the dwelling. Applying it to garage doors (openings.types) is a demo choice the code does not require", + "placeholder": false + } + }, + { + "id": "wall_equipment_above", + "label": "No box or vent above the battery", + "outcome": "pass", + "reason": "Nothing is mounted on the wall above the battery.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.wall_equipment_ft", + "source": "No public value; 0 means only overlap fails", + "placeholder": true + } + }, + { + "id": "facing_gap", + "label": "Open space in front", + "outcome": "pass", + "reason": "The gap in front of the battery is 7 ft 2 in (\u00b1 0 ft 0 in), clear of the 3 ft 0 in rule.", + "measured_ft": 7.166667, + "plus_minus_ft": 0.0, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": "facing[8]", + "rule": { + "key": "facing.min_ft", + "source": "Base help page: 3 ft from fences", + "placeholder": false + } + }, + { + "id": "headroom", + "label": "Headroom above", + "outcome": "pass", + "reason": "The headroom is 9 ft 0 in (\u00b1 0 ft 0 in), clear of the 6 ft 6 in rule.", + "measured_ft": 9.0, + "plus_minus_ft": 0.0, + "threshold_ft": 6.5, + "comparison": "at_least", + "subject": "overheads[8]", + "rule": { + "key": "headroom.min_ft", + "source": "NEC 110.26(A)(3) working-space headroom, applied to the battery as a demo choice", + "placeholder": true + } + }, + { + "id": "route_path", + "label": "Cable route along the wall", + "outcome": "pass", + "reason": "The cable runs along continuous, observed wall with nothing blocking it.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "route.crossing", + "source": "Demo rule: the cable can't cross a door, a garage or a stretch with no wall", + "placeholder": false + } + }, + { + "id": "route_length", + "label": "Cable run length", + "outcome": "pass", + "reason": "The cable run is 2 ft 4 in (\u00b1 1 ft 1 in), within the 15 ft 0 in confident reach.", + "measured_ft": 2.375, + "plus_minus_ft": 1.093333, + "threshold_ft": 20.0, + "comparison": "at_most", + "subject": null, + "rule": { + "key": "route.max_ft", + "source": "Base help page: within 20 ft of the meter. Review past 15 ft 0 in: No public value", + "placeholder": true + }, + "review_threshold_ft": 15.0 + } + ], + "nearest_considered": null, + "missing_evidence": [], + "objects_not_used": [], + "ends": { + "left": { + "kind": "limit", + "s_ft": -40.0, + "point": [ + -40.0, + 0.0 + ], + "beyond_reach": true + }, + "right": { + "kind": "limit", + "s_ft": 40.0, + "point": [ + 40.0, + 0.0 + ], + "beyond_reach": true + } + }, + "sweep": [ + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -40.0, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.242064, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.183532, + -25.625 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -25.541667, + -25.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + -25.208333, + -21.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w6", + "segment": 0, + "start_ft": [ + -21.208333, + -17.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -17.208333, + -14.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -14.875, + -13.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w8", + "segment": 0, + "start_ft": [ + -13.208333, + -9.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w9", + "segment": 0, + "start_ft": [ + -9.208333, + -5.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -5.208333, + -4.958333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -4.875, + -3.125 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -3.041667, + -1.291667 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + -1.270833, + 0.458333 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 0.541667, + 2.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 2.375, + 2.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w12", + "segment": 0, + "start_ft": [ + 2.791667, + 6.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w13", + "segment": 0, + "start_ft": [ + 6.791667, + 10.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 10.791667, + 12.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 12.375, + 14.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w15", + "segment": 0, + "start_ft": [ + 14.791667, + 18.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + 18.791667, + 22.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 22.791667, + 22.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 23.041667, + 24.600198 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 24.65873, + 24.658731 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + 24.658731, + 37.416667 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + } + ], + "stats": { + "candidates": 815, + "pass": 312, + "unsure": 444, + "fail": 59, + "elapsed_ms": 0, + "input_sha256": "d3a293f4c61b7fc3c40ac8275d32001c80425d0c774b155a06bc56724d77e71b" + } + }, + "surveyed wall~mirrored": { + "schema_version": "1.0", + "decision": "pass", + "summary": "The battery fits 3 ft 8 in left of the meter with a 2 ft 4 in cable run: every check passes. Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's.", + "reasons": [ + { + "code": "all_checks_pass", + "message": "A fully observed spot passes every check." + } + ], + "policy": { + "id": "demo", + "version": "2026-09-26", + "auto_approve": true, + "sources": [ + "public" + ], + "rules_sha256": "0df93f73b7c3176d33411888638189544a0848ffb51803219ab662a7ca51db66", + "notice": "Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's." + }, + "spot": { + "outcome": "pass", + "wall_id": "w11", + "segment": 0, + "span_ft": [ + -4.958333, + -2.375 + ], + "width_ft": 2.583333, + "depth_ft": 1.833333, + "height_ft": 3.291667, + "footprint": [ + [ + -4.958333, + 0.0 + ], + [ + -2.375, + 0.0 + ], + [ + -2.375, + 1.833333 + ], + [ + -4.958333, + 1.833333 + ] + ], + "center": [ + -3.666667, + 0.916667 + ], + "along": [ + 1.0, + 0.0 + ], + "outward": [ + -0.0, + 1.0 + ], + "meter_offset_ft": [ + -3.666667, + 0.916667 + ], + "route_length_ft": 2.375 + }, + "route": { + "outcome": "pass", + "length_ft": 2.375, + "plus_minus_ft": 1.093333, + "height_ft": 1.0, + "polyline": [ + [ + 0.0, + 0.0 + ], + [ + -2.375, + 0.0 + ] + ], + "detours": [], + "crossings": [] + }, + "checks": [ + { + "id": "wall_backing", + "label": "Flush against one straight wall", + "outcome": "pass", + "reason": "The whole footprint backs onto one straight, observed wall segment.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "battery.width_ft", + "source": "Base Core spec: 30.68 in wide, rounded up to 31 in (docs/04, What Base does today)", + "placeholder": false + } + }, + { + "id": "ground_surface", + "label": "Ground under the battery", + "outcome": "pass", + "reason": "The whole footprint stands on an allowed surface.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "ground.allowed", + "source": "Ground types from schemas/scene.schema.json; which surfaces are allowed is a demo choice", + "placeholder": true + } + }, + { + "id": "meter_working_space", + "label": "Clear of the meter's working space", + "outcome": "pass", + "reason": "The battery is 1 ft 1 in clear of the meter's 2 ft 6 in wide by 3 ft 0 in deep working space.", + "measured_ft": 1.125, + "plus_minus_ft": 1.093333, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": "meter", + "rule": { + "key": "meter_working_space.width_ft", + "source": "NEC 110.26(A)(2): 30 in wide working space", + "placeholder": false + } + }, + { + "id": "gas_clearance", + "label": "Distance from gas equipment", + "outcome": "pass", + "reason": "No gas meter or pipe within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.gas_ft", + "source": "Base help page: 3 ft from the gas meter; Austin Energy \u00a71.9; Texas Gas Service", + "placeholder": false + } + }, + { + "id": "ac_clearance", + "label": "Distance from AC units", + "outcome": "pass", + "reason": "No AC unit within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.ac_ft", + "source": "Base help page: 3 ft from AC units, fences and other batteries", + "placeholder": false + } + }, + { + "id": "drive_clearance", + "label": "Distance from the driveway", + "outcome": "pass", + "reason": "No drivable surface within 5 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 5.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.drive_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "pool_clearance", + "label": "Distance from a pool", + "outcome": "pass", + "reason": "No pool within 10 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 10.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.pool_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "opening_clearance", + "label": "Distance from doors and windows", + "outcome": "pass", + "reason": "No door or window within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.opening_ft", + "source": "IRC 2021 R328.4: 3 ft from doors and windows directly entering the dwelling. Applying it to garage doors (openings.types) is a demo choice the code does not require", + "placeholder": false + } + }, + { + "id": "wall_equipment_above", + "label": "No box or vent above the battery", + "outcome": "pass", + "reason": "Nothing is mounted on the wall above the battery.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.wall_equipment_ft", + "source": "No public value; 0 means only overlap fails", + "placeholder": true + } + }, + { + "id": "facing_gap", + "label": "Open space in front", + "outcome": "pass", + "reason": "The gap in front of the battery is 7 ft 2 in (\u00b1 0 ft 0 in), clear of the 3 ft 0 in rule.", + "measured_ft": 7.166667, + "plus_minus_ft": 0.0, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": "facing[10]", + "rule": { + "key": "facing.min_ft", + "source": "Base help page: 3 ft from fences", + "placeholder": false + } + }, + { + "id": "headroom", + "label": "Headroom above", + "outcome": "pass", + "reason": "The headroom is 9 ft 0 in (\u00b1 0 ft 0 in), clear of the 6 ft 6 in rule.", + "measured_ft": 9.0, + "plus_minus_ft": 0.0, + "threshold_ft": 6.5, + "comparison": "at_least", + "subject": "overheads[10]", + "rule": { + "key": "headroom.min_ft", + "source": "NEC 110.26(A)(3) working-space headroom, applied to the battery as a demo choice", + "placeholder": true + } + }, + { + "id": "route_path", + "label": "Cable route along the wall", + "outcome": "pass", + "reason": "The cable runs along continuous, observed wall with nothing blocking it.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "route.crossing", + "source": "Demo rule: the cable can't cross a door, a garage or a stretch with no wall", + "placeholder": false + } + }, + { + "id": "route_length", + "label": "Cable run length", + "outcome": "pass", + "reason": "The cable run is 2 ft 4 in (\u00b1 1 ft 1 in), within the 15 ft 0 in confident reach.", + "measured_ft": 2.375, + "plus_minus_ft": 1.093333, + "threshold_ft": 20.0, + "comparison": "at_most", + "subject": null, + "rule": { + "key": "route.max_ft", + "source": "Base help page: within 20 ft of the meter. Review past 15 ft 0 in: No public value", + "placeholder": true + }, + "review_threshold_ft": 15.0 + } + ], + "nearest_considered": null, + "missing_evidence": [], + "objects_not_used": [], + "ends": { + "left": { + "kind": "limit", + "s_ft": -40.0, + "point": [ + -40.0, + 0.0 + ], + "beyond_reach": true + }, + "right": { + "kind": "limit", + "s_ft": 40.0, + "point": [ + 40.0, + 0.0 + ], + "beyond_reach": true + } + }, + "sweep": [ + { + "wall_id": "w20", + "segment": 0, + "start_ft": [ + -40.0, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + -27.242064, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + -27.183532, + -25.625 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + -25.541667, + -25.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + -25.208333, + -21.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w15", + "segment": 0, + "start_ft": [ + -21.208333, + -17.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + -17.208333, + -14.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + -14.875, + -13.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w13", + "segment": 0, + "start_ft": [ + -13.208333, + -9.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w12", + "segment": 0, + "start_ft": [ + -9.208333, + -5.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + -5.208333, + -4.958333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + -4.875, + -3.125 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + -3.041667, + -1.291667 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -1.270833, + 0.458333 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + 0.541667, + 2.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + 2.375, + 2.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w9", + "segment": 0, + "start_ft": [ + 2.791667, + 6.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w8", + "segment": 0, + "start_ft": [ + 6.791667, + 10.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + 10.791667, + 12.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + 12.375, + 14.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w6", + "segment": 0, + "start_ft": [ + 14.791667, + 18.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + 18.791667, + 22.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + 22.791667, + 22.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + 23.041667, + 24.600198 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + 24.65873, + 24.658731 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w20", + "segment": 0, + "start_ft": [ + 24.658731, + 37.416667 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + } + ], + "stats": { + "candidates": 815, + "pass": 312, + "unsure": 444, + "fail": 59, + "elapsed_ms": 0, + "input_sha256": "4aee34ed47a051b645f3427c2ca421113b2b7441f8d70a4ac33a6e8a14b7d702" + } + }, + "surveyed wall~no coverage": { + "schema_version": "1.0", + "decision": "manual_review", + "summary": "More views are needed around the best spot, 3 ft 8 in left of the meter: 11 checks depend on areas the scan did not see. Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's.", + "reasons": [ + { + "code": "unsure_checks", + "checks": [ + "wall_backing", + "ground_surface", + "gas_clearance", + "ac_clearance", + "drive_clearance", + "pool_clearance", + "opening_clearance", + "wall_equipment_above", + "facing_gap", + "headroom", + "route_path" + ], + "message": "The best spot has checks nobody can settle from this scan: flush against one straight wall; ground under the battery; distance from gas equipment; distance from ac units; distance from the driveway; distance from a pool; distance from doors and windows; no box or vent above the battery; open space in front; headroom above; cable route along the wall." + }, + { + "code": "unobserved_area", + "checks": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ], + "message": "Part of the area the checks need was not seen." + } + ], + "policy": { + "id": "demo", + "version": "2026-09-26", + "auto_approve": true, + "sources": [ + "public" + ], + "rules_sha256": "0df93f73b7c3176d33411888638189544a0848ffb51803219ab662a7ca51db66", + "notice": "Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's." + }, + "spot": { + "outcome": "unsure", + "wall_id": "w10", + "segment": 0, + "span_ft": [ + -4.958333, + -2.375 + ], + "width_ft": 2.583333, + "depth_ft": 1.833333, + "height_ft": 3.291667, + "footprint": [ + [ + -4.958333, + 0.0 + ], + [ + -2.375, + 0.0 + ], + [ + -2.375, + 1.833333 + ], + [ + -4.958333, + 1.833333 + ] + ], + "center": [ + -3.666667, + 0.916667 + ], + "along": [ + 1.0, + 0.0 + ], + "outward": [ + -0.0, + 1.0 + ], + "meter_offset_ft": [ + -3.666667, + 0.916667 + ], + "route_length_ft": 2.375 + }, + "route": { + "outcome": "unsure", + "length_ft": 2.375, + "plus_minus_ft": 1.093333, + "height_ft": 1.0, + "polyline": [ + [ + 0.0, + 0.0 + ], + [ + -2.375, + 0.0 + ] + ], + "detours": [], + "crossings": [] + }, + "checks": [ + { + "id": "wall_backing", + "label": "Flush against one straight wall", + "outcome": "unsure", + "reason": "Part of the wall behind the battery was not seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "battery.width_ft", + "source": "Base Core spec: 30.68 in wide, rounded up to 31 in (docs/04, What Base does today)", + "placeholder": false + }, + "unsure_cause": "unobserved" + }, + { + "id": "ground_surface", + "label": "Ground under the battery", + "outcome": "unsure", + "reason": "The ground under the footprint was not seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "ground.allowed", + "source": "Ground types from schemas/scene.schema.json; which surfaces are allowed is a demo choice", + "placeholder": true + }, + "unsure_cause": "unobserved" + }, + { + "id": "meter_working_space", + "label": "Clear of the meter's working space", + "outcome": "pass", + "reason": "The battery is 1 ft 1 in clear of the meter's 2 ft 6 in wide by 3 ft 0 in deep working space.", + "measured_ft": 1.125, + "plus_minus_ft": 1.093333, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": "meter", + "rule": { + "key": "meter_working_space.width_ft", + "source": "NEC 110.26(A)(2): 30 in wide working space", + "placeholder": false + } + }, + { + "id": "gas_clearance", + "label": "Distance from gas equipment", + "outcome": "unsure", + "reason": "Not everything within 3 ft 0 in of the battery was seen, so a gas meter or pipe could hide there.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.gas_ft", + "source": "Base help page: 3 ft from the gas meter; Austin Energy \u00a71.9; Texas Gas Service", + "placeholder": false + }, + "unsure_cause": "unobserved" + }, + { + "id": "ac_clearance", + "label": "Distance from AC units", + "outcome": "unsure", + "reason": "Not everything within 3 ft 0 in of the battery was seen, so a AC unit could hide there.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.ac_ft", + "source": "Base help page: 3 ft from AC units, fences and other batteries", + "placeholder": false + }, + "unsure_cause": "unobserved" + }, + { + "id": "drive_clearance", + "label": "Distance from the driveway", + "outcome": "unsure", + "reason": "Not everything within 5 ft 0 in of the battery was seen, so a drivable surface could hide there.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 5.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.drive_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + }, + "unsure_cause": "unobserved" + }, + { + "id": "pool_clearance", + "label": "Distance from a pool", + "outcome": "unsure", + "reason": "Not everything within 10 ft 0 in of the battery was seen, so a pool could hide there.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 10.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.pool_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + }, + "unsure_cause": "unobserved" + }, + { + "id": "opening_clearance", + "label": "Distance from doors and windows", + "outcome": "unsure", + "reason": "Not everything within 3 ft 0 in of the battery was seen, so a door or window could hide there.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.opening_ft", + "source": "IRC 2021 R328.4: 3 ft from doors and windows directly entering the dwelling. Applying it to garage doors (openings.types) is a demo choice the code does not require", + "placeholder": false + }, + "unsure_cause": "unobserved" + }, + { + "id": "wall_equipment_above", + "label": "No box or vent above the battery", + "outcome": "unsure", + "reason": "The wall above the battery was not fully seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.wall_equipment_ft", + "source": "No public value; 0 means only overlap fails", + "placeholder": true + }, + "unsure_cause": "unobserved" + }, + { + "id": "facing_gap", + "label": "Open space in front", + "outcome": "unsure", + "reason": "The gap in front of the battery over the battery's stretch of wall was not measured everywhere.", + "measured_ft": 7.166667, + "plus_minus_ft": 0.0, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": "facing[8]", + "rule": { + "key": "facing.min_ft", + "source": "Base help page: 3 ft from fences", + "placeholder": false + }, + "unsure_cause": "unobserved" + }, + { + "id": "headroom", + "label": "Headroom above", + "outcome": "unsure", + "reason": "The headroom over the battery's stretch of wall was not measured everywhere.", + "measured_ft": 9.0, + "plus_minus_ft": 0.0, + "threshold_ft": 6.5, + "comparison": "at_least", + "subject": "overheads[8]", + "rule": { + "key": "headroom.min_ft", + "source": "NEC 110.26(A)(3) working-space headroom, applied to the battery as a demo choice", + "placeholder": true + }, + "unsure_cause": "unobserved" + }, + { + "id": "route_path", + "label": "Cable route along the wall", + "outcome": "unsure", + "reason": "Part of the wall the cable would run along was not seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "route.crossing", + "source": "Demo rule: the cable can't cross a door, a garage or a stretch with no wall", + "placeholder": false + }, + "unsure_cause": "unobserved" + }, + { + "id": "route_length", + "label": "Cable run length", + "outcome": "pass", + "reason": "The cable run is 2 ft 4 in (\u00b1 1 ft 1 in), within the 15 ft 0 in confident reach.", + "measured_ft": 2.375, + "plus_minus_ft": 1.093333, + "threshold_ft": 20.0, + "comparison": "at_most", + "subject": null, + "rule": { + "key": "route.max_ft", + "source": "Base help page: within 20 ft of the meter. Review past 15 ft 0 in: No public value", + "placeholder": true + }, + "review_threshold_ft": 15.0 + } + ], + "nearest_considered": null, + "missing_evidence": [ + { + "kind": "band", + "band": "wall", + "span_ft": [ + -9.056604, + 1.72327 + ], + "checks": [ + "gas_clearance", + "opening_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ], + "out_ft": 6.500001, + "message": "Show the wall from 9 ft 1 in left of the meter to 1 ft 9 in right of the meter, seen at least 6 ft 6 in up the wall." + }, + { + "kind": "band", + "band": "ground", + "span_ft": [ + -16.065046, + 8.731712 + ], + "checks": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "ground_surface", + "pool_clearance" + ], + "out_ft": 12.940046, + "message": "Show the ground in front of the wall from 16 ft 1 in left of the meter to 8 ft 9 in right of the meter, at least 12 ft 11 in out from the wall." + }, + { + "kind": "band", + "band": "facing", + "span_ft": [ + -4.958334, + -2.374999 + ], + "checks": [ + "facing_gap" + ], + "out_ft": 4.833334, + "message": "Show the gap in front of the wall from 4 ft 11 in left of the meter to 2 ft 4 in left of the meter, clear at least 4 ft 10 in out from the wall." + }, + { + "kind": "band", + "band": "overhead", + "span_ft": [ + -4.958334, + -2.374999 + ], + "checks": [ + "headroom" + ], + "out_ft": 6.500001, + "message": "Show the space overhead from 4 ft 11 in left of the meter to 2 ft 4 in left of the meter, clear at least 6 ft 6 in up." + } + ], + "objects_not_used": [], + "ends": { + "left": { + "kind": "unexplored", + "s_ft": -40.0, + "point": [ + -40.0, + 0.0 + ], + "beyond_reach": true + }, + "right": { + "kind": "unexplored", + "s_ft": 40.0, + "point": [ + 40.0, + 0.0 + ], + "beyond_reach": true + } + }, + "sweep": [ + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -40.0, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.242064, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.183532, + -25.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_length", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + -25.208333, + -21.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_length", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w6", + "segment": 0, + "start_ft": [ + -21.208333, + -17.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_length", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -17.208333, + -14.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_length", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -14.875, + -13.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w8", + "segment": 0, + "start_ft": [ + -13.208333, + -9.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w9", + "segment": 0, + "start_ft": [ + -9.208333, + -5.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -5.208333, + -4.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -4.875, + -3.125 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "meter_working_space", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -3.041667, + -2.604167 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -2.583333, + -1.291667 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + -1.270833, + 0.0 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 0.020833, + 0.458333 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 0.541667, + 2.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "meter_working_space", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 2.375, + 2.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w12", + "segment": 0, + "start_ft": [ + 2.791667, + 6.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w13", + "segment": 0, + "start_ft": [ + 6.791667, + 10.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 10.791667, + 12.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 12.375, + 14.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_length", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w15", + "segment": 0, + "start_ft": [ + 14.791667, + 18.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_length", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + 18.791667, + 22.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_length", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 22.791667, + 24.600198 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_length", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 24.65873, + 24.658731 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "ac_clearance", + "drive_clearance", + "facing_gap", + "gas_clearance", + "ground_surface", + "headroom", + "opening_clearance", + "pool_clearance", + "route_path", + "wall_backing", + "wall_equipment_above" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + 24.658731, + 37.416667 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + } + ], + "stats": { + "candidates": 815, + "pass": 0, + "unsure": 756, + "fail": 59, + "elapsed_ms": 0, + "input_sha256": "266238d71d75c0edf27c1b2e720c616f0315ee4154d6f57250a7268f90e5f09d" + } + }, + "surveyed wall~less coverage": { + "schema_version": "1.0", + "decision": "pass", + "summary": "The battery fits 3 ft 8 in left of the meter with a 2 ft 4 in cable run: every check passes. Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's.", + "reasons": [ + { + "code": "all_checks_pass", + "message": "A fully observed spot passes every check." + } + ], + "policy": { + "id": "demo", + "version": "2026-09-26", + "auto_approve": true, + "sources": [ + "public" + ], + "rules_sha256": "0df93f73b7c3176d33411888638189544a0848ffb51803219ab662a7ca51db66", + "notice": "Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's." + }, + "spot": { + "outcome": "pass", + "wall_id": "w10", + "segment": 0, + "span_ft": [ + -4.958333, + -2.375 + ], + "width_ft": 2.583333, + "depth_ft": 1.833333, + "height_ft": 3.291667, + "footprint": [ + [ + -4.958333, + 0.0 + ], + [ + -2.375, + 0.0 + ], + [ + -2.375, + 1.833333 + ], + [ + -4.958333, + 1.833333 + ] + ], + "center": [ + -3.666667, + 0.916667 + ], + "along": [ + 1.0, + 0.0 + ], + "outward": [ + -0.0, + 1.0 + ], + "meter_offset_ft": [ + -3.666667, + 0.916667 + ], + "route_length_ft": 2.375 + }, + "route": { + "outcome": "pass", + "length_ft": 2.375, + "plus_minus_ft": 1.093333, + "height_ft": 1.0, + "polyline": [ + [ + 0.0, + 0.0 + ], + [ + -2.375, + 0.0 + ] + ], + "detours": [], + "crossings": [] + }, + "checks": [ + { + "id": "wall_backing", + "label": "Flush against one straight wall", + "outcome": "pass", + "reason": "The whole footprint backs onto one straight, observed wall segment.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "battery.width_ft", + "source": "Base Core spec: 30.68 in wide, rounded up to 31 in (docs/04, What Base does today)", + "placeholder": false + } + }, + { + "id": "ground_surface", + "label": "Ground under the battery", + "outcome": "pass", + "reason": "The whole footprint stands on an allowed surface.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "ground.allowed", + "source": "Ground types from schemas/scene.schema.json; which surfaces are allowed is a demo choice", + "placeholder": true + } + }, + { + "id": "meter_working_space", + "label": "Clear of the meter's working space", + "outcome": "pass", + "reason": "The battery is 1 ft 1 in clear of the meter's 2 ft 6 in wide by 3 ft 0 in deep working space.", + "measured_ft": 1.125, + "plus_minus_ft": 1.093333, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": "meter", + "rule": { + "key": "meter_working_space.width_ft", + "source": "NEC 110.26(A)(2): 30 in wide working space", + "placeholder": false + } + }, + { + "id": "gas_clearance", + "label": "Distance from gas equipment", + "outcome": "pass", + "reason": "No gas meter or pipe within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.gas_ft", + "source": "Base help page: 3 ft from the gas meter; Austin Energy \u00a71.9; Texas Gas Service", + "placeholder": false + } + }, + { + "id": "ac_clearance", + "label": "Distance from AC units", + "outcome": "pass", + "reason": "No AC unit within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.ac_ft", + "source": "Base help page: 3 ft from AC units, fences and other batteries", + "placeholder": false + } + }, + { + "id": "drive_clearance", + "label": "Distance from the driveway", + "outcome": "pass", + "reason": "No drivable surface within 5 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 5.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.drive_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "pool_clearance", + "label": "Distance from a pool", + "outcome": "pass", + "reason": "No pool within 10 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 10.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.pool_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "opening_clearance", + "label": "Distance from doors and windows", + "outcome": "pass", + "reason": "No door or window within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.opening_ft", + "source": "IRC 2021 R328.4: 3 ft from doors and windows directly entering the dwelling. Applying it to garage doors (openings.types) is a demo choice the code does not require", + "placeholder": false + } + }, + { + "id": "wall_equipment_above", + "label": "No box or vent above the battery", + "outcome": "pass", + "reason": "Nothing is mounted on the wall above the battery.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.wall_equipment_ft", + "source": "No public value; 0 means only overlap fails", + "placeholder": true + } + }, + { + "id": "facing_gap", + "label": "Open space in front", + "outcome": "pass", + "reason": "The gap in front of the battery is 7 ft 2 in (\u00b1 0 ft 0 in), clear of the 3 ft 0 in rule.", + "measured_ft": 7.166667, + "plus_minus_ft": 0.0, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": "facing[8]", + "rule": { + "key": "facing.min_ft", + "source": "Base help page: 3 ft from fences", + "placeholder": false + } + }, + { + "id": "headroom", + "label": "Headroom above", + "outcome": "pass", + "reason": "The headroom is 9 ft 0 in (\u00b1 0 ft 0 in), clear of the 6 ft 6 in rule.", + "measured_ft": 9.0, + "plus_minus_ft": 0.0, + "threshold_ft": 6.5, + "comparison": "at_least", + "subject": "overheads[8]", + "rule": { + "key": "headroom.min_ft", + "source": "NEC 110.26(A)(3) working-space headroom, applied to the battery as a demo choice", + "placeholder": true + } + }, + { + "id": "route_path", + "label": "Cable route along the wall", + "outcome": "pass", + "reason": "The cable runs along continuous, observed wall with nothing blocking it.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "route.crossing", + "source": "Demo rule: the cable can't cross a door, a garage or a stretch with no wall", + "placeholder": false + } + }, + { + "id": "route_length", + "label": "Cable run length", + "outcome": "pass", + "reason": "The cable run is 2 ft 4 in (\u00b1 1 ft 1 in), within the 15 ft 0 in confident reach.", + "measured_ft": 2.375, + "plus_minus_ft": 1.093333, + "threshold_ft": 20.0, + "comparison": "at_most", + "subject": null, + "rule": { + "key": "route.max_ft", + "source": "Base help page: within 20 ft of the meter. Review past 15 ft 0 in: No public value", + "placeholder": true + }, + "review_threshold_ft": 15.0 + } + ], + "nearest_considered": null, + "missing_evidence": [], + "objects_not_used": [], + "ends": { + "left": { + "kind": "limit", + "s_ft": -40.0, + "point": [ + -40.0, + 0.0 + ], + "beyond_reach": true + }, + "right": { + "kind": "limit", + "s_ft": 40.0, + "point": [ + 40.0, + 0.0 + ], + "beyond_reach": true + } + }, + "sweep": [ + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -40.0, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.242064, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.183532, + -25.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + -25.208333, + -25.208333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + -25.125, + -21.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w6", + "segment": 0, + "start_ft": [ + -21.208333, + -17.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -17.208333, + -14.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -14.875, + -13.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w8", + "segment": 0, + "start_ft": [ + -13.208333, + -9.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w9", + "segment": 0, + "start_ft": [ + -9.208333, + -5.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -5.208333, + -4.958333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -4.875, + -3.125 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -3.041667, + -1.291667 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + -1.270833, + 0.458333 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 0.541667, + 2.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 2.375, + 2.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w12", + "segment": 0, + "start_ft": [ + 2.791667, + 6.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w13", + "segment": 0, + "start_ft": [ + 6.791667, + 10.708333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 10.791667, + 12.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 12.375, + 14.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w15", + "segment": 0, + "start_ft": [ + 14.791667, + 18.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + 18.791667, + 22.541667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + 22.625, + 22.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 22.791667, + 24.600198 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 24.65873, + 24.658731 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + 24.658731, + 37.416667 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + } + ], + "stats": { + "candidates": 815, + "pass": 312, + "unsure": 444, + "fail": 59, + "elapsed_ms": 0, + "input_sha256": "45f4346323d5517919839411fecc8c95bf07a62f2b08990f6b0b765e03fa9fc5" + } + }, + "surveyed wall~ground short of the largest clearance": { + "schema_version": "1.0", + "decision": "manual_review", + "summary": "More views are needed around the best spot, 3 ft 8 in left of the meter: 1 check depends on areas the scan did not see. Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's.", + "reasons": [ + { + "code": "unsure_checks", + "checks": [ + "pool_clearance" + ], + "message": "The best spot has checks nobody can settle from this scan: distance from a pool." + }, + { + "code": "unobserved_area", + "checks": [ + "pool_clearance" + ], + "message": "Part of the area the checks need was not seen." + } + ], + "policy": { + "id": "demo", + "version": "2026-09-26", + "auto_approve": true, + "sources": [ + "public" + ], + "rules_sha256": "0df93f73b7c3176d33411888638189544a0848ffb51803219ab662a7ca51db66", + "notice": "Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's." + }, + "spot": { + "outcome": "unsure", + "wall_id": "w10", + "segment": 0, + "span_ft": [ + -4.958333, + -2.375 + ], + "width_ft": 2.583333, + "depth_ft": 1.833333, + "height_ft": 3.291667, + "footprint": [ + [ + -4.958333, + 0.0 + ], + [ + -2.375, + 0.0 + ], + [ + -2.375, + 1.833333 + ], + [ + -4.958333, + 1.833333 + ] + ], + "center": [ + -3.666667, + 0.916667 + ], + "along": [ + 1.0, + 0.0 + ], + "outward": [ + -0.0, + 1.0 + ], + "meter_offset_ft": [ + -3.666667, + 0.916667 + ], + "route_length_ft": 2.375 + }, + "route": { + "outcome": "pass", + "length_ft": 2.375, + "plus_minus_ft": 1.093333, + "height_ft": 1.0, + "polyline": [ + [ + 0.0, + 0.0 + ], + [ + -2.375, + 0.0 + ] + ], + "detours": [], + "crossings": [] + }, + "checks": [ + { + "id": "wall_backing", + "label": "Flush against one straight wall", + "outcome": "pass", + "reason": "The whole footprint backs onto one straight, observed wall segment.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "battery.width_ft", + "source": "Base Core spec: 30.68 in wide, rounded up to 31 in (docs/04, What Base does today)", + "placeholder": false + } + }, + { + "id": "ground_surface", + "label": "Ground under the battery", + "outcome": "pass", + "reason": "The whole footprint stands on an allowed surface.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "ground.allowed", + "source": "Ground types from schemas/scene.schema.json; which surfaces are allowed is a demo choice", + "placeholder": true + } + }, + { + "id": "meter_working_space", + "label": "Clear of the meter's working space", + "outcome": "pass", + "reason": "The battery is 1 ft 1 in clear of the meter's 2 ft 6 in wide by 3 ft 0 in deep working space.", + "measured_ft": 1.125, + "plus_minus_ft": 1.093333, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": "meter", + "rule": { + "key": "meter_working_space.width_ft", + "source": "NEC 110.26(A)(2): 30 in wide working space", + "placeholder": false + } + }, + { + "id": "gas_clearance", + "label": "Distance from gas equipment", + "outcome": "pass", + "reason": "No gas meter or pipe within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.gas_ft", + "source": "Base help page: 3 ft from the gas meter; Austin Energy \u00a71.9; Texas Gas Service", + "placeholder": false + } + }, + { + "id": "ac_clearance", + "label": "Distance from AC units", + "outcome": "pass", + "reason": "No AC unit within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.ac_ft", + "source": "Base help page: 3 ft from AC units, fences and other batteries", + "placeholder": false + } + }, + { + "id": "drive_clearance", + "label": "Distance from the driveway", + "outcome": "pass", + "reason": "No drivable surface within 5 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 5.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.drive_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "pool_clearance", + "label": "Distance from a pool", + "outcome": "unsure", + "reason": "Not everything within 10 ft 0 in of the battery was seen, so a pool could hide there.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 10.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.pool_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + }, + "unsure_cause": "unobserved" + }, + { + "id": "opening_clearance", + "label": "Distance from doors and windows", + "outcome": "pass", + "reason": "No door or window within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.opening_ft", + "source": "IRC 2021 R328.4: 3 ft from doors and windows directly entering the dwelling. Applying it to garage doors (openings.types) is a demo choice the code does not require", + "placeholder": false + } + }, + { + "id": "wall_equipment_above", + "label": "No box or vent above the battery", + "outcome": "pass", + "reason": "Nothing is mounted on the wall above the battery.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.wall_equipment_ft", + "source": "No public value; 0 means only overlap fails", + "placeholder": true + } + }, + { + "id": "facing_gap", + "label": "Open space in front", + "outcome": "pass", + "reason": "The gap in front of the battery is 7 ft 2 in (\u00b1 0 ft 0 in), clear of the 3 ft 0 in rule.", + "measured_ft": 7.166667, + "plus_minus_ft": 0.0, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": "facing[8]", + "rule": { + "key": "facing.min_ft", + "source": "Base help page: 3 ft from fences", + "placeholder": false + } + }, + { + "id": "headroom", + "label": "Headroom above", + "outcome": "pass", + "reason": "The headroom is 9 ft 0 in (\u00b1 0 ft 0 in), clear of the 6 ft 6 in rule.", + "measured_ft": 9.0, + "plus_minus_ft": 0.0, + "threshold_ft": 6.5, + "comparison": "at_least", + "subject": "overheads[8]", + "rule": { + "key": "headroom.min_ft", + "source": "NEC 110.26(A)(3) working-space headroom, applied to the battery as a demo choice", + "placeholder": true + } + }, + { + "id": "route_path", + "label": "Cable route along the wall", + "outcome": "pass", + "reason": "The cable runs along continuous, observed wall with nothing blocking it.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "route.crossing", + "source": "Demo rule: the cable can't cross a door, a garage or a stretch with no wall", + "placeholder": false + } + }, + { + "id": "route_length", + "label": "Cable run length", + "outcome": "pass", + "reason": "The cable run is 2 ft 4 in (\u00b1 1 ft 1 in), within the 15 ft 0 in confident reach.", + "measured_ft": 2.375, + "plus_minus_ft": 1.093333, + "threshold_ft": 20.0, + "comparison": "at_most", + "subject": null, + "rule": { + "key": "route.max_ft", + "source": "Base help page: within 20 ft of the meter. Review past 15 ft 0 in: No public value", + "placeholder": true + }, + "review_threshold_ft": 15.0 + } + ], + "nearest_considered": null, + "missing_evidence": [ + { + "kind": "band", + "band": "ground", + "span_ft": [ + -9.961945, + 2.628612 + ], + "checks": [ + "pool_clearance" + ], + "out_ft": 12.940046, + "message": "Show the ground in front of the wall from 10 ft 0 in left of the meter to 2 ft 8 in right of the meter, at least 12 ft 11 in out from the wall." + } + ], + "objects_not_used": [], + "ends": { + "left": { + "kind": "limit", + "s_ft": -40.0, + "point": [ + -40.0, + 0.0 + ], + "beyond_reach": true + }, + "right": { + "kind": "limit", + "s_ft": 40.0, + "point": [ + 40.0, + 0.0 + ], + "beyond_reach": true + } + }, + "sweep": [ + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -40.0, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.242064, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -27.183532, + -25.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + -25.208333, + -21.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w6", + "segment": 0, + "start_ft": [ + -21.208333, + -17.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -17.208333, + -14.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -14.875, + -13.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w8", + "segment": 0, + "start_ft": [ + -13.208333, + -9.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w9", + "segment": 0, + "start_ft": [ + -9.208333, + -5.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -5.208333, + -4.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -4.875, + -3.125 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "pool_clearance" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -3.041667, + -1.291667 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + -1.270833, + 0.458333 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 0.541667, + 2.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "pool_clearance" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 2.375, + 2.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w12", + "segment": 0, + "start_ft": [ + 2.791667, + 6.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w13", + "segment": 0, + "start_ft": [ + 6.791667, + 10.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 10.791667, + 12.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 12.375, + 14.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w15", + "segment": 0, + "start_ft": [ + 14.791667, + 18.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + 18.791667, + 22.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 22.791667, + 24.600198 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 24.65873, + 24.658731 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + 24.658731, + 37.416667 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + } + ], + "stats": { + "candidates": 815, + "pass": 0, + "unsure": 756, + "fail": 59, + "elapsed_ms": 0, + "input_sha256": "11e367e90218bfe3b8663e25f78640d62daed9192d959bb8a599877aa7729684" + } + }, + "surveyed wall~more error": { + "schema_version": "1.0", + "decision": "manual_review", + "summary": "A person needs to check the best spot, 2 ft 6 in left of the meter: clear of the meter's working space; cable run length. Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's.", + "reasons": [ + { + "code": "unsure_checks", + "checks": [ + "meter_working_space", + "route_length" + ], + "message": "The best spot has checks nobody can settle from this scan: clear of the meter's working space; cable run length." + } + ], + "policy": { + "id": "demo", + "version": "2026-09-26", + "auto_approve": true, + "sources": [ + "public" + ], + "rules_sha256": "0df93f73b7c3176d33411888638189544a0848ffb51803219ab662a7ca51db66", + "notice": "Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's." + }, + "spot": { + "outcome": "unsure", + "wall_id": "w10", + "segment": 0, + "span_ft": [ + -3.833333, + -1.25 + ], + "width_ft": 2.583333, + "depth_ft": 1.833333, + "height_ft": 3.291667, + "footprint": [ + [ + -3.833333, + 0.0 + ], + [ + -1.25, + 0.0 + ], + [ + -1.25, + 1.833333 + ], + [ + -3.833333, + 1.833333 + ] + ], + "center": [ + -2.541667, + 0.916667 + ], + "along": [ + 1.0, + 0.0 + ], + "outward": [ + -0.0, + 1.0 + ], + "meter_offset_ft": [ + -2.541667, + 0.916667 + ], + "route_length_ft": 1.25 + }, + "route": { + "outcome": "unsure", + "length_ft": 1.25, + "plus_minus_ft": 14.1, + "height_ft": 1.0, + "polyline": [ + [ + 0.0, + 0.0 + ], + [ + -1.25, + 0.0 + ] + ], + "detours": [], + "crossings": [] + }, + "checks": [ + { + "id": "wall_backing", + "label": "Flush against one straight wall", + "outcome": "pass", + "reason": "The whole footprint backs onto one straight, observed wall segment.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "battery.width_ft", + "source": "Base Core spec: 30.68 in wide, rounded up to 31 in (docs/04, What Base does today)", + "placeholder": false + } + }, + { + "id": "ground_surface", + "label": "Ground under the battery", + "outcome": "pass", + "reason": "The whole footprint stands on an allowed surface.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "ground.allowed", + "source": "Ground types from schemas/scene.schema.json; which surfaces are allowed is a demo choice", + "placeholder": true + } + }, + { + "id": "meter_working_space", + "label": "Clear of the meter's working space", + "outcome": "unsure", + "reason": "The battery is within measurement error of the meter's 2 ft 6 in wide by 3 ft 0 in deep working space.", + "measured_ft": -0.0, + "plus_minus_ft": 14.1, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": "meter", + "rule": { + "key": "meter_working_space.width_ft", + "source": "NEC 110.26(A)(2): 30 in wide working space", + "placeholder": false + }, + "unsure_cause": "margin" + }, + { + "id": "gas_clearance", + "label": "Distance from gas equipment", + "outcome": "pass", + "reason": "No gas meter or pipe within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.gas_ft", + "source": "Base help page: 3 ft from the gas meter; Austin Energy \u00a71.9; Texas Gas Service", + "placeholder": false + } + }, + { + "id": "ac_clearance", + "label": "Distance from AC units", + "outcome": "pass", + "reason": "No AC unit within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.ac_ft", + "source": "Base help page: 3 ft from AC units, fences and other batteries", + "placeholder": false + } + }, + { + "id": "drive_clearance", + "label": "Distance from the driveway", + "outcome": "pass", + "reason": "No drivable surface within 5 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 5.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.drive_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "pool_clearance", + "label": "Distance from a pool", + "outcome": "pass", + "reason": "No pool within 10 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 10.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.pool_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "opening_clearance", + "label": "Distance from doors and windows", + "outcome": "pass", + "reason": "No door or window within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.opening_ft", + "source": "IRC 2021 R328.4: 3 ft from doors and windows directly entering the dwelling. Applying it to garage doors (openings.types) is a demo choice the code does not require", + "placeholder": false + } + }, + { + "id": "wall_equipment_above", + "label": "No box or vent above the battery", + "outcome": "pass", + "reason": "Nothing is mounted on the wall above the battery.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.wall_equipment_ft", + "source": "No public value; 0 means only overlap fails", + "placeholder": true + } + }, + { + "id": "facing_gap", + "label": "Open space in front", + "outcome": "pass", + "reason": "The gap in front of the battery is 7 ft 2 in (\u00b1 0 ft 0 in), clear of the 3 ft 0 in rule.", + "measured_ft": 7.166667, + "plus_minus_ft": 0.0, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": "facing[5]", + "rule": { + "key": "facing.min_ft", + "source": "Base help page: 3 ft from fences", + "placeholder": false + } + }, + { + "id": "headroom", + "label": "Headroom above", + "outcome": "pass", + "reason": "The headroom is 9 ft 0 in (\u00b1 0 ft 0 in), clear of the 6 ft 6 in rule.", + "measured_ft": 9.0, + "plus_minus_ft": 0.0, + "threshold_ft": 6.5, + "comparison": "at_least", + "subject": "overheads[5]", + "rule": { + "key": "headroom.min_ft", + "source": "NEC 110.26(A)(3) working-space headroom, applied to the battery as a demo choice", + "placeholder": true + } + }, + { + "id": "route_path", + "label": "Cable route along the wall", + "outcome": "pass", + "reason": "The cable runs along continuous, observed wall with nothing blocking it.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "route.crossing", + "source": "Demo rule: the cable can't cross a door, a garage or a stretch with no wall", + "placeholder": false + } + }, + { + "id": "route_length", + "label": "Cable run length", + "outcome": "unsure", + "reason": "The cable run is 1 ft 3 in (\u00b1 14 ft 1 in), within its error of the 15 ft 0 in confident reach or the 20 ft 0 in maximum: too close to call.", + "measured_ft": 1.25, + "plus_minus_ft": 14.1, + "threshold_ft": 20.0, + "comparison": "at_most", + "subject": null, + "rule": { + "key": "route.max_ft", + "source": "Base help page: within 20 ft of the meter. Review past 15 ft 0 in: No public value", + "placeholder": true + }, + "review_threshold_ft": 15.0, + "unsure_cause": "margin" + } + ], + "nearest_considered": null, + "missing_evidence": [], + "objects_not_used": [], + "ends": { + "left": { + "kind": "limit", + "s_ft": -40.0, + "point": [ + -40.0, + 0.0 + ], + "beyond_reach": true + }, + "right": { + "kind": "limit", + "s_ft": 40.0, + "point": [ + 40.0, + 0.0 + ], + "beyond_reach": true + } + }, + "sweep": [ + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -40.0, + -36.683334 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + }, + { + "wall_id": "w2", + "segment": 0, + "start_ft": [ + -36.683334, + -36.683334 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "ground_surface", + "opening_clearance", + "pool_clearance" + ] + }, + { + "wall_id": "w2", + "segment": 0, + "start_ft": [ + -36.683333, + -33.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "ground_surface", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w3", + "segment": 0, + "start_ft": [ + -33.208333, + -29.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "ground_surface", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -29.208333, + -26.429167 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "ground_surface", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w4", + "segment": 0, + "start_ft": [ + -26.4, + -25.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + -25.208333, + -23.458333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + -23.375, + -21.458333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "drive_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w5", + "segment": 0, + "start_ft": [ + -21.375, + -21.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w6", + "segment": 0, + "start_ft": [ + -21.208333, + -17.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w6", + "segment": 0, + "start_ft": [ + -17.875, + -17.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -17.208333, + -16.458333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w7", + "segment": 0, + "start_ft": [ + -16.375, + -13.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "route_length" + ] + }, + { + "wall_id": "w8", + "segment": 0, + "start_ft": [ + -13.208333, + -9.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "route_length" + ] + }, + { + "wall_id": "w9", + "segment": 0, + "start_ft": [ + -9.208333, + -5.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "route_length" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -5.208333, + -3.483333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "route_length" + ] + }, + { + "wall_id": "w10", + "segment": 0, + "start_ft": [ + -3.470833, + -1.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + -1.270833, + 0.8875 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w11", + "segment": 0, + "start_ft": [ + 0.9, + 2.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "route_length" + ] + }, + { + "wall_id": "w12", + "segment": 0, + "start_ft": [ + 2.791667, + 6.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "route_length" + ] + }, + { + "wall_id": "w13", + "segment": 0, + "start_ft": [ + 6.791667, + 10.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "route_length" + ] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 10.791667, + 13.791667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "route_length" + ] + }, + { + "wall_id": "w14", + "segment": 0, + "start_ft": [ + 13.875, + 14.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w15", + "segment": 0, + "start_ft": [ + 14.779167, + 15.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w15", + "segment": 0, + "start_ft": [ + 15.375, + 18.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + 18.791667, + 18.791667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + 18.875, + 20.791667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "drive_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w16", + "segment": 0, + "start_ft": [ + 20.875, + 22.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 22.791667, + 23.816667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w17", + "segment": 0, + "start_ft": [ + 23.845833, + 26.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "ground_surface", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w18", + "segment": 0, + "start_ft": [ + 26.791667, + 30.708333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "ground_surface", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w19", + "segment": 0, + "start_ft": [ + 30.791667, + 34.1 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "ground_surface", + "opening_clearance", + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w19", + "segment": 0, + "start_ft": [ + 34.100001, + 34.100001 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "ac_clearance", + "drive_clearance", + "gas_clearance", + "ground_surface", + "opening_clearance", + "pool_clearance" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + 34.100001, + 37.416667 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + } + ], + "stats": { + "candidates": 1095, + "pass": 0, + "unsure": 1093, + "fail": 2, + "elapsed_ms": 0, + "input_sha256": "89f16e3ba80ea5576ec49bca3db348b343d1a8f713417af402614a4ac1761505" + } + } +} diff --git a/verification/tests/fixtures/wide-public-pass.json b/verification/tests/fixtures/wide-public-pass.json new file mode 100644 index 000000000..b47031e2a --- /dev/null +++ b/verification/tests/fixtures/wide-public-pass.json @@ -0,0 +1,618 @@ +{ + "scene": { + "meter": { + "pos": [ + 0, + 4, + 0 + ], + "wall_id": "w1", + "plus_minus_ft": 0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -40, + 0 + ], + [ + 40, + 0 + ] + ], + "height_ft": 9 + } + ], + "objects": [], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -40, + 0 + ], + [ + 40, + 0 + ], + [ + 40, + 30 + ], + [ + -40, + 30 + ] + ], + "plus_minus_ft": 0 + } + ], + "overheads": [ + { + "wall_id": "w1", + "span_ft": [ + -40, + 40 + ], + "clearance_ft": 9, + "plus_minus_ft": 0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -40, + 40 + ], + "depth_ft": 9, + "plus_minus_ft": 0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -40, + 40 + ] + }, + { + "band": "ground", + "span_ft": [ + -40, + 40 + ], + "out_ft": 30 + }, + { + "band": "overhead", + "span_ft": [ + -40, + 40 + ] + }, + { + "band": "facing", + "span_ft": [ + -40, + 40 + ] + } + ] + } + }, + "result": { + "schema_version": "1.0", + "decision": "pass", + "summary": "The battery fits 3 ft 8 in left of the meter with a 2 ft 4 in cable run: every check passes. Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's.", + "reasons": [ + { + "code": "all_checks_pass", + "message": "A fully observed spot passes every check." + } + ], + "policy": { + "id": "demo", + "version": "2026-09-26", + "auto_approve": true, + "sources": [ + "public" + ], + "rules_sha256": "0df93f73b7c3176d33411888638189544a0848ffb51803219ab662a7ca51db66", + "notice": "Demo rules: public values and placeholders (pool 10 ft, drive 5 ft), not Base's." + }, + "spot": { + "outcome": "pass", + "wall_id": "w1", + "segment": 0, + "span_ft": [ + -4.958333, + -2.375 + ], + "width_ft": 2.583333, + "depth_ft": 1.833333, + "height_ft": 3.291667, + "footprint": [ + [ + -4.958333, + 0.0 + ], + [ + -2.375, + 0.0 + ], + [ + -2.375, + 1.833333 + ], + [ + -4.958333, + 1.833333 + ] + ], + "center": [ + -3.666667, + 0.916667 + ], + "along": [ + 1.0, + 0.0 + ], + "outward": [ + -0.0, + 1.0 + ], + "meter_offset_ft": [ + -3.666667, + 0.916667 + ], + "route_length_ft": 2.375 + }, + "route": { + "outcome": "pass", + "length_ft": 2.375, + "plus_minus_ft": 1.093333, + "height_ft": 1.0, + "polyline": [ + [ + 0.0, + 0.0 + ], + [ + -2.375, + 0.0 + ] + ], + "detours": [], + "crossings": [] + }, + "checks": [ + { + "id": "wall_backing", + "label": "Flush against one straight wall", + "outcome": "pass", + "reason": "The whole footprint backs onto one straight, observed wall segment.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "battery.width_ft", + "source": "Base Core spec: 30.68 in wide, rounded up to 31 in (docs/04, What Base does today)", + "placeholder": false + } + }, + { + "id": "ground_surface", + "label": "Ground under the battery", + "outcome": "pass", + "reason": "The whole footprint stands on an allowed surface.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "ground.allowed", + "source": "Ground types from schemas/scene.schema.json; which surfaces are allowed is a demo choice", + "placeholder": true + } + }, + { + "id": "meter_working_space", + "label": "Clear of the meter's working space", + "outcome": "pass", + "reason": "The battery is 1 ft 1 in clear of the meter's 2 ft 6 in wide by 3 ft 0 in deep working space.", + "measured_ft": 1.125, + "plus_minus_ft": 1.093333, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": "meter", + "rule": { + "key": "meter_working_space.width_ft", + "source": "NEC 110.26(A)(2): 30 in wide working space", + "placeholder": false + } + }, + { + "id": "gas_clearance", + "label": "Distance from gas equipment", + "outcome": "pass", + "reason": "No gas meter or pipe within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.gas_ft", + "source": "Base help page: 3 ft from the gas meter; Austin Energy \u00a71.9; Texas Gas Service", + "placeholder": false + } + }, + { + "id": "ac_clearance", + "label": "Distance from AC units", + "outcome": "pass", + "reason": "No AC unit within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.ac_ft", + "source": "Base help page: 3 ft from AC units, fences and other batteries", + "placeholder": false + } + }, + { + "id": "drive_clearance", + "label": "Distance from the driveway", + "outcome": "pass", + "reason": "No drivable surface within 5 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 5.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.drive_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "pool_clearance", + "label": "Distance from a pool", + "outcome": "pass", + "reason": "No pool within 10 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 10.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.pool_ft", + "source": "No public value (docs/04, Public rule values)", + "placeholder": true + } + }, + { + "id": "opening_clearance", + "label": "Distance from doors and windows", + "outcome": "pass", + "reason": "No door or window within 3 ft 0 in of the battery, and that whole area was seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.opening_ft", + "source": "IRC 2021 R328.4: 3 ft from doors and windows directly entering the dwelling. Applying it to garage doors (openings.types) is a demo choice the code does not require", + "placeholder": false + } + }, + { + "id": "wall_equipment_above", + "label": "No box or vent above the battery", + "outcome": "pass", + "reason": "Nothing is mounted on the wall above the battery.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 0.0, + "comparison": "at_least", + "subject": null, + "rule": { + "key": "clearances.wall_equipment_ft", + "source": "No public value; 0 means only overlap fails", + "placeholder": true + } + }, + { + "id": "facing_gap", + "label": "Open space in front", + "outcome": "pass", + "reason": "The gap in front of the battery is 7 ft 2 in (\u00b1 0 ft 0 in), clear of the 3 ft 0 in rule.", + "measured_ft": 7.166667, + "plus_minus_ft": 0.0, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": "facing[0]", + "rule": { + "key": "facing.min_ft", + "source": "Base help page: 3 ft from fences", + "placeholder": false + } + }, + { + "id": "headroom", + "label": "Headroom above", + "outcome": "pass", + "reason": "The headroom is 9 ft 0 in (\u00b1 0 ft 0 in), clear of the 6 ft 6 in rule.", + "measured_ft": 9.0, + "plus_minus_ft": 0.0, + "threshold_ft": 6.5, + "comparison": "at_least", + "subject": "overheads[0]", + "rule": { + "key": "headroom.min_ft", + "source": "NEC 110.26(A)(3) working-space headroom, applied to the battery as a demo choice", + "placeholder": true + } + }, + { + "id": "route_path", + "label": "Cable route along the wall", + "outcome": "pass", + "reason": "The cable runs along continuous, observed wall with nothing blocking it.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": null, + "comparison": null, + "subject": null, + "rule": { + "key": "route.crossing", + "source": "Demo rule: the cable can't cross a door, a garage or a stretch with no wall", + "placeholder": false + } + }, + { + "id": "route_length", + "label": "Cable run length", + "outcome": "pass", + "reason": "The cable run is 2 ft 4 in (\u00b1 1 ft 1 in), within the 15 ft 0 in confident reach.", + "measured_ft": 2.375, + "plus_minus_ft": 1.093333, + "threshold_ft": 20.0, + "comparison": "at_most", + "subject": null, + "rule": { + "key": "route.max_ft", + "source": "Base help page: within 20 ft of the meter. Review past 15 ft 0 in: No public value", + "placeholder": true + }, + "review_threshold_ft": 15.0 + } + ], + "nearest_considered": null, + "missing_evidence": [], + "objects_not_used": [], + "ends": { + "left": { + "kind": "limit", + "s_ft": -40.0, + "point": [ + -40.0, + 0.0 + ], + "beyond_reach": true + }, + "right": { + "kind": "limit", + "s_ft": 40.0, + "point": [ + 40.0, + 0.0 + ], + "beyond_reach": true + } + }, + "sweep": [ + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -40.0, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -27.242064, + -27.242064 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -27.183532, + -25.625 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -25.541667, + -14.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -14.875, + -4.958333 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -4.875, + -3.125 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + -3.041667, + 0.458333 + ], + "outcome": "fail", + "failing": [ + "meter_working_space" + ], + "unsure": [] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + 0.541667, + 2.291667 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "meter_working_space" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + 2.375, + 12.291667 + ], + "outcome": "pass", + "failing": [], + "unsure": [] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + 12.375, + 22.958333 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "route_length" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + 23.041667, + 24.600198 + ], + "outcome": "unsure", + "failing": [], + "unsure": [ + "pool_clearance", + "route_length" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + 24.65873, + 24.658731 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [ + "pool_clearance" + ] + }, + { + "wall_id": "w1", + "segment": 0, + "start_ft": [ + 24.658731, + 37.416667 + ], + "outcome": "fail", + "failing": [ + "route_length" + ], + "unsure": [] + } + ], + "stats": { + "candidates": 683, + "pass": 260, + "unsure": 368, + "fail": 55, + "input_sha256": "edc4a25462e15ff4c795730a42cc5c8fdcd838e672c262a4ed327f00bc288ea0", + "elapsed_ms": 0 + } + } +} diff --git a/verification/tests/segmented_wall.py b/verification/tests/segmented_wall.py new file mode 100644 index 000000000..a46904f2d --- /dev/null +++ b/verification/tests/segmented_wall.py @@ -0,0 +1,27 @@ +"""Equivalent surveyed straight walls for verifier representation regressions.""" + +import copy +import json +from pathlib import Path + + +def segmented_wall_scene(segments_per_wall: int) -> dict: + fixture = json.loads((Path(__file__).parent / "fixtures/wide-public-pass.json").read_text()) + scene = copy.deepcopy(fixture["scene"]) + scene["walls"] = [ + { + "id": f"w{i + 1}", + "baseline": [ + [-40 + 4 * i + 4 * j / segments_per_wall, 0] for j in range(segments_per_wall + 1) + ], + "height_ft": 9, + } + for i in range(20) + ] + scene["meter"]["wall_id"] = "w11" + for key in ("facing", "overheads"): + scene[key] = [ + dict(fixture["scene"][key][0], wall_id=f"w{i + 1}", span_ft=[-40 + 4 * i, -36 + 4 * i]) + for i in range(20) + ] + return scene diff --git a/verification/tests/test_a11yaudit.py b/verification/tests/test_a11yaudit.py index 6013a20e3..7a2f1fd03 100644 --- a/verification/tests/test_a11yaudit.py +++ b/verification/tests/test_a11yaudit.py @@ -1,3 +1,7 @@ +import argparse +import subprocess + +from hsverify import a11yaudit from hsverify.a11yaudit import audit_problems, audited_screens, parse_output, symbol_name_labels @@ -51,7 +55,7 @@ def test_zero_screens_is_not_evidence(): def test_harness_that_did_not_run_is_not_evidence(): problems = audit_problems(NINE, harness_ran=False, min_screens=9) - assert problems == ["the audit harness did not run (no TEST SUCCEEDED or TEST FAILED line)"] + assert problems == ["the audit harness did not finish successfully"] def test_too_few_distinct_screens(): @@ -79,3 +83,18 @@ def test_error_lines_from_the_harness_are_parsed_as_screens(): "screen 1: app not running", "no screen was audited", ] + + +def test_failed_xctest_cannot_be_clean_evidence(tmp_path, monkeypatch): + output = "\n".join(f'A11Y_SCREEN={{"index":{i},"issues":0}}' for i in range(1, 10)) + output += "\n** TEST SUCCEEDED **" + monkeypatch.setattr(a11yaudit.simrun, "wait_for_other_builds", lambda _: None) + monkeypatch.setattr( + a11yaudit.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess([], 65, output, "failed"), + ) + args = argparse.Namespace(build_wait=0, seconds=1, idle=1) + text, code = a11yaudit.run_harness("mock", "mock", args, [], tmp_path) + screens, _ = parse_output(text) + assert audit_problems(screens, harness_ran=code == 0, min_screens=9) diff --git a/verification/tests/test_build_scene.py b/verification/tests/test_build_scene.py index d2983bfb7..39ebcdabd 100644 --- a/verification/tests/test_build_scene.py +++ b/verification/tests/test_build_scene.py @@ -1,12 +1,15 @@ """The ETH3D facade scene builder replaces only its own outputs.""" import importlib.util +import json import sys from pathlib import Path import numpy as np import pytest +from hsverify.resultcheck import assumption_mismatches + SCRIPT = Path(__file__).resolve().parents[1] / "scenes" / "eth3d-facade" / "build_scene.py" spec = importlib.util.spec_from_file_location("build_scene", SCRIPT) build_scene = importlib.util.module_from_spec(spec) @@ -66,17 +69,68 @@ def seen_everywhere(kf, w, X, walls): [ ("wall", 3.29), # up to the battery's height, not the headroom an omitted view claims ("ground", build_scene.GROUND_OUT_FT), - ("overhead", build_scene.HEADROOM_FT), # not clear all the way up + ("overhead", 7.25), # the rules' headroom below, not clear all the way up ], ) def test_every_band_claims_only_the_reach_its_probes_showed(band, out_ft): entries, spans = build_scene.band_coverage( - band, [wall("w", 0.0, 2.0)], [], ["kf"], seen_everywhere + band, [wall("w", 0.0, 2.0)], [], ["kf"], seen_everywhere, headroom_ft=7.25 ) assert entries == [{"band": band, "span_ft": [0.0, 2.0], "out_ft": out_ft}] assert spans == [[0.0, 2.0]] +def test_the_overhead_band_is_probed_at_the_rules_headroom(): + heights = set() + + def record(kf, w, X, walls): + heights.add(round(float(X[1]), 6)) + return True + + build_scene.band_coverage("overhead", [wall("w", 0.0, 1.0)], [], ["kf"], record, 7.25) + assert max(heights) == 7.25 + with pytest.raises(ValueError, match=r"headroom\.min_ft"): + build_scene.band_coverage("overhead", [wall("w", 0.0, 1.0)], [], ["kf"], record) + + +def test_headroom_comes_from_rules_yaml_and_a_missing_one_is_loud(): + text = 'headroom:\n min_ft: {value: 7.0, source: "override"}\n' + assert build_scene.rules_headroom_ft(text, "rules.yaml") == 7.0 + with pytest.raises(SystemExit, match=r"rules\.yaml has no headroom\.min_ft\.value"): + build_scene.rules_headroom_ft("battery: {}\n", "rules.yaml") + + +@pytest.mark.parametrize(("value", "ok"), [(6.5, True), (12.0, False)]) +def test_a_headroom_above_the_scanned_overheads_is_refused(monkeypatch, value, ok): + """Overhead returns stop at OVERHEAD_MAX_FT (12 ft), so a headroom there or above would + read an obstruction above 12 ft as clear.""" + + def fake_git(args, **_): + out = "abc123\n" if args[1] == "rev-parse" else f"headroom:\n min_ft: {{value: {value}}}\n" + return build_scene.subprocess.CompletedProcess(args, 0, out, "") + + monkeypatch.setattr(build_scene.subprocess, "run", fake_git) + if ok: + assert build_scene.read_rules_headroom("origin/t3/server") == ("abc123", value) + return + with pytest.raises(SystemExit, match=r"headroom\.min_ft to 12 ft.*OVERHEAD_MAX_FT = 12 ft"): + build_scene.read_rules_headroom("origin/t3/server") + + +def test_the_case_pins_the_headroom_the_scene_was_built_with(): + """A server applying another headroom (changed rules or a private override) reports it as + the headroom check's threshold, and the case then stops judging a scene probed elsewhere.""" + case = json.loads((SCRIPT.parent / "case.json").read_text()) + assumed = case["rules_assumed"] + assert assumed["headroom"] == 6.5 + same = {"checks": [{"id": "headroom", "threshold_ft": 6.5}]} + assert assumption_mismatches(assumed, same) == [] + moved = {"checks": [{"id": "headroom", "threshold_ft": 7.0}]} + assert assumption_mismatches(assumed, moved) == [ + "case assumes headroom threshold 6.5, server's headroom uses 7.0" + ] + + def test_facing_claims_the_smallest_depth_probed_and_nothing_past_20_ft(): walls = [wall("a", 0.0, 1.0), wall("b", 1.0, 1.0)] # meet at a corner: one merged entry facing = [("a", 0.0, 1.0, 35.0), ("b", 1.0, 1.5, 12.345), ("b", 1.5, 2.0, 30.0)] @@ -92,3 +146,44 @@ def only_low(kf, w, X, walls): entries, _ = build_scene.band_coverage("wall", [wall("w", 0.0, 2.0)], [], ["kf"], only_low) assert entries == [] + + +@pytest.mark.parametrize("length", [2.2, 2.3, 0.05]) +def test_non_grid_wall_has_no_excess_or_missing_tail(length): + entries, spans = build_scene.band_coverage( + "wall", [wall("w", 0.0, length)], [], ["kf"], seen_everywhere + ) + assert entries[0]["span_ft"] == [0.0, length] + assert spans == [[0.0, length]] + + +@pytest.mark.parametrize("length", [2.2, 2.3, 0.05]) +def test_clipped_tail_probes_stay_within_the_wall(length): + probes = [] + + def within_wall(kf, w, point, walls): + probes.append(float(point[0])) + return 0.0 <= point[0] <= length + + entries, spans = build_scene.band_coverage( + "wall", [wall("w", 0.0, length)], [], ["kf"], within_wall + ) + assert probes and all(0.0 <= p <= length for p in probes) + assert spans == [[0.0, length]] + assert entries[0]["span_ft"] == [0.0, length] + + +def test_clipped_overhead_tail_retains_the_active_headroom(): + points = [] + + def within_wall(kf, w, point, walls): + points.append(point) + return 0.0 <= point[0] <= 2.2 + + entries, spans = build_scene.band_coverage( + "overhead", [wall("w", 0.0, 2.2)], [], ["kf"], within_wall, headroom_ft=7.25 + ) + assert max(float(point[1]) for point in points) == 7.25 + assert all(0.0 <= point[0] <= 2.2 for point in points) + assert entries == [{"band": "overhead", "span_ft": [0.0, 2.2], "out_ft": 7.25}] + assert spans == [[0.0, 2.2]] diff --git a/verification/tests/test_bundle_limits.py b/verification/tests/test_bundle_limits.py new file mode 100644 index 000000000..cc3a8f1aa --- /dev/null +++ b/verification/tests/test_bundle_limits.py @@ -0,0 +1,94 @@ +import ast +import io +import struct +import zipfile +from pathlib import Path + +import pytest + +from hsverify.bundle_limits import zip_directory_limits + + +def recon_guard(max_entries, max_bytes): + # Execute only the stdlib guard so this parity check needs no recon model dependencies. + source = Path(__file__).resolve().parents[2] / "recon" / "recon" / "capture.py" + tree = ast.parse(source.read_text()) + guard = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "_zip_directory_limits" + ) + namespace = { + "Path": Path, + "struct": struct, + "MAX_BUNDLE_ENTRIES": max_entries, + "MAX_BUNDLE_BYTES": max_bytes, + } + exec(compile(ast.Module(body=[guard], type_ignores=[]), str(source), "exec"), namespace) + return namespace["_zip_directory_limits"] + + +def ordinary_zip(): + data = io.BytesIO() + with zipfile.ZipFile(data, "w") as archive: + archive.writestr("scene.json", b"{}") + archive.writestr("frames/", b"") + archive.writestr("frames/a.jpg", b"image") + return bytearray(data.getvalue()) + + +@pytest.mark.parametrize( + "mutation", + [ + "ordinary", + "entries", + "bytes", + "false_count", + "comment", + "footer_in_comment", + "split", + "directory_offset", + "directory_record", + "zip64_locator", + "truncated", + ], +) +def test_admission_matches_recon_guard(tmp_path, mutation): + data = ordinary_zip() + entries, expanded = 3, 7 + footer = len(data) - 22 + directory = struct.unpack_from("oops", b"[1, 2]"): monkeypatch.setattr(e2e, "post", lambda *args, body=body, **kwargs: (200, body, 5.0)) @@ -484,3 +712,530 @@ def test_a_200_that_is_not_a_json_object_is_a_failure_not_a_crash(monkeypatch): assert "not a JSON object" in record["contract_problems"][0] hostile = judge_hostile("http://unused", endpoint, hostile_inputs()[3], SCHEMAS) assert hostile["status"] == "fail" and "not a JSON object" in hostile["problems"][0] + + +@pytest.mark.parametrize( + "extra", + [ + {"status": "skipped", "skip_reason": "missing scene"}, + {"status": "fail", "expectation_problems": ["wrong decision"], "contract_problems": []}, + ], +) +def test_run_acceptance_is_separate_from_contract_invariants(tmp_path, extra): + report = e2e.write_report(tmp_path, META, [CHECKED, {"name": "other", **extra}]) + assert not report["all_passed"] + assert report["contract_ok"] + + +def test_unexercised_large_input_probe_fails_contract(): + item = hostile_inputs()[-1] + item.bundle = b"archive" + record = judge_hostile("unused", Endpoint("/place", "application/json"), item, {}) + assert record["status"] == "fail" and record["contract_problems"] + + +def test_bundle_uses_exact_canonical_scene(tmp_path): + archive = tmp_path / "scene.zip" + with zipfile.ZipFile(archive, "w") as z: + z.writestr("old-scene.json", b"old") + z.writestr("scene.json", b"canonical") + assert e2e.read_bundle(archive)[0] == b"canonical" + + +def test_bundle_reads_one_wrapped_capture_and_ignores_macos_metadata(tmp_path): + archive = tmp_path / "scene.zip" + with zipfile.ZipFile(archive, "w") as z: + z.writestr("capture/", b"") + z.writestr("capture/scene.json", b"canonical") + z.writestr("capture/frame.jpg", b"photo") + z.writestr("__MACOSX/capture/._scene.json", b"metadata") + z.writestr("__MACOSX/capture/frame.jpg", b"metadata") + assert e2e.read_bundle(archive) == (b"canonical", {"frame.jpg": b"photo"}) + + +def test_bundle_accepts_public_api_finder_fixture_shape(tmp_path): + archive = tmp_path / "scene.zip" + with zipfile.ZipFile(archive, "w") as z: + z.writestr("capture-2026-09-26/scene.json", b"canonical") + z.writestr("__MACOSX/capture-2026-09-26/._scene.json", b"resource fork") + assert e2e.read_bundle(archive) == (b"canonical", {}) + + +@pytest.mark.parametrize("wrapped_scene", ["capture/scene.json", "a/b/scene.json"]) +def test_bundle_rejects_root_and_wrapped_scene_together(tmp_path, wrapped_scene): + archive = tmp_path / "scene.zip" + with zipfile.ZipFile(archive, "w") as z: + z.writestr("scene.json", b"canonical") + z.writestr(wrapped_scene, b"other") + with pytest.raises(SystemExit, match=r"ambiguous scene\.json"): + e2e.read_bundle(archive) + + +@pytest.mark.parametrize( + "names", + [ + ["old-scene.json"], + ["not-scene.json"], + ["scene.json", "scene.json"], + ["capture/scene.json", "capture/scene.json"], + ["first/scene.json", "second/scene.json"], + ["capture/scene.json", "other/frame.jpg"], + ["capture/nested/scene.json"], + ["capture/scene.json", "other/"], + ["capture/scene.json", "capture"], + ], +) +def test_bundle_rejects_missing_or_duplicate_canonical_scene(tmp_path, names): + archive = tmp_path / "scene.zip" + with zipfile.ZipFile(archive, "w") as z: + for name in names: + z.writestr(name, b"{}") + with pytest.raises( + SystemExit, match=r"exactly one canonical scene\.json|ambiguous scene\.json" + ): + e2e.read_bundle(archive) + + +def test_transformed_answer_checks_its_own_hash(monkeypatch): + calls = [] + original = {"decision": "pass", "stats": {}, "sweep": []} + monkeypatch.setattr( + e2e, "post", lambda *args, **kwargs: (200, json.dumps(original).encode(), 1) + ) + monkeypatch.setattr(e2e, "answer", lambda *args: (original, None)) + monkeypatch.setattr(e2e, "schema_errors", lambda *args: []) + + def check_hash(scene, result, sent=None, rules=None, *, meter=None): + calls.append((scene, sent)) + return ["stale input hash"] if sent is not None else [] + + monkeypatch.setattr(e2e, "invariant_problems", check_hash) + monkeypatch.setattr(e2e, "with_ground_short_of", lambda *args: None) + monkeypatch.setattr(e2e, "with_more_error", lambda *args: None) + monkeypatch.setattr(e2e, "tape_to_tap", lambda *args: None) + monkeypatch.setattr(e2e, "with_less_coverage", lambda *args: None) + scene = { + "walls": [{"id": "w1", "baseline": [[-1, 0], [1, 0]]}], + "meter": {"pos": [0, 1, 0], "wall_id": "w1"}, + } + probe = SceneInput(name="test", scene=scene, raw=json.dumps(scene).encode(), source="test") + failures = e2e.property_problems( + "unused", Endpoint("/p", "application/json"), probe, original, {"result": {}}, RULES + ) + assert calls and all(raw == json.dumps(scene, indent=1).encode() for scene, raw in calls) + assert any("stale input hash" in p for p in failures) + + +def test_connection_and_handshake_share_one_remaining_budget(monkeypatch): + import socket + import ssl + + clock = [0.0] + phases = [] + + class FakeSocket: + def settimeout(self, timeout): + phases.append(timeout) + + def connect(self, address): + clock[0] += 0.6 + + def do_handshake(self): + assert phases[-1] == pytest.approx(0.4) + clock[0] += 0.5 + raise TimeoutError + + def close(self): + pass + + def shutdown(self, how): + pass + + sock = FakeSocket() + + class Context: + def wrap_socket(self, raw, *, server_hostname, do_handshake_on_connect): + assert raw is sock and not do_handshake_on_connect + return sock + + monkeypatch.setattr(e2e.time, "perf_counter", lambda: clock[0]) + monkeypatch.setattr( + socket, + "getaddrinfo", + lambda *args, **kwargs: [(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("127.0.0.1", 443))], + ) + monkeypatch.setattr(socket, "socket", lambda *args: sock) + monkeypatch.setattr(ssl, "create_default_context", Context) + status, payload, ms = e2e.post( + "https://localhost", Endpoint("/p", "application/json"), item(), 1 + ) + assert status is None and payload == b"no answer within 1 s" + assert phases[:2] == pytest.approx([1.0, 0.4]) + assert ms == pytest.approx(1100) + + +@pytest.mark.parametrize("phase", ["dns", "tcp", "tls"]) +def test_deadline_bounds_blocked_connection_phase_and_cleans_worker(monkeypatch, phase): + import socket + import ssl + + entered, release, exited, closed = (threading.Event() for _ in range(4)) + + def block(): + entered.set() + assert release.wait(2) + raise TimeoutError + + class FakeSocket: + def settimeout(self, timeout): + assert 0 < timeout <= 0.1 + + def connect(self, address): + if phase == "tcp": + block() + + def do_handshake(self): + block() + + def shutdown(self, how): + release.set() + + def close(self): + closed.set() + + sock = FakeSocket() + + def lookup(*args, **kwargs): + if phase == "dns": + entered.set() + assert release.wait(2) + return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("127.0.0.1", 443))] + + class Context: + def wrap_socket(self, raw, **kwargs): + assert not kwargs["do_handshake_on_connect"] + return raw + + original_thread = threading.Thread + workers = [] + + def thread(*args, target, **kwargs): + def run(): + try: + target() + finally: + exited.set() + + worker = original_thread(*args, target=run, **kwargs) + workers.append(worker) + return worker + + monkeypatch.setattr(e2e.threading, "Thread", thread) + monkeypatch.setattr(socket, "getaddrinfo", lookup) + monkeypatch.setattr(socket, "socket", lambda *args: sock) + monkeypatch.setattr(ssl, "create_default_context", Context) + try: + status, payload, ms = e2e.post( + "https://localhost", Endpoint("/p", "application/json"), item(), 0.1 + ) + assert entered.is_set() + assert status is None and payload == b"no answer within 0.1 s" + assert 90 <= ms < 500 + if phase != "dns": + assert closed.is_set() + finally: + release.set() + for worker in workers: + worker.join(2) + assert exited.is_set() and all(not w.is_alive() for w in workers) + if phase == "dns": + assert not closed.is_set() # no socket can be allocated after late DNS completes + + +def test_stalled_local_tls_handshake_ends_with_overall_deadline(): + import socket + + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + listener.settimeout(2) + stopped, accepted, disconnected = (threading.Event() for _ in range(3)) + + def peer(): + with listener: + conn, _ = listener.accept() + with conn: + conn.settimeout(2) + accepted.set() + while conn.recv(4096): + pass # read ClientHello but never answer with ServerHello + disconnected.set() + stopped.set() + + server = threading.Thread(target=peer) + server.start() + try: + status, payload, ms = e2e.post( + f"https://127.0.0.1:{listener.getsockname()[1]}", + Endpoint("/p", "application/json"), + item(), + 0.2, + ) + assert accepted.is_set() + assert status is None and payload == b"no answer within 0.2 s" + assert 150 <= ms < 1000 + assert disconnected.wait(2) + finally: + server.join(3) + assert not server.is_alive() and stopped.is_set() + + +def hostile_records(): + return [ + { + "name": name, + "hostile": True, + "hostile_probe_executed": True, + "http_status": 422, + "latency_ms": 1.0, + "status": "pass", + "contract_problems": [], + } + for name in e2e.HOSTILE_PROBE_NAMES + ] + + +@pytest.mark.parametrize("missing", range(5)) +def test_every_required_hostile_probe_must_be_exercised(tmp_path, missing): + probes = hostile_records() + omitted = probes.pop(missing) + probes.append(probes[0].copy()) # duplicate evidence cannot replace the missing probe + report = e2e.write_report(tmp_path, META, [CHECKED, *probes]) + assert report["all_passed"] and report["contract_ok"] + assert not report["hostile_probes_complete"] and not report["full_contract_ok"] + assert omitted["name"] not in report["hostile_probes_executed"] + + +def test_probe_name_without_execution_is_not_full_evidence(tmp_path): + probes = hostile_records() + del probes[0]["hostile_probe_executed"] + assert not e2e.write_report(tmp_path, META, [CHECKED, *probes])["full_contract_ok"] + assert e2e.write_report(tmp_path, META, [CHECKED, *hostile_records()])["full_contract_ok"] + + +def test_hostile_execution_is_recorded_only_after_sending(monkeypatch): + monkeypatch.setattr(e2e, "post", lambda *args, **kwargs: (422, b"{}", 1.0)) + probe = SceneInput(e2e.HOSTILE_PROBE_NAMES[0], {}, b"{}", hostile=True) + record = judge("unused", Endpoint("/place", "application/json"), probe, SCHEMAS, RULES) + assert record["hostile_probe_executed"] is True + probe.bundle = b"archive" + record = judge("unused", Endpoint("/place", "application/json"), probe, SCHEMAS, RULES) + assert not record.get("hostile_probe_executed", False) + + +@pytest.mark.parametrize("no_hostile", [False, True]) +@pytest.mark.parametrize("unverified", [False, True]) +def test_cli_shortened_run_cannot_claim_full_contract( + tmp_path, monkeypatch, no_hostile, unverified +): + from pathlib import Path + + # Only public rules are read. All HTTP, git and hostile payload generation are mocked. + public_rules = (Path(__file__).resolve().parents[2] / "server" / "rules.yaml").read_text() + cases = tmp_path / "cases" + cases.mkdir() + (cases / "scene.json").write_text(json.dumps(SCENE)) + monkeypatch.setattr(e2e.gitref, "fetch", lambda: None) + monkeypatch.setattr(e2e.gitref, "resolve", lambda _: "a" * 40) + monkeypatch.setattr( + e2e.gitref, "show", lambda sha, path: public_rules if path.endswith("rules.yaml") else "{}" + ) + monkeypatch.setattr( + e2e.urllib.request, + "urlopen", + lambda *args, **kwargs: io.BytesIO(json.dumps(openapi({"/place": json_op()})).encode()), + ) + monkeypatch.setattr( + e2e, + "hostile_inputs", + lambda: [SceneInput(name, {}, b"{}", hostile=True) for name in e2e.HOSTILE_PROBE_NAMES], + ) + + def fake_judge(url, endpoint, probe, *args): + if probe.hostile: + return next(r for r in hostile_records() if r["name"] == probe.name) + return ( + CHECKED + | {"name": probe.name} + | ( + { + "status": "assumptions unverified", + "rule_checks_unverified": True, + "assumption_warnings": ["assumed public rules; server may override"], + } + if unverified + else {} + ) + ) + + monkeypatch.setattr(e2e, "judge", fake_judge) + monkeypatch.setattr(e2e, "peak_rss_mb", lambda: {}) + out = tmp_path / "out" + args = ["--server-url", "http://unused", "--cases", str(cases), "--out", str(out)] + if no_hostile: + args.append("--no-hostile") + assert e2e.main(args) == (2 if unverified else 0) + data = json.loads((out / "report.json").read_text()) + assert data["all_passed"] is (not unverified) + assert data["contract_ok"] is (not unverified) + assert data["full_contract_ok"] is (not no_hostile and not unverified) + assert data["hostile_probes_executed"] == ( + [] if no_hostile else sorted(e2e.HOSTILE_PROBE_NAMES) + ) + from hsverify import scoreboard + + metric = next(m for m in scoreboard.load_metrics()[1] if m.id == "S2-8") + metric.probe["glob"] = str(out / "report.json") + status = scoreboard.probe_report(metric.probe, data["sha"]).status + assert status == ("gap" if no_hostile or unverified else "met") + + +@pytest.mark.parametrize("reported_width", [None, 31 / 12, 4]) +@pytest.mark.parametrize("contract_failure", [False, True]) +def test_judge_distinguishes_reported_override_mismatch_and_unverified_defaults( + tmp_path, monkeypatch, reported_width, contract_failure +): + answer = { + "decision": "manual_review", + "spot": None, + "checks": [], + "sweep": [], + "policy": {"sources": ["private"], "rules_sha256": "f" * 64}, + } + if reported_width is not None: + answer["checks"] = [ + { + "id": "wall_backing", + "threshold_ft": reported_width, + "rule": {"key": "battery.width_ft"}, + } + ] + monkeypatch.setattr(e2e, "post", lambda *args: (200, json.dumps(answer).encode(), 1)) + monkeypatch.setattr(e2e, "schema_errors", lambda *args: []) + seen = [] + + def invariants(scene, result, sent=None, rules=None, *, meter=None): + seen.append(None if rules is None else rules.width_ft) + return ( + ["stats.input_sha256 does not match the scene bytes sent"] if contract_failure else [] + ) + + monkeypatch.setattr(e2e, "invariant_problems", invariants) + properties = [] + monkeypatch.setattr( + e2e, + "property_problems", + lambda *args, **kwargs: properties.append((args[-1].width_ft, kwargs["rule_checks"])) or [], + ) + expectations = [] + monkeypatch.setattr( + e2e, "expectation_problems", lambda *args, **kwargs: expectations.append(True) or [] + ) + item = SceneInput( + "override", + SCENE, + b"{}", + expect={"decision_in": ["manual_review"]}, + rules_assumed={"battery_width": 31 / 12}, + ) + record = judge( + "unused", Endpoint("/place", "application/json"), item, SCHEMAS, replace(RULES, width_ft=4) + ) + assert seen == [reported_width] + if contract_failure: + assert record["status"] == "fail" and record["contract_problems"] + assert record["problems"] == ["stats.input_sha256 does not match the scene bytes sent"] + else: + assert record["problems"] == [] + if reported_width is None: + assert record["status"] == ("fail" if contract_failure else "assumptions unverified") + assert "assumed public rules; server may override" in record["assumption_warnings"][0] + assert properties == ([] if contract_failure else [(4, False)]) + assert not expectations + elif reported_width == 31 / 12: + assert record["status"] == ("fail" if contract_failure else "pass") + assert properties == ([] if contract_failure else [(31 / 12, True)]) + assert expectations == [True] + else: + assert record["status"] == ("fail" if contract_failure else "assumption mismatch") + assert record["assumption_mismatches"] and not expectations + report = e2e.write_report(tmp_path, META, [record, *hostile_records()]) + assert report["all_passed"] is (reported_width == 31 / 12 and not contract_failure) + assert report["full_contract_ok"] is (reported_width == 31 / 12 and not contract_failure) + if reported_width is None: + assert not report["contract_ok"] + assert "assumed public rules; server may override" in (tmp_path / "report.md").read_text() + + +@pytest.mark.parametrize("width", [-1, 0, float("nan"), float("inf")]) +def test_vet_judge_rejects_invalid_width_before_rule_consumers(monkeypatch, width): + answer = { + "decision": "reject", + "spot": None, + "sweep": [], + "checks": [{"id": "width", "threshold_ft": width, "rule": {"key": "battery.width_ft"}}], + } + monkeypatch.setattr(e2e, "post", lambda *args: (200, json.dumps(answer).encode(), 1)) + monkeypatch.setattr(e2e, "schema_errors", lambda *args: []) + consumers = [] + + def no_adoption(*args): + pytest.fail("invalid dimensions reached rule adoption") + + monkeypatch.setattr(e2e, "reported_rules", no_adoption) + monkeypatch.setattr( + e2e, "invariant_problems", lambda *args, **kwargs: consumers.append("invariants") or [] + ) + monkeypatch.setattr( + e2e, "property_problems", lambda *args: consumers.append("properties") or [] + ) + item = SceneInput("invalid-width", SCENE, b"{}") + record = judge("unused", Endpoint("/place", "application/json"), item, SCHEMAS, RULES) + assert record["status"] == "fail" + assert "width must be finite and positive" in record["contract_problems"] + assert consumers == [] + + +@pytest.mark.parametrize("allowance", [-1, float("nan"), float("inf")]) +def test_breadth_invalid_corner_allowance_stops_before_rule_adoption(monkeypatch, allowance): + answer = { + "decision": "reject", + "spot": None, + "sweep": [], + "checks": [ + { + "id": "route_corner", + "threshold_ft": allowance, + "rule": {"key": "route.corner_allowance_ft"}, + } + ], + } + monkeypatch.setattr(e2e, "post", lambda *args: (200, json.dumps(answer).encode(), 1)) + monkeypatch.setattr(e2e, "schema_errors", lambda *args: []) + + def no_adoption(*args): + pytest.fail("invalid corner allowance reached rule adoption") + + monkeypatch.setattr(e2e, "reported_rules", no_adoption) + record = judge( + "unused", + Endpoint("/place", "application/json"), + SceneInput("corner", SCENE, b"{}"), + SCHEMAS, + RULES, + ) + assert record["status"] == "fail" + assert ( + "route_corner corner allowance must be finite and nonnegative" + in record["contract_problems"] + ) diff --git a/verification/tests/test_generate_cases.py b/verification/tests/test_generate_cases.py index 01c7cdfac..cd10314e2 100644 --- a/verification/tests/test_generate_cases.py +++ b/verification/tests/test_generate_cases.py @@ -39,3 +39,84 @@ def test_the_committed_cases_are_what_the_generator_writes(tmp_path): committed = SCRIPT.parent for f in tmp_path.glob("*.json"): assert f.read_text() == (committed / f.name).read_text(), f.name + + +def test_fixture_geometric_baselines_match_public_rules(): + import yaml + + rules = yaml.safe_load((SCRIPT.parents[3] / "server" / "rules.yaml").read_text()) + assert pytest.approx(rules["battery"]["width_ft"]["value"]) == generate.W + assert rules["route"]["corner_allowance_ft"]["value"] == generate.CORNER_ALLOWANCE + for case in generate.CASES: + assert case["rules_assumed"]["battery_width"] == generate.W + for cid in [ + "g03-negative-s-near-edge", + "g05-inside-corner-gas", + "g11b-both-ends-real-all-fail", + "g12b-exact-fit-last-start", + ]: + case = next(c for c in generate.CASES if c["id"] == cid) + assert case["rules_assumed"]["corner_allowance"] == generate.CORNER_ALLOWANCE + + +@pytest.mark.parametrize("origin", ["committed", "generated"]) +@pytest.mark.parametrize("override", ["width", "corner"]) +def test_real_case_rule_override_skips_stale_expectation(tmp_path, monkeypatch, origin, override): + from dataclasses import replace + + from hsverify import e2e + from hsverify.resultcheck import RuleSet, expectation_problems + + if origin == "generated": + generate.main(tmp_path) + folder = tmp_path + else: + folder = SCRIPT.parent + fixture = e2e.load_input(folder / "g12b-exact-fit-last-start.json") + public = (SCRIPT.parents[3] / "server" / "rules.yaml").read_text() + baseline = RuleSet.from_yaml(public) + rules = ( + replace(baseline, width_ft=4.0) + if override == "width" + else replace(baseline, corner_allowance_ft=10.0) + ) + # A wider battery cannot fit the exact-width segment; a high corner allowance loses reach. + answer = {"decision": "reject", "spot": None, "checks": [], "sweep": []} + assert expectation_problems(fixture.expect, answer) # stale fit assertion would fail + monkeypatch.setattr( + e2e, "post", lambda *args, **kwargs: (200, json.dumps(answer).encode(), 1.0) + ) + monkeypatch.setattr(e2e, "schema_errors", lambda *args: []) + monkeypatch.setattr(e2e, "invariant_problems", lambda *args, **kwargs: []) + monkeypatch.setattr(e2e, "property_problems", lambda *args, **kwargs: []) + + def forbidden(*args): + pytest.fail("a mismatched case must not apply its stale expectations") + + monkeypatch.setattr(e2e, "expectation_problems", forbidden) + record = e2e.judge( + "unused", + e2e.Endpoint("/place", "application/json"), + fixture, + {"scene": {}, "result": {}}, + rules, + ) + assert record["status"] == "assumptions unverified" and record["assumption_warnings"] + assert "assumed public rules; server may override" in record["assumption_warnings"][0] + assert record["rule_checks_unverified"] + assert record["contract_problems"] == [] + assert record["expectation_problems"] == [] + meta = { + "peak_memory": {}, + "server_ref": "public", + "server_sha": "a" * 40, + "started_at": "test", + "endpoint": "mock", + "load_average_1_5_15": [], + } + probes = [ + {"name": name, "hostile": True, "hostile_probe_executed": True, "status": "pass"} + for name in e2e.HOSTILE_PROBE_NAMES + ] + report = e2e.write_report(tmp_path, meta, [record, *probes]) + assert not report["all_passed"] and not report["full_contract_ok"] diff --git a/verification/tests/test_gitref.py b/verification/tests/test_gitref.py index da2c1511a..3903936d3 100644 --- a/verification/tests/test_gitref.py +++ b/verification/tests/test_gitref.py @@ -12,3 +12,10 @@ def test_two_uses_of_one_sha_get_separate_trees_and_both_clean_up(): def test_show_returns_none_for_a_missing_file(): assert gitref.show("HEAD", "no/such/file") is None assert gitref.show("HEAD", "verification/pyproject.toml") is not None + + +def test_fetch_prunes_deleted_refs(monkeypatch): + calls = [] + monkeypatch.setattr(gitref, "git", lambda *args: calls.append(args)) + gitref.fetch() + assert calls == [("fetch", "--quiet", "--prune", "origin")] diff --git a/verification/tests/test_intervalindex.py b/verification/tests/test_intervalindex.py new file mode 100644 index 000000000..36b6f327b --- /dev/null +++ b/verification/tests/test_intervalindex.py @@ -0,0 +1,113 @@ +import random + +import pytest + +from hsverify.intervalindex import ( + SweepIndex, + less_coverage_problems, + more_error_problems, + probe_starts, +) + + +class CountingMeter: + def __init__(self, limit=None): + self.samples = 0 + self.limit = limit + + def charge(self, n=1): + self.samples += n + if self.limit is not None and self.samples > self.limit: + raise WorkLimit + + +class WorkLimit(Exception): + pass + + +def result(*runs, decision="fail"): + return {"decision": decision, "sweep": list(runs)} + + +def run(a, b, outcome="fail", wall_id="w1"): + return {"wall_id": wall_id, "start_ft": [a, b], "outcome": outcome} + + +def test_thirty_thousand_runs_compare_with_linear_logarithmic_work(): + answer = result(*(run(i / 1e4, i / 1e4) for i in range(30_000))) + meter = CountingMeter() + assert more_error_problems(answer, answer, "more error", meter) == [] + assert 30_000 < meter.samples < 100 * 30_000 + + +def test_first_original_run_wins_overlaps_including_eps_endpoints(): + answer = result(run(2, 4, "fail"), run(0, 6, "pass"), run(4, 4, "unsure")) + index = SweepIndex(answer) + for start in (2 - 1e-6, 2, 3, 4, 4 + 1e-6): + assert index.outcome_at("w1", start) == "fail" + assert index.outcome_at("w1", 2 - 2e-6) == "pass" + assert index.outcome_at("w1", 4 + 2e-6) == "pass" + + +def test_index_agrees_with_original_linear_lookup_on_overlapping_runs(): + rng = random.Random(731) + runs = [ + run( + a := rng.randrange(-20, 20), + a + rng.randrange(0, 10), + rng.choice(["pass", "fail", "unsure"]), + ) + for _ in range(200) + ] + index = SweepIndex(result(*runs)) + points = {x for r in runs for edge in r["start_ft"] for x in (edge - 1e-6, edge, edge + 1e-6)} + points.update(rng.uniform(-30, 30) for _ in range(500)) + for start in points: + expected = next( + ( + r["outcome"] + for r in runs + if r["start_ft"][0] - 1e-6 <= start <= r["start_ft"][1] + 1e-6 + ), + None, + ) + assert index.outcome_at("w1", start) == expected + + +def test_gap_neighbours_and_equal_endpoint_ties_preserve_original_order(): + index = SweepIndex(result(run(0, 1, "fail"), run(0, 1, "pass"), run(3, 4, "unsure"))) + assert index.outcomes_near("w1", 2) == ({"fail", "unsure"}, "unswept beside fail and unsure") + assert index.outcomes_near("w1", -1) == ({"fail"}, "past the original sweep") + assert index.outcomes_near("w1", 5) == ({"fail"}, "past the original sweep") + assert index.outcomes_near("missing", 0) == (set(), "no run on its wall") + + +def test_newly_evaluated_wall_can_fail_but_cannot_pass(): + before = result(run(0, 1)) + assert more_error_problems(before, result(run(0, 1, "fail", "w2")), "error") == [] + assert more_error_problems(before, result(run(0, 1, "pass", "w2")), "error") + assert less_coverage_problems(before, result(run(0, 1, "pass", "w2")), "coverage") + + +def test_unswept_gap_can_keep_neighbour_pass_but_not_create_one(): + after = result(run(2, 2, "pass")) + assert more_error_problems(result(run(0, 1, "pass"), run(3, 4)), after, "error") == [] + assert less_coverage_problems(result(run(0, 1, "pass"), run(3, 4)), after, "coverage") == [] + assert less_coverage_problems(result(run(0, 1), run(3, 4)), after, "coverage") + + +@pytest.mark.parametrize("operation", [SweepIndex, probe_starts]) +def test_work_limit_propagates_from_index_or_probe_creation(operation): + with pytest.raises(WorkLimit): + operation(result(run(0, 1)), meter=CountingMeter(limit=0)) + + +@pytest.mark.parametrize("comparison", [more_error_problems, less_coverage_problems]) +def test_work_limit_propagates_from_comparison(comparison): + answer = result(run(0, 1, "pass")) + with pytest.raises(WorkLimit): + comparison(answer, answer, "change", meter=CountingMeter(limit=1)) + + +def test_probe_midpoint_does_not_overflow_finite_endpoints(): + assert probe_starts(result(run(1e308, 1e308))) == [("w1", 1e308)] diff --git a/verification/tests/test_judge_meter.py b/verification/tests/test_judge_meter.py new file mode 100644 index 000000000..77fea4f18 --- /dev/null +++ b/verification/tests/test_judge_meter.py @@ -0,0 +1,294 @@ +"""Hostile answers must still produce a report, with bounded verification work.""" + +import copy +import hashlib +import json +from pathlib import Path + +import pytest + +from hsverify import e2e +from hsverify.resultcheck import RuleSet, WorkMeter + +ROOT = Path(__file__).resolve().parents[2] +RULES = RuleSet.from_yaml((ROOT / "server/rules.yaml").read_text()) +SCHEMAS = { + "scene": {}, + "result": json.loads((ROOT / "server/schemas/result.schema.json").read_text()), +} +SCENE = { + "meter": {"pos": [0, 4, 0], "wall_id": "w1"}, + "walls": [{"id": "w1", "baseline": [[-30, 0], [30, 0]]}], +} +ENDPOINT = e2e.Endpoint("/place", "application/json") +META = { + "started_at": "2026-10-14", + "server_ref": "public test fixture", + "server_sha": "a" * 40, + "endpoint": "/place", + "load_average_1_5_15": [0, 0, 0], + "peak_memory": {}, +} + + +def probe(): + return e2e.SceneInput("hostile answer", SCENE, json.dumps(SCENE).encode()) + + +def response(): + return { + "schema_version": "1.0", + "decision": "manual_review", + "summary": "No spot chosen", + "reasons": [], + "policy": { + "id": "demo", + "version": "1", + "auto_approve": False, + "sources": ["public"], + "rules_sha256": "a" * 64, + }, + "spot": None, + "route": None, + "checks": [], + "missing_evidence": [], + "ends": { + side: {"kind": "limit", "s_ft": s, "point": [s, 0]} + for side, s in (("left", -30), ("right", 30)) + }, + "sweep": [], + "stats": { + "candidates": 0, + "pass": 0, + "unsure": 0, + "fail": 0, + "elapsed_ms": 1, + "input_sha256": hashlib.sha256(probe().raw).hexdigest(), + }, + } + + +def serve(monkeypatch, result): + assert not e2e.schema_errors(result, SCHEMAS["result"]) + monkeypatch.setattr(e2e, "post", lambda *args: (200, json.dumps(result).encode(), 1)) + + +def forbid_properties(*args, **kwargs): + raise AssertionError("a rejected answer reached property comparisons") + + +@pytest.mark.parametrize("exception", [OverflowError, ValueError, RuntimeError]) +def test_check_exception_is_written_to_report(monkeypatch, tmp_path, exception): + serve(monkeypatch, response()) + + def crash(*args, **kwargs): + raise exception("hostile response") + + monkeypatch.setattr(e2e, "invariant_problems", crash) + monkeypatch.setattr(e2e, "property_problems", forbid_properties) + record = e2e.judge("unused", ENDPOINT, probe(), SCHEMAS, RULES) + assert record["status"] == "fail" + assert record["problems"] == [f"invariants: {exception.__name__}: hostile response"] + report = e2e.write_report(tmp_path, META, [record]) + assert not report["contract_ok"] + assert json.loads((tmp_path / "report.json").read_text())["records"][0] == record + + +def test_failed_invariants_skip_properties(monkeypatch): + serve(monkeypatch, response()) + monkeypatch.setattr(e2e, "invariant_problems", lambda *args, **kwargs: ["bad invariant"]) + monkeypatch.setattr(e2e, "property_problems", forbid_properties) + record = e2e.judge("unused", ENDPOINT, probe(), SCHEMAS, RULES) + assert record["contract_problems"] == ["bad invariant"] + + +def test_original_and_properties_share_one_meter(monkeypatch): + serve(monkeypatch, response()) + meters = [] + + def invariants(*args, meter, **kwargs): + meters.append(meter) + meter.charge(4) + return [] + + def properties(*args, meter, **kwargs): + meters.append(meter) + meter.charge(5) + return [] + + monkeypatch.setattr(e2e, "invariant_problems", invariants) + monkeypatch.setattr(e2e, "property_problems", properties) + monkeypatch.setattr(e2e, "geometry_admission_problems", lambda *args, **kwargs: []) + meter = WorkMeter(limit=9) + record = e2e.judge("unused", ENDPOINT, probe(), SCHEMAS, RULES, meter=meter) + assert record["status"] == "pass" + assert record["verification_samples"] == 9 + assert meters == [meter, meter] + + +def test_meter_limit_is_one_specific_report_problem(monkeypatch, tmp_path): + serve(monkeypatch, response()) + + def invariants(*args, meter, **kwargs): + meter.charge(4) + return [] + + monkeypatch.setattr(e2e, "invariant_problems", invariants) + monkeypatch.setattr(e2e, "property_problems", forbid_properties) + monkeypatch.setattr(e2e, "geometry_admission_problems", lambda *args, **kwargs: []) + record = e2e.judge("unused", ENDPOINT, probe(), SCHEMAS, RULES, meter=WorkMeter(limit=3)) + assert record["problems"] == ["verification work limit exceeded (4 samples)"] + assert record["verification_samples"] == 4 + assert not e2e.write_report(tmp_path, META, [record])["contract_ok"] + + +def test_oversized_integer_is_an_admission_problem(monkeypatch, tmp_path): + result = response() + result["sweep"] = [ + {"wall_id": "w1", "start_ft": [0, 10**400], "outcome": "fail", "failing": [], "unsure": []} + ] + serve(monkeypatch, result) + monkeypatch.setattr(e2e, "property_problems", forbid_properties) + record = e2e.judge("unused", ENDPOINT, probe(), SCHEMAS, RULES) + assert record["status"] == "fail" + assert any("finite" in p for p in record["problems"]) + assert not any("OverflowError" in p for p in record["problems"]) + assert record["verification_samples"] == 0 + assert not e2e.write_report(tmp_path, META, [record])["contract_ok"] + + +def spot(wall_id): + return { + "outcome": "fail", + "wall_id": wall_id, + "segment": 0, + "span_ft": [1, 1 + RULES.width_ft], + "width_ft": RULES.width_ft, + "depth_ft": RULES.depth_ft, + "height_ft": RULES.battery_height_ft, + "footprint": [ + [1, 0], + [1 + RULES.width_ft, 0], + [1 + RULES.width_ft, RULES.depth_ft], + [1, RULES.depth_ft], + ], + "center": [1 + RULES.width_ft / 2, RULES.depth_ft / 2], + "along": [1, 0], + "outward": [0, 1], + "meter_offset_ft": [1 + RULES.width_ft / 2, RULES.depth_ft / 2], + "route_length_ft": None, + } + + +@pytest.mark.parametrize("field", ["sweep", "spot", "nearest_considered"]) +def test_unknown_wall_is_rejected_before_geometry(monkeypatch, tmp_path, field): + result = response() + if field == "sweep": + result[field] = [ + { + "wall_id": "missing", + "start_ft": [1, 2], + "outcome": "fail", + "failing": [], + "unsure": [], + } + ] + else: + result[field] = spot("missing") + serve(monkeypatch, result) + monkeypatch.setattr(e2e, "property_problems", forbid_properties) + record = e2e.judge("unused", ENDPOINT, probe(), SCHEMAS, RULES) + assert record["status"] == "fail" + assert any("unknown wall_id 'missing'" in p for p in record["problems"]) + assert record["verification_samples"] == 0 + assert not e2e.write_report(tmp_path, META, [record])["contract_ok"] + + +@pytest.mark.parametrize("failure", ["admission", "invariants"]) +def test_invalid_variant_never_reaches_comparisons(monkeypatch, failure): + original = response() + other = copy.deepcopy(original) + if failure == "admission": + other["nearest_considered"] = spot("missing") + serve(monkeypatch, original) + monkeypatch.setattr(e2e, "answer", lambda *args: (other, None)) + monkeypatch.setattr(e2e, "with_ground_short_of", lambda *args: None) + monkeypatch.setattr(e2e, "with_less_coverage", lambda *args: None) + monkeypatch.setattr(e2e, "with_more_error", lambda *args: copy.deepcopy(SCENE)) + monkeypatch.setattr(e2e, "tape_to_tap", lambda *args: None) + monkeypatch.setattr(e2e, "more_error_problems", forbid_properties) + monkeypatch.setattr(e2e, "less_coverage_problems", forbid_properties) + if failure == "admission": + monkeypatch.setattr(e2e, "invariant_problems", forbid_properties) + else: + monkeypatch.setattr(e2e, "invariant_problems", lambda *args, **kwargs: ["bad variant hash"]) + problems = e2e.property_problems("unused", ENDPOINT, probe(), original, SCHEMAS, RULES) + expected = "unknown wall_id" if failure == "admission" else "bad variant hash" + assert any(p.startswith("more error:") and expected in p for p in problems) + + +def test_variants_and_comparisons_charge_the_judge_meter(monkeypatch): + original = response() + serve(monkeypatch, original) + monkeypatch.setattr(e2e, "answer", lambda *args: (original, None)) + monkeypatch.setattr(e2e, "geometry_admission_problems", lambda *args, **kwargs: []) + monkeypatch.setattr(e2e, "with_ground_short_of", lambda *args: None) + monkeypatch.setattr(e2e, "with_less_coverage", lambda *args: None) + monkeypatch.setattr(e2e, "with_more_error", lambda *args: copy.deepcopy(SCENE)) + monkeypatch.setattr(e2e, "tape_to_tap", lambda *args: None) + received = [] + + def invariants(*args, meter, **kwargs): + received.append(meter) + meter.charge() + return [] + + def compare(*args, meter): + received.append(meter) + meter.charge(2) + return [] + + monkeypatch.setattr(e2e, "invariant_problems", invariants) + monkeypatch.setattr(e2e, "more_error_problems", compare) + meter = WorkMeter(limit=4) + problems = e2e.property_problems( + "unused", ENDPOINT, probe(), original, SCHEMAS, RULES, meter=meter + ) + assert not problems + assert received == [meter, meter, meter] + assert meter.samples == 4 + + +def test_unknown_rules_keep_rule_independent_properties(monkeypatch): + original = response() + changed = copy.deepcopy(original) + changed["summary"] = "Changed answer for the identical scene" + serve(monkeypatch, changed) + scene = copy.deepcopy(SCENE) + scene["coverage"] = {"observed": [{"band": "wall", "span_ft": [-30, 30]}]} + item = e2e.SceneInput("rule override", scene, json.dumps(scene).encode()) + variants = [] + checked_rules = [] + + def answer(url, endpoint, transformed): + variants.append(transformed.name.split("~")[-1]) + return original, None + + def invariants(*args, rules, **kwargs): + checked_rules.append(rules) + return [] + + def public_rule_transform(*args): + raise AssertionError("private override judged using a public-rule transform") + + monkeypatch.setattr(e2e, "answer", answer) + monkeypatch.setattr(e2e, "invariant_problems", invariants) + monkeypatch.setattr(e2e, "with_ground_short_of", public_rule_transform) + monkeypatch.setattr(e2e, "with_more_error", public_rule_transform) + monkeypatch.setattr(e2e, "tape_to_tap", public_rule_transform) + problems = e2e.property_problems( + "unused", ENDPOINT, item, original, SCHEMAS, RULES, rule_checks=False + ) + assert problems == ["sending the same scene twice gave different results"] + assert variants == ["mirrored", "no coverage", "less coverage"] + assert checked_rules == [None, None, None] diff --git a/verification/tests/test_judge_scene_cache.py b/verification/tests/test_judge_scene_cache.py new file mode 100644 index 000000000..b4e711770 --- /dev/null +++ b/verification/tests/test_judge_scene_cache.py @@ -0,0 +1,73 @@ +"""Recorded public responses exercise the complete judge without a server dependency.""" + +import copy +import hashlib +import json +from pathlib import Path + +import pytest +from segmented_wall import segmented_wall_scene + +from hsverify import e2e +from hsverify.resultcheck import MAX_RESULT_CHECKS, RuleSet, WorkMeter, ingestion_problems + +ROOT = Path(__file__).parents[2] +RULES = RuleSet.from_yaml((ROOT / "server/rules.yaml").read_text()) +SCHEMAS = { + kind: json.loads((ROOT / f"server/schemas/{kind}.schema.json").read_text()) + for kind in ("scene", "result") +} + + +def workload(monkeypatch, segments): + responses = json.loads( + (Path(__file__).parent / "fixtures/segmented-public-workload.json").read_text() + ) + calls = [] + + def post(url, endpoint, item): + calls.append(item.name) + answer = copy.deepcopy(responses[item.name]) + answer["stats"]["input_sha256"] = hashlib.sha256(item.raw).hexdigest() + return 200, json.dumps(answer).encode(), 0 + + monkeypatch.setattr(e2e, "post", post) + scene = segmented_wall_scene(segments) + item = e2e.SceneInput("surveyed wall", scene, json.dumps(scene).encode()) + meter = WorkMeter() + record = e2e.judge( + "unused", e2e.Endpoint("/place", "application/json"), item, SCHEMAS, RULES, meter=meter + ) + assert record["status"] == "pass", record["problems"] + assert calls.count("surveyed wall") == 2 + assert "surveyed wall~mirrored" in calls + return meter.samples + + +@pytest.mark.parametrize("segments", [13, 20]) +def test_full_judge_accepts_equivalent_segmented_walls(monkeypatch, segments): + baseline = workload(monkeypatch, 1) + actual = workload(monkeypatch, segments) + assert actual < baseline * 1.2 + + +def test_identical_resend_counts_are_checked_before_comparison(monkeypatch): + responses = json.loads( + (Path(__file__).parent / "fixtures/segmented-public-workload.json").read_text() + ) + original = copy.deepcopy(responses["surveyed wall"]) + oversized = original | {"checks": [None] * (MAX_RESULT_CHECKS + 1)} + replies = iter((original, oversized)) + scene = segmented_wall_scene(1) + item = e2e.SceneInput("surveyed wall", scene, json.dumps(scene).encode()) + original["stats"]["input_sha256"] = hashlib.sha256(item.raw).hexdigest() + monkeypatch.setattr(e2e, "post", lambda *args: (200, json.dumps(next(replies)).encode(), 0)) + # Other transforms are immaterial to the identical-resend admission control. + monkeypatch.setattr(e2e, "mirror_scene", lambda value: value) + monkeypatch.setattr(e2e, "answer", lambda *args: (None, "transform omitted by test")) + record = e2e.judge("unused", e2e.Endpoint("/place", "application/json"), item, SCHEMAS, RULES) + assert any( + "identical resend: result ingestion limit exceeded: checks" in problem + for problem in record["problems"] + ) + assert ingestion_problems(oversized, "result") diff --git a/verification/tests/test_metadata_bounds.py b/verification/tests/test_metadata_bounds.py new file mode 100644 index 000000000..12fd16e4a --- /dev/null +++ b/verification/tests/test_metadata_bounds.py @@ -0,0 +1,188 @@ +"""Metadata admission and lookup costs for large, schema-shaped responses.""" + +import ast +import copy +import inspect +import json +from pathlib import Path + +import pytest +from test_resultcheck import RULES, SCENE, check, result + +import hsverify.resultcheck as checks + + +class ComparedName(str): + comparisons = 0 + + def __eq__(self, other): + type(self).comparisons += 1 + if type(self).comparisons > 100_000: + raise AssertionError("metadata lookup performed more than 100,000 comparisons") + return super().__eq__(other) + + __hash__ = str.__hash__ + + +def test_distinct_evidence_ids_use_linear_name_lookups(): + names = [ComparedName(f"clearance_{i}") for i in range(12_000)] + answer = result(spot=False, checks=[], complete=False) + answer["reasons"] = [{"code": "unobserved_area", "message": ""}] + answer["checks"] = [ + check(outcome="unsure", measured=None, cause="unobserved") | {"id": name} for name in names + ] + answer["missing_evidence"] = [ + {"kind": "band", "band": "ground", "checks": [name], "message": ""} for name in names + ] + ComparedName.comparisons = 0 + assert checks.missing_evidence_problems(SCENE, answer) == [] + assert ComparedName.comparisons < 4 * len(names) + + +def test_invalid_width_message_deduplication_is_linear(): + # Instrument only the diagnostic buffer. The old list-membership search calls + # ComparedName.__eq__ for every previous diagnostic; hash deduplication does not. + class DiagnosticBuffer(list): + def append(self, message): + super().append(ComparedName(message)) + + class InstrumentBuffer(ast.NodeTransformer): + def visit_Assign(self, node): + if any(isinstance(t, ast.Name) and t.id == "problems" for t in node.targets): + node.value = ast.Call( + func=ast.Name(id="DiagnosticBuffer", ctx=ast.Load()), args=[], keywords=[] + ) + return node + + source = ast.parse(inspect.getsource(checks.reported_dimension_problems)) + source = ast.fix_missing_locations(InstrumentBuffer().visit(source)) + scope = checks.reported_dimension_problems.__globals__ | {"DiagnosticBuffer": DiagnosticBuffer} + exec(compile(source, "", "exec"), scope) + answer = result(spot=False, checks=[], complete=False) + answer["checks"] = [ + check(threshold=-1) | {"id": f"width_{i}", "rule": {"key": "battery.width_ft"}} + for i in range(4_000) + ] + ComparedName.comparisons = 0 + problems = scope["reported_dimension_problems"](answer, RULES) + assert len(problems) == 4_000 + assert problems[0] == "width_0 must be finite and positive" + assert problems[-1] == "width_3999 must be finite and positive" + assert ComparedName.comparisons < 4 * len(problems) + + +@pytest.mark.parametrize( + ("field", "limit"), + [ + ("sweep", checks.MAX_RESULT_RUNS), + ("checks", checks.MAX_RESULT_CHECKS), + ("missing_evidence", checks.MAX_EVIDENCE_REQUESTS), + ], +) +def test_result_count_caps_accept_boundary_and_reject_next(field, limit): + answer = {field: [None] * limit} + assert checks.ingestion_problems(answer, "result") == [] + answer[field].append(None) + assert checks.ingestion_problems(answer, "result") == [ + f"result ingestion limit exceeded: {field} exceeds {limit} entries" + ] + + +@pytest.mark.parametrize("field", ["sweep", "checks", "missing_evidence"]) +def test_named_result_caps_allow_schema_valid_responses(field): + fixture = json.loads((Path(__file__).parent / "fixtures/wide-public-pass.json").read_text()) + answer = copy.deepcopy(fixture["result"]) + request = {"kind": "band", "message": "", "band": "ground", "checks": ["ground_surface"]} + first_check = answer["checks"][0] + first_check.update(label="", reason="", rule={"key": "", "source": "", "placeholder": False}) + entries = { + "sweep": (answer["sweep"][0], checks.MAX_RESULT_RUNS), + "checks": (first_check, checks.MAX_RESULT_CHECKS), + "missing_evidence": (request, checks.MAX_EVIDENCE_REQUESTS), + } + entry, limit = entries[field] + answer[field] = ( + answer[field] + [entry] * (limit - len(answer[field])) + if field == "checks" + else [entry] * limit + ) + schema = json.loads( + (Path(__file__).parents[2] / "server/schemas/result.schema.json").read_text() + ) + assert checks.schema_errors(answer, schema) == [] + if field == "sweep": + answer["spot"] = answer["route"] = None + answer["decision"] = "reject" + elif field == "missing_evidence": + answer["decision"] = "manual_review" + rules = checks.RuleSet.from_yaml((Path(__file__).parents[2] / "server/rules.yaml").read_text()) + assert checks.invariant_problems(fixture["scene"], answer, rules=rules) == [] + + +def test_scene_geometry_counts_keep_existing_schema_caps(): + wall = {"id": "wall", "baseline": [[0, 0]] * 500} + scene = {"walls": [wall] * 50} + assert checks.ingestion_problems(scene, "scene") == [] + scene["walls"].append(wall) + assert "walls exceeds 50 entries" in checks.ingestion_problems(scene, "scene")[0] + assert ( + "baseline exceeds 500 vertices" + in checks.ingestion_problems({"walls": [wall | {"baseline": [[0, 0]] * 501}]}, "scene")[0] + ) + + +@pytest.mark.parametrize( + ("value", "message"), + [ + ([0] * checks.MAX_NUMERIC_LEAVES, "numeric leaves"), + ([None] * (checks.MAX_METADATA_NODES - 1), "JSON nodes"), + ("x" * checks.MAX_METADATA_CHARACTERS, "characters"), + ], +) +def test_generic_metadata_caps_accept_boundary_and_reject_next(value, message): + assert checks.ingestion_problems(value) == [] + over = value + ("x" if isinstance(value, str) else [value[0]]) + assert message in checks.ingestion_problems(over)[0] + + +def test_nesting_cap_precedes_recursive_schema_validation(monkeypatch): + nested = 0 + for _ in range(checks.MAX_METADATA_DEPTH): + nested = [nested] + assert checks.ingestion_problems(nested) == [] + + def unexpected_validator(*args): + pytest.fail("oversized metadata reached recursive schema validation") + + monkeypatch.setattr(checks, "Draft202012Validator", unexpected_validator) + assert checks.schema_errors([nested], {}) == [ + f"document ingestion limit exceeded: nesting exceeds {checks.MAX_METADATA_DEPTH}" + ] + + +def test_ingestion_failure_precedes_geometry(monkeypatch): + def unexpected_dimensions(*args): + pytest.fail("oversized metadata reached dimension validation") + + monkeypatch.setattr(checks, "reported_dimension_problems", unexpected_dimensions) + answer = {"checks": [None] * (checks.MAX_RESULT_CHECKS + 1)} + assert checks.geometry_admission_problems(SCENE, answer, RULES) == [ + f"result ingestion limit exceeded: checks exceeds {checks.MAX_RESULT_CHECKS} entries" + ] + + +def test_invalid_schema_stops_after_bounded_diagnostics(): + errors = checks.schema_errors([None] * 10_000, {"type": "array", "items": {"type": "number"}}) + assert len(errors) == checks.MAX_SCHEMA_PROBLEMS + + +def test_case_queries_have_a_fixed_scan_bound(): + at_cap = {f"rule_{i}": 0 for i in range(checks.MAX_CASE_RULE_QUERIES)} + assert checks.case_metadata_problems(at_cap) == [] + at_cap["one_more"] = 0 + assert checks.case_metadata_problems(at_cap) == [ + "case ingestion limit exceeded: more than 64 rule queries" + ] + assert checks.expectation_problems( + {"checks": [{"match": "gas"}] * (checks.MAX_CASE_RULE_QUERIES + 1)}, {} + ) == ["case ingestion limit exceeded: more than 64 rule queries"] diff --git a/verification/tests/test_numeric_cache_keys.py b/verification/tests/test_numeric_cache_keys.py new file mode 100644 index 000000000..d9cc49c0f --- /dev/null +++ b/verification/tests/test_numeric_cache_keys.py @@ -0,0 +1,72 @@ +import sys + +import pytest + +from hsverify.intervalindex import SweepIndex, numeric_key, probe_starts +from hsverify.resultcheck import WorkMeter, observed + + +class ComparedInt(int): + """Count equality work while retaining the hash of the JSON integer it represents.""" + + comparisons = 0 + __hash__ = int.__hash__ + + def __eq__(self, other): + type(self).comparisons += 1 + if type(self).comparisons > 100_000: + raise AssertionError("numeric keys exceeded 100,000 equality comparisons") + return super().__eq__(other) + + +def values(colliding, count=2_000): + step = sys.hash_info.modulus if colliding else 1 + return [ComparedInt(step * i) for i in range(1, count + 1)] + + +def equality_work(operation, colliding): + numbers = values(colliding) + ComparedInt.comparisons = 0 + operation(numbers) + return ComparedInt.comparisons + + +def test_numeric_keys_preserve_exact_numeric_equivalence(): + assert numeric_key(1) == numeric_key(1.0) + assert numeric_key(0) == numeric_key(-0.0) + assert numeric_key(2**60) == numeric_key(float(2**60)) + assert numeric_key(2**53 + 1) != numeric_key(float(2**53 + 1)) + assert numeric_key(0.1) != numeric_key(1) + + +@pytest.mark.parametrize("operation", ["observed", "input", "index", "probes"]) +def test_modulus_multiple_response_numbers_do_not_multiply_equality_work(operation): + def exercise(numbers): + meter = WorkMeter() + if operation == "observed": + scene = {"coverage": {"observed": []}} + for number in numbers: + assert observed(scene, "ground", number, meter=meter) == [] + assert len(meter.cache) == len(numbers) + elif operation == "input": + for number in numbers: + meter.input_key({"out_ft": number}) + assert len(meter.input_values) == len(numbers) + else: + result = { + "sweep": [ + {"wall_id": "w1", "start_ft": [number, number], "outcome": "fail"} + for number in numbers + ] + } + if operation == "index": + index = SweepIndex(result) + assert len(index.walls["w1"].starts) == len(numbers) + else: + assert len(probe_starts(result)) >= len(numbers) + + ordinary = equality_work(exercise, False) + colliding = equality_work(exercise, True) + # Count comparisons rather than wall time: sorting/deduplication may do a small + # multiple of the ordinary workload, but a numeric hash table does millions. + assert colliding <= 4 * ordinary + 20 * 2_000 diff --git a/verification/tests/test_policy_admission.py b/verification/tests/test_policy_admission.py new file mode 100644 index 000000000..c09f9859f --- /dev/null +++ b/verification/tests/test_policy_admission.py @@ -0,0 +1,163 @@ +import copy +import hashlib +import json +from dataclasses import replace +from pathlib import Path + +import pytest +from segmented_wall import segmented_wall_scene +from test_judge_meter import META + +from hsverify import e2e +from hsverify.resultcheck import RuleSet, public_dimension_problems, reported_rules + +ROOT = Path(__file__).parents[2] +RULES = RuleSet.from_yaml((ROOT / "server/rules.yaml").read_text()) +SCHEMAS = { + kind: json.loads((ROOT / f"server/schemas/{kind}.schema.json").read_text()) + for kind in ("scene", "result") +} + + +def replies(): + return json.loads( + (Path(__file__).parent / "fixtures/segmented-public-workload.json").read_text() + ) + + +@pytest.mark.parametrize("field", ["width_ft", "depth_ft", "height_ft"]) +def test_matching_public_policy_keeps_loaded_dimensions(field): + answer = replies()["surveyed wall"] + answer["spot"].pop("footprint") + answer["spot"].pop("span_ft") + answer["spot"][field] = 1 + assert reported_rules(answer, RULES) is RULES + + +@pytest.mark.parametrize("wrong", [False, True]) +def test_public_policy_width_is_checked_without_case_assumptions(monkeypatch, wrong): + answers = replies() + if wrong: + spot = answers["surveyed wall"]["spot"] + lo = spot["span_ft"][0] + spot.update( + width_ft=1, + span_ft=[lo, lo + 1], + footprint=[[lo, 0], [lo + 1, 0], [lo + 1, RULES.depth_ft], [lo, RULES.depth_ft]], + center=[lo + 0.5, RULES.depth_ft / 2], + meter_offset_ft=[lo + 0.5, RULES.depth_ft / 2], + ) + seen_rules = [] + original = e2e.invariant_problems + + def invariant(*args, rules=None, **kwargs): + seen_rules.append(rules) + return original(*args, rules=rules, **kwargs) + + def post(url, endpoint, item): + answer = copy.deepcopy(answers[item.name]) + answer["stats"]["input_sha256"] = hashlib.sha256(item.raw).hexdigest() + return 200, json.dumps(answer).encode(), 0 + + monkeypatch.setattr(e2e, "post", post) + monkeypatch.setattr(e2e, "invariant_problems", invariant) + scene = segmented_wall_scene(1) + item = e2e.SceneInput("surveyed wall", scene, json.dumps(scene).encode()) + record = e2e.judge("unused", e2e.Endpoint("/place", "application/json"), item, SCHEMAS, RULES) + assert record["status"] == ("fail" if wrong else "pass"), record["problems"] + assert seen_rules[0] is RULES + if wrong: + assert any("width_ft" in p and "public rules" in p for p in record["contract_problems"]) + + +@pytest.mark.parametrize("sources", [["public", "private"], ["public"]]) +def test_different_policy_keeps_validated_reported_width(sources): + answer = replies()["surveyed wall"] + answer["policy"].update(sources=sources, rules_sha256="f" * 64) + loaded = replace(RULES, width_ft=4) + assert reported_rules(answer, loaded).width_ft == answer["spot"]["width_ft"] + + +def test_malformed_cases_are_reported_and_next_case_runs(monkeypatch, tmp_path): + scene = segmented_wall_scene(1) + cases = [] + for name, metadata in ( + ("null-assumptions", {"rules_assumed": None}), + ("null-checks", {"expect": {"checks": None}}), + ): + path = tmp_path / f"{name}.json" + path.write_text(json.dumps({"id": name, "scene": scene, "expect": {}, **metadata})) + cases.append(e2e.load_input(path)) + cases.append(e2e.SceneInput("surveyed wall", scene, json.dumps(scene).encode())) + answers = replies() + + def post(url, endpoint, item): + assert item.name.startswith("surveyed wall") + answer = copy.deepcopy(answers[item.name]) + answer["stats"]["input_sha256"] = hashlib.sha256(item.raw).hexdigest() + return 200, json.dumps(answer).encode(), 0 + + monkeypatch.setattr(e2e, "post", post) + records = [ + e2e.judge("unused", e2e.Endpoint("/place", "application/json"), item, SCHEMAS, RULES) + for item in cases + ] + assert [r["status"] for r in records] == ["bad input", "bad input", "pass"] + assert "rules_assumed" in records[0]["problems"][0] + assert "expect.checks" in records[1]["problems"][0] + (tmp_path / "report").mkdir() + e2e.write_report(tmp_path / "report", META, records) + saved = json.loads((tmp_path / "report/report.json").read_text()) + assert len(saved["records"]) == 3 + assert not saved["all_passed"] + + +@pytest.mark.parametrize("name", ["spot", "nearest_considered"]) +@pytest.mark.parametrize("field", ["width_ft", "depth_ft", "height_ft"]) +def test_every_public_spot_dimension_is_compared(name, field): + answer = {"policy": {"rules_sha256": RULES.rules_sha256}, name: {field: 1}} + assert any(f"{name}.{field}" in p for p in public_dimension_problems(answer, RULES)) + + +def test_public_corner_allowance_and_route_height_are_compared(): + answer = { + "policy": {"rules_sha256": RULES.rules_sha256}, + "route": {"height_ft": 100}, + "checks": [ + {"id": "corner", "rule": {"key": "route.corner_allowance_ft"}, "threshold_ft": 100} + ], + } + problems = public_dimension_problems(answer, RULES) + assert any("route.height_ft" in p for p in problems) + assert any("corner" in p for p in problems) + + +def test_private_marker_cannot_override_matching_fingerprint(): + answer = replies()["surveyed wall"] + answer["policy"]["sources"].append("private") + assert reported_rules(answer, RULES) is RULES + + +def test_missing_policy_cannot_override_loaded_dimensions(): + answer = replies()["surveyed wall"] + answer.pop("policy") + assert reported_rules(answer, RULES) is RULES + + +def test_transformed_public_response_uses_original_loaded_rules(monkeypatch): + answers = replies() + loaded = replace(RULES, width_ft=4) + + def post(url, endpoint, item): + answer = copy.deepcopy(answers[item.name]) + answer["stats"]["input_sha256"] = hashlib.sha256(item.raw).hexdigest() + if not item.name.endswith("mirrored"): + answer["policy"].update(sources=["public", "private"], rules_sha256="f" * 64) + return 200, json.dumps(answer).encode(), 0 + + monkeypatch.setattr(e2e, "post", post) + scene = segmented_wall_scene(1) + item = e2e.SceneInput("surveyed wall", scene, json.dumps(scene).encode()) + record = e2e.judge("unused", e2e.Endpoint("/place", "application/json"), item, SCHEMAS, loaded) + assert record["status"] == "fail" + assert any("mirrored" in p and "public rules" in p for p in record["contract_problems"]) diff --git a/verification/tests/test_replaycheck.py b/verification/tests/test_replaycheck.py index dd830e2e3..ef07b710e 100644 --- a/verification/tests/test_replaycheck.py +++ b/verification/tests/test_replaycheck.py @@ -4,6 +4,7 @@ import numpy as np import pytest +from hsverify import replaycheck from hsverify.replaycheck import ( accuracy, check_format, @@ -136,3 +137,20 @@ def test_accuracy_separates_scale_from_alignment(tmp_path, scale): assert result["max_vertical_axis_mismatch"] == pytest.approx(0, abs=1e-9) expected_final = (1 - scale) * 39 assert result["anchored_error_m_final"] == pytest.approx(expected_final, abs=1e-3) + + +def test_oversized_integer_is_a_format_error(tmp_path): + session = session_with(tmp_path, [[0, 0, 0]]) + session["keyframes"][0]["intrinsics"][0] = 10**400 + errors, _ = check_format(tmp_path, session) + assert any("finite numbers" in e for e in errors) + + +def test_invalid_replay_with_ground_truth_still_reports(tmp_path, capsys): + session = session_with(tmp_path, [[0, 0, 0]]) + del session["keyframes"][0]["pose"] + (tmp_path / "session.json").write_text(json.dumps(session)) + (tmp_path / "ground_truth.json").write_text("{}") + assert replaycheck.main([str(tmp_path), "--json"]) == 1 + data = json.loads(capsys.readouterr().out) + assert data["format_errors"] and "pose_accuracy" not in data diff --git a/verification/tests/test_reported_rules.py b/verification/tests/test_reported_rules.py new file mode 100644 index 000000000..64442f0e2 --- /dev/null +++ b/verification/tests/test_reported_rules.py @@ -0,0 +1,115 @@ +"""Reported geometry and policy identity determine whether rule assumptions are verified.""" + +import copy +import hashlib +import json +from dataclasses import replace +from pathlib import Path + +import pytest +from segmented_wall import segmented_wall_scene +from test_judge_meter import META + +from hsverify import e2e +from hsverify.resultcheck import RuleSet, public_assumption_warnings + +ROOT = Path(__file__).parents[2] +RULES = RuleSet.from_yaml((ROOT / "server/rules.yaml").read_text()) +SCHEMAS = { + kind: json.loads((ROOT / f"server/schemas/{kind}.schema.json").read_text()) + for kind in ("scene", "result") +} + + +def test_public_fingerprint_matches_recorded_public_solver(): + fixture = json.loads( + (Path(__file__).parent / "fixtures/segmented-public-pass.json").read_text() + ) + assert RULES.rules_sha256 == fixture["result"]["policy"]["rules_sha256"] + + +@pytest.mark.parametrize("identity", ["private source", "different fingerprint"]) +def test_unreported_override_is_unverified_even_when_case_matches_public(identity): + answer = {"policy": {"sources": ["public"], "rules_sha256": RULES.rules_sha256}} + if identity == "private source": + answer["policy"]["sources"].append("private") + else: + answer["policy"]["rules_sha256"] = "f" * 64 + warnings = public_assumption_warnings({"battery_width": RULES.width_ft}, answer, RULES) + assert len(warnings) == 1 + assert "assumed public rules; server may override" in warnings[0] + + +@pytest.mark.parametrize("matches", [True, False]) +def test_judge_compares_reported_width_with_case_assumption(monkeypatch, matches): + answers = json.loads( + (Path(__file__).parent / "fixtures/segmented-public-workload.json").read_text() + ) + + def post(url, endpoint, item): + answer = copy.deepcopy(answers[item.name]) + answer["stats"]["input_sha256"] = hashlib.sha256(item.raw).hexdigest() + answer["policy"].update(sources=["public", "private"], rules_sha256="f" * 64) + return 200, json.dumps(answer).encode(), 0 + + monkeypatch.setattr(e2e, "post", post) + scene = segmented_wall_scene(1) + item = e2e.SceneInput( + "surveyed wall", + scene, + json.dumps(scene).encode(), + rules_assumed={"battery_width": RULES.width_ft if matches else 4}, + ) + record = e2e.judge( + "unused", + e2e.Endpoint("/place", "application/json"), + item, + SCHEMAS, + replace(RULES, width_ft=4), + ) + assert record["status"] == ("pass" if matches else "assumption mismatch") + assert bool(record.get("assumption_mismatches")) is not matches + + +@pytest.mark.parametrize("identity", ["private source", "different fingerprint"]) +def test_unreported_private_width_cannot_claim_contract(monkeypatch, tmp_path, identity): + fixture = json.loads( + (Path(__file__).parent / "fixtures/segmented-public-pass.json").read_text() + ) + answer = copy.deepcopy(fixture["result"]) + answer.update(decision="reject", spot=None, route=None) + answer["reasons"] = [{"code": "policy_not_approved", "message": "Private rules reject"}] + answer["sweep"] = [ + { + "wall_id": "w1", + "start_ft": [1, 1], + "outcome": "fail", + "failing": ["route_length"], + "unsure": [], + } + ] + answer["stats"].update(candidates=1, fail=1, unsure=0, **{"pass": 0}) + for check in answer["checks"]: + check.update(outcome="fail", measured_ft=None, plus_minus_ft=None) + if identity == "private source": + answer["policy"]["sources"].append("private") + else: + answer["policy"]["rules_sha256"] = "f" * 64 + + def post(url, endpoint, item): + value = copy.deepcopy(answer) + value["stats"]["input_sha256"] = hashlib.sha256(item.raw).hexdigest() + return 200, json.dumps(value).encode(), 0 + + monkeypatch.setattr(e2e, "post", post) + scene = segmented_wall_scene(1) + item = e2e.SceneInput( + "private reject", + scene, + json.dumps(scene).encode(), + rules_assumed={"battery_width": RULES.width_ft}, + ) + record = e2e.judge("unused", e2e.Endpoint("/place", "application/json"), item, SCHEMAS, RULES) + assert record["status"] == "assumptions unverified", record["problems"] + assert record["rule_checks_unverified"] + assert not e2e.write_report(tmp_path, META, [record])["contract_ok"] diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 5846b7a9a..ed6a626c1 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -6,9 +6,12 @@ import pytest from hsverify.resultcheck import ( + MAX_SWEEP_SAMPLES, GroundPlan, Need, RuleSet, + WorkLimitExceeded, + WorkMeter, assumption_mismatches, battery_error, chain_ends_s, @@ -28,7 +31,9 @@ observed, outcome_at, outcome_lengths, + public_assumption_warnings, reach_gaps, + reported_rules, run_starts, straight_pieces, tape_to_tap, @@ -412,7 +417,7 @@ def test_rules_come_from_rules_yaml_and_a_missing_one_is_loud(): openings: {exempt_bottom_above_ft: null} facing: {min_ft: {value: 3}} headroom: {min_ft: {value: 6.5}} -route: {height_ft: {value: 1}} +route: {height_ft: {value: 1}, corner_allowance_ft: {value: 0.5}} sweep: {step_ft: {value: 0.166666666667}, wall_join_ft: {value: 0.6}} """ rules = RuleSet.from_yaml(text) @@ -505,7 +510,8 @@ def test_outcome_not_rules_out_one_outcome_and_tolerates_no_runs(): ] } assert expectation_problems(bad, r) == [ - "sweep w1 starts [2.1, 3.0] is pass, which this case rules out (window)" + "sweep expectation w1 starts (0.5, 2.5) failed for 1 runs; " + "expected {'outcome_not': 'pass'}; examples: w1 starts [2.1, 3.0] is pass; reason: window" ] @@ -620,6 +626,64 @@ def test_less_coverage_never_creates_a_pass(): assert msgs and all("went from unsure to pass" in m for m in msgs) +def split(): + """An original sweep: fail up to 2.0, then pass from 2.1, nothing evaluated between.""" + return swept(run(0.0, 2.0, "fail"), run(2.1, 5.0, "pass")) + + +def test_less_coverage_may_move_a_boundary_into_an_unswept_gap(): + """A coverage edge that moved adds evaluated starts; one inside the original's gap between + a fail run and a pass run may pass, because the original allows either outcome there.""" + after = swept(run(0.0, 2.04, "fail"), run(2.05, 5.0, "pass")) + assert less_coverage_problems(split(), after, "x") == [] + + +def test_less_coverage_cannot_pass_a_gap_between_runs_that_did_not(): + before = swept(run(0.0, 2.0, "fail"), run(2.1, 5.0, "unsure")) + after = swept(run(0.0, 2.0, "fail"), run(2.05, 2.05, "pass"), run(2.1, 5.0, "unsure")) + assert less_coverage_problems(before, after, "x") == [ + "x: start w1 2.050 went from unswept beside fail and unsure to pass" + ] + + +def test_less_coverage_cannot_pass_where_the_original_swept_nothing(): + """The decision guard does not catch this when the original already passed elsewhere.""" + before = swept(run(0.0, 4.0, "pass"), decision="pass") + after = swept(run(0.0, 4.0, "pass"), run(10.0, 12.0, "pass", wall="w2"), decision="pass") + assert less_coverage_problems(before, after, "x") == [ + f"x: start w2 {s:.3f} went from no run on its wall to pass" for s in (10.0, 11.0, 12.0) + ] + past_end = swept(run(0.0, 4.0, "pass"), run(4.1, 6.0, "fail"), decision="pass") + longer = swept(run(0.0, 4.0, "pass"), run(4.1, 6.0, "fail"), run(6.1, 6.1, "pass")) + assert less_coverage_problems(past_end, longer, "x") == [ + "x: start w1 6.100 went from past the original sweep to pass" + ] + + +def test_a_new_pass_past_the_original_sweep_is_not_read_from_its_last_run(): + """The original ends on a pass run; nothing it evaluated brackets 4.1 to 6.0.""" + before = swept(run(0.0, 4.0, "pass"), decision="pass") + after = swept(run(0.0, 4.0, "pass"), run(4.1, 6.0, "pass"), decision="pass") + expected = [ + f"x: start w1 {s:.3f} went from past the original sweep to pass" for s in (4.1, 5.05, 6.0) + ] + assert less_coverage_problems(before, after, "x") == expected + assert more_error_problems(before, after, "x") == expected + # More error widens the reach limit, so a longer sweep may fail or turn unsure out there. + wider = swept(run(0.0, 4.0, "unsure"), run(4.1, 5.0, "fail"), run(5.1, 6.0, "unsure")) + assert more_error_problems(before, wider, "x") == [] + + +def test_more_error_reads_an_unswept_start_from_its_neighbours(): + moved = swept(run(0.0, 2.04, "fail"), run(2.05, 5.0, "pass")) + assert more_error_problems(split(), moved, "x") == [] + flipped = swept(run(0.0, 2.0, "fail"), run(2.05, 2.05, "pass"), run(2.1, 5.0, "fail")) + before = swept(run(0.0, 2.0, "fail"), run(2.1, 5.0, "fail")) + assert more_error_problems(before, flipped, "x") == [ + "x: start w1 2.050 went from unswept beside fail and fail to pass" + ] + + def test_transforms(): scene = copy.deepcopy(SCENE) scene["objects"].append({"type": "ac", "wall_id": "w1", "span_ft": [5, 6], "source": "tape"}) @@ -752,9 +816,9 @@ def test_an_overhead_seen_clear_all_the_way_up_settles_headroom(): def test_battery_clearance_is_needed_when_its_rule_reaches_past_gas_or_a_battery_is_marked(): - # BAND_RULES set battery_clearance's radius (100) above gas's (1): the server evaluates it, + # BAND_RULES set battery_clearance's radius (2) above gas's (1): the server evaluates it, # so a result that leaves it out is flagged and its coverage is needed anyway. - wide = replace(RULES, needs=RULES.needs | {"battery_clearance": (Need("ground", 100.0),)}) + wide = replace(RULES, needs=RULES.needs | {"battery_clearance": (Need("ground", 2.0),)}) r = result(checks=[check()], sweep=[PASS_RUN]) msgs = invariant_problems(SCENE, r, rules=wide) assert "the result leaves out check battery_clearance, which the rules define" in msgs @@ -875,8 +939,8 @@ def test_each_start_in_a_sweep_run_needs_only_its_own_reach(): def test_sweep_run_starts_are_sampled_at_the_rules_step_plus_the_ends(): - assert run_starts([1.0, 1.5], 1 / 6) == pytest.approx([1.0, 7 / 6, 8 / 6, 1.5]) - assert run_starts([2.0, 2.0], 1 / 6) == [2.0] + assert list(run_starts([1.0, 1.5], 1 / 6)) == pytest.approx([1.0, 7 / 6, 8 / 6, 1.5]) + assert list(run_starts([2.0, 2.0], 1 / 6)) == [2.0] with pytest.raises(ValueError, match=r"sweep\.step_ft"): run_starts([1.0, 2.0], None) @@ -1083,7 +1147,7 @@ def test_a_captured_request_claims_only_the_reach_it_asked_for(): def test_a_sweep_run_shorter_than_a_step_keeps_its_first_start(): # Review 4114352251: run_starts([0, 0.4], 1) returned only 0.4. - assert run_starts([0.0, 0.4], 1.0) == [0.0, 0.4] + assert list(run_starts([0.0, 0.4], 1.0)) == [0.0, 0.4] def test_a_passing_spot_must_pass_in_the_sweep_too(): @@ -1222,9 +1286,9 @@ def test_a_passing_spot_must_stand_on_the_scanned_wall(): ], ) r["spot"]["span_ft"] = [11.0, 11.0 + 31 / 12] + r["spot"]["footprint"] = [[11, 0], [11 + 31 / 12, 0], [11 + 31 / 12, 22 / 12], [11, 22 / 12]] msgs = invariant_problems(broadly_seen(), r, rules=RULES) - assert any("spot [11.00, 13.58] runs past its straight wall [-12.00, 12.00]" in m for m in msgs) - assert any("sweep pass start 11.00 runs past its straight wall" in m for m in msgs) + assert msgs == ["spot: span_ft exceeds scanned range [-12.0, 12.0]"] def test_walls_meeting_in_a_straight_line_are_one_stretch_and_a_corner_is_not(): @@ -1335,7 +1399,7 @@ def test_a_passing_footprint_keeps_the_wall_error_clear_of_its_stretch_end(): ] r = result(spot=False, sweep=[pass_run(6.846, 10.0 - 31 / 12)]) assert footprint_problems(scene, r, RULES) == [ - "sweep pass start 7.42 ends within its error (0.60 ft) of its straight wall's end at 10.00" + "sweep pass start 7.01 ends within its error (0.60 ft) of its straight wall's end at 10.00" ] r["sweep"][0]["outcome"] = "unsure" # unsure may stand that close assert footprint_problems(scene, r, RULES) == [] @@ -1479,3 +1543,377 @@ def test_a_band_gap_under_the_coverage_tolerance_is_rounding(): assert gaps_over([(-6.5, 18.0)], 9.94, 18.011) == [(18.0, 18.011)] assert gaps_over([(0.0, 1.0), (1.009, 3.0)], 0.5, 2.0) == [] assert gaps_over([(0.0, 1.0), (1.011, 3.0)], 0.5, 2.0) == [(1.0, 1.011)] + + +@pytest.mark.parametrize("outcome", ["pass", "unsure"]) +def test_interior_sweep_start_cannot_cross_a_corner(outcome): + scene = copy.deepcopy(SCENE) + scene["walls"][0]["baseline"] = [[-12.0, 0.0], [5.0, 0.0], [5.0, 7.0]] + result = {"sweep": [{"wall_id": "w1", "start_ft": [1.0, 8.0], "outcome": outcome}]} + assert footprint_problems(scene, result, RULES) + + +def test_ground_trimming_keeps_omitted_reach_at_zero(): + scene = copy.deepcopy(SCENE) + scene["coverage"]["observed"] = [ + {"band": "ground", "span_ft": [-12, 0]}, + {"band": "ground", "span_ft": [0, 12], "out_ft": 100}, + ] + trimmed = with_ground_short_of(scene, RULES) + assert trimmed["coverage"]["observed"][0]["out_ft"] == 0 + + +def test_geometric_assumptions_use_reported_override_instead_of_public_defaults(): + public = replace(RULES, width_ft=4, corner_allowance_ft=10) + answer = result() + answer["policy"] = {"sources": ["private"], "rules_sha256": "f" * 64} + answer["checks"].append( + { + "id": "corner_allowance", + "threshold_ft": 0.5, + "rule": {"key": "route.corner_allowance_ft"}, + } + ) + assumed = {"battery_width": RULES.width_ft, "corner_allowance": 0.5} + assert assumption_mismatches(assumed, answer) == [] + assert public_assumption_warnings(assumed, answer, public) == [] + active = reported_rules(answer, public) + assert (active.width_ft, active.corner_allowance_ft) == (RULES.width_ft, 0.5) + assert (public.width_ft, public.corner_allowance_ft) == (4, 10) + + +def test_geometric_assumptions_detect_a_real_reported_mismatch(): + answer = { + "spot": {"width_ft": 4}, + "checks": [ + { + "id": "route_corner", + "threshold_ft": 10, + "rule": {"key": "route.corner_allowance_ft"}, + } + ], + } + assert ( + len( + assumption_mismatches( + {"battery_width": RULES.width_ft, "corner_allowance": 0.5}, answer + ) + ) + == 2 + ) + + +def test_unreported_geometric_override_is_a_warning_not_a_mismatch(): + public = replace(RULES, width_ft=4, corner_allowance_ft=10) + assumed = {"battery_width": RULES.width_ft, "corner_allowance": 0.5} + assert assumption_mismatches(assumed, {}) == [] + warnings = public_assumption_warnings(assumed, {}, public) + assert len(warnings) == 2 + assert all("assumed public rules; server may override" in text for text in warnings) + assert ( + public_assumption_warnings({"battery_width": 4, "corner_allowance": 10}, {}, public) == [] + ) + + +@pytest.mark.parametrize("source", ["spot", "nearest_considered", "check"]) +def test_reported_width_sources_override_public_width(source): + answer = ( + { + "checks": [ + { + "id": "wall_backing", + "threshold_ft": RULES.width_ft, + "rule": {"key": "battery.width_ft"}, + } + ] + } + if source == "check" + else {source: {"width_ft": RULES.width_ft}} + ) + answer["policy"] = {"sources": ["private"], "rules_sha256": "f" * 64} + assert reported_rules(answer, replace(RULES, width_ft=4)).width_ft == RULES.width_ft + assert assumption_mismatches({"battery_width": RULES.width_ft}, answer) == [] + + +def test_other_width_thresholds_do_not_report_battery_width(): + answer = { + "checks": [ + { + "id": "meter_working_space", + "threshold_ft": 4, + "rule": {"key": "meter_working_space.width_ft"}, + } + ] + } + assert reported_rules(answer, RULES).width_ft == RULES.width_ft + + +@pytest.mark.parametrize("outcome", ["pass", "unsure"]) +@pytest.mark.parametrize("span", [[0, 1e9], [-1e9, 0], [1e9, 1e9]]) +def test_out_of_scan_sweep_fails_before_sampling(monkeypatch, outcome, span): + def no_sampling(*args): + pytest.fail("out-of-scan sweep reached sampling") + + monkeypatch.setattr("hsverify.resultcheck.run_starts", no_sampling) + r = result(spot=False, sweep=[pass_run(*span) | {"outcome": outcome}]) + message = f"sweep {outcome} starts {span}: start_ft span exceeds scanned range [-12.0, 12.0]" + for judge in ( + invariant_problems, + footprint_problems, + coverage_problems, + declared_height_problems, + ): + assert judge(SCENE, r, rules=RULES) == [message] + + +def test_sweep_sampling_is_lazy_and_keeps_the_limit_endpoint(): + samples = run_starts([0, MAX_SWEEP_SAMPLES - 1], 1) + assert iter(samples) is samples + assert next(samples) == 0 + remaining = list(samples) + assert len(remaining) == MAX_SWEEP_SAMPLES - 1 + assert remaining[-1] == MAX_SWEEP_SAMPLES - 1 + assert list(run_starts([0, 1e-9], 1)) == [0, 1e-9] + + +@pytest.mark.parametrize( + "span,step", + [([0, 1e9], 1 / 6), ([0, MAX_SWEEP_SAMPLES], 1), ([0, 1], 1e-300), ([-1e308, 1e308], 1)], +) +def test_sweep_sample_limit_is_checked_before_iteration(span, step): + with pytest.raises(ValueError, match="10000 sample limit"): + run_starts(span, step) + + +@pytest.mark.parametrize("span", [[2, 1], [0, float("inf")], [float("nan"), 1]]) +def test_sweep_sampler_refuses_invalid_spans(span): + with pytest.raises(ValueError, match="finite and ordered"): + run_starts(span, 1) + + +@pytest.mark.parametrize("step", [None, 0, -1, float("inf"), float("nan")]) +def test_sweep_sampler_refuses_invalid_steps(step): + with pytest.raises(ValueError, match="finite positive"): + run_starts([0, 1], step) + + +def test_in_scan_sweep_with_tiny_rule_step_fails_loudly(): + r = result(spot=False, sweep=[pass_run(0, 1) | {"outcome": "unsure"}]) + assert invariant_problems(SCENE, r, rules=replace(RULES, step_ft=1e-300)) == [ + "sweep unsure starts [0, 1]: sweep sampling exceeds the 10000 sample limit" + ] + + +def test_reported_override_prevents_a_false_footprint_failure(): + public = replace(RULES, width_ft=4) + answer = result(spot=False, sweep=[pass_run(9, 9) | {"outcome": "unsure"}]) + answer["policy"] = {"sources": ["private"], "rules_sha256": "f" * 64} + backing = next(c for c in answer["checks"] if c["id"] == "wall_backing") + backing["threshold_ft"] = RULES.width_ft + backing["rule"]["key"] = "battery.width_ft" + assert footprint_problems(SCENE, answer, public) + assert footprint_problems(SCENE, answer, reported_rules(answer, public)) == [] + + +@pytest.mark.parametrize("field", ["width_ft", "depth_ft", "height_ft"]) +@pytest.mark.parametrize("value", [-1, 0, float("nan"), float("inf")]) +def test_vet_reported_dimensions_refuse_invalid_numbers(field, value): + answer = result() + answer["spot"][field] = value + with pytest.raises(ValueError, match=f"spot.{field} must be finite and positive"): + reported_rules(answer, RULES) + + +@pytest.mark.parametrize("source", ["spot", "nearest_considered", "check"]) +def test_vet_reported_width_sources_must_agree(source): + answer = result() + answer["nearest_considered"] = copy.deepcopy(answer["spot"]) + answer["checks"].append( + {"id": "width", "threshold_ft": RULES.width_ft, "rule": {"key": "battery.width_ft"}} + ) + if source == "check": + answer["checks"][-1]["threshold_ft"] = 4 + else: + answer[source]["width_ft"] = 4 + with pytest.raises(ValueError, match="reported battery widths disagree"): + reported_rules(answer, RULES) + + +@pytest.mark.parametrize("source", ["spot", "nearest_considered"]) +@pytest.mark.parametrize("value", [4, 1e9]) +def test_vet_reported_width_must_match_span_and_footprint(source, value): + answer = result() + answer[source] = copy.deepcopy(answer["spot"]) + answer[source]["width_ft"] = value + if source == "nearest_considered": + answer["spot"] = None + with pytest.raises(ValueError, match="does not match"): + reported_rules(answer, RULES) + + +def test_vet_reported_depth_must_match_footprint(): + answer = result() + answer["spot"]["depth_ft"] = 4 + with pytest.raises(ValueError, match="depth_ft does not match footprint"): + reported_rules(answer, RULES) + + +@pytest.mark.parametrize("value", [-1, 0, float("nan"), float("inf")]) +def test_vet_reported_width_threshold_must_be_finite_positive(value): + answer = { + "checks": [{"id": "width", "threshold_ft": value, "rule": {"key": "battery.width_ft"}}] + } + with pytest.raises(ValueError, match="width must be finite and positive"): + reported_rules(answer, RULES) + + +@pytest.mark.parametrize("outcome", ["fail", "unsure"]) +@pytest.mark.parametrize("old_wall_has_run", [False, True]) +def test_vet_more_error_may_add_nonpassing_run_on_previously_unswept_wall( + outcome, old_wall_has_run +): + before = swept(run(0, 4, "fail")) if old_wall_has_run else swept() + after = swept(*before["sweep"], run(10, 12, outcome, wall="w2")) + assert more_error_problems(before, after, "more error") == [] + + +def test_vet_more_error_cannot_add_pass_on_previously_unswept_wall(): + before = swept(run(0, 4, "pass"), decision="pass") + after = swept(*before["sweep"], run(10, 12, "pass", wall="w2"), decision="pass") + assert more_error_problems(before, after, "more error") == [ + f"more error: start w2 {s:.3f} went from no run on its wall to pass" for s in (10, 11, 12) + ] + + +@pytest.mark.parametrize("outcome", ["pass", "unsure", "fail"]) +@pytest.mark.parametrize("span", [[float("nan"), 1], [0, float("inf")], [2, 1]]) +def test_vet_all_sweep_spans_are_validated_before_downstream(monkeypatch, outcome, span): + def no_downstream(*args): + pytest.fail("invalid geometry reached downstream sampling") + + monkeypatch.setattr("hsverify.resultcheck.run_starts", no_downstream) + answer = result(spot=False, sweep=[pass_run(*span) | {"outcome": outcome}]) + assert any("finite and ordered" in p for p in invariant_problems(SCENE, answer, rules=RULES)) + + +def test_vet_identical_unsure_runs_are_sampled_only_once(monkeypatch): + calls = [] + + def starts(*args): + calls.append(True) + return iter([1.0]) + + monkeypatch.setattr("hsverify.resultcheck.run_starts", starts) + run = pass_run(1, 9) | {"outcome": "unsure"} + answer = result(spot=False, sweep=[copy.deepcopy(run) for _ in range(10_000)]) + assert footprint_problems(SCENE, answer, RULES) == [] + assert len(calls) == 1 + + +def test_vet_many_distinct_admissible_runs_exceed_one_response_budget(): + scene = copy.deepcopy(SCENE) + scene["walls"][0]["baseline"] = [[-2000, 0], [2000, 0]] + runs = [pass_run(i / 1000, 1000 + i / 1000) | {"outcome": "unsure"} for i in range(5000)] + answer = result(spot=False, sweep=runs) + meter = WorkMeter(1000) + with pytest.raises(WorkLimitExceeded): + invariant_problems(scene, answer, rules=RULES, meter=meter) + assert meter.samples == 1001 + + +def test_vet_huge_spot_with_empty_sweep_stops_before_geometry(monkeypatch): + def no_geometry(*args): + pytest.fail("invalid spot reached the ground grid") + + monkeypatch.setattr(GroundPlan, "_samples", no_geometry) + answer = result(sweep=[]) + answer["spot"]["span_ft"] = [1, 1e9] + problems = invariant_problems(SCENE, answer, rules=RULES) + assert problems and any("span_ft" in p for p in problems) + + +def test_vet_unused_spot_grid_does_not_consume_predicted_work(): + answer = result(sweep=[]) + huge_reach = replace(RULES, needs={"gas_clearance": (Need("ground", 1e9),)}) + meter = WorkMeter(1000) + problems = invariant_problems(SCENE, answer, rules=huge_reach, meter=meter) + assert any("check gas_clearance passes" in p for p in problems) + assert meter.samples < 1000 + + +def test_vet_check_only_width_does_not_predict_unused_geometry(): + answer = result(spot=False, sweep=[]) + answer["checks"].append( + {"id": "width", "outcome": "pass", "threshold_ft": 1e9, "rule": {"key": "battery.width_ft"}} + ) + meter = WorkMeter(1000) + assert invariant_problems(SCENE, answer, rules=RULES, meter=meter) == [] + assert meter.samples < 1000 + + +def test_vet_ground_samples_are_lazy_and_preserve_grid_edges_and_centres(): + points = GroundPlan._samples(0, 0.1) + assert iter(points) is points + assert list(points) == pytest.approx([0, 0.025, 0.05, 0.075, 0.1]) + + +def test_vet_repeated_spot_checks_reuse_geometry(): + one = result(sweep=[], checks=[passing("ground_surface")]) + many = result(sweep=[], checks=[passing("ground_surface") for _ in range(1000)]) + broad = replace(RULES, needs={"ground_surface": (Need("ground", 10),)}) + one_meter, many_meter = WorkMeter(100_000), WorkMeter(100_000) + assert coverage_problems(SCENE, many, broad, meter=many_meter) == coverage_problems( + SCENE, one, broad, meter=one_meter + ) + assert many_meter.samples == one_meter.samples + + +def test_vet_runtime_samples_include_midpoint_direction_switches(): + scene = copy.deepcopy(SCENE) + scene["meter"]["plus_minus_ft"] = 8.8 + scene["walls"] = [ + {"id": "w1", "baseline": [[0, 0], [10, 0]], "plus_minus_ft": 0}, + {"id": "w2", "baseline": [[10, 0], [9, 0]], "plus_minus_ft": 0}, + {"id": "w3", "baseline": [[9, 0], [109, 0]], "plus_minus_ft": 0}, + ] + broad = replace( + RULES, width_ft=10, depth_ft=2, step_ft=0.1, needs={"ground_surface": (Need("ground", 0),)} + ) + answer = result(spot=False, sweep=[pass_run(0, 10)]) + meter = WorkMeter(1000) + with pytest.raises(WorkLimitExceeded): + invariant_problems(scene, answer, rules=broad, meter=meter) + assert meter.samples == 1001 + + +@pytest.mark.parametrize("allowance", [-1, float("nan"), float("inf"), -float("inf")]) +def test_breadth_corner_allowance_must_be_finite_nonnegative(allowance): + answer = { + "checks": [ + { + "id": "route_corner", + "threshold_ft": allowance, + "rule": {"key": "route.corner_allowance_ft"}, + } + ] + } + with pytest.raises(ValueError, match="corner allowance must be finite and nonnegative"): + reported_rules(answer, RULES) + assert "route_corner corner allowance must be finite and nonnegative" in invariant_problems( + SCENE, answer, rules=RULES + ) + + +@pytest.mark.parametrize("allowance", [0, 0.5]) +def test_breadth_zero_or_positive_corner_allowance_is_valid(allowance): + answer = { + "checks": [ + { + "id": "route_corner", + "threshold_ft": allowance, + "rule": {"key": "route.corner_allowance_ft"}, + } + ] + } + answer["policy"] = {"sources": ["private"], "rules_sha256": "f" * 64} + assert reported_rules(answer, RULES).corner_allowance_ft == allowance diff --git a/verification/tests/test_runtime_meter.py b/verification/tests/test_runtime_meter.py new file mode 100644 index 000000000..cc7cef084 --- /dev/null +++ b/verification/tests/test_runtime_meter.py @@ -0,0 +1,209 @@ +import copy +import json +from dataclasses import replace +from pathlib import Path + +import pytest +from test_resultcheck import RULES, SCENE, check, result + +import hsverify.resultcheck as checks +from hsverify.resultcheck import ( + GroundPlan, + Need, + RuleSet, + WorkLimitExceeded, + WorkMeter, + finite, + geometry_admission_problems, + invariant_problems, + missing_evidence_problems, + reach_gaps, + schema_errors, +) + + +def test_oversized_integer_fails_admission_before_geometry(): + answer = result( + spot=False, sweep=[{"wall_id": "w1", "start_ft": [0, 10**400], "outcome": "fail"}] + ) + assert not finite(10**400) + meter = WorkMeter(0) + problems = invariant_problems(SCENE, answer, rules=RULES, meter=meter) + assert any("start_ft must be finite and ordered" in problem for problem in problems) + assert meter.samples == 0 + + +def test_unknown_nearest_wall_is_rejected_before_geometry(): + answer = result() + answer["nearest_considered"] = answer["spot"] | {"wall_id": "missing"} + meter = WorkMeter(0) + assert any( + "nearest_considered names unknown wall_id 'missing'" in problem + for problem in invariant_problems(SCENE, answer, rules=RULES, meter=meter) + ) + assert meter.samples == 0 + + +@pytest.mark.parametrize("has_unobserved_reason", [False, True]) +def test_identical_unsure_checks_do_not_repeat_geometry(has_unobserved_reason): + unsure = check(outcome="unsure", measured=None, cause="unobserved") | {"id": "pool_clearance"} + answer = result(checks=[unsure], complete=False) + answer["spot"]["outcome"] = "unsure" + answer["reasons"] = [{"code": "unobserved_area"}] if has_unobserved_reason else [] + answer["missing_evidence"] = [{"kind": "band", "band": "ground", "checks": ["pool_clearance"]}] + rules = replace(RULES, needs={"pool_clearance": (Need("ground", 10),)}) + meter = WorkMeter(100_000) + once = missing_evidence_problems(SCENE, answer, rules, meter=meter) + many = copy.deepcopy(answer) + many["checks"] *= 10_000 + duplicate_meter = WorkMeter(100_000) + assert missing_evidence_problems(SCENE, many, rules, meter=duplicate_meter) == once + assert duplicate_meter.samples == meter.samples + assert 0 < meter.samples < 100_000 + + +def test_reach_cache_includes_geometry_and_parameters(): + meter = WorkMeter(10_000) + first = reach_gaps(SCENE, RULES, "gas_clearance", "w1", 1, 3, meter=meter) + count = meter.samples + assert reach_gaps(SCENE, RULES, "gas_clearance", "w1", 1, 3, meter=meter) == first + assert meter.samples == count + reach_gaps(SCENE, RULES, "gas_clearance", "w1", 1, 4, meter=meter) + assert meter.samples > count + count = meter.samples + changed = copy.deepcopy(SCENE) + changed["coverage"]["observed"][1]["out_ft"] = 0.1 + reach_gaps(changed, RULES, "gas_clearance", "w1", 1, 3, meter=meter) + assert meter.samples > count + + +def test_ground_grid_stops_at_shared_runtime_limit(): + meter = WorkMeter() + plan = GroundPlan(SCENE, RULES, meter=meter) + setup = meter.samples + meter.limit = setup + 30 + with pytest.raises(WorkLimitExceeded) as caught: + plan.covers(2, 1, 3, 0, lambda _: 2) + assert caught.value.samples == meter.samples == setup + 31 + assert str(caught.value) == f"verification work limit exceeded ({setup + 31} samples)" + + +def test_valid_wide_direct_coverage_uses_only_visited_work(): + rules = RuleSet.from_yaml((Path(__file__).parents[2] / "server" / "rules.yaml").read_text()) + scene = copy.deepcopy(SCENE) + scene["walls"][0]["baseline"] = [[-30, 0], [30, 0]] + scene["objects"] = [] + scene["ground"] = [{"type": "concrete", "polygon": [[-30, 0], [30, 0], [30, 20], [-30, 20]]}] + scene["coverage"]["observed"] = [ + {"band": band, "span_ft": [-30, 30], "out_ft": 20} + for band in ("ground", "wall", "facing", "overhead") + ] + answer = result(decision="pass") + answer["summary"] = "All public-rule checks pass with coverage beyond every required reach." + lo, hi = 2, 2 + rules.width_ft + answer["spot"].update( + span_ft=[lo, hi], + center=[(lo + hi) / 2, rules.depth_ft / 2], + meter_offset_ft=[(lo + hi) / 2, rules.depth_ft / 2], + footprint=[[lo, 0], [hi, 0], [hi, rules.depth_ft], [lo, rules.depth_ft]], + route_length_ft=2, + ) + answer["sweep"][0]["start_ft"] = [2, 12] + answer["route"] = { + "outcome": "pass", + "length_ft": 2, + "plus_minus_ft": 0, + "height_ft": rules.route_height_ft, + "polyline": [[0, 0], [2, 0]], + "detours": [], + "crossings": [], + } + answer["ends"] = { + "left": {"kind": "limit", "s_ft": -30, "point": [-30, 0]}, + "right": {"kind": "limit", "s_ft": 30, "point": [30, 0]}, + } + for kind, value in (("scene", scene), ("result", answer)): + schema = json.loads( + (Path(__file__).parents[2] / "server" / "schemas" / f"{kind}.schema.json").read_text() + ) + assert schema_errors(value, schema) == [] + assert len(json.dumps(answer).encode()) == 4420 + meter = WorkMeter(100_000) + assert geometry_admission_problems(scene, answer, rules) == [] + assert invariant_problems(scene, answer, rules=rules, meter=meter) == [] + assert 0 < meter.samples < 100_000 + + +def test_public_solver_80_foot_wall_uses_direct_coverage(monkeypatch): + # Recorded from public server/rules.yaml with solve(parse_scene(...)) on 2026-10-14. + # Elapsed time is normalized to zero. Keep the response here so verification + # needs no solver or Shapely dependency. + fixture = json.loads((Path(__file__).parent / "fixtures/wide-public-pass.json").read_text()) + scene, answer = fixture["scene"], fixture["result"] + rules = RuleSet.from_yaml((Path(__file__).parents[2] / "server/rules.yaml").read_text()) + for kind, value in (("scene", scene), ("result", answer)): + schema = json.loads( + (Path(__file__).parents[2] / "server/schemas" / f"{kind}.schema.json").read_text() + ) + assert schema_errors(value, schema) == [] + assert answer["decision"] == "pass" + assert answer["stats"]["candidates"] == 683 + + def unexpected_grid(*args, **kwargs): + pytest.fail("complete direct coverage must not sample the ground grid") + + monkeypatch.setattr(GroundPlan, "_samples", unexpected_grid) + meter = WorkMeter() + assert invariant_problems(scene, answer, rules=rules, meter=meter) == [] + # Actual footprint, interval and metadata visits are below 1% of the shared limit. + assert 0 < meter.samples < checks.MAX_WORK_SAMPLES // 100 + + +def test_invariant_geometry_exception_names_its_check(monkeypatch): + def broken(*args, **kwargs): + raise RuntimeError("damaged geometry") + + monkeypatch.setattr("hsverify.resultcheck.coverage_problems", broken) + problems = invariant_problems(SCENE, result(), rules=RULES) + assert "coverage: RuntimeError: damaged geometry" in problems + + +def test_each_candidate_wall_piece_charges_the_shared_meter(monkeypatch): + monkeypatch.setattr(checks, "geometry_admission_problems", lambda *args, **kwargs: []) + monkeypatch.setattr( + checks, + "straight_pieces", + lambda *args, **kwargs: [(i * 10, i * 10 + 5) for i in range(1000)], + ) + answer = result( + spot=False, sweep=[{"wall_id": "w1", "start_ft": [9991, 9991], "outcome": "unsure"}] + ) + meter = WorkMeter(100) + with pytest.raises(WorkLimitExceeded): + checks.footprint_problems(SCENE, answer, RULES, meter=meter) + assert meter.samples == 101 + + +def test_finite_endpoints_with_overflowing_length_stop_admission(): + answer = result( + spot=False, sweep=[{"wall_id": "w1", "start_ft": [-1e308, 1e308], "outcome": "fail"}] + ) + meter = WorkMeter(0) + problems = invariant_problems(SCENE, answer, rules=RULES, meter=meter) + assert any("span length must be finite" in p for p in problems) + assert meter.samples == 0 + + +def test_equal_scene_copies_share_geometry_cache_keys(): + meter = WorkMeter() + assert meter.input_key(SCENE) == meter.input_key(copy.deepcopy(SCENE)) + assert isinstance(meter.input_key(SCENE), int) + before = meter.samples + for _ in range(10_000): + meter.input_key(SCENE) + assert meter.samples == before + + +def test_ground_samples_do_not_overflow_finite_span(): + points = GroundPlan._samples(0, 1e306) + assert all(finite(next(points)) for _ in range(1000)) diff --git a/verification/tests/test_scene_cache.py b/verification/tests/test_scene_cache.py new file mode 100644 index 000000000..af18ea263 --- /dev/null +++ b/verification/tests/test_scene_cache.py @@ -0,0 +1,81 @@ +import copy +import json +from dataclasses import replace +from pathlib import Path + +import pytest +from segmented_wall import segmented_wall_scene +from test_resultcheck import RULES, SCENE + +from hsverify.resultcheck import ( + RuleSet, + WorkMeter, + ground_plan, + invariant_problems, + schema_errors, + segments_s, + slide, + wall_spans_s, +) + + +@pytest.mark.parametrize("segments_per_wall", [13, 20]) +def test_public_pass_is_independent_of_redundant_baseline_vertices(segments_per_wall): + # Public solve(parse_scene(...), rules_from_dict(public YAML)) produced this + # 815-candidate, 26-run response. The geometry is unchanged by extra vertices. + fixture = json.loads( + (Path(__file__).parent / "fixtures/segmented-public-pass.json").read_text() + ) + rules = RuleSet.from_yaml((Path(__file__).parents[2] / "server/rules.yaml").read_text()) + answer = fixture["result"] + baseline_meter = WorkMeter() + assert invariant_problems(fixture["scene"], answer, rules=rules, meter=baseline_meter) == [] + scene = segmented_wall_scene(segments_per_wall) + for kind, value in (("scene", scene), ("result", answer)): + schema = json.loads( + (Path(__file__).parents[2] / f"server/schemas/{kind}.schema.json").read_text() + ) + assert schema_errors(value, schema) == [] + meter = WorkMeter() + assert invariant_problems(scene, answer, rules=rules, meter=meter) == [] + # Metadata grows with the input, but actual geometry work must not multiply + # with redundant baseline vertices. Leave room for the bounded input scan. + assert meter.samples < baseline_meter.samples * 1.2 + assert all(key[0] != "ground_strips" for key in meter.cache) + + +def test_scene_derivatives_are_built_once_per_judgment(): + scene = segmented_wall_scene(20) + meter = WorkMeter() + segments = segments_s(scene, RULES, meter=meter) + spans = wall_spans_s(scene, RULES, meter=meter) + plan = ground_plan(scene, RULES, meter) + once = meter.samples + assert segments_s(scene, RULES, meter=meter) is segments + assert wall_spans_s(scene, RULES, meter=meter) is spans + assert ground_plan(scene, RULES, meter) is plan + assert meter.samples == once + assert len(segments) == 20 + assert len({segment[0] for segment in segments}) == 20 + assert all(key[0] != "ground_strips" for key in meter.cache) + + +def test_collinear_merge_keeps_actual_turns_and_wall_error_sources(): + scene = copy.deepcopy(SCENE) + scene["walls"] = [ + {"id": "w1", "baseline": [[0, 0], [2, 0], [4, 0], [4, 4]], "plus_minus_ft": 0.1}, + {"id": "w2", "baseline": [[4, 4], [4, 6], [4, 8]], "plus_minus_ft": 0.2}, + ] + scene["meter"].update(pos=[0, 4, 0], wall_id="w1", plus_minus_ft=0) + rules = replace(RULES, wall_join_ft=0.1) + meter = WorkMeter() + segments = segments_s(scene, rules, meter=meter) + assert [(wid, a, b) for wid, a, b, _, _ in segments] == [ + ("w1", 0, 4), + ("w1", 4, 8), + ("w2", 8, 12), + ] + assert slide(scene, rules, 9, 10, meter=meter) == pytest.approx(0.1 * 2**0.5) + once = meter.samples + assert slide(scene, rules, 9, 10, meter=meter) == pytest.approx(0.1 * 2**0.5) + assert meter.samples == once diff --git a/verification/tests/test_scoreboard.py b/verification/tests/test_scoreboard.py index 21292cb27..f8b2f5e44 100644 --- a/verification/tests/test_scoreboard.py +++ b/verification/tests/test_scoreboard.py @@ -2,6 +2,7 @@ import datetime as dt import json import os +import subprocess import textwrap import time @@ -675,3 +676,39 @@ def test_tail_keeps_last_three_lines_with_spaces_collapsed(): sb._tail("a\n\n... [ 82%]\nb\n 87 passed in 1s \n") == "... [ 82%] / b / 87 passed in 1s" ) + + +def test_pr_inventory_exceeds_default_page(monkeypatch): + prs = [{"number": n} for n in range(1, 42)] + + def fake_run(cmd, **kwargs): + assert int(cmd[cmd.index("--limit") + 1]) > 30 + assert kwargs["timeout"] == sb.PR_LIST_TIMEOUT_S + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(prs)) + + monkeypatch.setattr(sb.subprocess, "run", fake_run) + assert sb.list_prs() == prs + + +def test_s2_contract_credit_requires_successful_full_run(tmp_path): + _, metrics = sb.load_metrics() + metric = next(m for m in metrics if m.id == "S2-8") + metric.probe["glob"] = str(tmp_path / "*" / "report.json") + write_report( + tmp_path / "r" / "report.json", + {"sha": SHA, "contract_ok": True, "all_passed": False}, + 100, + ) + assert sb.probe_report(metric.probe, SHA).status == "gap" + + +@pytest.mark.parametrize("full", [None, False, True]) +def test_s2_requires_hostile_execution_evidence_from_report(tmp_path, full): + metric = real_metric("S2-8") + metric.probe["glob"] = str(tmp_path / "*" / "report.json") + data = {"sha": SHA, "contract_ok": True, "all_passed": True} + if full is not None: + data["full_contract_ok"] = full + write_report(tmp_path / "r" / "report.json", data, 100) + status = sb.probe_report(metric.probe, SHA).status + assert (status == "met") is (full is True) diff --git a/verification/tests/test_sim_app_flow.py b/verification/tests/test_sim_app_flow.py new file mode 100644 index 000000000..c29de6ef7 --- /dev/null +++ b/verification/tests/test_sim_app_flow.py @@ -0,0 +1,93 @@ +"""`make sim-app` passes only a run that walked the capture flow to its result.""" + +import argparse +import re +from pathlib import Path + +import pytest + +from hsverify.simrun import ShotRecord, out_of_order, state_pair, unanswered + + +def shot(state: str) -> ShotRecord: + return ShotRecord(1, state, 0.0, "01.png", False, f"STATE={state}") + + +def sim_app_problems(sequence: str) -> list[str]: + """What `make sim-app`'s state requirements say about a run that logged `sequence`.""" + makefile = (Path(__file__).resolve().parents[1] / "Makefile").read_text() + recipe = makefile.split("\nsim-app:\n", 1)[1].split("\n\n", 1)[0] + orders = re.findall(r"--require-order (\S+)", recipe) + pairs = re.findall(r"--require-then (\S+)", recipe) + assert orders and pairs, recipe + states = [shot(name) for name in sequence.split(">")] + problems = [p for order in orders for p in out_of_order(order.split(","), states)] + closing = frozenset(order.split(",")[-1] for order in orders) + return problems + unanswered([state_pair(p) for p in pairs], states, closing) + + +# State sequences of recorded `make sim-app` runs on t3/ios-mvf (report.json in +# ~/house-scanning-data/reports/sim/): one upload at a39d0a5, a recapture round at f14947e, and +# recapture plus spot checks at c820cf1. +RECORDED_RUNS = [ + "onboarding>findMeter>meterCloseUp>wallWalk>markFeatures>gapRequest>uploading>result>resultAR" + ">result", + "onboarding>findMeter>meterCloseUp>wallWalk>markFeatures>gapRequest>uploading>gapRequest" + ">uploading>result>resultAR>result", + "onboarding>findMeter>meterCloseUp>wallWalk>markFeatures>gapRequest>uploading>gapRequest" + ">uploading>spotConfirm>uploading>spotConfirm>uploading>spotConfirm>result>resultAR>result", +] + + +@pytest.mark.parametrize("sequence", RECORDED_RUNS) +def test_the_recorded_flows_pass(sequence): + assert sim_app_problems(sequence) == [] + + +@pytest.mark.parametrize( + ("sequence", "problem"), + [ + # A cached or prebuilt scan shown straight away: no capture, no upload. + ("onboarding>result>resultAR>result", "result before findMeter"), + # Captured, then a result with no upload behind it. + ( + "onboarding>findMeter>meterCloseUp>wallWalk>markFeatures>result", + "result before uploading", + ), + # Uploaded before the walk that should have produced the scan. + ( + "onboarding>uploading>findMeter>meterCloseUp>wallWalk>markFeatures>result", + "result before uploading", + ), + # The answer asked for more, the app showed the request, then the result without sending. + ( + "onboarding>findMeter>meterCloseUp>wallWalk>markFeatures>uploading>gapRequest>result", + "state 7 gapRequest was not followed by uploading before result", + ), + # The request's upload came only after the app had already shown a result. + ( + "onboarding>findMeter>meterCloseUp>wallWalk>markFeatures>uploading>gapRequest>result" + ">uploading>result", + "state 7 gapRequest was not followed by uploading before result", + ), + # Two requests, one upload: the first request was never sent on its own. + ( + "onboarding>findMeter>meterCloseUp>wallWalk>markFeatures>gapRequest>gapRequest" + ">uploading>result", + "state 6 gapRequest was not followed by uploading before gapRequest", + ), + # The request at the end of the run was never answered at all. + ( + "onboarding>findMeter>meterCloseUp>wallWalk>markFeatures>uploading>result>gapRequest", + "state 8 gapRequest was never followed by uploading", + ), + ], +) +def test_a_shortcut_to_the_result_fails(sequence, problem): + problems = sim_app_problems(sequence) + assert any(problem in p for p in problems), problems + + +def test_a_malformed_pair_is_refused(): + with pytest.raises(argparse.ArgumentTypeError, match="FIRST:THEN"): + state_pair("gapRequest") diff --git a/verification/tests/test_simrun.py b/verification/tests/test_simrun.py index 515f29828..d39e57436 100644 --- a/verification/tests/test_simrun.py +++ b/verification/tests/test_simrun.py @@ -1,11 +1,15 @@ +import contextlib import datetime as dt import io import json import queue import subprocess +import sys from pathlib import Path -from hsverify import simrun +import pytest + +from hsverify import gitref, simrun from hsverify.simrun import ( ShotRecord, app_export, @@ -105,10 +109,23 @@ def test_missing_required_state_is_a_problem(): assert missing_required([], []) == [] -def test_sim_app_requires_the_result_state(): - makefile = (Path(__file__).resolve().parents[1] / "Makefile").read_text() - recipe = makefile.split("\nsim-app:\n", 1)[1].split("\n\n", 1)[0] - assert "--require result" in recipe +def test_required_states_reject_reversed_complete_journey(): + required = ["onboarding", "capture", "uploading", "result"] + assert missing_required(required, [shot(name) for name in reversed(required)]) == [ + "required states did not appear in order: onboarding -> capture -> uploading -> result" + ] + + +def test_required_states_allow_other_states_and_a_repeated_journey(): + required = ["capture", "uploading", "result"] + states = ["result", "capture", "capture", "review", "uploading", "result"] + assert missing_required(required, [shot(name) for name in states]) == [] + + +def test_repeated_required_state_needs_a_separate_occurrence(): + assert missing_required(["capture", "capture"], [shot("capture")]) == [ + "required states did not appear in order: capture -> capture" + ] # --- app export ------------------------------------------------------------------------------ @@ -186,3 +203,102 @@ def fake_build(cmd, **kwargs): assert (app / "Info.plist").read_text() == "run A" simrun.discard_app(app) assert not app.parent.exists() and out.exists() + + +def test_shared_output_keeps_both_app_copies(tmp_path, monkeypatch): + derived = tmp_path / "derived" + built = derived / "Build" / "Products" / "Debug-iphonesimulator" / "HouseScan.app" + monkeypatch.setattr(simrun, "DERIVED_DATA", derived) + monkeypatch.setattr(simrun, "BUILD_LOCK", tmp_path / "lock") + monkeypatch.setattr(simrun, "wait_for_other_builds", lambda _: 0.0) + + def fake_build(cmd, **kwargs): + built.mkdir(parents=True, exist_ok=True) + (built / "Info.plist").write_text("app") + kwargs["stdout"].write("warning: Run script build phase\n/path/File: warning: warning\n") + return subprocess.CompletedProcess(cmd, 0) + + monkeypatch.setattr(simrun.subprocess, "run", fake_build) + tree, out = tmp_path / "tree", tmp_path / "report" + (tree / "ios" / "HouseScan.xcodeproj").mkdir(parents=True) + out.mkdir() + result, first = simrun.build_app(tree, out, 0, "ios/HouseScan.xcodeproj", "HouseScan") + _, second = simrun.build_app(tree, out, 0, "ios/HouseScan.xcodeproj", "HouseScan") + assert first != second and first.exists() and second.exists() + assert len(result["warnings"]) == 2 + simrun.discard_app(first) + assert not first.exists() and second.exists() + + +@pytest.mark.parametrize("failure", ["device", "bundle", "install"]) +def test_failed_preparation_discards_app_copy(tmp_path, monkeypatch, failure): + out = tmp_path / "out" + app = out / "built-app-unique" / "HouseScan.app" + app.mkdir(parents=True) + monkeypatch.setattr(simrun, "DEVICE_LOCK", tmp_path / "device.lock") + monkeypatch.setattr(gitref, "fetch", lambda: None) + monkeypatch.setattr(gitref, "resolve", lambda _: "a" * 40) + monkeypatch.setattr( + gitref, "detached_worktree", lambda *args, **kwargs: contextlib.nullcontext(tmp_path) + ) + monkeypatch.setattr(simrun, "static_c4_check", lambda _: {}) + monkeypatch.setattr(simrun, "build_app", lambda *args: ({"ok": True}, app)) + + def fail(): + raise RuntimeError("preparation failed") + + monkeypatch.setattr( + simrun, "ensure_device", lambda _: fail() if failure == "device" else {"udid": "mock"} + ) + monkeypatch.setattr(simrun, "bundle_id_of", lambda _: fail() if failure == "bundle" else "mock") + monkeypatch.setattr(simrun, "prepare_display", lambda *args: None) + monkeypatch.setattr(simrun, "simctl", lambda *args: fail() if args[0] == "install" else "") + monkeypatch.setattr( + simrun.subprocess, "run", lambda *args, **kwargs: subprocess.CompletedProcess([], 0) + ) + monkeypatch.setattr(simrun, "peak_rss_mb", lambda: {}) + monkeypatch.setattr(simrun.report, "write_sim_report", lambda *args: None) + with pytest.raises(RuntimeError, match="preparation failed"): + simrun.main(["--out", str(out)]) + assert not app.parent.exists() + + +def test_device_lock_excludes_another_runner_and_releases(tmp_path, monkeypatch): + monkeypatch.setattr(simrun, "DEVICE_LOCK", tmp_path / "device.lock") + script = """import fcntl, sys +with open(sys.argv[1]) as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + sys.exit(43) +""" + cmd = [sys.executable, "-c", script, str(simrun.DEVICE_LOCK)] + with simrun.device_lock(): + assert subprocess.run(cmd, timeout=5, check=False).returncode == 43 + assert subprocess.run(cmd, timeout=5, check=False).returncode == 0 + + +@pytest.mark.parametrize("prefix", [[], ["findMeter"], ["findMeter", "uploading"]]) +def test_early_result_cannot_be_hidden_by_a_later_complete_flow(prefix): + order = ["findMeter", "meterCloseUp", "wallWalk", "markFeatures", "uploading", "result"] + states = [shot(name) for name in [*prefix, "result", *order]] + assert simrun.out_of_order(order, states) + assert simrun.out_of_order(order, [shot(name) for name in order]) == [] + + +def test_order_still_allows_intermediate_states_and_completed_recapture(): + order = ["findMeter", "uploading", "result"] + states = [ + shot(name) + for name in [ + "onboarding", + "findMeter", + "wallWalk", + "uploading", + "gapRequest", + "markFeatures", + "uploading", + "result", + ] + ] + assert simrun.out_of_order(order, states) == [] diff --git a/verification/tests/test_sweep_sampling_contract.py b/verification/tests/test_sweep_sampling_contract.py new file mode 100644 index 000000000..852eed588 --- /dev/null +++ b/verification/tests/test_sweep_sampling_contract.py @@ -0,0 +1,47 @@ +import copy +from dataclasses import replace + +import pytest +from test_resultcheck import RULES, SCENE, pass_run, result + +from hsverify import resultcheck as checks + + +@pytest.mark.parametrize( + "span,expected", [([0, 2.4], [0, 1, 2, 2.4]), ([0, 0.4], [0, 0.4]), ([2, 2], [2])] +) +def test_pre_end_count_plus_endpoint_matches_every_yield(span, expected): + samples = list(checks.run_starts(span, 1)) + assert samples == expected + assert len(samples) == checks.run_start_count(span, 1) + 1 + + +@pytest.mark.parametrize("span,failing_start", [([0, 2.4], 2), ([0, 0.4], 0)]) +def test_coverage_rejects_a_failure_at_the_required_pre_end_start(monkeypatch, span, failing_start): + rules = replace(RULES, needs={"ground_surface": ()}, step_ft=1) + scene = copy.deepcopy(SCENE) + seen = [] + + def gap(scene, rules, check, wall_id, lo, hi, meter=None): + seen.append(lo) + return [(None, "unobserved at required pre-end start")] if lo == failing_start else [] + + monkeypatch.setattr(checks, "reach_gaps", gap) + answer = result(spot=False, sweep=[pass_run(*span)]) + problems = checks.coverage_problems(scene, answer, rules) + assert seen == list(checks.run_starts(span, 1)) + assert any("unobserved at required pre-end start" in problem for problem in problems) + + +def test_coverage_charges_once_for_each_yielded_start(monkeypatch): + rules = replace(RULES, needs={"ground_surface": ()}, step_ft=1) + sampled = [] + + def gap(scene, rules, check, wall_id, lo, hi, meter=None): + sampled.append((lo, meter.samples)) + return [] + + monkeypatch.setattr(checks, "reach_gaps", gap) + checks.coverage_problems(SCENE, result(spot=False, sweep=[pass_run(0, 2.4)]), rules) + assert [start for start, _ in sampled] == [0, 1, 2, 2.4] + assert [count - sampled[0][1] for _, count in sampled] == [0, 1, 2, 3] diff --git a/verification/tests/test_unreported_rule_properties.py b/verification/tests/test_unreported_rule_properties.py new file mode 100644 index 000000000..6f1424a8b --- /dev/null +++ b/verification/tests/test_unreported_rule_properties.py @@ -0,0 +1,144 @@ +"""Unknown rule overrides do not hide nondeterminism in identical requests.""" + +import copy +import hashlib +import json +from dataclasses import replace +from pathlib import Path + +from test_judge_meter import response + +from hsverify import e2e +from hsverify.resultcheck import RuleSet, WorkMeter, expected_checks, schema_errors + +ROOT = Path(__file__).parents[2] +RULES = RuleSet.from_yaml((ROOT / "server/rules.yaml").read_text()) +SCENE = { + "schema_version": "1.0", + "meter": {"pos": [0, 4, 0], "wall_id": "w1"}, + "walls": [{"id": "w1", "baseline": [[-30, 0], [30, 0]]}], +} +SCHEMAS = { + kind: json.loads((ROOT / f"server/schemas/{kind}.schema.json").read_text()) + for kind in ("scene", "result") +} + + +def test_unreported_width_keeps_identical_input_comparison(monkeypatch): + raw = json.dumps(SCENE).encode() + answer = { + "schema_version": "1.0", + "decision": "manual_review", + "summary": "Review required", + "reasons": [{"code": "policy_not_approved", "message": "Review required"}], + "policy": { + "id": "demo", + "version": "1", + "auto_approve": False, + "sources": ["public"], + "rules_sha256": "a" * 64, + }, + "spot": None, + "route": None, + "checks": [], + "missing_evidence": [], + "ends": { + side: {"kind": "limit", "s_ft": s, "point": [s, 0]} + for side, s in (("left", -30), ("right", 30)) + }, + "sweep": [], + "stats": { + "candidates": 0, + "pass": 0, + "unsure": 0, + "fail": 0, + "elapsed_ms": 1, + "input_sha256": hashlib.sha256(raw).hexdigest(), + }, + } + assert schema_errors(answer, SCHEMAS["result"]) == [] + changed = copy.deepcopy(answer) | {"summary": "Different answer to identical input"} + calls = [] + + def post(url, endpoint, item): + calls.append(item.raw) + value = answer if len(calls) == 1 else changed + value = copy.deepcopy(value) + value["stats"]["input_sha256"] = hashlib.sha256(item.raw).hexdigest() + return 200, json.dumps(value).encode(), 1 + + monkeypatch.setattr(e2e, "post", post) + item = e2e.SceneInput( + "unknown rules", SCENE, raw, rules_assumed={"battery_width": RULES.width_ft} + ) + record = e2e.judge( + "unused", + e2e.Endpoint("/place", "application/json"), + item, + SCHEMAS, + replace(RULES, width_ft=4), + ) + assert record["rule_checks_unverified"] + assert record["status"] == "fail" + assert "sending the same scene twice gave different results" in record["contract_problems"] + assert calls[:2] == [raw, raw] + + +def test_reported_width_uses_runtime_meter_without_predicting_unused_grid(monkeypatch): + answer = response() + answer["decision"] = "reject" + answer["sweep"] = [ + { + "wall_id": "w1", + "start_ft": [1, 1], + "outcome": "fail", + "failing": ["route_length"], + "unsure": [], + } + ] + answer["stats"].update(candidates=1, fail=1) + answer["checks"] = [ + { + "id": key, + "label": key, + "outcome": "fail", + "reason": "No fitting footprint", + "measured_ft": None, + "plus_minus_ft": None, + "threshold_ft": None, + "comparison": None, + "subject": None, + "rule": {"key": key, "source": "public fixture", "placeholder": False}, + } + for key in sorted( + expected_checks(SCENE, RULES) | {"meter_working_space", "route_path", "route_length"} + ) + ] + answer["checks"].append( + { + "id": "width", + "label": "width", + "outcome": "fail", + "reason": "No fitting footprint", + "measured_ft": None, + "plus_minus_ft": None, + "threshold_ft": 1e9, + "comparison": None, + "rule": {"key": "battery.width_ft", "source": "public fixture", "placeholder": False}, + } + ) + assert schema_errors(answer, SCHEMAS["result"]) == [] + + def post(url, endpoint, item): + value = copy.deepcopy(answer) + value["stats"]["input_sha256"] = hashlib.sha256(item.raw).hexdigest() + return 200, json.dumps(value).encode(), 1 + + monkeypatch.setattr(e2e, "post", post) + meter = WorkMeter(2000) + item = e2e.SceneInput("large reported width", SCENE, json.dumps(SCENE).encode()) + record = e2e.judge( + "unused", e2e.Endpoint("/place", "application/json"), item, SCHEMAS, RULES, meter=meter + ) + assert record["status"] == "pass", record["problems"] + assert 0 < record["verification_samples"] == meter.samples < 2000