From 1cd340d345b5f637d6c73ff33c58f4dfa7f843e3 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 02:47:01 -0500 Subject: [PATCH 01/59] verification: add Simulator runner, STATE probe and UX checklist The runner builds a ref in a /tmp worktree, runs it on its own Simulator with the contract C4 launch arguments, follows STATE= log markers and saves a screenshot per state into a report outside git. The probe app logs a known sequence so the runner can be checked before the real app emits markers. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/Makefile | 22 + verification/README.md | 34 ++ .../StateProbe.xcodeproj/project.pbxproj | 286 +++++++++++ .../contents.xcworkspacedata | 7 + .../StateProbe/StateProbeApp.swift | 37 ++ verification/fixtures/state-probe/project.yml | 21 + verification/hsverify/__init__.py | 0 verification/hsverify/gitref.py | 66 +++ verification/hsverify/report.py | 113 ++++ verification/hsverify/simrun.py | 485 ++++++++++++++++++ verification/hsverify/statelog.py | 65 +++ verification/pyproject.toml | 30 ++ verification/tests/test_statelog.py | 61 +++ verification/uv.lock | 317 ++++++++++++ verification/ux/checklist.md | 127 +++++ 15 files changed, 1671 insertions(+) create mode 100644 verification/Makefile create mode 100644 verification/README.md create mode 100644 verification/fixtures/state-probe/StateProbe.xcodeproj/project.pbxproj create mode 100644 verification/fixtures/state-probe/StateProbe.xcodeproj/project.xcworkspace/contents.xcworkspacedata create mode 100644 verification/fixtures/state-probe/StateProbe/StateProbeApp.swift create mode 100644 verification/fixtures/state-probe/project.yml create mode 100644 verification/hsverify/__init__.py create mode 100644 verification/hsverify/gitref.py create mode 100644 verification/hsverify/report.py create mode 100644 verification/hsverify/simrun.py create mode 100644 verification/hsverify/statelog.py create mode 100644 verification/pyproject.toml create mode 100644 verification/tests/test_statelog.py create mode 100644 verification/uv.lock create mode 100644 verification/ux/checklist.md diff --git a/verification/Makefile b/verification/Makefile new file mode 100644 index 000000000..65ea1c81b --- /dev/null +++ b/verification/Makefile @@ -0,0 +1,22 @@ +# Independent verification. Every target runs from this folder; reports go to +# ~/house-scanning-data/reports/ (outside git) because runs can show dataset frames. + +REF ?= origin/t3/ios-mvf +ARGS ?= + +.PHONY: test sim sim-probe e2e scoreboard + +test: + uv run ruff check . + uv run ruff format --check . + uv run pytest -q + +# Build REF for the Simulator, run it with ARGS (e.g. ARGS="--replay --autopilot"), +# screenshot every STATE. +sim: + uv run python -m hsverify.simrun --ref $(REF) $(ARGS) + +# Self-test of the runner against a fixture app that logs a known STATE sequence. +sim-probe: + uv run python -m hsverify.simrun --ref HEAD --project verification/fixtures/state-probe/StateProbe.xcodeproj \ + --scheme StateProbe --until probe_done --extra-arg -probeArg --label probe diff --git a/verification/README.md b/verification/README.md new file mode 100644 index 000000000..80346d610 --- /dev/null +++ b/verification/README.md @@ -0,0 +1,34 @@ +# Verification (S4) + +Independent checks of what works end to end, run against the exact commits the other +workstreams push. Nothing here fixes their code; findings go back to the owning thread. + +| Check | Command | Output | +| --- | --- | --- | +| Unit tests for these tools | `make test` | pass or fail | +| App in the Simulator, a screenshot per `STATE` | `make sim REF=origin/t3/ios-mvf ARGS="--replay --autopilot --server-url "` | `~/house-scanning-data/reports/sim//index.html` | +| Runner self-test | `make sim-probe` | same, for `fixtures/state-probe` | + +Requirements: uv, Xcode 26 or newer with an iOS Simulator runtime. + +## Simulator runner + +`hsverify/simrun.py` checks the ref out into a detached worktree under `/tmp`, builds it for +the Simulator, installs it on its own device ("HouseScan Verify", so it never touches a +Simulator another worker is using), and launches it with the contract C4 arguments. It +attaches `log stream` before launch and follows `STATE=` lines from subsystem +`dev.housescanning.housescan`, category `state`. + +A state is screenshotted after it has been on screen for `--settle` seconds (default 1.2). If +the next state arrives first, the screenshot is taken at once and marked *transient*, since it +may already show the next screen. The run ends at an `--until` state, after `--idle` seconds +with no new state, or at `--timeout`. It reports a build failure, a crash, markers logged as +``, and a run with no markers at all as problems. + +Builds wait while any other `xcodebuild` runs, because four workers share this Mac. + +The report folder holds `index.html` (screenshots in state order), `report.json`, +`report.md`, the build log, the raw log stream and the app's stdout and stderr. + +`fixtures/state-probe` is a ten-line app that logs a fixed sequence, including a 0.3 s state, +so the runner can be checked without the real app. diff --git a/verification/fixtures/state-probe/StateProbe.xcodeproj/project.pbxproj b/verification/fixtures/state-probe/StateProbe.xcodeproj/project.pbxproj new file mode 100644 index 000000000..dfd18cc2b --- /dev/null +++ b/verification/fixtures/state-probe/StateProbe.xcodeproj/project.pbxproj @@ -0,0 +1,286 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 77; + objects = { + +/* Begin PBXBuildFile section */ + 6522E1C65A1BACAEC71F045D /* StateProbeApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4E18E6F4F55A0C4DDD1D9F7B /* StateProbeApp.swift */; }; +/* End PBXBuildFile section */ + +/* Begin PBXFileReference section */ + 4E18E6F4F55A0C4DDD1D9F7B /* StateProbeApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StateProbeApp.swift; sourceTree = ""; }; + C9CD2DF097F422B75CA9BF54 /* StateProbe.app */ = {isa = PBXFileReference; includeInIndex = 0; lastKnownFileType = wrapper.application; path = StateProbe.app; sourceTree = BUILT_PRODUCTS_DIR; }; +/* End PBXFileReference section */ + +/* Begin PBXGroup section */ + 25367929A4B1815698729180 /* StateProbe */ = { + isa = PBXGroup; + children = ( + 4E18E6F4F55A0C4DDD1D9F7B /* StateProbeApp.swift */, + ); + path = StateProbe; + sourceTree = ""; + }; + 8BB70B547481E3A64DB58AAE = { + isa = PBXGroup; + children = ( + 25367929A4B1815698729180 /* StateProbe */, + F150664F13B4D3D8F233DCF1 /* Products */, + ); + sourceTree = ""; + }; + F150664F13B4D3D8F233DCF1 /* Products */ = { + isa = PBXGroup; + children = ( + C9CD2DF097F422B75CA9BF54 /* StateProbe.app */, + ); + name = Products; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + FAEE147748CA52747B658B0E /* StateProbe */ = { + isa = PBXNativeTarget; + buildConfigurationList = A136F875CF6480DCB5470BE1 /* Build configuration list for PBXNativeTarget "StateProbe" */; + buildPhases = ( + D5123BEFD882A7E9BDB4CBD1 /* Sources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = StateProbe; + packageProductDependencies = ( + ); + productName = StateProbe; + productReference = C9CD2DF097F422B75CA9BF54 /* StateProbe.app */; + productType = "com.apple.product-type.application"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 1C039B21D02E15301E6003C9 /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = YES; + LastUpgradeCheck = 1430; + TargetAttributes = { + }; + }; + buildConfigurationList = 9C450EBC261D9A1F5EB7EBF5 /* Build configuration list for PBXProject "StateProbe" */; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + Base, + en, + ); + mainGroup = 8BB70B547481E3A64DB58AAE; + minimizedProjectReferenceProxies = 1; + preferredProjectObjectVersion = 77; + productRefGroup = F150664F13B4D3D8F233DCF1 /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + FAEE147748CA52747B658B0E /* StateProbe */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXSourcesBuildPhase section */ + D5123BEFD882A7E9BDB4CBD1 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 6522E1C65A1BACAEC71F045D /* StateProbeApp.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin XCBuildConfiguration section */ + 559C3159F38CBBF3A6D4DEC2 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CODE_SIGNING_ALLOWED = NO; + CODE_SIGN_IDENTITY = "iPhone Developer"; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_UILaunchScreen_Generation = YES; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = dev.housescanning.stateprobe; + SDKROOT = iphoneos; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; + A8B6B380BE9E234A02662996 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 26.0; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = iphoneos; + SWIFT_COMPILATION_MODE = wholemodule; + SWIFT_OPTIMIZATION_LEVEL = "-O"; + SWIFT_STRICT_CONCURRENCY = complete; + SWIFT_VERSION = 6.0; + }; + name = Release; + }; + AEE11670B0C02BAA2879B728 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "$(inherited)", + "DEBUG=1", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 26.0; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = iphoneos; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_STRICT_CONCURRENCY = complete; + SWIFT_VERSION = 6.0; + }; + name = Debug; + }; + BF4C25D52E56316274082339 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CODE_SIGNING_ALLOWED = NO; + CODE_SIGN_IDENTITY = "iPhone Developer"; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_UILaunchScreen_Generation = YES; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = dev.housescanning.stateprobe; + SDKROOT = iphoneos; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 9C450EBC261D9A1F5EB7EBF5 /* Build configuration list for PBXProject "StateProbe" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + AEE11670B0C02BAA2879B728 /* Debug */, + A8B6B380BE9E234A02662996 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Debug; + }; + A136F875CF6480DCB5470BE1 /* Build configuration list for PBXNativeTarget "StateProbe" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + BF4C25D52E56316274082339 /* Debug */, + 559C3159F38CBBF3A6D4DEC2 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Debug; + }; +/* End XCConfigurationList section */ + }; + rootObject = 1C039B21D02E15301E6003C9 /* Project object */; +} diff --git a/verification/fixtures/state-probe/StateProbe.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/verification/fixtures/state-probe/StateProbe.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 000000000..919434a62 --- /dev/null +++ b/verification/fixtures/state-probe/StateProbe.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/verification/fixtures/state-probe/StateProbe/StateProbeApp.swift b/verification/fixtures/state-probe/StateProbe/StateProbeApp.swift new file mode 100644 index 000000000..feaab5290 --- /dev/null +++ b/verification/fixtures/state-probe/StateProbe/StateProbeApp.swift @@ -0,0 +1,37 @@ +import OSLog +import SwiftUI + +/// Walks through a fixed list of states, logging each as `STATE=` the way contract C4 +/// asks the real app to. `quick` lasts 0.3 s so the runner's transient flag is exercised. +/// Launch arguments are echoed on screen so the runner's argument passing can be checked. +@main +struct StateProbeApp: App { + var body: some Scene { + WindowGroup { ProbeView() } + } +} + +private let log = Logger(subsystem: "dev.housescanning.housescan", category: "state") +private let steps: [(name: String, seconds: Double)] = [ + ("probe_start", 2), ("quick", 0.3), ("probe_middle", 2), ("probe_done", 2), +] + +struct ProbeView: View { + @State private var current = "" + + var body: some View { + VStack(spacing: 12) { + Text(current).font(.largeTitle.bold()) + Text(CommandLine.arguments.dropFirst().joined(separator: " ")) + .font(.footnote.monospaced()) + } + .padding() + .task { + for step in steps { + current = step.name + log.info("STATE=\(step.name, privacy: .public)") + try? await Task.sleep(for: .seconds(step.seconds)) + } + } + } +} diff --git a/verification/fixtures/state-probe/project.yml b/verification/fixtures/state-probe/project.yml new file mode 100644 index 000000000..6d5ef05ab --- /dev/null +++ b/verification/fixtures/state-probe/project.yml @@ -0,0 +1,21 @@ +# Test fixture for hsverify.simrun, not a product. It logs a fixed STATE sequence (contract C4) +# so the runner's log following and screenshots can be checked without the real app. +name: StateProbe +options: + deploymentTarget: + iOS: "26.0" +settings: + base: + SWIFT_VERSION: "6.0" + SWIFT_STRICT_CONCURRENCY: complete +targets: + StateProbe: + type: application + platform: iOS + sources: [StateProbe] + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: dev.housescanning.stateprobe + GENERATE_INFOPLIST_FILE: YES + INFOPLIST_KEY_UILaunchScreen_Generation: YES + CODE_SIGNING_ALLOWED: NO diff --git a/verification/hsverify/__init__.py b/verification/hsverify/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/verification/hsverify/gitref.py b/verification/hsverify/gitref.py new file mode 100644 index 000000000..7a6fff74f --- /dev/null +++ b/verification/hsverify/gitref.py @@ -0,0 +1,66 @@ +"""Read-only access to other branches through throwaway worktrees under /tmp. + +Verification never checks out another thread's branch in its own worktree and never +pushes to one. A detached worktree at the exact commit keeps the evidence tied to a SHA. +""" + +from __future__ import annotations + +import contextlib +import shutil +import subprocess +from collections.abc import Iterator +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +TMP_ROOT = Path("/tmp") + + +def git(*args: str, cwd: Path = REPO) -> str: + return subprocess.run( + ["git", *args], cwd=cwd, check=True, capture_output=True, text=True + ).stdout.strip() + + +def fetch() -> None: + git("fetch", "--quiet", "origin") + + +def resolve(ref: str) -> str: + """Full SHA of a ref. Raises CalledProcessError with git's message if it does not exist.""" + return git("rev-parse", "--verify", f"{ref}^{{commit}}") + + +def ref_exists(ref: str) -> bool: + try: + resolve(ref) + except subprocess.CalledProcessError: + return False + return True + + +def show(ref: str, path: str) -> str | None: + """File contents at a ref, or None when the file does not exist there.""" + try: + return git("show", f"{ref}:{path}") + except subprocess.CalledProcessError: + return None + + +@contextlib.contextmanager +def detached_worktree(sha: str, keep: bool = False) -> Iterator[Path]: + """A detached worktree of `sha` at /tmp/hs-verify-, removed afterwards unless `keep`. + + An existing worktree at that path is reused, since it holds the same commit. + """ + path = TMP_ROOT / f"hs-verify-{sha[:12]}" + if not (path / ".git").exists(): + if path.exists(): + shutil.rmtree(path) + git("worktree", "add", "--detach", "--force", str(path), sha) + # The landing page submodule is not needed for any check here. + try: + yield path + finally: + if not keep: + git("worktree", "remove", "--force", str(path)) diff --git a/verification/hsverify/report.py b/verification/hsverify/report.py new file mode 100644 index 000000000..fa3611fcb --- /dev/null +++ b/verification/hsverify/report.py @@ -0,0 +1,113 @@ +"""Static HTML and Markdown pages for run folders. No external assets, so a folder opens offline.""" + +from __future__ import annotations + +import html +from pathlib import Path + +_CSS = """ +:root { color-scheme: light dark; --fg: #1d1d1f; --muted: #6e6e73; --line: #d2d2d7; + --bad: #b3261e; --warn: #8a5a00; --ok: #1b6e3a; --card: #f5f5f7; } +@media (prefers-color-scheme: dark) { :root { --fg: #f5f5f7; --muted: #a1a1a6; + --line: #3a3a3c; --card: #1c1c1e; --bad: #ff8a80; --warn: #ffcc66; --ok: #7ee2a0; } } +body { font: 15px/1.45 -apple-system, BlinkMacSystemFont, sans-serif; color: var(--fg); + margin: 32px auto; max-width: 1180px; padding: 0 20px; } +h1 { font-size: 22px; margin: 0 0 4px; } h2 { font-size: 17px; margin: 28px 0 8px; } +.meta { color: var(--muted); font-size: 13px; } +code { font: 12.5px ui-monospace, SFMono-Regular, monospace; } +.problems li { color: var(--bad); } +.grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(220px, 1fr)); gap: 18px; } +figure { margin: 0; } +figure img { width: 100%; border-radius: 18px; border: 1px solid var(--line); } +figcaption { font-size: 13px; margin-top: 6px; } figcaption b { font-weight: 600; } +.tag { font-size: 11px; padding: 1px 6px; border-radius: 6px; background: var(--card); + color: var(--warn); margin-left: 4px; } +table { border-collapse: collapse; font-size: 13px; } td, th { text-align: left; + padding: 3px 12px 3px 0; border-bottom: 1px solid var(--line); vertical-align: top; } +.ok { color: var(--ok); } .no { color: var(--bad); } +""" + + +def _esc(value: object) -> str: + return html.escape(str(value)) + + +def _yes(value: bool) -> str: + return 'yes' if value else 'no' + + +def write_sim_report(out: Path, data: dict) -> None: + build = data["build"] + states = data["states"] + parts = [ + "", + f"Simulator run {_esc(data['ref'])} {_esc(data['sha'][:8])}", + f"", + f"

Simulator run: {_esc(data['ref'])} at " + f"{_esc(data['sha'][:12])}

", + f"

{_esc(data['started_at'])} · {_esc(data['device'].get('runtime', ''))}" + f" · {_esc(data['device'].get('appearance', ''))}" + f" · text size {_esc(data['device'].get('content_size') or 'default')}" + f" · launch args {_esc(' '.join(data['launch_arguments']) or '(none)')}

", + f"

{len(states)} states. Ended: {_esc(data['end_reason'])}. Build " + f"{'passed' if build.get('ok') else 'failed'} in {_esc(build.get('seconds', '?'))} s with " + f"{len(build.get('warnings', []))} warnings.

", + ] + if data["problems"]: + parts.append("

Problems

    ") + parts += [f"
  • {_esc(p)}
    • " for p in data["problems"]] + parts.append("
") + parts.append("

States

") + for s in states: + tag = ( + 'transient' + if s["transient"] + else "" + ) + parts.append( + f"
{_esc(s[" + f"
{s['index']}. {_esc(s['state'])}{tag}
" + f"+{s['seconds_after_launch']} s
" + ) + if data.get("final_screenshot"): + parts.append( + f"
final screen" + "
Final screen
" + ) + parts.append("

Contract C4 in source

") + parts += [ + f"" for k, v in data["static_c4"].items() + ] + parts.append("
{_esc(k)}{_yes(v)}
") + if build.get("warnings") or build.get("errors"): + parts.append("

Build diagnostics

    ") + parts += [ + f"
  • {_esc(x)}
    • " + for x in build.get("errors", []) + build.get("warnings", []) + ] + parts.append("
") + parts.append( + "

Files: report.json · " + "build.log · state.ndjson · " + "app-stdout.log · app-stderr.log" + "

" + ) + (out / "index.html").write_text("\n".join(parts) + "\n") + + lines = [ + f"# Simulator run: `{data['ref']}` at `{data['sha'][:12]}`", + "", + f"- Build: {'passed' if build.get('ok') else 'failed'}, {build.get('seconds', '?')} s, " + f"{len(build.get('warnings', []))} warnings", + f"- Launch arguments: `{' '.join(data['launch_arguments']) or '(none)'}`", + f"- States: {len(states)}; ended: {data['end_reason']}", + ] + lines += [f"- Problem: {p}" for p in data["problems"]] + lines += ["", "| # | State | +s | Screenshot |", "| --- | --- | --- | --- |"] + lines += [ + f"| {s['index']} | {s['state']}{' (transient)' if s['transient'] else ''} | " + f"{s['seconds_after_launch']} | {s['screenshot']} |" + for s in states + ] + (out / "report.md").write_text("\n".join(lines) + "\n") diff --git a/verification/hsverify/simrun.py b/verification/hsverify/simrun.py new file mode 100644 index 000000000..3315d43cb --- /dev/null +++ b/verification/hsverify/simrun.py @@ -0,0 +1,485 @@ +"""Build the iOS app from a git ref, run it in a dedicated Simulator, screenshot every STATE. + + uv run python -m hsverify.simrun --ref origin/t3/ios-mvf \ + --replay ~/house-scanning-data/replays/ --autopilot --server-url http://127.0.0.1:8000 + +The app is launched with the contract C4 arguments (`-replay `, `-autopilot`, +`-serverURL `) and the runner follows its `STATE=` log markers. Each state gets +a screenshot once it has been on screen for `--settle` seconds; a state replaced sooner is +captured immediately and flagged `transient`, because its image may already show the next +state. The run ends at `--until`, after `--idle` seconds without a new state, or at +`--timeout`. Reports go outside git (default ~/house-scanning-data/reports/sim/) because a +replay can put dataset frames on screen. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import fcntl +import json +import os +import plistlib +import queue +import re +import shlex +import subprocess +import sys +import threading +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path + +from hsverify import gitref, report +from hsverify.statelog import LOG_PREDICATE, RedactedStateError, parse_ndjson_line + +DEVICE_NAME = "HouseScan Verify" +DEVICE_TYPE = "com.apple.CoreSimulator.SimDeviceType.iPhone-17" +DEFAULT_REPORTS = Path.home() / "house-scanning-data" / "reports" / "sim" +DERIVED_DATA = Path("/tmp/hs-verify-derived-data") +BUILD_LOCK = Path("/tmp/hs-verify-xcodebuild.lock") + + +@dataclass +class ShotRecord: + index: int + state: str + seconds_after_launch: float + screenshot: str + transient: bool + log_message: str + + +@dataclass +class RunReport: + ref: str + sha: str + started_at: str + command: list[str] + device: dict + build: dict + launch_arguments: list[str] + static_c4: dict + states: list[ShotRecord] = field(default_factory=list) + end_reason: str = "" + problems: list[str] = field(default_factory=list) + crash_reports: list[str] = field(default_factory=list) + final_screenshot: str | None = None + + +def run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess: + return subprocess.run(cmd, check=True, capture_output=True, text=True, **kwargs) + + +def simctl(*args: str) -> str: + return run(["xcrun", "simctl", *args]).stdout.strip() + + +# --- Simulator ------------------------------------------------------------------------------- + + +def latest_ios_runtime() -> str: + runtimes = json.loads(simctl("list", "runtimes", "-j"))["runtimes"] + ios = [r for r in runtimes if r.get("platform") == "iOS" and r.get("isAvailable")] + if not ios: + raise SystemExit("No available iOS Simulator runtime. Install one in Xcode > Settings.") + ios.sort(key=lambda r: tuple(int(p) for p in r["version"].split("."))) + return ios[-1]["identifier"] + + +def ensure_device() -> dict: + """The runner's own Simulator, so it never disturbs devices other workers are using.""" + runtime = latest_ios_runtime() + devices = json.loads(simctl("list", "devices", "-j"))["devices"].get(runtime, []) + for device in devices: + if device["name"] == DEVICE_NAME and device.get("isAvailable", True): + udid = device["udid"] + break + else: + udid = simctl("create", DEVICE_NAME, DEVICE_TYPE, runtime) + state = json.loads(simctl("list", "devices", "-j"))["devices"][runtime] + booted = any(d["udid"] == udid and d["state"] == "Booted" for d in state) + if not booted: + simctl("boot", udid) + simctl("bootstatus", udid, "-b") + return {"name": DEVICE_NAME, "udid": udid, "runtime": runtime, "type": DEVICE_TYPE} + + +def prepare_display(udid: str, appearance: str, content_size: str | None) -> None: + # A fixed status bar keeps screenshots comparable between runs. + simctl( + "status_bar", + udid, + "override", + "--time", + "9:41", + "--batteryState", + "charged", + "--batteryLevel", + "100", + "--cellularBars", + "4", + "--wifiBars", + "3", + ) + simctl("ui", udid, "appearance", appearance) + simctl("ui", udid, "content_size", content_size or "large") + + +# --- Build ----------------------------------------------------------------------------------- + + +def wait_for_other_builds(max_wait_s: float) -> float: + """The Mac is shared: wait while any other xcodebuild runs, up to `max_wait_s`.""" + start = time.monotonic() + announced = False + while True: + busy = subprocess.run(["pgrep", "-x", "xcodebuild"], capture_output=True).returncode == 0 + if not busy: + return time.monotonic() - start + if time.monotonic() - start > max_wait_s: + raise SystemExit(f"Another xcodebuild has run for over {max_wait_s:.0f} s; try later.") + if not announced: + print("Waiting for another xcodebuild to finish (shared Mac)...", flush=True) + announced = True + time.sleep(5) + + +def build_app( + tree: Path, out: Path, max_wait_s: float, project_path: str, scheme: str +) -> tuple[dict, Path | None]: + project = tree / project_path + if not project.exists(): + return {"ok": False, "error": f"{project.relative_to(tree)} not found"}, None + log_path = out / "build.log" + cmd = [ + "xcodebuild", + "-project", + str(project), + "-scheme", + scheme, + "-configuration", + "Debug", + "-sdk", + "iphonesimulator", + "-destination", + "generic/platform=iOS Simulator", + "-derivedDataPath", + str(DERIVED_DATA), + "ARCHS=arm64", + "ONLY_ACTIVE_ARCH=YES", + "CODE_SIGNING_ALLOWED=NO", + "build", + ] + BUILD_LOCK.touch() + with BUILD_LOCK.open() as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + waited = wait_for_other_builds(max_wait_s) + start = time.monotonic() + with log_path.open("w") as log: + code = subprocess.run(cmd, stdout=log, stderr=subprocess.STDOUT).returncode + seconds = time.monotonic() - start + text = log_path.read_text(errors="replace") + warnings = sorted(set(re.findall(r"^.*: warning: .*$", text, re.M))) + errors = sorted(set(re.findall(r"^.*: error: .*$", text, re.M))) + result = { + "ok": code == 0, + "seconds": round(seconds, 1), + "waited_for_other_builds_s": round(waited, 1), + "command": shlex.join(cmd), + "warnings": [shorten(w, tree) for w in warnings], + "errors": [shorten(e, tree) for e in errors], + "log": log_path.name, + } + app = DERIVED_DATA / "Build" / "Products" / "Debug-iphonesimulator" / f"{scheme}.app" + return result, (app if code == 0 and app.exists() else None) + + +def shorten(line: str, tree: Path) -> str: + return line.replace(str(tree) + "/", "") + + +# --- Static contract check ------------------------------------------------------------------- + + +def static_c4_check(tree: Path) -> dict: + """Which C4 pieces appear in the app's Swift source. Presence is not proof they work.""" + sources = "\n".join( + p.read_text(errors="replace") for p in (tree / "ios").rglob("*.swift") if p.is_file() + ) + return { + "replay_argument": "-replay" in sources or '"replay"' in sources, + "autopilot_argument": "-autopilot" in sources or '"autopilot"' in sources, + "server_url_argument": "-serverURL" in sources or '"serverURL"' in sources, + "state_subsystem": "dev.housescanning.housescan" in sources, + "state_marker": "STATE=" in sources, + "public_privacy": "privacy: .public" in sources, + } + + +# --- Run ------------------------------------------------------------------------------------- + + +def stream_states(udid: str, events: queue.Queue, stop: threading.Event, raw: Path) -> None: + cmd = [ + "xcrun", + "simctl", + "spawn", + udid, + "log", + "stream", + "--style", + "ndjson", + "--level", + "debug", + "--predicate", + LOG_PREDICATE, + ] + with raw.open("w") as sink: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) + events.put(("ready", None)) + assert proc.stdout is not None + for line in proc.stdout: + sink.write(line) + sink.flush() + try: + event = parse_ndjson_line(line) + except RedactedStateError as exc: + events.put(("redacted", str(exc))) + continue + if event is not None: + events.put(("state", event)) + if stop.is_set(): + break + proc.terminate() + + +def app_running(udid: str, bundle_id: str) -> bool: + out = subprocess.run( + ["xcrun", "simctl", "spawn", udid, "launchctl", "list"], capture_output=True, text=True + ).stdout + return f"UIKitApplication:{bundle_id}" in out + + +def screenshot(udid: str, path: Path) -> None: + simctl("io", udid, "screenshot", "--type=png", str(path)) + + +def slug(text: str) -> str: + return re.sub(r"[^A-Za-z0-9_.-]+", "-", text).strip("-")[:60] + + +def follow(args, udid: str, bundle_id: str, out: Path, rep: RunReport) -> None: + events: queue.Queue = queue.Queue() + stop = threading.Event() + reader = threading.Thread( + target=stream_states, args=(udid, events, stop, out / "state.ndjson"), daemon=True + ) + reader.start() + events.get(timeout=30) # log stream is attached before launch, so no early state is lost + time.sleep(1.0) + + launch = [ + "xcrun", + "simctl", + "launch", + "--terminate-running-process", + f"--stdout={out / 'app-stdout.log'}", + f"--stderr={out / 'app-stderr.log'}", + udid, + bundle_id, + *rep.launch_arguments, + ] + run(launch) + launched = time.monotonic() + last_event = launched + pending = None # (StateEvent, arrival time) waiting for its settled screenshot + redaction_noted = False + last_alive_check = launched + + def shoot(event, arrived: float, transient: bool) -> None: + index = len(rep.states) + 1 + name = f"{index:02d}-{slug(event.name)}.png" + screenshot(udid, out / name) + rep.states.append( + ShotRecord( + index, event.name, round(arrived - launched, 2), name, transient, event.message + ) + ) + + while True: + now = time.monotonic() + if now - launched > args.timeout: + rep.end_reason = f"timeout after {args.timeout:.0f} s" + break + wait = 0.25 + if pending is not None: + wait = max(0.0, min(wait, pending[1] + args.settle - now)) + try: + kind, payload = events.get(timeout=wait) + except queue.Empty: + kind, payload = None, None + + if kind == "redacted" and not redaction_noted: + rep.problems.append( + "STATE markers are logged as ; log the name with privacy: .public " + f"(saw: {payload!r})" + ) + redaction_noted = True + if kind == "state": + if pending is not None: + shoot(pending[0], pending[1], transient=True) + pending = (payload, time.monotonic()) + last_event = pending[1] + print(f" STATE={payload.name}", flush=True) + continue + + now = time.monotonic() + if pending is not None and now - pending[1] >= args.settle: + shoot(pending[0], pending[1], transient=False) + finished = args.until and pending[0].name in args.until + pending = None + if finished: + rep.end_reason = f"reached {rep.states[-1].state}" + break + if pending is None and now - last_event > args.idle: + rep.end_reason = f"no new state for {args.idle:.0f} s" + break + if pending is None and now - last_alive_check > 5: + last_alive_check = now + if not app_running(udid, bundle_id): + rep.end_reason = "app is no longer running" + rep.problems.append("The app exited or crashed during the run.") + break + + if pending is not None: + shoot(pending[0], pending[1], transient=False) + stop.set() + final = out / "final.png" + screenshot(udid, final) + rep.final_screenshot = final.name + if not rep.states: + rep.problems.append( + "No STATE markers arrived. Either C4 logging is not implemented on this ref or " + "the app never reached its first screen; final.png shows what was on screen." + ) + + +def collect_crashes(since: float, out: Path) -> list[str]: + found = [] + reports = Path.home() / "Library" / "Logs" / "DiagnosticReports" + for path in reports.glob("HouseScan*"): + if path.stat().st_mtime >= since: + target = out / path.name + target.write_bytes(path.read_bytes()) + found.append(path.name) + return found + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--ref", default="origin/main", help="git ref to build") + parser.add_argument("--project", default="ios/HouseScan.xcodeproj", help="path in the ref") + parser.add_argument("--scheme", default="HouseScan", help="also the .app name") + parser.add_argument("--replay", type=Path, help="replay session folder (contract C3)") + parser.add_argument("--autopilot", action="store_true") + parser.add_argument("--server-url") + parser.add_argument("--extra-arg", action="append", default=[], help="more launch args") + parser.add_argument("--until", action="append", default=[], help="state that ends the run") + parser.add_argument("--settle", type=float, default=1.2, help="seconds before a screenshot") + parser.add_argument("--idle", type=float, default=25.0, help="stop after this long quiet") + parser.add_argument("--timeout", type=float, default=300.0) + parser.add_argument("--appearance", choices=["light", "dark"], default="light") + parser.add_argument( + "--content-size", help="Dynamic Type size, e.g. accessibility-extra-extra-extra-large" + ) + parser.add_argument("--out", type=Path, help="report folder (default under reports/sim)") + parser.add_argument("--label", default="", help="short name added to the report folder") + parser.add_argument("--keep-worktree", action="store_true") + parser.add_argument("--keep-booted", action="store_true", help="leave the Simulator on") + parser.add_argument("--build-wait", type=float, default=1200.0) + args = parser.parse_args(argv) + + gitref.fetch() + sha = gitref.resolve(args.ref) + stamp = dt.datetime.now().strftime("%Y%m%d-%H%M%S") + name = "-".join( + filter(None, [stamp, slug(args.ref.removeprefix("origin/")), sha[:8], slug(args.label)]) + ) + out = (args.out or DEFAULT_REPORTS / name).expanduser() + out.mkdir(parents=True, exist_ok=True) + + launch_args: list[str] = [] + if args.replay: + replay = args.replay.expanduser().resolve() + if not replay.exists(): + raise SystemExit(f"Replay not found: {replay}") + launch_args += ["-replay", str(replay)] + if args.autopilot: + launch_args.append("-autopilot") + if args.server_url: + launch_args += ["-serverURL", args.server_url] + launch_args += args.extra_arg + + print(f"Report: {out}\nRef {args.ref} at {sha[:12]}", flush=True) + with gitref.detached_worktree(sha, keep=args.keep_worktree) as tree: + static = static_c4_check(tree) + print("Building for the Simulator...", flush=True) + build, app = build_app(tree, out, args.build_wait, args.project, args.scheme) + print( + f" build ok={build['ok']} in {build.get('seconds')} s, " + f"{len(build.get('warnings', []))} warnings", + flush=True, + ) + + device = ensure_device() + rep = RunReport( + ref=args.ref, + sha=sha, + started_at=dt.datetime.now().isoformat(timespec="seconds"), + command=[sys.executable, "-m", "hsverify.simrun", *(argv or sys.argv[1:])], + device=device | {"appearance": args.appearance, "content_size": args.content_size}, + build=build, + launch_arguments=launch_args, + static_c4=static, + ) + started = time.time() + try: + if app is None: + rep.end_reason = "build failed" + rep.problems.append("The app did not build; see build.log.") + else: + with (app / "Info.plist").open("rb") as f: + bundle_id = plistlib.load(f)["CFBundleIdentifier"] + rep.device["bundle_id"] = bundle_id + udid = device["udid"] + prepare_display(udid, args.appearance, args.content_size) + subprocess.run(["xcrun", "simctl", "uninstall", udid, bundle_id], capture_output=True) + simctl("install", udid, str(app)) + # The Simulator has no camera, but granting access keeps the permission prompt from + # hiding the replay flow. The prompt itself is reviewed on a device. + subprocess.run( + ["xcrun", "simctl", "privacy", udid, "grant", "camera", bundle_id], + capture_output=True, + ) + print("Running...", flush=True) + follow(args, udid, bundle_id, out, rep) + rep.crash_reports = collect_crashes(started, out) + if rep.crash_reports: + rep.problems.append(f"Crash reports: {', '.join(rep.crash_reports)}") + finally: + if not args.keep_booted: + subprocess.run(["xcrun", "simctl", "shutdown", device["udid"]], capture_output=True) + data = asdict(rep) + (out / "report.json").write_text(json.dumps(data, indent=2) + "\n") + report.write_sim_report(out, data) + print(f"{len(rep.states)} states; end: {rep.end_reason}") + for problem in rep.problems: + print(f"PROBLEM: {problem}") + print(f"Open {out / 'index.html'}") + return 0 if build["ok"] and not rep.problems else 1 + + +if __name__ == "__main__": + os.environ.setdefault("PYTHONUNBUFFERED", "1") + sys.exit(main()) diff --git a/verification/hsverify/statelog.py b/verification/hsverify/statelog.py new file mode 100644 index 000000000..1e2951ba6 --- /dev/null +++ b/verification/hsverify/statelog.py @@ -0,0 +1,65 @@ +"""Reading the app's STATE markers (contract C4) out of `log stream --style ndjson` output. + +The app logs `STATE=` under subsystem `dev.housescanning.housescan`, category `state`. +Everything here is pure so the parsing rules can be tested without a Simulator. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass + +SUBSYSTEM = "dev.housescanning.housescan" +CATEGORY = "state" +LOG_PREDICATE = f'subsystem == "{SUBSYSTEM}" AND category == "{CATEGORY}"' + +# A state name is an identifier-like token. Anything after it on the line is detail. +_STATE_RE = re.compile(r"STATE=([A-Za-z0-9_.\-]+)") +_REDACTED = "" + + +@dataclass(frozen=True) +class StateEvent: + name: str + timestamp: str # as printed by `log stream`, kept verbatim for the report + message: str + + +class RedactedStateError(ValueError): + """The app logged the marker as a private string, so `log stream` shows ``. + + The fix is on the app side: log the name with `privacy: .public`. + """ + + +def parse_ndjson_line(line: str) -> StateEvent | None: + """Return the STATE event on one `log stream --style ndjson` line, or None. + + Lines that are not JSON (the stream prints a banner first), entries from other + subsystems or categories, and messages without a marker all return None. + Raises RedactedStateError when the message is the redaction placeholder, because + silently dropping it would make a working app look like it logs nothing. + """ + line = line.strip() + if not line.startswith("{"): + return None + try: + entry = json.loads(line) + except json.JSONDecodeError: + return None + if entry.get("subsystem") != SUBSYSTEM or entry.get("category") != CATEGORY: + return None + message = entry.get("eventMessage") or "" + match = _STATE_RE.search(message) + if match is None: + if _REDACTED in message: + raise RedactedStateError(message) + return None + return StateEvent(name=match.group(1), timestamp=entry.get("timestamp", ""), message=message) + + +def parse_plain_line(line: str) -> str | None: + """Return the state name from a plain text line such as the app's stdout, or None.""" + match = _STATE_RE.search(line) + return match.group(1) if match else None diff --git a/verification/pyproject.toml b/verification/pyproject.toml new file mode 100644 index 000000000..ce6997073 --- /dev/null +++ b/verification/pyproject.toml @@ -0,0 +1,30 @@ +[project] +name = "house-scanning-verification" +version = "0.1.0" +description = "Independent end-to-end checks: Simulator runs, server round trips, scoreboard" +requires-python = ">=3.12" +dependencies = [ + "jsonschema>=4.23", + "pyyaml>=6.0", +] + +[dependency-groups] +dev = [ + "pytest>=9.1.1", + "ruff>=0.16.9", +] + +[tool.uv] +package = false +required-version = ">=0.11.15,<0.12" + +[tool.ruff] +line-length = 100 + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "B", "UP", "SIM", "RUF"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] +addopts = ["--strict-markers", "--strict-config"] diff --git a/verification/tests/test_statelog.py b/verification/tests/test_statelog.py new file mode 100644 index 000000000..eed95836f --- /dev/null +++ b/verification/tests/test_statelog.py @@ -0,0 +1,61 @@ +import json + +import pytest + +from hsverify.statelog import ( + CATEGORY, + SUBSYSTEM, + RedactedStateError, + parse_ndjson_line, + parse_plain_line, +) + + +def entry(message: str, subsystem: str = SUBSYSTEM, category: str = CATEGORY) -> str: + return json.dumps( + { + "timestamp": "2026-09-26 02:50:00.123456-0500", + "subsystem": subsystem, + "category": category, + "eventMessage": message, + } + ) + + +def test_reads_state_name(): + event = parse_ndjson_line(entry("STATE=wall_walk")) + assert event is not None + assert event.name == "wall_walk" + assert event.timestamp.startswith("2026-09-26") + + +def test_detail_after_the_name_is_not_part_of_it(): + event = parse_ndjson_line(entry("STATE=gap_request wall=w1 from=-4.0")) + assert event is not None + assert event.name == "gap_request" + + +@pytest.mark.parametrize( + "line", + [ + "Filtering the log data using ...", # banner printed before the JSON lines + "", + "{not json", + entry("STATE=x", subsystem="com.apple.arkit"), + entry("STATE=x", category="network"), + entry("tracking normal"), + ], +) +def test_ignores_everything_else(line): + assert parse_ndjson_line(line) is None + + +@pytest.mark.parametrize("message", ["", "STATE="]) +def test_redacted_marker_is_loud(message): + with pytest.raises(RedactedStateError): + parse_ndjson_line(entry(message)) + + +def test_plain_line(): + assert parse_plain_line("2026 app[12] STATE=result_reveal") == "result_reveal" + assert parse_plain_line("no marker here") is None diff --git a/verification/uv.lock b/verification/uv.lock new file mode 100644 index 000000000..896b5cc7f --- /dev/null +++ b/verification/uv.lock @@ -0,0 +1,317 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "house-scanning-verification" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "jsonschema" }, + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "jsonschema", specifier = ">=4.23" }, + { name = "pyyaml", specifier = ">=6.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=9.1.1" }, + { name = "ruff", specifier = ">=0.16.9" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" }, + { url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" }, + { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, + { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, + { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, + { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, + { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, + { url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, + { url = "https://files.pythonhosted.org/packages/a4/9e/b818ee580026ec578138e961027a68820c40afeb1ec8f6819b54fb99e196/rpds_py-2026.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223", size = 343012, upload-time = "2026-06-30T07:15:36.005Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6b/686d9dc4359a8f163cfbbf89ee0b4e586431de22fe8248edb63a8cf50d49/rpds_py-2026.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f", size = 338203, upload-time = "2026-06-30T07:15:37.462Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, + { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, + { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, + { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, + { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, + { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/adab103321c0a6565d5ae1c2998349bc3ee175b82ccc5ae8fc04cc413075/rpds_py-2026.6.3-cp313-cp313-win32.whl", hash = "sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0", size = 201710, upload-time = "2026-06-30T07:15:53.894Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ed/a03b09668e74e5dabbf2e211f6468e1820c0552f7b0500082da31841bf7b/rpds_py-2026.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80", size = 219454, upload-time = "2026-06-30T07:15:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, + { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, + { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, + { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, + { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, + { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, + { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, + { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, + { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, + { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, + { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, + { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, + { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, + { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, + { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, + { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, +] + +[[package]] +name = "ruff" +version = "0.16.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/bf/c935ca98e73fe8ce65b87ef08a280c0c1e85295d569228c15e87d8fdfaf1/ruff-0.16.9.tar.gz", hash = "sha256:12b625c6cfba78d285d9f48eda5f053374f1e53cb10ef17342a383750db99161", size = 4948764, upload-time = "2026-09-24T20:37:49.416Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/26/df51322b52ee1ada7eff2d071ea09d11d5c2d1dcc9f02594f5785c4d1635/ruff-0.16.9-py3-none-linux_armv6l.whl", hash = "sha256:95e6f022090368ab3b824c36276839c53b2adf1a3f4c09fefc33dfc400f6da96", size = 10082922, upload-time = "2026-09-24T20:37:13.045Z" }, + { url = "https://files.pythonhosted.org/packages/a5/27/7bf51f5a7aa375e9f339280a303aab44ca75dc1525f1cdc5991761685b0f/ruff-0.16.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:a5f27be168556594a86d2f415db0cf43f5291917849318f873c7e2791f7a8c67", size = 10236360, upload-time = "2026-09-24T20:37:16.049Z" }, + { url = "https://files.pythonhosted.org/packages/b6/63/09659283f92f02dff45809da194a70da2f688d87c55d8875c4fae3536072/ruff-0.16.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1632eb1d6197f33bd00b1acbc5b71009e89a8895c158e2d2b03a834fac964ab6", size = 9892940, upload-time = "2026-09-24T20:37:17.957Z" }, + { url = "https://files.pythonhosted.org/packages/24/58/98de1b72ec172f5f8f1731236fe21585b3998bf7dfe9fcc44ae9ba626012/ruff-0.16.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3f951b14d865d5952c89d40a5ca07e87abe24fa5453299878411e127748fb1c", size = 10032114, upload-time = "2026-09-24T20:37:19.942Z" }, + { url = "https://files.pythonhosted.org/packages/c8/7d/f1e17c54ab59d4bad1dce8ee3e22a7a1d0ef4745240decacdcf3832b5bb2/ruff-0.16.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:447fc07e1573afff7cb02803462b12b6c8ece7cf10e2cd78565fa6d7a1c0bf8d", size = 9910227, upload-time = "2026-09-24T20:37:21.872Z" }, + { url = "https://files.pythonhosted.org/packages/34/19/436f647a65075bbd3bab2668b3bdaa5120559b294694018cdcefabbbf30b/ruff-0.16.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a3e039a6a40ed976c491722b60e0ae4a4aa1a86057f540ee7a37a5d19ae9120", size = 10547484, upload-time = "2026-09-24T20:37:24.229Z" }, + { url = "https://files.pythonhosted.org/packages/03/59/38430a6bc2f6d8095447ac39625cf8b6e9344a47e6c26225c8ba1bff3ffb/ruff-0.16.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4684dded7db60aa57cb118fa158630f5feade4af5782903b6053484bdf9bd129", size = 11412367, upload-time = "2026-09-24T20:37:26.307Z" }, + { url = "https://files.pythonhosted.org/packages/c8/bd/bbb6d7fc7f208c8b8c50dd5dc8206e4cfdb1e7a8fb852606a3adf370c880/ruff-0.16.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d29c934357e45642fda2f34c0b1f4025b4a6c01e15e4bf0016879d60078a142c", size = 10869787, upload-time = "2026-09-24T20:37:28.35Z" }, + { url = "https://files.pythonhosted.org/packages/bc/b8/9c543074918061abbefc3bd139bee22de35abedb00dde0f2d27288838962/ruff-0.16.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a21713e629d3e5bdb2f5c2def1cc7f04f47fa8e1a7eb0571b4a28e1da64bc728", size = 10406494, upload-time = "2026-09-24T20:37:30.624Z" }, + { url = "https://files.pythonhosted.org/packages/35/7a/5a8851bd146e7ccf8fd4b003f6c75c11f8fbdb0e60673b45097986b6bf41/ruff-0.16.9-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:7baa24ef5fc8e77aa93879e1d3f43754a01ae488e869f1ae30cf431afd4d2452", size = 10590083, upload-time = "2026-09-24T20:37:32.439Z" }, + { url = "https://files.pythonhosted.org/packages/87/f0/4c3467188f23f806960b46fa76575a7cd0514c9ba90562650b71efc96980/ruff-0.16.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a41aac6230aadfaa133bdfa1614488531ffa3e0837567ae04c0da2058a9c0f9e", size = 10119151, upload-time = "2026-09-24T20:37:34.581Z" }, + { url = "https://files.pythonhosted.org/packages/15/34/5a4def5adea572ce6aea0bb64f21f928ee317b80e0db747d7979b01d7261/ruff-0.16.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:c2529fb5896d49115b0e9aa8f887490b34bbe76baf879ec2264ac59406869ce7", size = 9911544, upload-time = "2026-09-24T20:37:36.796Z" }, + { url = "https://files.pythonhosted.org/packages/62/5d/d15ebea7499eef9373318c0ee6ca127832927c6529731f6d48e18dce7ca9/ruff-0.16.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:41e3870277694177429b56406d65dfbdb2c2802c52b715edaf6a0b829c69d4ee", size = 10269884, upload-time = "2026-09-24T20:37:38.857Z" }, + { url = "https://files.pythonhosted.org/packages/d1/56/c5d3cd119ded7a3c7aba0e961b69cb3df701c91662c98ad694467d060ce1/ruff-0.16.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:8adbe4e58af167f767d7b2ba5e83c42e878350796cf78c2f5e14ab9903a92588", size = 10749366, upload-time = "2026-09-24T20:37:41.042Z" }, + { url = "https://files.pythonhosted.org/packages/ac/fe/734ec7527029ac757ecf821f143c9f3fcf69149c21f53a044a899b430f5a/ruff-0.16.9-py3-none-win32.whl", hash = "sha256:0e1dbc2073624dee6618d41d0098690a7244654af746704b64759e12b6b6b385", size = 10152355, upload-time = "2026-09-24T20:37:43.025Z" }, + { url = "https://files.pythonhosted.org/packages/14/21/26e4643629b3ebb44f0a06f9c9a53058d63d989415f63a9a3c28e2ee7f22/ruff-0.16.9-py3-none-win_amd64.whl", hash = "sha256:6bd40fec8cd4c8a3d4dd589bd8ad4e6320c13c29234159bfd959a40d529d597b", size = 10592965, upload-time = "2026-09-24T20:37:44.944Z" }, + { url = "https://files.pythonhosted.org/packages/51/60/5fb1a39dbb5ae314d5f59bc7348a63c1d5c20f3cd83914c4b5cb0be31d2d/ruff-0.16.9-py3-none-win_arm64.whl", hash = "sha256:ed1a252039200f57a59eebc063b54beabea67bfbaaca0eeaa7f54b5fbcda2284", size = 10458649, upload-time = "2026-09-24T20:37:46.882Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] diff --git a/verification/ux/checklist.md b/verification/ux/checklist.md new file mode 100644 index 000000000..f0a6e8800 --- /dev/null +++ b/verification/ux/checklist.md @@ -0,0 +1,127 @@ +# HouseScan capture app: UX review checklist + +A reviewer uses this list to judge each screen state of the iOS capture app from three kinds of evidence. The first is Simulator screenshots. The second is the ordered `STATE=` log from contract C4. The third is the SwiftUI source. The product priority is a first-time homeowner finishing the scan on the first try, with a result that doesn't overstate what the phone saw. + +## How to use it + +1. Collect evidence for each state in the per-state table at the end. For each state, take a screenshot at the default text size and a second one at the largest accessibility text size (AX5, `accessibilityExtraExtraExtraLarge`). Also capture the `STATE=` log for the whole run and the Accessibility Inspector readout of each screen. +2. For each item, record **Pass**, **Fail** or **Not verified**, with the evidence file or `path:line`. Mark anything you could not observe as Not verified. Never infer a Pass. +3. The verdict is **Block** if any blocker fails, **Incomplete** if any blocker is Not verified, and **Approve** otherwise. List major and minor failures as work to do. + +Severity: **blocker** means the homeowner can't finish or gets a wrong result. **Major** means likely confusion or retakes. **Minor** means polish. + +Observed by: `SS` screenshot, `SEQ` STATE log order, `SRC` SwiftUI source, `VO` VoiceOver or Accessibility Inspector, `DT` Dynamic Type run at AX5, `RM` Reduce Motion run, `GRAY` screenshot converted to grayscale. + +The Simulator cannot run ARKit world tracking. Screens that depend on it (walk, fog, tracking loss, relocalization, reveal) can be screenshotted only if the app has a preview or fixture mode that injects those states. Without one, check those items from `SRC` and mark the runtime part Not verified. + +Sources: `HLD` is `docs/05-live-guided-survey-hld.md`. `FTC` is `docs/research/t3-first-try-capture.md` on `origin/t3/research`. Skill names refer to the design and accessibility skills the checklist draws on. + +Numeric limits marked *(reviewer threshold)* are review conventions. No user testing backs them. + +## R. First-try reliability + +| ID | Check | Observed by | Pass when | Sev | Source | +|---|---|---|---|---|---| +| R1 | The walk can't end with a silent gap | SEQ, SS | Every strip cell on the last screen before the upload or finish state is green, marked "can't access", or listed by name as open for installer review | blocker | FTC finish rule; HLD §2 | +| R2 | Missing coverage never shows as done | SRC, SS | The cell-to-style mapping has one explicit case per state (unseen, seen but not enough, enough, failed, skipped) with no `default:` that falls through to green. Any progress count counts only green cells | blocker | HLD §2 coverage display | +| R3 | Tracking loss stops coverage credit | SRC, SS | Code that clears fog or marks cells runs only when `trackingState == .normal`. The strip in the tracking-lost screenshot is unchanged from the screenshot before it | blocker | HLD §9 tracking row; FTC recovery | +| R4 | An interruption resumes the walk | SEQ, SS | Backgrounding mid-walk logs walk, then a relocalizing state, then walk again (or tap meter). It never returns to onboarding, and the captured count and strip survive | blocker | FTC recovery; HLD §9 session resumed | +| R5 | Failed relocalization starts a new frame | SEQ, SRC | After relocalization fails, the app asks to tap the meter again and resets the strip's spatial cells. It doesn't keep drawing old world-anchored overlays | blocker | FTC failed relocalization | +| R6 | Finish waits for server acknowledgment | SEQ, SS | Finish or reveal is reachable only after every upload is acknowledged. Offline, the screen says the scan is saved on the phone and not yet sent, and it doesn't say done | blocker | FTC finish rule 3 | +| R7 | Upload progress is separate from evidence completeness | SS | Upload status is its own element with its own wording (for example "3 photos waiting to send"), placed apart from the coverage strip, and it never changes a strip cell | major | HLD §2, §9 | +| R8 | Offline keeps local guidance running | SS | Offline screenshots still show the instruction and strip. Findings that need the server are labelled pending, not pass or fail | major | HLD §9 connectivity row | +| R9 | Camera denied has a way forward | SEQ, SS | A `cameraAccessDenied` state explains in one sentence why the camera is needed and has an "Open Settings" button that opens the app's Settings page | blocker | HLD §2 device check | +| R10 | AR unavailable is explained | SEQ, SS | On a device without world tracking, the app shows a plain explanation and what to do next. There's no black camera view, spinner loop or crash | blocker | HLD §2 device check | +| R11 | Each auto-capture is acknowledged | SS, SRC | Each saved capture triggers a brief visual mark, such as a flash on the reticle or a counter tick, and the capture count goes up | major | HLD §2 | + +## I. One clear instruction per screen + +| ID | Check | Observed by | Pass when | Sev | Source | +|---|---|---|---|---|---| +| I1 | One primary instruction | SS | The largest text is a single instruction sentence. Any other text is visibly smaller and isn't a second command | major | HLD §2; holistic-ux cognitive load | +| I2 | Standing and aiming cues are distinct | SS, SRC | The where-to-stand cue (a ground marker or "step back" text) and the where-to-aim cue (the reticle or target) differ in shape and position. Neither is a recolored copy of the other | major | HLD §2 | +| I3 | Plain words | SRC | A grep of user-visible strings finds none of: raycast, plane, keyframe, anchor, tracking, LiDAR, epoch, relocaliz, OCR, UNSURE, PASS, FAIL | major | frontend-design writing; apple-design simplicity | +| I4 | One question at a time with two large buttons | SS | A question screen shows exactly one question and two full-width buttons, each at least 44 pt tall. Button labels state the answer ("It opens", "It stays shut"), not Yes and No | major | FTC what the homeowner sees 6 | +| I5 | No shutter where auto-capture applies | SS, SRC | The meter close-up, panel shots and walk have no shutter button. Tapping to set the meter anchor is allowed, because that tap marks a location rather than taking a photo | major | HLD §2; FTC 3 | +| I6 | The instruction stays stable | SEQ | The log never shows an A, B, A change of instruction within 3 s unless a task was completed *(reviewer threshold)* | major | HLD §4 next-view planning | +| I7 | No arrow through unseen ground | SS | Guidance toward an unobserved area is text ("Walk around the corner if you can"), not a world-anchored arrow | major | HLD §4 | +| I8 | Anchoring never guesses a depth | SEQ, SS | Tapping the meter with no detected wall plane shows "step closer" and doesn't advance to the step-back state | blocker | FTC 4 | +| I9 | Panel instructions include the safety wording | SS | The panel screen says to open only the hinged door, not to unscrew the cover and not to touch the breakers, and offers Skip | blocker | FTC opening the panel | +| I10 | Recapture requests give a reason and a target | SS | The request is one line naming what to show and where, for example "Show the ground below this part of the wall" | major | HLD §4 capture request | + +## T. Recovery from tracking loss and gaps + +| ID | Check | Observed by | Pass when | Sev | Source | +|---|---|---|---|---|---| +| T1 | Tracking-loss wording matches the cause | SRC, SS | `excessiveMotion` shows "Slow down". `insufficientFeatures` shows "Aim at a corner or somewhere with more texture". These are separate `case` branches with separate strings | major | FTC recovery | +| T2 | No modal on tracking loss | SRC, SS | No `.alert`, `.sheet` or `.fullScreenCover` is bound to tracking state, and the camera stays visible behind the prompt | major | FTC recovery | +| T3 | World-anchored overlays are hidden while tracking is limited | SRC, SS | Arrows, the fog and the candidate box hide or freeze when tracking isn't `.normal`, and they come back only after it returns | major | HLD §9 tracking row | +| T4 | "I can't access this area" is available | SS | The button is on the walk and gap-request screens and takes one tap. Afterwards the cell shows a distinct skipped mark (not green) and the next task starts | major | HLD §2 | +| T5 | "Can't get a clear shot" appears after two tries | SEQ, SS | The option appears after the second failed close-up attempt, not earlier and never absent. Choosing it moves on and doesn't loop | blocker | FTC close-up gates | +| T6 | Close-up failures name the fix | SS, SRC | Blur, glare and a cut-off label each have their own message saying what to do (hold still, tilt away from the glare, fit the whole label in the box) | major | HLD §9 blur row | +| T7 | Relocalization shows the saved meter view | SS | The screen shows the saved meter image and "Point at the meter like this." | major | FTC recovery | + +## H. Result reveal honesty + +| ID | Check | Observed by | Pass when | Sev | Source | +|---|---|---|---|---|---| +| H1 | UNSURE reads as needing review, with a reason | SS, SRC | Each unsure check shows "An installer will check this" plus a one-line reason. Its style is distinct from pass, and it's never hidden or folded into a pass | blocker | FTC intro; HLD §7 | +| H2 | Borderline numbers show the error | SS | A result near its limit shows the measured value, the rule and the error, as in FTC's example ("3 ft 2 in ... rule is 3 ft ... off by about 4 in") | blocker | FTC borderline reveal | +| H3 | Green coverage doesn't imply a pass | SS | The screen shown when coverage is complete uses no approval words ("approved", "all set", "you qualify"). The verdict appears only on the reveal, in wording separate from coverage | blocker | HLD §2 green row | +| H4 | The reveal is anchored to the meter | SRC, SS | The candidate box and cable line are placed relative to the meter's anchor, and they hide with an explanation if tracking isn't normal at reveal | major | AGENTS.md hard rules; FTC 9 | +| H5 | A failure isn't a dead end | SS | A failed spot names the rule and distance, then shows another candidate or says an installer will look for one | major | holistic-ux peak-end | +| H6 | An unseen side is disclosed | SS | If the other side of the meter wasn't seen, the reveal says so ("A closer spot may exist on the left") | minor | FTC early stop | +| H7 | The panel review needs are stated | SS | The reveal says the panel still needs an electrician's review whenever the server marks it that way | major | FTC borderline reveal | + +## A. Accessibility + +| ID | Check | Observed by | Pass when | Sev | Source | +|---|---|---|---|---|---| +| A1 | Every control has a label | VO, SRC | The inspector reads a name for every button. Each icon-only `Button { Image(systemName:) }` has `.accessibilityLabel`. A primary action without one is a blocker | blocker | better-accessibility names | +| A2 | The AR overlay state is exposed | VO | One element summarises coverage in words ("Wall: 4 of 10 sections done, 2 need another look") and the tracking status | major | better-accessibility; HLD §2 | +| A3 | Instruction changes are announced | VO, SRC | A new instruction posts `AccessibilityNotification.Announcement` or updates a focused element's value | major | better-accessibility live regions | +| A4 | Dynamic Type reaches AX sizes | DT, SRC | At AX5 the full instruction wraps with no "…", both answer buttons stay on screen, and there's no `.lineLimit(1)` or `.dynamicTypeSize(...)` cap below AX5 on instruction text | major | apple-design typography | +| A5 | Text stays readable over the camera | SRC, SS | Instruction text sits on a solid or `.regularMaterial` backing and reaches 4.5:1 contrast against both a white and a black backdrop. Text drawn straight on the camera image fails | major | apple-design materials; holistic-ux | +| A6 | Targets are at least 44 pt | SRC, SS | Each tappable element is at least 44 by 44 pt (132 px in a @3x screenshot), and neighbouring targets don't overlap | major | holistic-ux Fitts; better-accessibility | +| A7 | Reduce Motion is honoured | RM, SRC | The code reads `accessibilityReduceMotion`. With it on, fog clearing, strip fills and the reveal crossfade or cut instead of sliding or scaling, and nothing pulses in a loop | major | apple-design §14; emil-design-eng | +| A8 | Haptics are paired with a visual | SRC | Every `.sensoryFeedback` or feedback-generator call fires with a visible change on the same state update. No signal is haptic-only | major | apple-design multimodal | +| A9 | Coverage isn't colour-only | GRAY, SS | In grayscale, unseen, not enough, enough, failed and skipped cells stay distinguishable by fill pattern or icon, and a text legend or label exists | major | better-accessibility colour; HLD §2 | +| A10 | Reduce Transparency and Increase Contrast are honoured | SRC | With either setting on, materials become opaque | minor | apple-design §14 | +| A11 | VoiceOver order is instruction first | VO | Swiping from the top reads the instruction, then the primary action, then secondary controls | minor | better-accessibility structure | + +## V. Visual and motion quality + +| ID | Check | Observed by | Pass when | Sev | Source | +|---|---|---|---|---|---| +| V1 | The camera dominates capture screens | SS | Chrome covers no more than about a third of the screen during the walk and close-ups *(reviewer threshold)* | minor | frontend-design restraint | +| V2 | At most one prominent button per screen | SS | Only one filled or prominent button is visible. Recovery options such as "I can't access this area" use a secondary style | minor | apple-design simplicity | +| V3 | No stacked translucency | SS | No translucent panel sits on another translucent panel | minor | apple-design materials | +| V4 | No decorative animation during capture | SRC, SS | Capture states have no `.repeatForever`, shimmer, confetti or idle pulse. The motion that remains is state feedback: capture acknowledgment, fog clearing, strip update | major | emil-design-eng purpose | +| V5 | Motion never blocks input | SRC | No `.disabled`, `sleep` or `asyncAfter` gates a button while a transition plays | minor | apple-design interruptibility | +| V6 | UI transitions take 300 ms or less | SRC | Screen-to-screen and overlay animations take 300 ms or less. Only the result reveal may take longer | minor | emil-design-eng duration | +| V7 | Buttons respond to presses | SRC | Buttons use a style with a visible pressed state (a system style, or a scale of 0.95 to 0.98) | minor | emil-design-eng | +| V8 | One name per action | SRC | An action keeps its name through the flow: the button that says "I'm here" isn't called "Continue" on the next screen | minor | frontend-design writing | + +## Per-state table + +The state names below are placeholders. Match them to the app's real `STATE=` names once C4 logging lands. As of 2026-09-26 the local `t3/ios-mvf` worktree has none and isn't pushed. Its tracking cases (`normal`, `limited` with `excessiveMotion`, `insufficientFeatures`, `initializing` or `relocalizing`, `notAvailable`, `cameraAccessDenied`) should map onto the rows below. + +| State (placeholder) | Must show | Key items | +|---|---|---| +| onboarding | What the scan does and roughly how long it takes, in one or two sentences, and one start button | I1, I3, A1, A4, V2 | +| camera_permission / denied | Why the camera is needed. When denied, "Open Settings" | R9, I3, A1 | +| ar_unavailable | A plain explanation and what to do instead | R10, I3 | +| go_to_meter | "Go to your electric meter" and **I'm here** | I1, V8, A4 | +| meter_closeup | A reticle, no shutter, a named fix on failure, "Can't get a clear shot" after two tries | I5, T5, T6, R11, A8 | +| tap_meter_anchor | "Tap the meter". With no plane, "step closer" | I8, I2, A6 | +| step_back | A standing cue distinct from the aiming cue | I2, I1 | +| wall_walk (strip, fog) | One instruction, the strip with non-colour cell marks, upload status kept separate, "I can't access this area" | R1, R2, R3, R7, I6, I7, T4, A2, A9, V1, V4 | +| tracking_lost | A cause-specific prompt, no modal, coverage and arrows frozen | T1, T2, T3, R3, A3 | +| relocalizing | The saved meter view and "Point at the meter like this.", with a fallback to tapping the meter again | T7, R4, R5 | +| gap_recapture | One line with the reason and target, a skip option | I10, T4, R1 | +| panel | Safety wording and Skip, auto-capture with no shutter | I9, I5, T5 | +| feature_marking | The saved frame with one instruction for marking a missed object and one confirm button | I1, A1, A6, H1 | +| homeowner_question | One question and two large answer buttons | I4, A4, A6 | +| upload_processing | Upload count and pending state, no approval wording | R6, R7, H3 | +| result_reveal | The box on the meter anchor, numbers with error on tap, unsure checks as installer review | H1 to H7, A7 | +| offline_error | Saved on the phone and not sent, local guidance still running, retry | R6, R8, I3 | From 05f70dcbfdef571ac7a687c95301138725fd300b Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 02:48:47 -0500 Subject: [PATCH 02/59] verification: time states by the log clock; fix probe argument Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/Makefile | 2 +- verification/hsverify/simrun.py | 21 ++++++++++++++++++++- verification/tests/test_simrun.py | 18 ++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 verification/tests/test_simrun.py diff --git a/verification/Makefile b/verification/Makefile index 65ea1c81b..89787714c 100644 --- a/verification/Makefile +++ b/verification/Makefile @@ -19,4 +19,4 @@ sim: # Self-test of the runner against a fixture app that logs a known STATE sequence. sim-probe: uv run python -m hsverify.simrun --ref HEAD --project verification/fixtures/state-probe/StateProbe.xcodeproj \ - --scheme StateProbe --until probe_done --extra-arg -probeArg --label probe + --scheme StateProbe --until probe_done --extra-arg=-probeArg --label probe diff --git a/verification/hsverify/simrun.py b/verification/hsverify/simrun.py index 3315d43cb..71720e2f3 100644 --- a/verification/hsverify/simrun.py +++ b/verification/hsverify/simrun.py @@ -265,6 +265,19 @@ def screenshot(udid: str, path: Path) -> None: simctl("io", udid, "screenshot", "--type=png", str(path)) +def seconds_since(launch_wall: float, log_timestamp: str, fallback: float) -> float: + """Seconds from launch to the log entry, by the log's own clock. + + Arrival time at the runner lags behind while a screenshot is being taken, so the log + timestamp is preferred; the arrival time is used only if the timestamp does not parse. + """ + try: + logged = dt.datetime.strptime(log_timestamp, "%Y-%m-%d %H:%M:%S.%f%z").timestamp() + except ValueError: + return round(fallback, 2) + return round(logged - launch_wall, 2) + + def slug(text: str) -> str: return re.sub(r"[^A-Za-z0-9_.-]+", "-", text).strip("-")[:60] @@ -292,6 +305,7 @@ def follow(args, udid: str, bundle_id: str, out: Path, rep: RunReport) -> None: ] run(launch) launched = time.monotonic() + launched_wall = time.time() last_event = launched pending = None # (StateEvent, arrival time) waiting for its settled screenshot redaction_noted = False @@ -303,7 +317,12 @@ def shoot(event, arrived: float, transient: bool) -> None: screenshot(udid, out / name) rep.states.append( ShotRecord( - index, event.name, round(arrived - launched, 2), name, transient, event.message + index, + event.name, + seconds_since(launched_wall, event.timestamp, arrived - launched), + name, + transient, + event.message, ) ) diff --git a/verification/tests/test_simrun.py b/verification/tests/test_simrun.py new file mode 100644 index 000000000..aa68306c7 --- /dev/null +++ b/verification/tests/test_simrun.py @@ -0,0 +1,18 @@ +import datetime as dt + +from hsverify.simrun import seconds_since, slug + + +def test_state_time_uses_the_log_clock(): + stamp = "2026-09-26 02:47:25.977214-0500" + logged = dt.datetime.strptime(stamp, "%Y-%m-%d %H:%M:%S.%f%z").timestamp() + assert seconds_since(logged - 2.5, stamp, fallback=9.0) == 2.5 + + +def test_unparseable_timestamp_falls_back_to_arrival_time(): + assert seconds_since(0.0, "", fallback=1.234) == 1.23 + + +def test_slug_is_filename_safe(): + assert slug("origin/t3/ios-mvf") == "origin-t3-ios-mvf" + assert slug("wall walk: gap!") == "wall-walk-gap" From 78d6553991bd669bfd62c2616209f0c8b84d057f Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 03:06:15 -0500 Subject: [PATCH 03/59] verification: check replay sessions against C3 and ground truth Validates the Measure Lab v2 fields, JPEG sizes, rigid poses and the spacing gate, then compares ARKit poses with ground truth after a heading-and-origin alignment, reporting the path-length ratio separately. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/hsverify/replaycheck.py | 248 +++++++++++++++++++++++++ verification/pyproject.toml | 1 + verification/tests/test_replaycheck.py | 118 ++++++++++++ verification/uv.lock | 75 ++++++++ 4 files changed, 442 insertions(+) create mode 100644 verification/hsverify/replaycheck.py create mode 100644 verification/tests/test_replaycheck.py diff --git a/verification/hsverify/replaycheck.py b/verification/hsverify/replaycheck.py new file mode 100644 index 000000000..87d1ab05f --- /dev/null +++ b/verification/hsverify/replaycheck.py @@ -0,0 +1,248 @@ +"""Check a replay session (contract C3, Measure Lab session format v2) before anything trusts it. + + uv run python -m hsverify.replaycheck ~/house-scanning-data/replays/ + +Two parts: + +1. Format: the fields the app and server read are present, every listed JPEG exists with the + stated size, poses are rigid transforms, timestamps increase, and each `motion` keyframe + respects the spacing gate the session declares. +2. Pose accuracy, when `ground_truth.json` sits beside `session.json`. The two worlds share + gravity but not heading or origin, so the ground truth is aligned with a rotation about + the vertical axis plus a translation. Reported two ways: + - best fit over all keyframes: the error left after the best possible alignment; + - anchored at the first keyframe: aligned using only the first keyframe's pose, the way + the app anchors everything to the meter. This is the error a result placed from the + start of the walk would carry, reported against distance walked (ground-truth feet). + The ratio of ARKit's path length to the ground truth's is reported separately, since a + scale disagreement dominates both errors when present. + +Exits non-zero when a format check fails. Accuracy numbers are reported, not judged. +""" + +from __future__ import annotations + +import argparse +import json +import struct +import sys +from pathlib import Path + +import numpy as np + +REQUIRED_KEYFRAME_FIELDS = ("id", "img", "w", "h", "intrinsics", "pose", "timestamp", "tracking") + + +def pose_matrix(pose: list[float]) -> np.ndarray: + """16 numbers, column by column (simd_float4x4 layout), to a 4x4 array.""" + return np.asarray(pose, dtype=float).reshape(4, 4).T + + +def rigid_problems(m: np.ndarray, tol: float = 1e-3) -> list[str]: + problems = [] + r = m[:3, :3] + if not np.allclose(r.T @ r, np.eye(3), atol=tol): + problems.append("rotation is not orthonormal") + if abs(np.linalg.det(r) - 1) > tol: + problems.append(f"rotation determinant {np.linalg.det(r):.4f}, expected 1") + if not np.allclose(m[3], [0, 0, 0, 1], atol=tol): + problems.append("bottom row is not [0, 0, 0, 1]") + return problems + + +def rotation_angle_deg(a: np.ndarray, b: np.ndarray) -> float: + rel = a[:3, :3].T @ b[:3, :3] + cos = np.clip((np.trace(rel) - 1) / 2, -1.0, 1.0) + return float(np.degrees(np.arccos(cos))) + + +def jpeg_size(path: Path) -> tuple[int, int]: + """(width, height) from a JPEG's start-of-frame marker.""" + data = path.read_bytes() + if data[:2] != b"\xff\xd8": + raise ValueError("not a JPEG") + i = 2 + while i + 9 < len(data): + if data[i] != 0xFF: + i += 1 + continue + marker = data[i + 1] + if marker in (0xD8, 0x01) or 0xD0 <= marker <= 0xD7: + i += 2 + continue + length = struct.unpack(">H", data[i + 2 : i + 4])[0] + if 0xC0 <= marker <= 0xCF and marker not in (0xC4, 0xC8, 0xCC): + height, width = struct.unpack(">HH", data[i + 5 : i + 9]) + return width, height + i += 2 + length + raise ValueError("no start-of-frame marker") + + +def check_format(folder: Path, session: dict) -> tuple[list[str], dict]: + errors: list[str] = [] + if session.get("format") != "measure-lab-session" or session.get("formatVersion") != 2: + errors.append( + f"format {session.get('format')!r} version {session.get('formatVersion')!r}, " + "expected 'measure-lab-session' version 2" + ) + for key in ("session", "gates", "keyframes"): + if key not in session: + errors.append(f"missing top-level field {key!r}") + keyframes = session.get("keyframes", []) + gates = session.get("gates", {}) + spacing_m = gates.get("keyframeSpacingMeters") + spacing_deg = gates.get("keyframeSpacingDegrees") + previous = None + previous_motion = None + gate_violations = 0 + tracking_counts: dict[str, int] = {} + for kf in keyframes: + name = kf.get("id", "?") + missing = [f for f in REQUIRED_KEYFRAME_FIELDS if f not in kf] + if missing: + errors.append(f"{name}: missing {', '.join(missing)}") + continue + tracking_counts[kf["tracking"]] = tracking_counts.get(kf["tracking"], 0) + 1 + image = folder / kf["img"] + if not image.exists(): + errors.append(f"{name}: image {kf['img']} not found") + else: + try: + size = jpeg_size(image) + if size != (kf["w"], kf["h"]): + errors.append(f"{name}: JPEG is {size}, session says {(kf['w'], kf['h'])}") + except ValueError as exc: + errors.append(f"{name}: {exc}") + fx, fy, cx, cy = kf["intrinsics"] + if not (fx > 0 and fy > 0 and 0 < cx < kf["w"] and 0 < cy < kf["h"]): + errors.append(f"{name}: intrinsics {kf['intrinsics']} outside the image") + m = pose_matrix(kf["pose"]) + errors += [f"{name}: {p}" for p in rigid_problems(m)] + if previous is not None and kf["timestamp"] <= previous["timestamp"]: + errors.append(f"{name}: timestamp does not increase") + if kf.get("reason") == "motion" and previous_motion is not None and spacing_m: + moved = float(np.linalg.norm(m[:3, 3] - previous_motion[:3, 3])) + turned = rotation_angle_deg(previous_motion, m) + if moved < spacing_m * 0.99 and turned < spacing_deg * 0.99: + gate_violations += 1 + if kf.get("reason") == "motion": + previous_motion = m + previous = kf + if gate_violations: + errors.append(f"{gate_violations} motion keyframes closer than the declared spacing gate") + summary = { + "keyframes": len(keyframes), + "tracking": tracking_counts, + "taps": len(session.get("taps", [])), + "walls": len(session.get("walls", [])), + } + return errors, summary + + +def yaw_translation_fit(src: np.ndarray, dst: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """Rotation about +y and translation that best map src points onto dst (least squares).""" + sc, dc = src.mean(axis=0), dst.mean(axis=0) + a, b = src - sc, dst - dc + # Rotation about y: x' = c x + s z, z' = -s x + c z. Maximise sum b . R a. + num = np.sum(a[:, 0] * b[:, 2] - a[:, 2] * b[:, 0]) + den = np.sum(a[:, 0] * b[:, 0] + a[:, 2] * b[:, 2]) + theta = np.arctan2(-num, den) + c, s = np.cos(theta), np.sin(theta) + r = np.array([[c, 0, s], [0, 1, 0], [-s, 0, c]]) + return r, dc - r @ sc + + +def yaw_of(r: np.ndarray) -> np.ndarray: + """The rotation about +y closest to `r`, from where it sends the camera's forward axis.""" + forward = r @ np.array([0.0, 0.0, -1.0]) + theta = np.arctan2(-forward[0], -forward[2]) + c, s = np.cos(theta), np.sin(theta) + return np.array([[c, 0, s], [0, 1, 0], [-s, 0, c]]) + + +def accuracy(session: dict, truth: dict) -> dict: + by_id = {entry["keyframe"]: entry for entry in truth["keyframes"]} + ar, gt = [], [] + for kf in session["keyframes"]: + if kf["id"] in by_id: + ar.append(pose_matrix(kf["pose"])) + gt.append(pose_matrix(by_id[kf["id"]]["pose"])) + if len(ar) < 2: + return {"matched_keyframes": len(ar)} + ar_pos = np.array([m[:3, 3] for m in ar]) + gt_pos = np.array([m[:3, 3] for m in gt]) + + # Both worlds claim gravity alignment: each camera axis should have the same vertical part. + tilt = [float(np.max(np.abs(a[1, :3] - g[1, :3]))) for a, g in zip(ar, gt, strict=True)] + + r, t = yaw_translation_fit(gt_pos, ar_pos) + fit_err = np.linalg.norm((gt_pos @ r.T + t) - ar_pos, axis=1) + + # Anchor at the first keyframe: heading from its orientations, origin from its position. + r0 = yaw_of(ar[0][:3, :3]) @ yaw_of(gt[0][:3, :3]).T + t0 = ar_pos[0] - r0 @ gt_pos[0] + anchored = np.linalg.norm((gt_pos @ r0.T + t0) - ar_pos, axis=1) + walked = np.concatenate([[0.0], np.cumsum(np.linalg.norm(np.diff(gt_pos, axis=0), axis=1))]) + + def at(distance_m: float) -> float | None: + idx = np.searchsorted(walked, distance_m) + return None if idx >= len(walked) else round(float(anchored[idx]), 3) + + ar_walked = float(np.sum(np.linalg.norm(np.diff(ar_pos, axis=0), axis=1))) + # Same fit after scaling the ground truth to ARKit's path length: what remains is shape + # and heading drift, with the scale disagreement taken out. + scale = ar_walked / walked[-1] + rs, ts = yaw_translation_fit(gt_pos * scale, ar_pos) + scaled_err = np.linalg.norm((gt_pos * scale @ rs.T + ts) - ar_pos, axis=1) + + ft = 0.3048 + return { + "matched_keyframes": len(ar), + "path_length_m": round(float(walked[-1]), 2), + "arkit_path_length_m": round(ar_walked, 2), + "arkit_to_truth_length_ratio": round(float(scale), 3), + "max_vertical_axis_mismatch": round(max(tilt), 4), + "best_fit_error_m": { + "median": round(float(np.median(fit_err)), 3), + "p90": round(float(np.percentile(fit_err, 90)), 3), + "max": round(float(fit_err.max()), 3), + }, + "best_fit_error_after_scale_m": { + "median": round(float(np.median(scaled_err)), 3), + "p90": round(float(np.percentile(scaled_err, 90)), 3), + }, + "anchored_error_m_after_ft_walked": { + f"{d}ft": at(d * ft) for d in (3, 10, 20, 30, 60, 100) + }, + "anchored_error_m_final": round(float(anchored[-1]), 3), + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("folder", type=Path, help="session folder containing session.json") + parser.add_argument("--json", action="store_true", help="print the report as JSON") + args = parser.parse_args(argv) + folder = args.folder.expanduser() + session = json.loads((folder / "session.json").read_text()) + errors, summary = check_format(folder, session) + report: dict = {"session": str(folder), "format_errors": errors, "summary": summary} + truth_path = folder / "ground_truth.json" + if truth_path.exists(): + report["pose_accuracy"] = accuracy(session, json.loads(truth_path.read_text())) + if args.json: + print(json.dumps(report, indent=2)) + else: + print(f"{folder.name}: {summary}") + print("Format: " + ("ok" if not errors else f"{len(errors)} problems")) + for e in errors[:30]: + print(f" - {e}") + if "pose_accuracy" in report: + print("Pose accuracy against ground truth:") + for k, v in report["pose_accuracy"].items(): + print(f" {k}: {v}") + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/verification/pyproject.toml b/verification/pyproject.toml index ce6997073..e8669fc2a 100644 --- a/verification/pyproject.toml +++ b/verification/pyproject.toml @@ -5,6 +5,7 @@ description = "Independent end-to-end checks: Simulator runs, server round trips requires-python = ">=3.12" dependencies = [ "jsonschema>=4.23", + "numpy>=2.5.3", "pyyaml>=6.0", ] diff --git a/verification/tests/test_replaycheck.py b/verification/tests/test_replaycheck.py new file mode 100644 index 000000000..a87681934 --- /dev/null +++ b/verification/tests/test_replaycheck.py @@ -0,0 +1,118 @@ +import json +import struct + +import numpy as np +import pytest + +from hsverify.replaycheck import ( + accuracy, + check_format, + jpeg_size, + pose_matrix, + rigid_problems, + yaw_translation_fit, +) + + +def yaw(deg: float) -> np.ndarray: + t = np.radians(deg) + return np.array([[np.cos(t), 0, np.sin(t)], [0, 1, 0], [-np.sin(t), 0, np.cos(t)]]) + + +def flat(m: np.ndarray) -> list[float]: + return [float(x) for x in m.T.reshape(-1)] # column by column + + +def pose(r: np.ndarray, t) -> np.ndarray: + m = np.eye(4) + m[:3, :3], m[:3, 3] = r, t + return m + + +def write_jpeg_header(path, w: int, h: int) -> None: + # SOI, then a baseline SOF0 segment with the size; enough for jpeg_size. + sof = b"\xff\xc0" + struct.pack(">HBHHB", 11, 8, h, w, 1) + b"\x01\x11\x00" + path.write_bytes(b"\xff\xd8" + sof + b"\xff\xd9") + + +def test_pose_is_read_column_by_column(): + m = pose(yaw(30), [1, 2, 3]) + assert np.allclose(pose_matrix(flat(m)), m) + + +def test_rigid_problems(): + assert rigid_problems(pose(yaw(10), [0, 0, 0])) == [] + bad = pose(yaw(10), [0, 0, 0]) + bad[:3, :3] *= 1.1 + assert rigid_problems(bad) + + +def test_yaw_fit_recovers_known_transform(): + rng = np.random.default_rng(0) + src = rng.normal(size=(20, 3)) * 5 + dst = src @ yaw(123).T + np.array([4.0, -1.0, 2.0]) + r, t = yaw_translation_fit(src, dst) + assert np.allclose(r, yaw(123), atol=1e-9) + assert np.allclose(t, [4.0, -1.0, 2.0], atol=1e-9) + + +def test_jpeg_size(tmp_path): + write_jpeg_header(tmp_path / "a.jpg", 1280, 720) + assert jpeg_size(tmp_path / "a.jpg") == (1280, 720) + + +def session_with(tmp_path, positions, spacing=0.5): + (tmp_path / "keyframes").mkdir() + keyframes = [] + for i, p in enumerate(positions, 1): + write_jpeg_header(tmp_path / "keyframes" / f"k{i}.jpg", 640, 480) + keyframes.append( + { + "id": f"k{i}", "img": f"keyframes/k{i}.jpg", "w": 640, "h": 480, + "intrinsics": [500, 500, 320, 240], "pose": flat(pose(yaw(0), p)), + "timestamp": float(i), "tracking": "normal", "reason": "motion", + } + ) # fmt: skip + return { + "format": "measure-lab-session", "formatVersion": 2, "session": {}, + "gates": {"keyframeSpacingMeters": spacing, "keyframeSpacingDegrees": 15.0}, + "keyframes": keyframes, + } # fmt: skip + + +def test_valid_session_has_no_format_errors(tmp_path): + s = session_with(tmp_path, [[0, 0, -0.6 * i] for i in range(5)]) + errors, summary = check_format(tmp_path, s) + assert errors == [] + assert summary["keyframes"] == 5 + + +def test_format_errors_are_specific(tmp_path): + s = session_with(tmp_path, [[0, 0, -0.6 * i] for i in range(3)]) + s["keyframes"][1]["w"] = 1920 # disagrees with the JPEG + s["keyframes"][2]["pose"] = flat(pose(yaw(0), [0, 0, -0.7])) # 0.1 m after the previous + (tmp_path / "keyframes" / "k1.jpg").unlink() + errors, _ = check_format(tmp_path, s) + joined = "\n".join(errors) + assert "k1: image keyframes/k1.jpg not found" in joined + assert "k2: JPEG is (640, 480)" in joined + assert "1 motion keyframes closer than the declared spacing gate" in joined + + +@pytest.mark.parametrize("scale", [1.0, 0.64]) +def test_accuracy_separates_scale_from_alignment(tmp_path, scale): + truth_pos = [[0.0, 0.0, -float(i)] for i in range(40)] # straight walk along -z + s = session_with(tmp_path, [[p[0], p[1], p[2] * scale] for p in truth_pos]) + # Ground truth in its own world: turned 90 degrees and shifted. + truth = { + "keyframes": [ + {"keyframe": kf["id"], "pose": flat(pose(yaw(90), yaw(90) @ np.array(p) + [7, 0, 3]))} + for kf, p in zip(s["keyframes"], truth_pos, strict=True) + ] + } + result = accuracy(json.loads(json.dumps(s)), truth) + assert result["arkit_to_truth_length_ratio"] == pytest.approx(scale, abs=1e-3) + assert result["best_fit_error_after_scale_m"]["p90"] == pytest.approx(0, abs=1e-6) + assert result["max_vertical_axis_mismatch"] == pytest.approx(0, abs=1e-9) + expected_final = (1 - scale) * 39 + assert result["anchored_error_m_final"] == pytest.approx(expected_final, abs=1e-3) diff --git a/verification/uv.lock b/verification/uv.lock index 896b5cc7f..09fa3648b 100644 --- a/verification/uv.lock +++ b/verification/uv.lock @@ -26,6 +26,7 @@ version = "0.1.0" source = { virtual = "." } dependencies = [ { name = "jsonschema" }, + { name = "numpy" }, { name = "pyyaml" }, ] @@ -38,6 +39,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "jsonschema", specifier = ">=4.23" }, + { name = "numpy", specifier = ">=2.5.3" }, { name = "pyyaml", specifier = ">=6.0" }, ] @@ -83,6 +85,79 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] +[[package]] +name = "numpy" +version = "2.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/01/11703282db468b85f6f7b8c7f22d058de5970d5c7e60a3a8aaa313c3de36/numpy-2.5.3.tar.gz", hash = "sha256:df2d5874ff183595a4ba404edd04f6bd9b5505c1d7708573f6a6c17489a67563", size = 20791231, upload-time = "2026-09-06T16:27:47.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/50/8fdbb16af64895706a45f06a4068e29db732ec180f3c1375f14123359138/numpy-2.5.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cb189f09db39283b26bfd061ec16189e14f71c6755207f72a0f7540867afe5b9", size = 16994982, upload-time = "2026-09-06T16:24:29.244Z" }, + { url = "https://files.pythonhosted.org/packages/60/39/789131c1188c078dcb3a1692e72e1e050c68b88ffe72c9ccaac9bcd7a9cd/numpy-2.5.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f59a878c33d6b88122d80d239bb3b845d58708750b0cb06a09aebb9b18ec696c", size = 12009327, upload-time = "2026-09-06T16:24:32.491Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/a312e95696e5f601914dd8b6dd844692ba61670807417e24b68e337b5c70/numpy-2.5.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:a72f874bc9e10e4b8f80426fb49716d5141f64442a0c8418065093ec8017fbb0", size = 5445405, upload-time = "2026-09-06T16:24:35.071Z" }, + { url = "https://files.pythonhosted.org/packages/30/d0/5623a1707ed4fe16e3909fe3cf5ee3da004ae677ad23d83bbf3adf1a6faf/numpy-2.5.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:fc36dc566135b5eceec4cf89758fcb719266a019ef07dae1754ae7c9f617ef3e", size = 6783213, upload-time = "2026-09-06T16:24:37.253Z" }, + { url = "https://files.pythonhosted.org/packages/f1/32/84146fc020ad3c25f805f70ab60da46fe3c540a21369754a7e4369754b6f/numpy-2.5.3-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:76c2c1e6bfa5c84adc6434dfbf013aa92096a7985221762c8f11fedfd20fff58", size = 15687872, upload-time = "2026-09-06T16:24:39.751Z" }, + { url = "https://files.pythonhosted.org/packages/65/af/aa78d1a88805456e212b65461354cd943197fb9acecc4c90fd12295123a3/numpy-2.5.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7e18c623bb5c95acb3b3328861272816ba199fb531921c5d6d0b675f1fde9e3", size = 16717410, upload-time = "2026-09-06T16:24:42.745Z" }, + { url = "https://files.pythonhosted.org/packages/3b/24/faa79d865e69a97ba17473b23a1b74094b2259c03e820c70297293b9ea49/numpy-2.5.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4f8929ee6c96bfbd7b4ed2032e0c03af86fe1826740ab61ddabf9072d06e57ff", size = 17040975, upload-time = "2026-09-06T16:24:45.961Z" }, + { url = "https://files.pythonhosted.org/packages/62/4a/8877e629445a7176297dffcaf9c485faa96a95d81728a62521ad55bd4c0f/numpy-2.5.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b5d93cf48f687479941d12b69c873ad2cc76bbd487f0091c2200636497f34034", size = 18476479, upload-time = "2026-09-06T16:24:49.35Z" }, + { url = "https://files.pythonhosted.org/packages/c8/db/35e1c2d38b04cbd5b731f9d71495e055e813197669d22b612f11748d2ff9/numpy-2.5.3-cp312-cp312-win32.whl", hash = "sha256:bf63afbe037eb5d2fe87fbcc7778e61da53ebaf21d938a4515aa73b62532a5d4", size = 6133378, upload-time = "2026-09-06T16:24:51.915Z" }, + { url = "https://files.pythonhosted.org/packages/3c/a1/accf6d4f0c80c5d9ba9735d6b1550e444180599f34dec69ca01360f717ad/numpy-2.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:0a59a421a32580a009e8a1751345bf829631b990dc1794b80514ab722b435def", size = 12567828, upload-time = "2026-09-06T16:24:54.255Z" }, + { url = "https://files.pythonhosted.org/packages/22/43/1764aff32e4652526ae2f71fa8b3efd8d25c8a3d6926914454e47138ed1e/numpy-2.5.3-cp312-cp312-win_arm64.whl", hash = "sha256:ccb32e0525d29e8b0572eb84c9a57af0e7a4e615726927506f55063c62414034", size = 10485432, upload-time = "2026-09-06T16:24:57.278Z" }, + { url = "https://files.pythonhosted.org/packages/79/e5/8fb89cd46d14e35699d13bf943a5f5f441ecee8667120a1f6105ab89e349/numpy-2.5.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:66a78fe4556c60aceda5916f9eacd638b18e9e681016ec302dcb4682d6d4d034", size = 16991061, upload-time = "2026-09-06T16:25:00.411Z" }, + { url = "https://files.pythonhosted.org/packages/2f/06/9dc9e48b5e5e941c8b10350c5ff2d721da42a20517d911d15544246775ff/numpy-2.5.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92f30e89b8ee0ecf363033576c422b2f58fed6a80bed0aa48dff6d14c654663e", size = 12003676, upload-time = "2026-09-06T16:25:03.475Z" }, + { url = "https://files.pythonhosted.org/packages/ab/2a/98282aa5b8f58b1157d440bb6282eed47e3632a5de53a714fbab17e659fe/numpy-2.5.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f9a2353b37a1a9e78fd82b27ad7e2a32a2d036604d18f02b05e3136c62ca3b09", size = 5439695, upload-time = "2026-09-06T16:25:05.978Z" }, + { url = "https://files.pythonhosted.org/packages/a1/f9/b6533d777be9d6ffd29dc1be0867e563e6e8cc9a220ff1b716adc317f060/numpy-2.5.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:ccbc4665079665c3cf3bab4db9f6b095370cd6437d66be549b6c2a1fd19e1958", size = 6779395, upload-time = "2026-09-06T16:25:08.599Z" }, + { url = "https://files.pythonhosted.org/packages/73/85/735720d04ec197c5dcfacdfc9922667c7f1f5f496a279b7ba4d7c74c4cc7/numpy-2.5.3-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c76d5dde9f445058f83d0c02af00557a4db91de9a9a57c0df87d1535001d654b", size = 15681750, upload-time = "2026-09-06T16:25:11.173Z" }, + { url = "https://files.pythonhosted.org/packages/3a/1b/3b16a9bc514a440a7a0883684111dcb1ef1aee960af2ca95da8fc775f124/numpy-2.5.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5fa86b80fd24bcd1aff83ad23be44ea323de3f787be8f8b15d4a65621e25321", size = 16708577, upload-time = "2026-09-06T16:25:14.171Z" }, + { url = "https://files.pythonhosted.org/packages/69/c4/386f397831b07328b639c96c5b62719346cf4baf07c68d927239752b1534/numpy-2.5.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bd4cb9ad3c7889b9b3fe0a9a9fb5d2ed26f9879bff2608d9f01aed147a20d231", size = 17042047, upload-time = "2026-09-06T16:25:17.582Z" }, + { url = "https://files.pythonhosted.org/packages/5f/3e/a700ecbf36e85ae8328fd3b0e12eeddc22ed6358a64cb2bd913e0d195d65/numpy-2.5.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1302b90c0e52281681b2975adfe8a860cb7b12216a27b4b0b4207c44bf7bccf0", size = 18465724, upload-time = "2026-09-06T16:25:20.949Z" }, + { url = "https://files.pythonhosted.org/packages/41/ee/38e785e88a4045f6ad1d1f2808dcdfafdca48c760260c0587bf171e29fc9/numpy-2.5.3-cp313-cp313-win32.whl", hash = "sha256:1c80eabb4035ecf4ca9cd49cde8a9fdd69a729e63e6474887d1523ade7aa277f", size = 6129003, upload-time = "2026-09-06T16:25:23.664Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ec/100f2b1794ede74a9b3d7ec6b9736927f56713414c1dfe19ab6c383494bf/numpy-2.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:71cad2b2a7451ab79d8f5e71b453485b6775963d5cf794179144a7463fe6e8ec", size = 12560965, upload-time = "2026-09-06T16:25:26.602Z" }, + { url = "https://files.pythonhosted.org/packages/80/b1/7dc825ca94c12acebbce4c37caa5e198695eb31424bc579679f32b1bb49d/numpy-2.5.3-cp313-cp313-win_arm64.whl", hash = "sha256:8e4dd766076855b5ff7ea52fa5f07ce26286726e0f8bff446b7739d02e6ea204", size = 10482343, upload-time = "2026-09-06T16:25:29.772Z" }, + { url = "https://files.pythonhosted.org/packages/70/78/cf416f15dc29375a229d9dfebf8db6e313f291580b39fa1a568b6052bb07/numpy-2.5.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:350ba9783ce969cf9f7ce6e6a9a58e1a6e2a19ca025b7ee448c4db727706212a", size = 16998686, upload-time = "2026-09-06T16:25:33.171Z" }, + { url = "https://files.pythonhosted.org/packages/9e/59/abcc2d8def4fd60eec7d87f92d27c13448ffd9ab14339bcc63a0d7a2fdea/numpy-2.5.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:012e66aca395d795496446e52aeeb5866312a5d4d3f27da270e5a0b43f70dc5c", size = 12013862, upload-time = "2026-09-06T16:25:36.748Z" }, + { url = "https://files.pythonhosted.org/packages/94/75/4640d2d6e4b64a049e48425a82728a41ef4adb61332d2cba68055774878b/numpy-2.5.3-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:adc1ada2662f8a5f960b8a10d9986897e7499ef07e06d4cfe7197f8cce923c07", size = 5449793, upload-time = "2026-09-06T16:25:39.476Z" }, + { url = "https://files.pythonhosted.org/packages/96/cd/625b57ae33d4ca560f32cc0b47b4a5922146d9beb998ddf773900d440a73/numpy-2.5.3-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:54a115e5a73b8fc44f0cebef486365a1894b5c9760685d4558b72b7c3eb846e0", size = 6785176, upload-time = "2026-09-06T16:25:42.069Z" }, + { url = "https://files.pythonhosted.org/packages/9c/72/12918652e7912ef9751e8694c88820fcd1908e0618cb23f5f3caa6004b7b/numpy-2.5.3-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be5a8381859b6da607c84f4f7d6847725f1cf1853ef8a2c9e115b7d58bef47dc", size = 15703377, upload-time = "2026-09-06T16:25:45.135Z" }, + { url = "https://files.pythonhosted.org/packages/45/8f/9beacf79ca7c650688ad0baa80931adb988fe6e6e5d5903c23cc3dbd70eb/numpy-2.5.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b0521d0f4aebb6e06189451025fa17a913287b13c03d5fe05c017333b654ea5b", size = 16711928, upload-time = "2026-09-06T16:25:48.461Z" }, + { url = "https://files.pythonhosted.org/packages/09/8d/41d0a56e1ac4c87495c897a211b1368691b7237aadabec8b3b8f3a74d48f/numpy-2.5.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9deb49575e5b0b94ed72c8a64ec4d033381adc27e9060ae842971f697ba96104", size = 17059507, upload-time = "2026-09-06T16:25:51.873Z" }, + { url = "https://files.pythonhosted.org/packages/08/1e/0dfbc5cc251d54e2af790f254d24ec38637fa97ec7d5d11de7ffed787098/numpy-2.5.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b00eefbcf0f292945c4b4dec2ae845389ef5bcdcd596e6e4328051db5b5ba694", size = 18471002, upload-time = "2026-09-06T16:25:55.233Z" }, + { url = "https://files.pythonhosted.org/packages/b5/2c/dfa40f6991f8185c8c30ffd023dfcbb11888e823cfab9557b920f3bb7bed/numpy-2.5.3-cp314-cp314-win32.whl", hash = "sha256:c2381f82999704f818e2c987a865050e285ec3621262c66d40f5a96c8f899f8e", size = 6180485, upload-time = "2026-09-06T16:25:58.157Z" }, + { url = "https://files.pythonhosted.org/packages/a4/73/d2c08231e4fde7e415501fd02c715d96e98599b2d8384445933944152984/numpy-2.5.3-cp314-cp314-win_amd64.whl", hash = "sha256:2c25dfa72943e4336ddb6b0ee4277b47a0c85bede0807530ec68103bf58e2c10", size = 12698179, upload-time = "2026-09-06T16:26:00.789Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e9/dcdcc9b95cf5f49815055573aee1b11cfbf5299f38a180e437ded050810f/numpy-2.5.3-cp314-cp314-win_arm64.whl", hash = "sha256:15aa985ac73a8db02db7663381aa109510449d3819d37206caed27b33a65a8a6", size = 10769383, upload-time = "2026-09-06T16:26:04.011Z" }, + { url = "https://files.pythonhosted.org/packages/49/c4/af8bc08a7ef4e1529a7c0cf24969accce316b783999802089a581ec99272/numpy-2.5.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ac7bb1c52d445bd4f8f7f97fefe6abc3a084dc4d63df50d79b17fa2b78e89297", size = 12132668, upload-time = "2026-09-06T16:26:07.138Z" }, + { url = "https://files.pythonhosted.org/packages/c5/ae/0f15eb56d4ec5e13c1f7ff04ff407f997d1acbadb45d3e1f2e2645a8f43c/numpy-2.5.3-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e6ab667ba76450084eb64013762c438ea76d9d29cc676dcd6c2e9892ba37f841", size = 5568580, upload-time = "2026-09-06T16:26:09.828Z" }, + { url = "https://files.pythonhosted.org/packages/23/fb/c72a8f25d4b6e96c354e7ab45ace3b27dc11e5d6a13b6c7d0cd6b08bf112/numpy-2.5.3-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:f7fabeb6cea87d65f3b926de33d03fb016cfdc29314c90974383b5582ae72891", size = 6882634, upload-time = "2026-09-06T16:26:12.524Z" }, + { url = "https://files.pythonhosted.org/packages/07/a9/968c90ed2ab15060c338e8137f1215b5a60756ae07328e0a60d1c6734df4/numpy-2.5.3-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1fb6f8fb9ff0b3a69f52c66ce397b0246583e9f28616231b0e32ca49259a5fa6", size = 15748923, upload-time = "2026-09-06T16:26:15.092Z" }, + { url = "https://files.pythonhosted.org/packages/59/08/9df04103947b95e3b6b1f2ed1a70521f325647a31b82da6a2aae3a485508/numpy-2.5.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:93e1f5447e2b1e479d7bd74701e84746b86450cff1fc368b132d195e2b8f8211", size = 16746748, upload-time = "2026-09-06T16:26:18.43Z" }, + { url = "https://files.pythonhosted.org/packages/41/a0/14c8d5fe5b53a334aabb653deb391c0fef49558f491880ea300ed6785224/numpy-2.5.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c00abe94c1a69d75d827dcf1c025b25c8a45d230b3bcd77a9020883a1b047653", size = 17111561, upload-time = "2026-09-06T16:26:22.113Z" }, + { url = "https://files.pythonhosted.org/packages/c4/a6/d7e96e42f01522e154c32489640f16dfc4f6181d165d05fc3bec8c2c4999/numpy-2.5.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:536f963710a4e63934d80ac0dc4f478804a83e9a84b6828018f25d09953ada33", size = 18513945, upload-time = "2026-09-06T16:26:25.401Z" }, + { url = "https://files.pythonhosted.org/packages/25/39/3453afb7119d0449ef11c886874120ff180e2c337760e0e2d88f70f1a945/numpy-2.5.3-cp314-cp314t-win32.whl", hash = "sha256:4c8a6d2ebce6305fd82fbefca827775437147052a976ee7c94b36a0c1b52ac6c", size = 6335421, upload-time = "2026-09-06T16:26:28.175Z" }, + { url = "https://files.pythonhosted.org/packages/99/01/22815d2b19a1a746b1d45205cffebb3fe511a18acb75fba6c88491fc9894/numpy-2.5.3-cp314-cp314t-win_amd64.whl", hash = "sha256:9a37475425b431b4d060f23b4f52cd2f3aef6bc7c654bd760adf0040eec9d435", size = 12896420, upload-time = "2026-09-06T16:26:31.265Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ee/a7cbba67eeaff038dc29ca8b98a88396c8b0cc9c89d4924f4a27a5c9150b/numpy-2.5.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2d8240cb4c16fd831074aa2b2cf9fc54664d826341d61c372245b96a74a49a9a", size = 10857177, upload-time = "2026-09-06T16:26:34.167Z" }, + { url = "https://files.pythonhosted.org/packages/45/56/78194492883ff5eec90423fe56a3a44b154da047d88a6307f629713c584f/numpy-2.5.3-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:a6391fafaba97500887132cd582abc6e19452b1ac775a47caa7b24490e152058", size = 16996531, upload-time = "2026-09-06T16:26:37.287Z" }, + { url = "https://files.pythonhosted.org/packages/11/39/dd55c0af90bbab564b09ae3b0aa60ec5c02b900fa4f1ba23440525c8b32d/numpy-2.5.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:09d5a423c71ad5feb5625844ad58050e35df43871004b52ac9c0ad44a56775be", size = 12012569, upload-time = "2026-09-06T16:26:40.707Z" }, + { url = "https://files.pythonhosted.org/packages/b6/51/04f67d32e4862b281b1cb84ceeaed3421189a84fb6fb51a391cd6d5009f7/numpy-2.5.3-cp315-cp315-macosx_14_0_arm64.whl", hash = "sha256:f9579f383d1bf9df80081e72760e84960a7fd4f88cf0c9e535a8597c9bb646f5", size = 5448498, upload-time = "2026-09-06T16:26:43.435Z" }, + { url = "https://files.pythonhosted.org/packages/a3/c9/25b4dc0dd1344ec26c7319e84fd4e9809d2b5628f4e12decd618036e5178/numpy-2.5.3-cp315-cp315-macosx_14_0_x86_64.whl", hash = "sha256:86bff898a431c0fb71f7610b75726e75a54d47b37edc9d537f48de63bb3c0b90", size = 6783026, upload-time = "2026-09-06T16:26:46.374Z" }, + { url = "https://files.pythonhosted.org/packages/fc/c7/29285be1e5232a6e7ee3268a33c85843f5a8ee93350c6465cddd66ebbf76/numpy-2.5.3-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f3ed25271581281f2fccb1adcedfcde4c07362eec69189b50baf6f90e3ae159", size = 15697322, upload-time = "2026-09-06T16:26:49.415Z" }, + { url = "https://files.pythonhosted.org/packages/55/49/bbad5335fb4996a16881f853ff3e0ba582f01720e55c89b1c06b8fc42a90/numpy-2.5.3-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffdc76bfcae6b255dff75202c5e7feaf95b40246bc0a17944facc1fecf9f79ab", size = 16708995, upload-time = "2026-09-06T16:26:53.127Z" }, + { url = "https://files.pythonhosted.org/packages/ef/e9/1df35483760b04a65ea44669f89dc64f30e5aca098b48ceb8b1310b0e0fe/numpy-2.5.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:116f96cadd935c6122e9228d676fe7ede19e741f5c8bb1c3cddbe0c51ccebea2", size = 17052508, upload-time = "2026-09-06T16:26:56.464Z" }, + { url = "https://files.pythonhosted.org/packages/b8/99/66e54da8265cc8be8a7382bf96edce17aaa2837d6f484432025932a3caa5/numpy-2.5.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:09ffa5d903faeaa5c4dd05009cf81c8bab9f2cb37c548b8d39b65b4cfa7c97f7", size = 18468224, upload-time = "2026-09-06T16:26:59.966Z" }, + { url = "https://files.pythonhosted.org/packages/01/bc/b5e90a91c115168d793dfd2ad9c69c438c2fe7a13a437e770bc5b078e732/numpy-2.5.3-cp315-cp315-win32.whl", hash = "sha256:e01c918ac3d48e18a927cf7b14a26a3e29ff2bdf2eacb976da0aecd6a43ed034", size = 6179919, upload-time = "2026-09-06T16:27:03.166Z" }, + { url = "https://files.pythonhosted.org/packages/37/ea/780748fd3985109075514ef8fc64cd25f943e40dde13a6d59141eb268fc8/numpy-2.5.3-cp315-cp315-win_amd64.whl", hash = "sha256:e931e4f499e0dc7ef29d269a8e5b35dd722e5d14be07df6240166ea7c6532fae", size = 12697656, upload-time = "2026-09-06T16:27:06.153Z" }, + { url = "https://files.pythonhosted.org/packages/b3/16/407be69a2a87c8cab64d95975a8977a426a29e138f07e276ec258f0fe4e5/numpy-2.5.3-cp315-cp315-win_arm64.whl", hash = "sha256:26e15e4aecd8617dfbaecb37d223e365d7b39411fba20454be2670a96aa74cb5", size = 10767601, upload-time = "2026-09-06T16:27:09.297Z" }, + { url = "https://files.pythonhosted.org/packages/44/bf/a97ffb01e41d50a32a9177aef942a4d0e389a3daf451d04e5f38ef6afb87/numpy-2.5.3-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:6cef4bb1706dfec49243c05d921eefb4e190d41e2528b30d8035ea1f36b4c24a", size = 17090092, upload-time = "2026-09-06T16:27:12.907Z" }, + { url = "https://files.pythonhosted.org/packages/d1/24/136c02f2c2af9a067a84d0c3aa10c99012c0476fa5066732fa4a4202557d/numpy-2.5.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:d1c89973648c85069c5046ad460f7b8a00218b29a2e42359ac8cc63e9ab94832", size = 12129429, upload-time = "2026-09-06T16:27:16.089Z" }, + { url = "https://files.pythonhosted.org/packages/fe/6c/b47582d6597789bf946d5efbeb6b9e56fd8bcbd5efc6fbf51dbe1ea31eb3/numpy-2.5.3-cp315-cp315t-macosx_14_0_arm64.whl", hash = "sha256:214045a5bf00113a146ab9ee9730c44501af6723cdf1f6830932f7b5ef2e7af0", size = 5565452, upload-time = "2026-09-06T16:27:19.868Z" }, + { url = "https://files.pythonhosted.org/packages/be/b4/ef3cc6da73774202d4deae16bb321fd8298a4e0561e3539f8c4be237d916/numpy-2.5.3-cp315-cp315t-macosx_14_0_x86_64.whl", hash = "sha256:8617bbfae4486cf99c9f899966699428d19da931d06ca94ad3da986c76e15997", size = 6876736, upload-time = "2026-09-06T16:27:22.232Z" }, + { url = "https://files.pythonhosted.org/packages/9e/24/e3813329498596cb842703dcacac1741612ed9fb9c4e6a3e0c7e2ebbc597/numpy-2.5.3-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:595d020938c84e320bcf40ad71089e108eac0d377cd018e14a8c094f39e98d85", size = 15745777, upload-time = "2026-09-06T16:27:25.181Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9e/4e7a07fd0776dc2210cdacf2010be8665194d094defc10c419d7dea794cc/numpy-2.5.3-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f24021b9f22bc6301c37b196974a92c1c18dccedb6fef3dd252e95f2d6adbe4", size = 16746949, upload-time = "2026-09-06T16:27:28.576Z" }, + { url = "https://files.pythonhosted.org/packages/91/db/01674c0e20335057813a00c2ebd546ed25bff9ed7914f9bced00f8c55d94/numpy-2.5.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:71b39d9f935b6ec0f8753e3e2afb51e3efba6f2e05b68b32a40754d24bcd4a3c", size = 17108994, upload-time = "2026-09-06T16:27:31.946Z" }, + { url = "https://files.pythonhosted.org/packages/45/7a/584c5e71f8d378e57cac0b033891ed65c683ef90573ba4854e8c28203db0/numpy-2.5.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:6b05c171afb3aa07adbd20abc00aea86fe375beb0fdb9ef780ec5b7f63bab1c0", size = 18512266, upload-time = "2026-09-06T16:27:35.196Z" }, + { url = "https://files.pythonhosted.org/packages/a1/d2/4e1014173aa3c55e6a756e0e567290743a6ab33a288460374d7ef6bcd239/numpy-2.5.3-cp315-cp315t-win32.whl", hash = "sha256:f54660b0eb6b0b9f36e7fe1cdfdff472028dd0d14acd9b9b65098efbad059469", size = 6330292, upload-time = "2026-09-06T16:27:38.149Z" }, + { url = "https://files.pythonhosted.org/packages/6c/b0/ff5658a58199b7bcaad87bf260eef6713d9d42cca4e028f935b4fc5fbac6/numpy-2.5.3-cp315-cp315t-win_amd64.whl", hash = "sha256:1aad64d99730d013cfc6debafed22783b4fc5a7f4b8bc744d2d8cf7dcc880551", size = 12884918, upload-time = "2026-09-06T16:27:40.965Z" }, + { url = "https://files.pythonhosted.org/packages/fb/0b/b12a2df5d1b774bd9007a6fdff9381145b6223d37f11afc9c37ab0efd9a1/numpy-2.5.3-cp315-cp315t-win_arm64.whl", hash = "sha256:befa1ae5bd6030b3f512b43ff3fa5290bbed6b84411a44244b14adf835f5b89d", size = 10850807, upload-time = "2026-09-06T16:27:43.868Z" }, +] + [[package]] name = "packaging" version = "26.3" From 2a6d3f7b0c7c7593ef890669ec75e400bb6d2641 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 03:13:10 -0500 Subject: [PATCH 04/59] verification: add end-to-end server check Starts the server from a ref (or uses a running one), finds its scene endpoint from OpenAPI, validates scenes and results against the published C1 and C2 schemas, and checks decision consistency, the C5 margin rule and coverage on every answer. Each scene is also resent unchanged, mirrored left to right and without coverage. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/hsverify/e2e.py | 469 +++++++++++++++++++++++++ verification/hsverify/resultcheck.py | 329 +++++++++++++++++ verification/tests/test_e2e.py | 189 ++++++++++ verification/tests/test_replaycheck.py | 20 +- verification/tests/test_resultcheck.py | 298 ++++++++++++++++ 5 files changed, 1299 insertions(+), 6 deletions(-) create mode 100644 verification/hsverify/e2e.py create mode 100644 verification/hsverify/resultcheck.py create mode 100644 verification/tests/test_e2e.py create mode 100644 verification/tests/test_resultcheck.py diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py new file mode 100644 index 000000000..b718b9aa9 --- /dev/null +++ b/verification/hsverify/e2e.py @@ -0,0 +1,469 @@ +"""Send scenes through the placement server's HTTP API and judge every answer. + + uv run python -m hsverify.e2e --server-ref origin/t3/server # cases/ + real scenes + uv run python -m hsverify.e2e --server-url http://127.0.0.1:8000 --scene path/to/scene.json + +The server under test is either started from a git ref (a detached worktree under /tmp, +`uv sync --locked`, then uvicorn on a free port) or reached at `--server-url`. Its scene +endpoint is read from its OpenAPI document; with more than one candidate the run stops and +asks for `--endpoint` instead of guessing. + +Each scene is first validated against the scene schema published at the same ref, so a bad +input is reported as an input problem, not a server bug. Each response must validate against +the result schema and pass the invariants in `resultcheck`. Case files add the outcomes their +geometry forces. Every scene is also sent three more ways: again (the result must be +identical apart from timing), mirrored left to right (same decision, same length of passing, +unsure and failing wall), and without coverage (never a pass). + +Reports go to ~/house-scanning-data/reports/e2e// (outside git; real scenes can +reference dataset images). +""" + +from __future__ import annotations + +import argparse +import contextlib +import datetime as dt +import hashlib +import io +import json +import re +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.request +import uuid +import zipfile +from collections.abc import Iterator +from dataclasses import dataclass, field +from pathlib import Path + +from hsverify import gitref +from hsverify.resultcheck import ( + assumption_mismatches, + comparable, + expectation_problems, + invariant_problems, + mirror_scene, + outcome_lengths, + schema_errors, +) + +HERE = Path(__file__).resolve().parents[1] +DEFAULT_CASES = HERE / "e2e" / "cases" +DEFAULT_REPORTS = Path.home() / "house-scanning-data" / "reports" / "e2e" +SCENE_SCHEMA = "server/schemas/scene.schema.json" +RESULT_SCHEMA = "server/schemas/result.schema.json" +# The plan's S2 metric: a real-derived scene returns a result in under 1 s. +LATENCY_BUDGET_MS = 1000.0 +MIRROR_LENGTH_TOL_FT = 0.5 + + +# --- Inputs ---------------------------------------------------------------------------------- + + +@dataclass +class SceneInput: + name: str + scene: dict + raw: bytes # exactly the bytes sent, so the server's input hash can be checked + images: dict[str, bytes] = field(default_factory=dict) + expect: dict | None = None + rules_assumed: dict[str, float] = field(default_factory=dict) + real: bool = False + source: str = "" + skip_reason: str | None = None + + +def load_input(path: Path, real: bool = False) -> SceneInput: + """A case file (scene + expect), a bare scene.json, or a C1 zip bundle.""" + if path.suffix == ".zip": + with zipfile.ZipFile(path) as bundle: + names = bundle.namelist() + scene_name = next((n for n in names if n.endswith("scene.json")), None) + if scene_name is None: + raise SystemExit(f"{path}: no scene.json in the bundle") + raw = bundle.read(scene_name) + images = {Path(n).name: bundle.read(n) for n in names if n.lower().endswith(".jpg")} + return SceneInput(path.stem, json.loads(raw), raw, images, real=real, source=str(path)) + data = json.loads(path.read_text()) + if "scene" in data and "expect" in data: + raw = json.dumps(data["scene"], indent=1).encode() + return SceneInput( + data.get("id", path.stem), + data["scene"], + raw, + expect=data["expect"], + rules_assumed=data.get("rules_assumed", {}), + real=bool(data.get("real", real)), + source=data.get("source", str(path)), + skip_reason=data.get("skip_reason"), + ) + return SceneInput(path.stem, data, path.read_bytes(), real=real, source=str(path)) + + +# --- Server ---------------------------------------------------------------------------------- + + +def free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def find_app(server_dir: Path) -> str: + """`module:variable` of the FastAPI app. Exactly one is required.""" + found = [] + for path in sorted(server_dir.glob("*.py")): + for match in re.finditer(r"^(\w+)\s*=\s*FastAPI\(", path.read_text(), re.M): + found.append(f"{path.stem}:{match.group(1)}") + if len(found) != 1: + raise SystemExit( + f"Expected one `app = FastAPI(...)` in {server_dir}, found {found or 'none'}. " + "Start the server yourself and pass --server-url." + ) + return found[0] + + +@contextlib.contextmanager +def server_from_ref(sha: str, log_path: Path) -> Iterator[str]: + with gitref.detached_worktree(sha) as tree: + server_dir = tree / "server" + subprocess.run(["uv", "sync", "--locked", "--quiet"], cwd=server_dir, check=True) + app = find_app(server_dir) + port = free_port() + url = f"http://127.0.0.1:{port}" + with log_path.open("w") as log: + proc = subprocess.Popen( + [ + "uv", + "run", + "--quiet", + "uvicorn", + app, + "--host", + "127.0.0.1", + "--port", + str(port), + ], + cwd=server_dir, + stdout=log, + stderr=subprocess.STDOUT, + ) + try: + wait_until_up(url, proc) + yield url + finally: + proc.terminate() + proc.wait(timeout=10) + + +def wait_until_up(url: str, proc: subprocess.Popen, timeout_s: float = 60) -> None: + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + if proc.poll() is not None: + raise SystemExit(f"The server exited with code {proc.returncode}; see server.log.") + try: + urllib.request.urlopen(f"{url}/openapi.json", timeout=2).read() + return + except (urllib.error.URLError, ConnectionError, TimeoutError): + time.sleep(0.5) + raise SystemExit(f"The server did not answer at {url} within {timeout_s:.0f} s.") + + +@dataclass +class Endpoint: + path: str + content_type: str # "application/json" or "multipart/form-data" + file_field: str | None = None + + +def discover_endpoint(openapi: dict, override: str | None = None) -> Endpoint: + """The POST operation that takes a scene: JSON body, or a multipart upload of the bundle.""" + candidates = [] + for path, ops in openapi.get("paths", {}).items(): + body = ops.get("post", {}).get("requestBody", {}).get("content", {}) + for kind in ("application/json", "multipart/form-data"): + if kind in body: + candidates.append((path, kind, body[kind].get("schema", {}))) + if override: + candidates = [c for c in candidates if c[0] == override] + if not candidates: + raise SystemExit(f"--endpoint {override} is not a POST with a JSON or multipart body") + elif len(candidates) > 1: + named = [c for c in candidates if re.search(r"scene|place|solve|placement", c[0])] + candidates = named if len(named) == 1 else candidates + if len(candidates) != 1: + listed = ", ".join(f"{p} ({k})" for p, k, _ in candidates) or "none" + raise SystemExit(f"Cannot tell which endpoint takes a scene: {listed}. Pass --endpoint.") + path, kind, schema = candidates[0] + file_field = None + if kind == "multipart/form-data": + props = resolve_ref(openapi, schema).get("properties", {}) + binary = [ + k + for k, v in props.items() + if v.get("format") == "binary" + or v.get("contentMediaType") == "application/octet-stream" + ] + if len(binary) != 1: + raise SystemExit(f"{path}: expected one file field in the upload form, got {binary}") + file_field = binary[0] + return Endpoint(path, kind, file_field) + + +def resolve_ref(openapi: dict, schema: dict) -> dict: + ref = schema.get("$ref") + if not ref: + return schema + node = openapi + for part in ref.removeprefix("#/").split("/"): + node = node[part] + return node + + +def bundle_zip(item: SceneInput) -> bytes: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z: + z.writestr("scene.json", item.raw) + for name, data in item.images.items(): + z.writestr(name, data) + return buf.getvalue() + + +def post(url: str, endpoint: Endpoint, item: SceneInput) -> tuple[int, bytes, float]: + if endpoint.content_type == "application/json": + body, ctype = item.raw, "application/json" + else: + boundary = uuid.uuid4().hex + body = ( + ( + f'--{boundary}\r\nContent-Disposition: form-data; name="{endpoint.file_field}"; ' + f'filename="scene.zip"\r\nContent-Type: application/zip\r\n\r\n' + ).encode() + + bundle_zip(item) + + f"\r\n--{boundary}--\r\n".encode() + ) + ctype = f"multipart/form-data; boundary={boundary}" + request = urllib.request.Request( + url + endpoint.path, data=body, headers={"Content-Type": ctype}, method="POST" + ) + start = time.perf_counter() + try: + with urllib.request.urlopen(request, timeout=60) as response: + status, payload = response.status, response.read() + except urllib.error.HTTPError as exc: + status, payload = exc.code, exc.read() + return status, payload, (time.perf_counter() - start) * 1000 + + +# --- Judging --------------------------------------------------------------------------------- + + +def variant(item: SceneInput, name: str, scene: dict) -> SceneInput: + raw = json.dumps(scene, indent=1).encode() + return SceneInput(f"{item.name}~{name}", scene, raw, real=item.real, source=item.source) + + +def judge(url: str, endpoint: Endpoint, item: SceneInput, schemas: dict) -> dict: + record: dict = {"name": item.name, "source": item.source, "real": item.real} + if item.skip_reason: + return record | {"status": "skipped", "problems": [], "skip_reason": item.skip_reason} + input_errors = schema_errors(item.scene, schemas["scene"]) + if input_errors: + return record | {"status": "bad input", "problems": input_errors[:20]} + + status, payload, ms = post(url, endpoint, item) + record |= {"http_status": status, "latency_ms": round(ms, 1)} + if status != 200: + return record | {"status": "fail", "problems": [f"HTTP {status}: {payload[:500]!r}"]} + result = json.loads(payload) + record["result"] = result + problems = [f"result schema: {e}" for e in schema_errors(result, schemas["result"])] + if problems: + return record | {"status": "fail", "problems": problems[:20]} + + problems += invariant_problems(item.scene, result, sent=item.raw) + if item.real and ms > LATENCY_BUDGET_MS: + problems.append(f"real-derived scene took {ms:.0f} ms, budget {LATENCY_BUDGET_MS:.0f}") + mismatches = assumption_mismatches(item.rules_assumed, result) + if item.expect and not mismatches: + problems += expectation_problems(item.expect, result) + problems += property_problems(url, endpoint, item, result) + + record["decision"] = result["decision"] + record["outcome_lengths_ft"] = outcome_lengths(result) + if mismatches: + record["assumption_mismatches"] = mismatches + record["problems"] = problems + record["status"] = "fail" if problems else "assumption mismatch" if mismatches else "pass" + return record + + +def property_problems(url: str, endpoint: Endpoint, item: SceneInput, result: dict) -> list[str]: + problems = [] + # 1. Same input, same answer. + status, payload, _ = post(url, endpoint, item) + if status != 200 or comparable(json.loads(payload)) != comparable(result): + problems.append("sending the same scene twice gave different results") + # 2. Mirrored left to right: same decision, same amount of each outcome along the wall. + mirrored = variant(item, "mirror", mirror_scene(item.scene)) + status, payload, _ = post(url, endpoint, mirrored) + if status != 200: + problems.append(f"mirrored scene: HTTP {status}") + else: + other = json.loads(payload) + if other["decision"] != result["decision"]: + problems.append( + f"mirrored scene decided {other['decision']}, original {result['decision']}" + ) + a, b = outcome_lengths(result), outcome_lengths(other) + for outcome in a: + if abs(a[outcome] - b[outcome]) > MIRROR_LENGTH_TOL_FT: + problems.append( + f"mirrored scene: {outcome} starts cover {b[outcome]:.2f} ft, " + f"original {a[outcome]:.2f} ft" + ) + # 3. Without coverage nothing was observed, so nothing may pass. + if "coverage" in item.scene: + bare = {k: v for k, v in item.scene.items() if k != "coverage"} + status, payload, _ = post(url, endpoint, variant(item, "no-coverage", bare)) + if status != 200: + problems.append(f"scene without coverage: HTTP {status}") + else: + other = json.loads(payload) + if other["decision"] == "pass" or any( + r["outcome"] == "pass" for r in other.get("sweep", []) + ): + problems.append("scene without coverage still produced a pass") + return problems + + +# --- Report ---------------------------------------------------------------------------------- + + +def write_report(out: Path, meta: dict, records: list[dict]) -> dict: + counts: dict[str, int] = {} + for r in records: + counts[r["status"]] = counts.get(r["status"], 0) + 1 + real = [r["latency_ms"] for r in records if r.get("real") and "latency_ms" in r] + report = meta | { + "counts": counts, + "all_passed": all(r["status"] in ("pass", "skipped") for r in records) and bool(records), + "latency_ms": max(real) if real else None, + "real_scene_passed": any(r.get("real") and r["status"] == "pass" for r in records), + "records": records, + } + (out / "report.json").write_text(json.dumps(report, indent=2) + "\n") + lines = [ + f"# End-to-end run against `{meta['server_ref']}` at `{meta['server_sha'][:12]}`", + "", + f"{meta['started_at']}. Endpoint `{meta['endpoint']}`. " + + ", ".join(f"{v} {k}" for k, v in sorted(counts.items())), + "", + "| Scene | Status | Decision | ms | Problems |", + "| --- | --- | --- | --- | --- |", + ] + for r in records: + issues = "; ".join(r.get("problems", []) + r.get("assumption_mismatches", [])) + issues = issues or r.get("skip_reason", "") + lines.append( + f"| {r['name']}{' (real)' if r.get('real') else ''} | {r['status']} | " + f"{r.get('decision', '')} | {r.get('latency_ms', '')} | {issues.replace('|', '/')} |" + ) + (out / "report.md").write_text("\n".join(lines) + "\n") + return report + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + where = parser.add_mutually_exclusive_group() + where.add_argument("--server-ref", default="origin/t3/server") + where.add_argument("--server-url", help="use a running server instead of starting one") + parser.add_argument("--schema-ref", help="ref for the schemas (default: --server-ref)") + parser.add_argument("--endpoint", help="POST path, when discovery is ambiguous") + parser.add_argument("--cases", type=Path, default=DEFAULT_CASES) + parser.add_argument( + "--scene", + type=Path, + action="append", + default=[], + help="extra case file, scene.json or scene.zip", + ) + parser.add_argument( + "--real", + type=Path, + action="append", + default=[], + help="scene derived from real data; held to the latency budget", + ) + parser.add_argument("--out", type=Path) + args = parser.parse_args(argv) + + gitref.fetch() + schema_ref = args.schema_ref or (args.server_ref if not args.server_url else "origin/t3/server") + schema_sha = gitref.resolve(schema_ref) + schemas = {} + for key, path in (("scene", SCENE_SCHEMA), ("result", RESULT_SCHEMA)): + text = gitref.show(schema_sha, path) + if text is None: + raise SystemExit(f"{path} does not exist at {schema_ref}") + schemas[key] = json.loads(text) + + items = ( + [load_input(p) for p in sorted(args.cases.glob("*.json"))] if args.cases.exists() else [] + ) + items += [load_input(p) for p in args.scene] + items += [load_input(p, real=True) for p in args.real] + if not items: + raise SystemExit("No scenes: add case files to e2e/cases or pass --scene/--real.") + + server_sha = schema_sha if not args.server_url else "(running server)" + if not args.server_url: + server_sha = gitref.resolve(args.server_ref) + stamp = dt.datetime.now().strftime("%Y%m%d-%H%M%S") + out = (args.out or DEFAULT_REPORTS / f"{stamp}-{server_sha[:8]}").expanduser() + out.mkdir(parents=True, exist_ok=True) + + with contextlib.ExitStack() as stack: + url = args.server_url or stack.enter_context( + server_from_ref(server_sha, out / "server.log") + ) + openapi = json.loads(urllib.request.urlopen(f"{url}/openapi.json", timeout=10).read()) + endpoint = discover_endpoint(openapi, args.endpoint) + print(f"Server {url}, endpoint POST {endpoint.path} ({endpoint.content_type})") + records = [] + for item in items: + record = judge(url, endpoint, item, schemas) + records.append(record) + print( + f" {record['status']:<20} {item.name}" + + ( + f" [{record.get('decision')}, {record.get('latency_ms')} ms]" + if "decision" in record + else "" + ) + ) + for problem in record.get("problems", [])[:6]: + print(f" - {problem}") + + meta = { + "started_at": dt.datetime.now().astimezone().isoformat(timespec="seconds"), + "server_ref": args.server_url or args.server_ref, + "server_sha": server_sha, + "schema_ref": schema_ref, + "schema_sha": schema_sha, + "sha": server_sha, + "endpoint": f"POST {endpoint.path} ({endpoint.content_type})", + "command": " ".join([sys.executable, "-m", "hsverify.e2e", *(argv or sys.argv[1:])]), + "cases_sha256": hashlib.sha256(b"".join(i.raw for i in items)).hexdigest(), + } + report = write_report(out, meta, records) + print(f"Report: {out / 'report.md'}") + return 0 if report["all_passed"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py new file mode 100644 index 000000000..76ead2d6e --- /dev/null +++ b/verification/hsverify/resultcheck.py @@ -0,0 +1,329 @@ +"""Judge a placement result (contract C2) against its scene (C1) and the decision rule (C5). + +Everything here is pure: a scene and a result go in, a list of problems comes out. The +checks hold for any policy and any rule values, so they can run on every response: + +- decision consistency: what `pass`, `reject` and `manual_review` each require of the rest + of the record; +- the margin rule: a check's outcome must follow from its measurement, error and threshold + (PASS only when the margin beats the error, FAIL only when past the threshold by more + than the error, otherwise UNSURE); +- coverage: no battery position whose wall stretch, cable route or ground was never observed + may pass, and nothing may ask for photos of an area the scene says was observed; +- bookkeeping: counts add up, the input hash matches what was sent, the spot's offset from + the meter matches its centre. + +Case expectations (a scene plus the outcomes geometry forces) and the mirror transform used +for the left-right symmetry check also live here. +""" + +from __future__ import annotations + +import copy +import hashlib +from typing import Any + +from jsonschema import Draft202012Validator + +EPS = 1e-6 +# Spot centres and offsets are rounded by the server; a hundredth of a foot is 1/8 inch. +OFFSET_TOL_FT = 0.01 +# Sweep start ranges are sampled; a pass run may reach one 2 in step past a covered edge. +SWEEP_STEP_FT = 2 / 12 +DEFAULT_BATTERY_WIDTH_FT = 31 / 12 + + +def schema_errors(instance: Any, schema: dict) -> list[str]: + validator = Draft202012Validator(schema) + errors = sorted(validator.iter_errors(instance), key=lambda e: list(e.absolute_path)) + return [f"{'/'.join(map(str, e.absolute_path)) or '(root)'}: {e.message}" for e in errors] + + +# --- Coverage -------------------------------------------------------------------------------- + + +def observed(scene: dict, band: str, min_out_ft: float = 0.0) -> list[tuple[float, float]]: + """Merged observed s-intervals for a band. Ground intervals must reach `min_out_ft` out.""" + spans = [] + for entry in scene.get("coverage", {}).get("observed", []): + if entry["band"] != band: + continue + if band == "ground" and entry.get("out_ft", 0.0) + EPS < min_out_ft: + continue + spans.append(tuple(sorted(entry["span_ft"]))) + spans.sort() + merged: list[list[float]] = [] + for a, b in spans: + if merged and a <= merged[-1][1] + EPS: + merged[-1][1] = max(merged[-1][1], b) + else: + merged.append([a, b]) + return [(a, b) for a, b in merged] + + +def covers(intervals: list[tuple[float, float]], a: float, b: float, slack: float = 0.0) -> bool: + return any(lo - slack <= a + EPS and b - EPS <= hi + slack for lo, hi in intervals) + + +# --- Invariants ------------------------------------------------------------------------------ + + +def margin_problem(check: dict) -> str | None: + """The C5 rule for one check, when its numbers are all present.""" + m, e, t, cmp = ( + check.get(k) for k in ("measured_ft", "plus_minus_ft", "threshold_ft", "comparison") + ) + if m is None or e is None or t is None or cmp is None: + return None + if cmp == "at_least": + clear_pass, clear_fail = m - e > t + EPS, m + e < t - EPS + else: + clear_pass, clear_fail = m + e < t - EPS, m - e > t + EPS + expected = "pass" if clear_pass else "fail" if clear_fail else "unsure" + if check["outcome"] != expected: + return ( + f"check {check['id']}: measured {m} ± {e} {cmp} {t} should be {expected}, " + f"server says {check['outcome']}" + ) + return None + + +def invariant_problems(scene: dict, result: dict, sent: bytes | None = None) -> list[str]: + problems: list[str] = [] + decision = result["decision"] + spot = result.get("spot") + checks = result.get("checks", []) + + if decision == "pass": + if spot is None or spot.get("outcome") != "pass": + problems.append("decision pass without a passing spot") + not_passing = [c["id"] for c in checks if c["outcome"] != "pass"] + if not_passing: + problems.append(f"decision pass with checks not passing: {not_passing}") + if result.get("missing_evidence"): + problems.append("decision pass while asking for more evidence") + if not result["policy"].get("auto_approve"): + problems.append("decision pass although policy.auto_approve is false") + if decision == "reject": + if spot is not None: + problems.append("decision reject but a spot is given") + if result.get("missing_evidence"): + problems.append("decision reject while asking for more evidence") + if decision == "manual_review" and not result.get("reasons"): + problems.append("decision manual_review without reasons") + if (spot is None) != (result.get("route") is None): + problems.append("spot and route must be both null or both present") + + for check in checks: + problem = margin_problem(check) + if problem: + problems.append(problem) + cause = check.get("unsure_cause") + if check["outcome"] == "unsure" and cause is None: + problems.append(f"check {check['id']}: unsure without unsure_cause") + if check["outcome"] != "unsure" and cause is not None: + problems.append(f"check {check['id']}: unsure_cause on a {check['outcome']} check") + + stats = result["stats"] + if stats["pass"] + stats["unsure"] + stats["fail"] != stats["candidates"]: + problems.append( + f"stats: pass {stats['pass']} + unsure {stats['unsure']} + fail {stats['fail']} " + f"!= candidates {stats['candidates']}" + ) + if sent is not None and stats["input_sha256"] != hashlib.sha256(sent).hexdigest(): + problems.append("stats.input_sha256 does not match the scene bytes sent") + + if spot is not None: + mx, _, mz = scene["meter"]["pos"] + want = [spot["center"][0] - mx, spot["center"][1] - mz] + got = spot["meter_offset_ft"] + if max(abs(want[0] - got[0]), abs(want[1] - got[1])) > OFFSET_TOL_FT: + problems.append(f"spot.meter_offset_ft {got} != centre - meter {want}") + + problems += coverage_problems(scene, result) + return problems + + +def coverage_problems(scene: dict, result: dict) -> list[str]: + """Missing coverage is never a pass (C5); photo requests only for unobserved areas.""" + problems: list[str] = [] + spot = result.get("spot") + width = spot["width_ft"] if spot else DEFAULT_BATTERY_WIDTH_FT + depth = spot["depth_ft"] if spot else 22 / 12 + wall = observed(scene, "wall") + ground = observed(scene, "ground", min_out_ft=depth) + + if "coverage" not in scene and result["decision"] == "pass": + problems.append("decision pass for a scene with no coverage at all") + + for run in result.get("sweep", []): + if run["outcome"] != "pass": + continue + lo, hi = run["start_ft"][0], run["start_ft"][1] + width + route_lo, route_hi = min(0.0, lo), max(0.0, hi) + if not covers(wall, route_lo, route_hi, slack=SWEEP_STEP_FT): + problems.append( + f"sweep pass for starts {run['start_ft']} but the wall and cable route " + f"[{route_lo:.2f}, {route_hi:.2f}] were not all observed" + ) + if not covers(ground, lo, hi, slack=SWEEP_STEP_FT): + problems.append( + f"sweep pass for starts {run['start_ft']} but the ground under " + f"[{lo:.2f}, {hi:.2f}] was not observed {depth:.2f} ft out" + ) + + for request in result.get("missing_evidence", []): + if request["kind"] != "band" or "span_ft" not in request or "band" not in request: + continue + a, b = sorted(request["span_ft"]) + if b - a > EPS and covers(observed(scene, request["band"]), a, b): + problems.append( + f"missing_evidence asks for {request['band']} {request['span_ft']}, " + "which the scene lists as observed" + ) + return problems + + +# --- Case expectations ----------------------------------------------------------------------- + + +def assumption_mismatches(assumed: dict[str, float], result: dict) -> list[str]: + out = [] + for key, value in assumed.items(): + for check in result.get("checks", []): + threshold = check.get("threshold_ft") + if key in check["id"] and threshold is not None and abs(threshold - value) > EPS: + out.append( + f"case assumes {key} threshold {value}, server's {check['id']} uses " + f"{check['threshold_ft']}" + ) + return out + + +def expectation_problems(expect: dict, result: dict) -> list[str]: + problems: list[str] = [] + decision = result["decision"] + if "decision_in" in expect and decision not in expect["decision_in"]: + problems.append(f"decision {decision}, expected one of {expect['decision_in']}") + if decision in expect.get("decision_not", []): + problems.append(f"decision {decision} is ruled out for this case") + + if "spot" in expect: + want, spot = expect["spot"], result.get("spot") + if want is None and spot is not None: + problems.append(f"expected no spot, got {spot['wall_id']} {spot['span_ft']}") + if want is not None: + if spot is None: + problems.append("expected a spot, got none") + else: + a, b = want["span_within"] + s0, s1 = spot["span_ft"] + if spot["wall_id"] != want["wall_id"] or s0 < a - 1 / 12 or s1 > b + 1 / 12: + problems.append( + f"spot {spot['wall_id']} {spot['span_ft']} outside expected " + f"{want['wall_id']} {want['span_within']}" + ) + + for rule in expect.get("sweep_runs", []): + a, b = rule["start_ft"] + runs = [ + r + for r in result.get("sweep", []) + if r["wall_id"] == rule["wall_id"] and r["start_ft"][0] < b and r["start_ft"][1] > a + ] + if not runs: + problems.append(f"no sweep run overlaps {rule['wall_id']} starts ({a}, {b})") + for run in runs: + if run["outcome"] != rule["outcome"]: + problems.append( + f"sweep {run['wall_id']} starts {run['start_ft']} is {run['outcome']}, " + f"expected {rule['outcome']}" + ) + for field, key in (("failing", "failing_match"), ("unsure", "unsure_match")): + if key in rule and not any(rule[key] in c for c in run[field]): + problems.append( + f"sweep {run['wall_id']} starts {run['start_ft']}: no {field} check " + f"matching {rule[key]!r} (has {run[field]})" + ) + + for rule in expect.get("checks", []): + matching = [c for c in result.get("checks", []) if rule["match"] in c["id"]] + if not matching: + problems.append(f"no check matching {rule['match']!r}") + for check in matching: + for field in ("outcome", "unsure_cause"): + if field in rule and check.get(field) != rule[field]: + problems.append( + f"check {check['id']}: {field} {check.get(field)}, expected {rule[field]}" + ) + + if "missing_evidence_empty" in expect: + empty = not result.get("missing_evidence") + if empty != expect["missing_evidence_empty"]: + problems.append( + "missing_evidence " + + ("is empty" if empty else "is not empty") + + f", expected {'empty' if expect['missing_evidence_empty'] else 'requests'}" + ) + return problems + + +# --- Transforms for black-box properties ----------------------------------------------------- + + +def _flip_span(span: list[float]) -> list[float]: + return [-span[1], -span[0]] + + +def mirror_scene(scene: dict) -> dict: + """Reflect the scene left to right (x -> -x) so s becomes -s. + + Baselines are reversed after reflecting so they still run left to right as seen from + outside, which keeps every wall's outward side. Keyframes, stills and heading are dropped, + since a reflected camera pose is not a rotation. + """ + m = copy.deepcopy(scene) + for key in ("keyframes", "stills", "heading"): + m.pop(key, None) + x, y, z = m["meter"]["pos"] + m["meter"]["pos"] = [-x, y, z] + m["walls"] = [ + wall | {"baseline": [[-px, pz] for px, pz in reversed(wall["baseline"])]} + for wall in reversed(m["walls"]) + ] + for obj in m.get("objects", []): + obj["span_ft"] = _flip_span(obj["span_ft"]) + if "footprint" in obj: + obj["footprint"] = [[-px, pz] for px, pz in obj["footprint"]] + for area in m.get("ground", []): + area["polygon"] = [[-px, pz] for px, pz in area["polygon"]] + for key in ("overheads", "facing"): + for item in m.get(key, []): + item["span_ft"] = _flip_span(item["span_ft"]) + coverage = m.get("coverage") + if coverage: + for entry in coverage.get("observed", []): + entry["span_ft"] = _flip_span(entry["span_ft"]) + ends = coverage.get("ends") + if ends: + coverage["ends"] = { + side: ends[other] + for side, other in (("left", "right"), ("right", "left")) + if other in ends + } + return m + + +def outcome_lengths(result: dict) -> dict[str, float]: + """Total length of battery start positions per outcome, from the sweep.""" + totals = {"pass": 0.0, "unsure": 0.0, "fail": 0.0} + for run in result.get("sweep", []): + totals[run["outcome"]] += run["start_ft"][1] - run["start_ft"][0] + return totals + + +def comparable(result: dict) -> dict: + """The result with the fields that may legitimately differ between identical runs removed.""" + r = copy.deepcopy(result) + r["stats"].pop("elapsed_ms", None) + return r diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py new file mode 100644 index 000000000..54c44f9e9 --- /dev/null +++ b/verification/tests/test_e2e.py @@ -0,0 +1,189 @@ +import hashlib +import json +import re +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer + +import pytest + +from hsverify.e2e import Endpoint, SceneInput, discover_endpoint, judge, load_input + +SCENE = { + "meter": {"pos": [0.0, 4.0, 0.0], "wall_id": "w1"}, + "walls": [{"id": "w1", "baseline": [[-10.0, 0.0], [10.0, 0.0]]}], + "coverage": {"observed": [{"band": "wall", "span_ft": [-10.0, 10.0]}]}, +} + + +def openapi(paths: dict) -> dict: + return { + "paths": paths, + "components": { + "schemas": { + "Upload": {"properties": {"bundle": {"type": "string", "format": "binary"}}} + } + }, + } + + +def json_op() -> dict: + return {"post": {"requestBody": {"content": {"application/json": {"schema": {}}}}}} + + +def test_single_json_endpoint_is_found(): + assert discover_endpoint(openapi({"/place": json_op()})) == Endpoint( + "/place", "application/json" + ) + + +def test_multipart_endpoint_names_its_file_field(): + op = { + "post": { + "requestBody": { + "content": { + "multipart/form-data": {"schema": {"$ref": "#/components/schemas/Upload"}} + } + } + } + } + assert discover_endpoint(openapi({"/scenes": op})) == Endpoint( + "/scenes", "multipart/form-data", "bundle" + ) + + +def test_a_scene_named_path_wins_among_several(): + found = discover_endpoint(openapi({"/login": json_op(), "/placement": json_op()})) + assert found.path == "/placement" + + +def test_ambiguity_is_loud(): + with pytest.raises(SystemExit, match="Pass --endpoint"): + discover_endpoint(openapi({"/a": json_op(), "/b": json_op()})) + with pytest.raises(SystemExit, match="Pass --endpoint"): + discover_endpoint(openapi({})) + assert discover_endpoint(openapi({"/a": json_op(), "/b": json_op()}), "/b").path == "/b" + + +def test_case_file_is_loaded_with_its_expectations(tmp_path): + path = tmp_path / "c.json" + path.write_text( + json.dumps( + { + "id": "c1", + "scene": SCENE, + "expect": {"decision_not": ["pass"]}, + "rules_assumed": {"gas": 3.0}, + } + ) + ) + item = load_input(path) + assert item.name == "c1" + assert item.expect == {"decision_not": ["pass"]} + assert json.loads(item.raw) == SCENE + + +class FakeServer(BaseHTTPRequestHandler): + """Answers every scene with a consistent manual_review; `decide_pass` breaks C5.""" + + decide_pass = False + + def do_POST(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + if self.headers["Content-Type"].startswith("multipart/"): + start = body.index(b"PK") # the zip starts after the part headers + import io + import zipfile + + raw = zipfile.ZipFile(io.BytesIO(body[start:])).read("scene.json") + else: + raw = body + scene = json.loads(raw) + passing = self.decide_pass + result = { + "schema_version": "1.0", + "decision": "pass" if passing else "manual_review", + "summary": "", + "reasons": [{"code": "policy_not_approved", "message": ""}], + "policy": { + "id": None, + "version": None, + "auto_approve": passing, + "sources": [], + "rules_sha256": "0" * 64, + }, + "spot": None, + "route": None, + "checks": [], + "missing_evidence": [], + "ends": {}, + "sweep": [ + { + "wall_id": scene["walls"][0]["id"], + "start_ft": [1.0, 2.0], + "outcome": "pass" if passing else "unsure", + "failing": [], + "unsure": [], + } + ], + "stats": { + "candidates": 1, + "pass": int(passing), + "unsure": int(not passing), + "fail": 0, + "elapsed_ms": 1.0, + "input_sha256": hashlib.sha256(raw).hexdigest(), + }, + } + payload = json.dumps(result).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): + pass + + +@pytest.fixture +def fake_url(): + server = HTTPServer(("127.0.0.1", 0), FakeServer) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield f"http://127.0.0.1:{server.server_port}" + server.shutdown() + FakeServer.decide_pass = False + + +SCHEMAS = {"scene": {}, "result": {}} + + +def item() -> SceneInput: + raw = json.dumps(SCENE).encode() + return SceneInput("fake", SCENE, raw, expect={"decision_not": ["pass"]}) + + +@pytest.mark.parametrize( + "endpoint", + [Endpoint("/place", "application/json"), Endpoint("/place", "multipart/form-data", "file")], +) +def test_consistent_server_passes(fake_url, endpoint): + record = judge(fake_url, endpoint, item(), SCHEMAS) + assert record["status"] == "pass", record["problems"] + assert record["decision"] == "manual_review" + + +def test_server_breaking_coverage_rule_fails(fake_url): + FakeServer.decide_pass = True + record = judge(fake_url, Endpoint("/place", "application/json"), item(), SCHEMAS) + assert record["status"] == "fail" + joined = "\n".join(record["problems"]) + assert "ruled out for this case" in joined + assert "without coverage still produced a pass" in joined + assert re.search(r"ground under .* was not observed", joined) + + +def test_bad_input_is_not_blamed_on_the_server(fake_url): + strict = {"scene": {"required": ["nothing_has_this"]}, "result": {}} + record = judge(fake_url, Endpoint("/place", "application/json"), item(), strict) + assert record["status"] == "bad input" diff --git a/verification/tests/test_replaycheck.py b/verification/tests/test_replaycheck.py index a87681934..d4a3c2be3 100644 --- a/verification/tests/test_replaycheck.py +++ b/verification/tests/test_replaycheck.py @@ -68,16 +68,24 @@ def session_with(tmp_path, positions, spacing=0.5): write_jpeg_header(tmp_path / "keyframes" / f"k{i}.jpg", 640, 480) keyframes.append( { - "id": f"k{i}", "img": f"keyframes/k{i}.jpg", "w": 640, "h": 480, - "intrinsics": [500, 500, 320, 240], "pose": flat(pose(yaw(0), p)), - "timestamp": float(i), "tracking": "normal", "reason": "motion", + "id": f"k{i}", + "img": f"keyframes/k{i}.jpg", + "w": 640, + "h": 480, + "intrinsics": [500, 500, 320, 240], + "pose": flat(pose(yaw(0), p)), + "timestamp": float(i), + "tracking": "normal", + "reason": "motion", } - ) # fmt: skip + ) return { - "format": "measure-lab-session", "formatVersion": 2, "session": {}, + "format": "measure-lab-session", + "formatVersion": 2, + "session": {}, "gates": {"keyframeSpacingMeters": spacing, "keyframeSpacingDegrees": 15.0}, "keyframes": keyframes, - } # fmt: skip + } def test_valid_session_has_no_format_errors(tmp_path): diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py new file mode 100644 index 000000000..045dccd0e --- /dev/null +++ b/verification/tests/test_resultcheck.py @@ -0,0 +1,298 @@ +import copy +import hashlib +import json + +import pytest + +from hsverify.resultcheck import ( + assumption_mismatches, + comparable, + expectation_problems, + invariant_problems, + margin_problem, + mirror_scene, + observed, + outcome_lengths, +) + +SCENE = { + "meter": {"pos": [0.0, 4.0, 0.0], "wall_id": "w1"}, + "walls": [{"id": "w1", "baseline": [[-12.0, 0.0], [12.0, 0.0]]}], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [-6.0, -5.0], + "source": "tap", + "footprint": [[-6.0, 0.5], [-5.0, 1.0]], + }, + ], + "coverage": { + "ends": {"left": {"kind": "limit"}, "right": {"kind": "unexplored"}}, + "observed": [ + {"band": "wall", "span_ft": [-12.0, 10.0]}, + {"band": "ground", "span_ft": [-12.0, 4.0], "out_ft": 3.0}, + {"band": "ground", "span_ft": [4.0, 10.0], "out_ft": 1.0}, + ], + }, +} + + +def check(outcome="pass", measured=5.0, error=0.3, threshold=3.0, cmp="at_least", cause=None): + c = { + "id": "gas_clearance", + "label": "Gas", + "outcome": outcome, + "reason": "", + "measured_ft": measured, + "plus_minus_ft": error, + "threshold_ft": threshold, + "comparison": cmp, + "rule": {"key": "gas_clearance_ft", "source": "", "placeholder": False}, + } + if cause: + c["unsure_cause"] = cause + return c + + +def result(decision="manual_review", spot=True, checks=None, sweep=None, sent=b"{}"): + s = None + if spot: + s = { + "outcome": "pass", + "wall_id": "w1", + "segment": 0, + "span_ft": [1.0, 1.0 + 31 / 12], + "width_ft": 31 / 12, + "depth_ft": 22 / 12, + "height_ft": 3.3, + "footprint": [[1, 0], [3.58, 0], [3.58, 1.83], [1, 1.83]], + "center": [2.29, 0.92], + "along": [1, 0], + "outward": [0, 1], + "meter_offset_ft": [2.29, 0.92], + "route_length_ft": 1.0, + } + return { + "schema_version": "1.0", + "decision": decision, + "summary": "", + "reasons": [] if decision == "pass" else [{"code": "policy_not_approved", "message": ""}], + "policy": { + "id": None, + "version": None, + "auto_approve": decision != "manual_review", + "sources": ["public"], + "rules_sha256": "0" * 64, + }, + "spot": s, + "route": None if s is None else {"outcome": "pass"}, + "checks": [check()] if checks is None else checks, + "missing_evidence": [], + "ends": {}, + "sweep": sweep + if sweep is not None + else [ + { + "wall_id": "w1", + "start_ft": [1.0, 1.0], + "outcome": "pass", + "failing": [], + "unsure": [], + }, + ], + "stats": { + "candidates": 1, + "pass": 1, + "unsure": 0, + "fail": 0, + "elapsed_ms": 3.0, + "input_sha256": hashlib.sha256(sent).hexdigest(), + }, + } + + +def test_consistent_result_has_no_problems(): + assert invariant_problems(SCENE, result(), sent=b"{}") == [] + + +@pytest.mark.parametrize( + ("cmp", "measured", "error", "threshold", "expected"), + [ + ("at_least", 3.31, 0.3, 3.0, "pass"), + ("at_least", 3.3, 0.3, 3.0, "unsure"), # margin equals the error + ("at_least", 2.7, 0.3, 3.0, "unsure"), + ("at_least", 2.69, 0.3, 3.0, "fail"), + ("at_most", 19.69, 0.3, 20.0, "pass"), + ("at_most", 19.7, 0.3, 20.0, "unsure"), + ("at_most", 20.31, 0.3, 20.0, "fail"), + ], +) +def test_margin_rule(cmp, measured, error, threshold, expected): + for outcome in ("pass", "fail", "unsure"): + problem = margin_problem(check(outcome, measured, error, threshold, cmp)) + assert (problem is None) == (outcome == expected), (outcome, problem) + + +def test_margin_rule_skips_checks_without_numbers(): + assert margin_problem(check("unsure", measured=None, cause="unobserved")) is None + + +def test_decision_consistency(): + bad_pass = result("pass", checks=[check("unsure", 3.2, cause="margin")]) + msgs = invariant_problems(SCENE, bad_pass) + assert any("checks not passing" in m for m in msgs) + reject_with_spot = result("reject") + assert any("reject but a spot" in m for m in invariant_problems(SCENE, reject_with_spot)) + auto_off = result("pass") + auto_off["policy"]["auto_approve"] = False + assert any("auto_approve" in m for m in invariant_problems(SCENE, auto_off)) + + +def test_unsure_cause_bookkeeping(): + msgs = invariant_problems(SCENE, result(checks=[check("unsure", 3.2)])) + assert any("unsure without unsure_cause" in m for m in msgs) + msgs = invariant_problems(SCENE, result(checks=[check("pass", cause="margin")])) + assert any("unsure_cause on a pass check" in m for m in msgs) + + +def test_counts_hash_and_offset(): + r = result() + r["stats"]["fail"] = 2 + r["spot"]["meter_offset_ft"] = [2.0, 0.92] + msgs = invariant_problems(SCENE, r, sent=b"other bytes") + assert any(m.startswith("stats: pass") for m in msgs) + assert any("input_sha256" in m for m in msgs) + assert any("meter_offset_ft" in m for m in msgs) + + +def test_observed_merges_and_filters_ground_depth(): + assert observed(SCENE, "wall") == [(-12.0, 10.0)] + assert observed(SCENE, "ground", min_out_ft=22 / 12) == [(-12.0, 4.0)] + assert observed({"meter": {}}, "wall") == [] + + +def test_pass_over_unobserved_ground_is_flagged(): + # Starts [3, 3] put the battery over s = [3, 5.58]; ground past s = 4 was seen only 1 ft out. + sweep = [ + {"wall_id": "w1", "start_ft": [3.0, 3.0], "outcome": "pass", "failing": [], "unsure": []} + ] + msgs = invariant_problems(SCENE, result(sweep=sweep)) + assert any("ground under" in m for m in msgs) + + +def test_pass_whose_route_crosses_unobserved_wall_is_flagged(): + scene = copy.deepcopy(SCENE) + scene["coverage"]["observed"][0]["span_ft"] = [0.5, 10.0] # the wall at s = [0, 0.5] unseen + msgs = invariant_problems(scene, result()) + assert any("wall and cable route" in m for m in msgs) + + +def test_no_coverage_never_passes(): + scene = {k: v for k, v in SCENE.items() if k != "coverage"} + msgs = invariant_problems(scene, result("pass")) + assert any("no coverage at all" in m for m in msgs) + + +def test_photo_request_for_an_observed_area_is_flagged(): + r = result() + r["missing_evidence"] = [ + {"kind": "band", "band": "wall", "span_ft": [-3.0, -1.0], "message": ""} + ] + assert any("lists as observed" in m for m in invariant_problems(SCENE, r)) + + +def test_expectations(): + r = result( + sweep=[ + { + "wall_id": "w1", + "start_ft": [-9.0, -4.0], + "outcome": "fail", + "failing": ["gas_clearance"], + "unsure": [], + }, + { + "wall_id": "w1", + "start_ft": [1.0, 1.0], + "outcome": "pass", + "failing": [], + "unsure": [], + }, + ] + ) + ok = { + "decision_not": ["pass"], + "spot": {"wall_id": "w1", "span_within": [0.0, 4.0]}, + "sweep_runs": [ + {"wall_id": "w1", "start_ft": [-8, -5], "outcome": "fail", "failing_match": "gas"} + ], + "checks": [{"match": "gas", "outcome": "pass"}], + "missing_evidence_empty": True, + } + assert expectation_problems(ok, r) == [] + bad = { + "decision_in": ["reject"], + "spot": None, + "sweep_runs": [{"wall_id": "w1", "start_ft": [0, 2], "outcome": "fail"}], + "checks": [{"match": "route", "outcome": "pass"}], + } + msgs = expectation_problems(bad, r) + assert len(msgs) == 4, msgs + + +def test_assumption_mismatch(): + assert assumption_mismatches({"gas": 3.0}, result()) == [] + assert assumption_mismatches({"gas": 4.0}, result()) + + +def test_mirror_negates_s_and_keeps_baselines_left_to_right(): + m = mirror_scene(SCENE) + assert m["walls"][0]["baseline"] == [[-12.0, 0.0], [12.0, 0.0]] + assert m["objects"][0]["span_ft"] == [5.0, 6.0] + assert m["objects"][0]["footprint"] == [[6.0, 0.5], [5.0, 1.0]] + assert m["coverage"]["observed"][0]["span_ft"] == [-10.0, 12.0] + assert m["coverage"]["ends"] == {"left": {"kind": "unexplored"}, "right": {"kind": "limit"}} + assert mirror_scene(m) == json.loads(json.dumps(SCENE)) # an involution + assert SCENE["objects"][0]["span_ft"] == [-6.0, -5.0] # input untouched + + +def test_mirror_reverses_a_wall_chain(): + scene = { + "meter": {"pos": [0.0, 4.0, 0.0], "wall_id": "w1"}, + "walls": [ + {"id": "w1", "baseline": [[-4.0, 0.0], [4.0, 0.0]]}, + {"id": "w2", "baseline": [[4.0, 0.0], [4.0, 12.0]]}, + ], + } + m = mirror_scene(scene) + assert [w["id"] for w in m["walls"]] == ["w2", "w1"] + assert m["walls"][0]["baseline"] == [[-4.0, 12.0], [-4.0, 0.0]] + assert m["walls"][1]["baseline"] == [[-4.0, 0.0], [4.0, 0.0]] + # Chain stays connected: w2 ends where w1 starts. + assert m["walls"][0]["baseline"][-1] == m["walls"][1]["baseline"][0] + + +def test_outcome_lengths_and_comparable(): + r = result( + sweep=[ + { + "wall_id": "w1", + "start_ft": [0.0, 2.0], + "outcome": "pass", + "failing": [], + "unsure": [], + }, + { + "wall_id": "w1", + "start_ft": [2.0, 3.0], + "outcome": "fail", + "failing": [], + "unsure": [], + }, + ] + ) + assert outcome_lengths(r) == {"pass": 2.0, "unsure": 0.0, "fail": 1.0} + other = copy.deepcopy(r) + other["stats"]["elapsed_ms"] = 99.0 + assert comparable(r) == comparable(other) From 42b9fdcc779b4ed615c794e8372402aa83b43d57 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 03:15:32 -0500 Subject: [PATCH 05/59] verification: probe checks the Simulator can read a host-path replay; document checks Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/README.md | 38 +++++++++++++++++++ .../StateProbe/StateProbeApp.swift | 13 +++++++ 2 files changed, 51 insertions(+) diff --git a/verification/README.md b/verification/README.md index 80346d610..71e1fb33e 100644 --- a/verification/README.md +++ b/verification/README.md @@ -8,6 +8,8 @@ workstreams push. Nothing here fixes their code; findings go back to the owning | Unit tests for these tools | `make test` | pass or fail | | App in the Simulator, a screenshot per `STATE` | `make sim REF=origin/t3/ios-mvf ARGS="--replay --autopilot --server-url "` | `~/house-scanning-data/reports/sim//index.html` | | Runner self-test | `make sim-probe` | same, for `fixtures/state-probe` | +| A replay session (C3) is well formed, and how far its poses are from ground truth | `uv run python -m hsverify.replaycheck ` | printed | +| Scenes through the server's HTTP API, judged against C1, C2 and C5 | `uv run python -m hsverify.e2e --server-ref origin/t3/server [--real scene.zip]` | `~/house-scanning-data/reports/e2e//report.md` | Requirements: uv, Xcode 26 or newer with an iOS Simulator runtime. @@ -32,3 +34,39 @@ The report folder holds `index.html` (screenshots in state order), `report.json` `fixtures/state-probe` is a ten-line app that logs a fixed sequence, including a 0.3 s state, so the runner can be checked without the real app. + +## Replay check + +`hsverify/replaycheck.py` reads a Measure Lab v2 session and checks what the app and server +rely on: every listed JPEG exists at the stated size, poses are rigid, timestamps increase, +and motion keyframes respect the spacing gate the session declares. With `ground_truth.json` +beside it, it aligns ground truth to ARKit by heading and origin (both worlds are gravity +aligned) and reports the error after the best fit, after also removing scale, and when +anchored at the first keyframe, as the app anchors on the meter. The ARKit-to-truth path +length ratio is printed on its own because a scale difference dominates the other numbers. +It does not say which track is right: ADVIO's ground truth has its own scale error (see +`experiments/evals` on `t3/evals`). + +## End-to-end server check + +`hsverify/e2e.py` starts the server from a ref in a `/tmp` worktree, or uses `--server-url`, +and reads the scene endpoint from its OpenAPI document. It stops rather than guesses when +more than one POST could take a scene. Every scene is validated against the scene schema at +the same ref first, so bad input is not blamed on the server. Every answer must validate +against the result schema and pass `hsverify/resultcheck.py`: + +- `pass` needs a passing spot, every check passing, no photo request and an approved policy; + `reject` has no spot and no photo request; `manual_review` gives reasons. +- Each check's outcome follows from its numbers (C5): at least T needs measured − error > T + to pass and measured + error < T to fail; anything else is unsure. At most T is mirrored. +- No battery start passes unless the wall and cable route back to the meter, and the ground + under the battery out to its depth, were observed. No photo request covers an observed area. +- Counts add up, `input_sha256` is the hash of the bytes sent, and the spot's offset from the + meter equals its centre minus the meter. + +Each scene is also sent again (identical result apart from timing), mirrored left to right +(same decision and the same length of passing, unsure and failing starts), and without its +coverage (no pass anywhere). Case files in `e2e/cases/` add the outcomes their geometry +forces. Each one names the thresholds it assumes, and a case whose assumption differs from the +server's rules is reported as an assumption mismatch, not as a pass or fail. Scenes passed +with `--real` must answer within 1 s. diff --git a/verification/fixtures/state-probe/StateProbe/StateProbeApp.swift b/verification/fixtures/state-probe/StateProbe/StateProbeApp.swift index feaab5290..7daf0a53e 100644 --- a/verification/fixtures/state-probe/StateProbe/StateProbeApp.swift +++ b/verification/fixtures/state-probe/StateProbe/StateProbeApp.swift @@ -4,6 +4,8 @@ import SwiftUI /// Walks through a fixed list of states, logging each as `STATE=` the way contract C4 /// asks the real app to. `quick` lasts 0.3 s so the runner's transient flag is exercised. /// Launch arguments are echoed on screen so the runner's argument passing can be checked. +/// With `-replay `, it also logs whether the app can read `/session.json` +/// from the host path, which is how the real app receives a replay in the Simulator. @main struct StateProbeApp: App { var body: some Scene { @@ -16,6 +18,12 @@ private let steps: [(name: String, seconds: Double)] = [ ("probe_start", 2), ("quick", 0.3), ("probe_middle", 2), ("probe_done", 2), ] +private func replayFolder() -> String? { + let args = CommandLine.arguments + guard let i = args.firstIndex(of: "-replay"), i + 1 < args.count else { return nil } + return args[i + 1] +} + struct ProbeView: View { @State private var current = "" @@ -27,6 +35,11 @@ struct ProbeView: View { } .padding() .task { + if let replay = replayFolder() { + let url = URL(fileURLWithPath: replay).appendingPathComponent("session.json") + let readable = (try? Data(contentsOf: url)).map { !$0.isEmpty } ?? false + log.info("STATE=\(readable ? "replay_readable" : "replay_unreadable", privacy: .public)") + } for step in steps { current = step.name log.info("STATE=\(step.name, privacy: .public)") From e0959c2d3f6e3d802cf0af4495b60ce2270b7718 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 03:19:06 -0500 Subject: [PATCH 06/59] verification: solver shim, per-use worktrees, margin rule for other unsure causes The shim serves a server ref's solve() over HTTP so the end-to-end checks run before that branch has an API. Worktrees get a unique path per use because a path shared by SHA let concurrent checks delete each other's tree. An unsure check caused by an unobserved area may rest on passing numbers, never on numbers that clearly fail. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/hsverify/e2e.py | 37 +++++++------ verification/hsverify/gitref.py | 21 ++++--- verification/hsverify/resultcheck.py | 23 +++++--- verification/hsverify/solver_shim.py | 77 ++++++++++++++++++++++++++ verification/tests/test_gitref.py | 14 +++++ verification/tests/test_resultcheck.py | 9 +++ 6 files changed, 147 insertions(+), 34 deletions(-) create mode 100644 verification/hsverify/solver_shim.py create mode 100644 verification/tests/test_gitref.py diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index b718b9aa9..86c570647 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -127,27 +127,24 @@ def find_app(server_dir: Path) -> str: return found[0] +SHIM = Path(__file__).with_name("solver_shim.py") + + @contextlib.contextmanager -def server_from_ref(sha: str, log_path: Path) -> Iterator[str]: +def server_from_ref(sha: str, log_path: Path, via_shim: bool = False) -> Iterator[str]: + """The server at `sha` on a free port: its own FastAPI app, or `solver_shim` around `solve`.""" with gitref.detached_worktree(sha) as tree: server_dir = tree / "server" subprocess.run(["uv", "sync", "--locked", "--quiet"], cwd=server_dir, check=True) - app = find_app(server_dir) port = free_port() url = f"http://127.0.0.1:{port}" + if via_shim: + target = ["python", str(SHIM), "--server-dir", str(server_dir), "--port", str(port)] + else: + target = ["uvicorn", find_app(server_dir), "--host", "127.0.0.1", "--port", str(port)] with log_path.open("w") as log: proc = subprocess.Popen( - [ - "uv", - "run", - "--quiet", - "uvicorn", - app, - "--host", - "127.0.0.1", - "--port", - str(port), - ], + ["uv", "run", "--quiet", *target], cwd=server_dir, stdout=log, stderr=subprocess.STDOUT, @@ -360,7 +357,7 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: lines = [ f"# End-to-end run against `{meta['server_ref']}` at `{meta['server_sha'][:12]}`", "", - f"{meta['started_at']}. Endpoint `{meta['endpoint']}`. " + f"{meta['started_at']}. Endpoint `{meta['endpoint']}` via {meta['transport']}. " + ", ".join(f"{v} {k}" for k, v in sorted(counts.items())), "", "| Scene | Status | Decision | ms | Problems |", @@ -384,6 +381,11 @@ def main(argv: list[str] | None = None) -> int: where.add_argument("--server-url", help="use a running server instead of starting one") parser.add_argument("--schema-ref", help="ref for the schemas (default: --server-ref)") parser.add_argument("--endpoint", help="POST path, when discovery is ambiguous") + parser.add_argument( + "--via-shim", + action="store_true", + help="serve the ref's solver.solve through solver_shim (before the ref has an API)", + ) parser.add_argument("--cases", type=Path, default=DEFAULT_CASES) parser.add_argument( "--scene", @@ -420,16 +422,14 @@ def main(argv: list[str] | None = None) -> int: if not items: raise SystemExit("No scenes: add case files to e2e/cases or pass --scene/--real.") - server_sha = schema_sha if not args.server_url else "(running server)" - if not args.server_url: - server_sha = gitref.resolve(args.server_ref) + server_sha = "(running server)" if args.server_url else gitref.resolve(args.server_ref) stamp = dt.datetime.now().strftime("%Y%m%d-%H%M%S") out = (args.out or DEFAULT_REPORTS / f"{stamp}-{server_sha[:8]}").expanduser() out.mkdir(parents=True, exist_ok=True) with contextlib.ExitStack() as stack: url = args.server_url or stack.enter_context( - server_from_ref(server_sha, out / "server.log") + server_from_ref(server_sha, out / "server.log", via_shim=args.via_shim) ) openapi = json.loads(urllib.request.urlopen(f"{url}/openapi.json", timeout=10).read()) endpoint = discover_endpoint(openapi, args.endpoint) @@ -457,6 +457,7 @@ def main(argv: list[str] | None = None) -> int: "schema_sha": schema_sha, "sha": server_sha, "endpoint": f"POST {endpoint.path} ({endpoint.content_type})", + "transport": "solver shim (HTTP layer not tested)" if args.via_shim else "server API", "command": " ".join([sys.executable, "-m", "hsverify.e2e", *(argv or sys.argv[1:])]), "cases_sha256": hashlib.sha256(b"".join(i.raw for i in items)).hexdigest(), } diff --git a/verification/hsverify/gitref.py b/verification/hsverify/gitref.py index 7a6fff74f..e711e295d 100644 --- a/verification/hsverify/gitref.py +++ b/verification/hsverify/gitref.py @@ -9,6 +9,7 @@ import contextlib import shutil import subprocess +import tempfile from collections.abc import Iterator from pathlib import Path @@ -49,18 +50,20 @@ def show(ref: str, path: str) -> str | None: @contextlib.contextmanager def detached_worktree(sha: str, keep: bool = False) -> Iterator[Path]: - """A detached worktree of `sha` at /tmp/hs-verify-, removed afterwards unless `keep`. + """A detached worktree of `sha` under /tmp, removed afterwards unless `keep`. - An existing worktree at that path is reused, since it holds the same commit. + The path is unique per call: several checks (and the scoreboard) run at once on this Mac, + and a path shared by SHA let one run delete a tree another was still using. """ - path = TMP_ROOT / f"hs-verify-{sha[:12]}" - if not (path / ".git").exists(): - if path.exists(): - shutil.rmtree(path) - git("worktree", "add", "--detach", "--force", str(path), sha) - # The landing page submodule is not needed for any check here. + path = Path(tempfile.mkdtemp(prefix=f"hs-verify-{sha[:12]}-", dir=TMP_ROOT)) + path.rmdir() # git creates it + git("worktree", "add", "--detach", "--force", str(path), sha) try: yield path finally: if not keep: - git("worktree", "remove", "--force", str(path)) + try: + git("worktree", "remove", "--force", str(path)) + except subprocess.CalledProcessError: + shutil.rmtree(path, ignore_errors=True) + git("worktree", "prune") diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 76ead2d6e..2c73cf49a 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -69,7 +69,13 @@ def covers(intervals: list[tuple[float, float]], a: float, b: float, slack: floa def margin_problem(check: dict) -> str | None: - """The C5 rule for one check, when its numbers are all present.""" + """The C5 rule for one check, when its numbers are all present. + + PASS needs a clear pass and FAIL a clear fail. UNSURE from the margin needs neither. UNSURE + for another cause (an unobserved area, an unknown attribute, a review rule) may sit on + numbers that would pass, but not on numbers that clearly fail: a measured object inside + the clearance by more than the error fails whatever else went unseen. + """ m, e, t, cmp = ( check.get(k) for k in ("measured_ft", "plus_minus_ft", "threshold_ft", "comparison") ) @@ -80,12 +86,15 @@ def margin_problem(check: dict) -> str | None: else: clear_pass, clear_fail = m + e < t - EPS, m - e > t + EPS expected = "pass" if clear_pass else "fail" if clear_fail else "unsure" - if check["outcome"] != expected: - return ( - f"check {check['id']}: measured {m} ± {e} {cmp} {t} should be {expected}, " - f"server says {check['outcome']}" - ) - return None + outcome, cause = check["outcome"], check.get("unsure_cause") + other_cause = outcome == "unsure" and cause not in (None, "margin") + if outcome == expected or (other_cause and not clear_fail): + return None + because = f" ({cause})" if cause else "" + return ( + f"check {check['id']}: measured {m} ± {e} {cmp} {t} should be {expected}, " + f"server says {outcome}{because}" + ) def invariant_problems(scene: dict, result: dict, sent: bytes | None = None) -> list[str]: diff --git a/verification/hsverify/solver_shim.py b/verification/hsverify/solver_shim.py new file mode 100644 index 000000000..bf69d67e1 --- /dev/null +++ b/verification/hsverify/solver_shim.py @@ -0,0 +1,77 @@ +"""Serve the server branch's `solve` over HTTP until that branch has its own API. + +Runs inside the server's own environment (`uv run --project /server python this.py`), +imports its `rules`, `scene` and `solver` modules unchanged, and exposes one POST endpoint +plus a minimal OpenAPI document so `hsverify.e2e` can discover it. Only the public rules load: +the worktree has no `private/` folder and the override variable is cleared. + +Results through this shim test the solver and the ingest code, not the server's HTTP layer. +Stdlib only, because it runs in someone else's environment. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +OPENAPI = { + "openapi": "3.1.0", + "info": {"title": "hsverify solver shim", "version": "1"}, + "paths": { + "/shim/scene": {"post": {"requestBody": {"content": {"application/json": {"schema": {}}}}}} + }, +} + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--server-dir", required=True) + parser.add_argument("--port", type=int, required=True) + args = parser.parse_args() + os.environ.pop("HOUSESCAN_PRIVATE_RULES", None) + sys.path.insert(0, args.server_dir) + from rules import load_rules + from scene import SceneError, parse_scene + from solver import solve + + loaded = load_rules(None) + + class Handler(BaseHTTPRequestHandler): + def _send(self, status: int, payload: object) -> None: + body = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: + if self.path == "/openapi.json": + self._send(200, OPENAPI) + else: + self._send(404, {"detail": "not found"}) + + def do_POST(self) -> None: + if self.path != "/shim/scene": + self._send(404, {"detail": "not found"}) + return + body = self.rfile.read(int(self.headers["Content-Length"])) + try: + scene = parse_scene(json.loads(body), loaded.rules, input_bytes=body) + self._send(200, solve(scene, loaded)) + except (SceneError, ValueError) as exc: + self._send(422, {"detail": str(exc)}) + except Exception as exc: # a solver crash is a finding, not a shim failure + self._send(500, {"detail": f"{type(exc).__name__}: {exc}"}) + + def log_message(self, *args: object) -> None: + pass + + ThreadingHTTPServer(("127.0.0.1", args.port), Handler).serve_forever() + + +if __name__ == "__main__": + main() diff --git a/verification/tests/test_gitref.py b/verification/tests/test_gitref.py new file mode 100644 index 000000000..da2c1511a --- /dev/null +++ b/verification/tests/test_gitref.py @@ -0,0 +1,14 @@ +from hsverify import gitref + + +def test_two_uses_of_one_sha_get_separate_trees_and_both_clean_up(): + sha = gitref.resolve("HEAD") + with gitref.detached_worktree(sha) as a, gitref.detached_worktree(sha) as b: + assert a != b + assert (a / "verification").is_dir() and (b / "verification").is_dir() + assert not a.exists() and not b.exists() + + +def test_show_returns_none_for_a_missing_file(): + assert gitref.show("HEAD", "no/such/file") is None + assert gitref.show("HEAD", "verification/pyproject.toml") is not None diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 045dccd0e..299734058 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -134,6 +134,15 @@ def test_margin_rule(cmp, measured, error, threshold, expected): assert (problem is None) == (outcome == expected), (outcome, problem) +def test_unsure_for_another_cause_may_sit_on_passing_numbers_but_not_failing_ones(): + assert margin_problem(check("unsure", 5.9, 0.6, 3.0, cause="unobserved")) is None + assert margin_problem(check("unsure", 3.1, 0.6, 3.0, cause="unknown_attribute")) is None + problem = margin_problem(check("unsure", 1.0, 0.6, 3.0, cause="unobserved")) + assert problem is not None and "should be fail" in problem + # A margin-caused unsure must really be inside the band. + assert margin_problem(check("unsure", 5.9, 0.6, 3.0, cause="margin")) is not None + + def test_margin_rule_skips_checks_without_numbers(): assert margin_problem(check("unsure", measured=None, cause="unobserved")) is None From 328373d81f280496380c0a9737f1dd0cc3ac50bb Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 03:19:28 -0500 Subject: [PATCH 07/59] verification: record load average with e2e latency Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/hsverify/e2e.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index 86c570647..e63baefe9 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -27,6 +27,7 @@ import hashlib import io import json +import os import re import socket import subprocess @@ -358,6 +359,7 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: f"# End-to-end run against `{meta['server_ref']}` at `{meta['server_sha'][:12]}`", "", f"{meta['started_at']}. Endpoint `{meta['endpoint']}` via {meta['transport']}. " + f"Load average {meta['load_average_1_5_15']} on {os.cpu_count()} cores. " + ", ".join(f"{v} {k}" for k, v in sorted(counts.items())), "", "| Scene | Status | Decision | ms | Problems |", @@ -459,6 +461,8 @@ def main(argv: list[str] | None = None) -> int: "endpoint": f"POST {endpoint.path} ({endpoint.content_type})", "transport": "solver shim (HTTP layer not tested)" if args.via_shim else "server API", "command": " ".join([sys.executable, "-m", "hsverify.e2e", *(argv or sys.argv[1:])]), + # Latency on this shared Mac depends on what else runs; keep the load with the numbers. + "load_average_1_5_15": [round(x, 1) for x in os.getloadavg()], "cases_sha256": hashlib.sha256(b"".join(i.raw for i in items)).hexdigest(), } report = write_report(out, meta, records) From 2ae5a802b41793f6135e56d6c22f33e3109e7761 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 03:21:34 -0500 Subject: [PATCH 08/59] verification: add the scoreboard; e2e reports app exports make scoreboard lists every open PR with its head and CI state, and each overnight-plan metric with the evidence behind its status, read from the branches, the PRs and the local run reports. The e2e runner takes --app-export for scenes the app exported from a replay, and records whether they validated and passed. The UX checklist's state table now uses the app's phase names. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/Makefile | 6 + verification/SCOREBOARD.md | 80 +++ verification/hsverify/e2e.py | 25 +- verification/hsverify/scoreboard.py | 773 ++++++++++++++++++++++++ verification/scoreboard/.gitignore | 1 + verification/scoreboard/metrics.yaml | 274 +++++++++ verification/scoreboard/scoreboard.json | 545 +++++++++++++++++ verification/tests/test_scoreboard.py | 593 ++++++++++++++++++ verification/ux/checklist.md | 35 +- 9 files changed, 2310 insertions(+), 22 deletions(-) create mode 100644 verification/SCOREBOARD.md create mode 100644 verification/hsverify/scoreboard.py create mode 100644 verification/scoreboard/.gitignore create mode 100644 verification/scoreboard/metrics.yaml create mode 100644 verification/scoreboard/scoreboard.json create mode 100644 verification/tests/test_scoreboard.py diff --git a/verification/Makefile b/verification/Makefile index 89787714c..79e8d3cf6 100644 --- a/verification/Makefile +++ b/verification/Makefile @@ -20,3 +20,9 @@ sim: sim-probe: uv run python -m hsverify.simrun --ref HEAD --project verification/fixtures/state-probe/StateProbe.xcodeproj \ --scheme StateProbe --until probe_done --extra-arg=-probeArg --label probe + +# Every open PR with its CI, and every overnight-plan metric with its evidence or gap. Writes +# SCOREBOARD.md and scoreboard/scoreboard.json. ARGS=--slow also runs the slow test probes; +# ARGS="--slow S2-1" runs only that one. +scoreboard: + uv run python -m hsverify.scoreboard $(ARGS) diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md new file mode 100644 index 000000000..aefb762ce --- /dev/null +++ b/verification/SCOREBOARD.md @@ -0,0 +1,80 @@ +# Scoreboard + +Generated 2026-09-26 03:21 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. + +## Open pull requests + +| # | Branch | Head | Draft | CI | Updated | +| --- | --- | --- | --- | --- | --- | +| [#14](https://github.com/SamGu-NRX/house-scanning/pull/14) | `t3/docs-cleanup` | `9c9ceb2a` | no | 4 success | 2026-09-26 03:15 CDT | +| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `328373d8` | yes | 4 success | 2026-09-26 03:19 CDT | +| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `b0b8a274` | yes | 4 success | 2026-09-26 03:11 CDT | +| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `fcd45741` | yes | 4 success, 1 pending | 2026-09-26 03:20 CDT | +| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `6885b7b5` | yes | 5 success | 2026-09-26 03:07 CDT | +| [#9](https://github.com/SamGu-NRX/house-scanning/pull/9) | `codex/capture-contract-probe` | `4b6c4309` | yes | 4 success | 2026-09-26 00:08 CDT | +| [#8](https://github.com/SamGu-NRX/house-scanning/pull/8) | `t3/testflight` | `b896d59f` | no | 4 success | 2026-09-26 03:18 CDT | +| [#7](https://github.com/SamGu-NRX/house-scanning/pull/7) | `t3/measure-lab` | `45770a3b` | no | 6 success | 2026-09-26 03:19 CDT | +| [#6](https://github.com/SamGu-NRX/house-scanning/pull/6) | `t3/research` | `a7d91f16` | no | 4 success | 2026-09-26 02:11 CDT | +| [#4](https://github.com/SamGu-NRX/house-scanning/pull/4) | `t3/scoring-harness` | `77ce830d` | no | 5 success | 2026-09-26 03:17 CDT | + +## S1 Evals on real data + +`origin/t3/evals` at [`b0b8a274`](https://github.com/SamGu-NRX/house-scanning/tree/b0b8a2745e3ea67cb8f70df3943177e146f591b9). 1 met, 1 partial, 2 gap, 1 not run. + +| Metric | Text | Status | Evidence | +| --- | --- | --- | --- | +| S1-1 | One command regenerates every number from a clean checkout | gap | not yet reviewed | +| S1-2 | Dataset URLs and sha256 recorded | partial | found ADVIO; missing ETH3D [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/b0b8a2745e3ea67cb8f70df3943177e146f591b9/experiments/evals/README.md) | +| S1-3 | Metric code tested on hand-computed cases | not run | slow; run with --slow | +| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/b0b8a2745e3ea67cb8f70df3943177e146f591b9/experiments/evals/results/advio_drift.md) | +| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | gap | none found in 2 files; missing 1-3 m row, 3-10 m row, scale error, verdict | + +## S2 Server and solver + +`origin/t3/server` at [`fcd45741`](https://github.com/SamGu-NRX/house-scanning/tree/fcd457411a68171509d53b87b11c115f55a2a765). 3 met, 1 gap, 2 not run, 1 no evidence yet. + +| Metric | Text | Status | Evidence | +| --- | --- | --- | --- | +| S2-1 | pytest green | not run | slow; run with --slow | +| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/tests/test_golden_public.py) | +| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | gap | none found in 5 files; missing missing coverage, equality unsure, mirror, larger clearance | +| S2-4 | Schemas published | met | present at fcd45741 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/schemas/result.schema.json) | +| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/tests/test_schemas.py) | +| S2-6 | A real-derived scene returns a result in under 1 s | no evidence yet | no report matches ~/house-scanning-data/reports/e2e/*/report.json | +| S2-7 | ruff clean | not run | slow; run with --slow | + +## S3 iOS capture app + +`origin/t3/ios-mvf` at [`6885b7b5`](https://github.com/SamGu-NRX/house-scanning/tree/6885b7b5ad872df7f73a6b915e5837cd38a2034d). 3 met, 5 gap, 1 no evidence yet. + +| Metric | Text | Status | Evidence | +| --- | --- | --- | --- | +| S3-1 | CI build green | met | iOS build: success at 6885b7b5 [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | +| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/6885b7b5ad872df7f73a6b915e5837cd38a2034d/ios/project.yml) | +| S3-3 | Zero build warnings | met | build.warnings = length 0 (want 0): ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json | +| S3-4 | Unit tests for coverage, auto-capture and guidance | gap | none found in 1 files; missing coverage, auto-capture, guidance | +| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | gap | none found in 1 files; missing replay argument, autopilot argument, gap loop | +| S3-6 | Accessibility audit on every screen | gap | 0 of 1 matches needed, searched 1 files | +| S3-7 | Screenshots of each state in the PR | gap | 0 of 9 matches in the PR body [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | +| S3-8 | Exported scene.json validates against C1 | no evidence yet | no report matches ~/house-scanning-data/reports/e2e/*/report.json | +| S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | + +## S4 Verification + +`origin/t3/verification` at [`328373d8`](https://github.com/SamGu-NRX/house-scanning/tree/328373d81f280496380c0a9737f1dd0cc3ac50bb), `origin/t3/ios-mvf` at [`6885b7b5`](https://github.com/SamGu-NRX/house-scanning/tree/6885b7b5ad872df7f73a6b915e5837cd38a2034d). 3 gap, 1 no evidence yet. + +| Metric | Text | Status | Evidence | +| --- | --- | --- | --- | +| S4-1 | End to end from a real replay to a placement result passes | no evidence yet | no report matches ~/house-scanning-data/reports/e2e/*/report.json | +| S4-2 | Screenshot report per app state | gap | states = length 0 (want at least 9): ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json | +| S4-3 | Product description written from the code and checked against screenshots | gap | not yet reviewed | +| S4-4 | Scoreboard of every PR | gap | missing at 328373d8: verification/SCOREBOARD.md | + +## M Maintenance + +`origin/t3/docs-cleanup` at [`9c9ceb2a`](https://github.com/SamGu-NRX/house-scanning/tree/9c9ceb2a608e9762bd002f20f1b979694bb0c92f). 1 met, 1 gap. + +| Metric | Text | Status | Evidence | +| --- | --- | --- | --- | +| M-1 | AGENTS.md shorter than on main | met | exit 0: AGENTS.md 39 lines, main 45 | +| M-2 | AGENTS.md accurate, every path and command real | gap | not yet reviewed | diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index e63baefe9..5fac903d7 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -76,6 +76,7 @@ class SceneInput: real: bool = False source: str = "" skip_reason: str | None = None + app_export: bool = False # exported by the iOS app (from a replay), the S4 end-to-end path def load_input(path: Path, real: bool = False) -> SceneInput: @@ -266,7 +267,12 @@ def variant(item: SceneInput, name: str, scene: dict) -> SceneInput: def judge(url: str, endpoint: Endpoint, item: SceneInput, schemas: dict) -> dict: - record: dict = {"name": item.name, "source": item.source, "real": item.real} + record: dict = { + "name": item.name, + "source": item.source, + "real": item.real, + "app_export": item.app_export, + } if item.skip_reason: return record | {"status": "skipped", "problems": [], "skip_reason": item.skip_reason} input_errors = schema_errors(item.scene, schemas["scene"]) @@ -347,11 +353,17 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: for r in records: counts[r["status"]] = counts.get(r["status"], 0) + 1 real = [r["latency_ms"] for r in records if r.get("real") and "latency_ms" in r] + exported = [r for r in records if r.get("app_export")] report = meta | { "counts": counts, "all_passed": all(r["status"] in ("pass", "skipped") for r in records) and bool(records), "latency_ms": max(real) if real else None, "real_scene_passed": any(r.get("real") and r["status"] == "pass" for r in records), + # Null when the run had no app export, so the scoreboard looks for an older run. + "app_export_scene_valid": all(r["status"] != "bad input" for r in exported) + if exported + else None, + "app_export_passed": all(r["status"] == "pass" for r in exported) if exported else None, "records": records, } (out / "report.json").write_text(json.dumps(report, indent=2) + "\n") @@ -403,6 +415,13 @@ def main(argv: list[str] | None = None) -> int: default=[], help="scene derived from real data; held to the latency budget", ) + parser.add_argument( + "--app-export", + type=Path, + action="append", + default=[], + help="scene.json or bundle the iOS app exported from a replay (real, latency budget)", + ) parser.add_argument("--out", type=Path) args = parser.parse_args(argv) @@ -421,6 +440,10 @@ def main(argv: list[str] | None = None) -> int: ) items += [load_input(p) for p in args.scene] items += [load_input(p, real=True) for p in args.real] + for path in args.app_export: + exported = load_input(path, real=True) + exported.app_export = True + items.append(exported) if not items: raise SystemExit("No scenes: add case files to e2e/cases or pass --scene/--real.") diff --git a/verification/hsverify/scoreboard.py b/verification/hsverify/scoreboard.py new file mode 100644 index 000000000..d5297d0eb --- /dev/null +++ b/verification/hsverify/scoreboard.py @@ -0,0 +1,773 @@ +"""Scoreboard for the morning meeting: every open PR and every overnight-plan metric. + + uv run python -m hsverify.scoreboard # fast probes; slow ones show their cache + uv run python -m hsverify.scoreboard --slow # also run slow command probes + uv run python -m hsverify.scoreboard --slow S2-1 # only this slow probe + +Writes verification/SCOREBOARD.md and verification/scoreboard/scoreboard.json and prints the +Markdown. PRs come from `gh pr list`; CI is summarised from each PR's status-check rollup. + +Metrics live in verification/scoreboard/metrics.yaml. Each names a git ref and exactly one +probe, which is evaluated against the ref's current SHA so the evidence is tied to a commit: + +- file_at_ref {path | paths}: met when every file exists at the ref. +- grep_at_ref {paths, pattern, min_matches, each}: `paths` are fnmatch globs over the files + at the ref (`*` also matches `/`). Without `each`, met when the pattern matches at least + `min_matches` times. With `each` (a list, or a mapping of label to regex fragment), the + fragment replaces `{each}` in the pattern and every one must match: met when all do, + partial when some do, gap when none do. +- command {run, cwd, timeout_s, slow}: runs in a detached worktree of the SHA under /tmp; + met on exit 0. Slow probes run only with --slow; otherwise the last cached result for the + same SHA is shown with status "not run". +- report {glob, key, equals | less_than | length_equals | length_at_least, + require_current_sha}: judges the newest matching report.json (by mtime). With + require_current_sha only a report whose `sha` equals the ref's SHA counts; an older one + makes the metric "stale". +- pr_check {name}: the named CI check on the ref's open PR. +- pr_body {pattern, min_matches}: the ref's open PR description. +- manual {status, evidence}: a judgment the lead records by editing the YAML. + +Any probe may set `max_status: partial` with a `cap_reason`, for probes that can only show +that something exists, not that it works or is complete. + +A ref that does not exist (branch not pushed) gives "ref missing" for every probe that needs +it; nothing about a missing ref stops the rest of the board. +""" + +from __future__ import annotations + +import argparse +import contextlib +import datetime as dt +import fnmatch +import glob as globlib +import json +import os +import re +import signal +import subprocess +import sys +from collections.abc import Callable, Iterator +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Protocol + +import yaml + +from hsverify import gitref + +GITHUB_REPO = "SamGu-NRX/house-scanning" +BLOB_URL = f"https://github.com/{GITHUB_REPO}/blob" +TREE_URL = f"https://github.com/{GITHUB_REPO}/tree" +PR_URL = f"https://github.com/{GITHUB_REPO}/pull" +VERIFICATION = Path(__file__).resolve().parents[1] +SCOREBOARD_DIR = VERIFICATION / "scoreboard" +METRICS_PATH = SCOREBOARD_DIR / "metrics.yaml" +CACHE_PATH = SCOREBOARD_DIR / ".cache.json" +JSON_PATH = SCOREBOARD_DIR / "scoreboard.json" +MARKDOWN_PATH = VERIFICATION / "SCOREBOARD.md" +COMMAND = "uv run python -m hsverify.scoreboard" +PR_FIELDS = "number,title,headRefName,headRefOid,isDraft,updatedAt,statusCheckRollup,body" + +STATUSES = ("met", "partial", "gap", "stale", "not run", "no evidence yet", "ref missing") +MANUAL_STATUSES = ("met", "partial", "gap") +COMPARATORS = ("equals", "less_than", "length_equals", "length_at_least") + +# Keys each probe accepts, with defaults for the optional ones (None marks a required key). +PROBE_KEYS: dict[str, dict[str, Any]] = { + "file_at_ref": {"path": "", "paths": []}, + "grep_at_ref": {"paths": None, "pattern": None, "min_matches": 1, "each": None}, + "command": {"run": None, "cwd": ".", "timeout_s": 600, "slow": True}, + "report": { + "glob": None, + "key": None, + **dict.fromkeys(COMPARATORS), + "require_current_sha": False, + }, + "pr_check": {"name": None}, + "pr_body": {"pattern": None, "min_matches": 1}, + "manual": {"status": None, "evidence": None}, +} +COMMON_PROBE_KEYS = {"max_status": None, "cap_reason": None} +METRIC_KEYS = {"id", "workstream", "ref", "text"} + +# CI rollup buckets. CheckRun entries carry status + conclusion, StatusContext entries state. +CHECK_CONCLUSIONS = { + "SUCCESS": "success", + "NEUTRAL": "success", + "SKIPPED": "skipped", + "STALE": "skipped", + "FAILURE": "failure", + "CANCELLED": "failure", + "TIMED_OUT": "failure", + "ACTION_REQUIRED": "failure", + "STARTUP_FAILURE": "failure", +} +CONTEXT_STATES = { + "SUCCESS": "success", + "FAILURE": "failure", + "ERROR": "failure", + "PENDING": "pending", + "EXPECTED": "pending", +} +CI_BUCKETS = ("success", "failure", "pending", "skipped") + + +class MetricsError(ValueError): + """metrics.yaml does not match the probe vocabulary.""" + + +# --- metrics file ---------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Metric: + id: str + workstream: str + ref: str + text: str + kind: str + probe: dict[str, Any] + + +def load_metrics(path: Path = METRICS_PATH) -> tuple[dict[str, str], list[Metric]]: + """(workstream id -> title, metrics). Raises MetricsError naming the offending metric.""" + data = yaml.safe_load(path.read_text()) + workstreams = data.get("workstreams") or {} + metrics = [parse_metric(raw, workstreams) for raw in data.get("metrics") or []] + ids = [m.id for m in metrics] + dupes = sorted({i for i in ids if ids.count(i) > 1}) + if dupes: + raise MetricsError(f"duplicate metric ids: {', '.join(dupes)}") + return {k: v["title"] for k, v in workstreams.items()}, metrics + + +def parse_metric(raw: dict, workstreams: dict) -> Metric: + mid = raw.get("id", "") + missing = METRIC_KEYS - raw.keys() + if missing: + raise MetricsError(f"{mid}: missing {', '.join(sorted(missing))}") + if raw["workstream"] not in workstreams: + raise MetricsError(f"{mid}: unknown workstream {raw['workstream']!r}") + kinds = [k for k in raw if k not in METRIC_KEYS] + if len(kinds) != 1 or kinds[0] not in PROBE_KEYS: + raise MetricsError( + f"{mid}: needs exactly one probe out of {', '.join(PROBE_KEYS)}; got {kinds}" + ) + kind = kinds[0] + given = raw[kind] or {} + allowed = PROBE_KEYS[kind] | COMMON_PROBE_KEYS + unknown = set(given) - allowed.keys() + if unknown: + raise MetricsError(f"{mid}: {kind} does not take {', '.join(sorted(unknown))}") + required = {k for k, v in PROBE_KEYS[kind].items() if v is None and k not in COMPARATORS} + required -= {"each"} + absent = required - given.keys() + if absent: + raise MetricsError(f"{mid}: {kind} needs {', '.join(sorted(absent))}") + probe = {k: v for k, v in allowed.items() if v is not None or k in given} | given + _check_probe(mid, kind, probe) + return Metric(mid, raw["workstream"], raw["ref"], raw["text"], kind, probe) + + +def _check_probe(mid: str, kind: str, probe: dict) -> None: + if kind == "file_at_ref" and bool(probe.get("path")) == bool(probe.get("paths")): + raise MetricsError(f"{mid}: file_at_ref needs exactly one of path, paths") + if kind == "report": + chosen = [c for c in COMPARATORS if probe.get(c) is not None] + if len(chosen) != 1: + raise MetricsError(f"{mid}: report needs exactly one of {', '.join(COMPARATORS)}") + if kind == "manual" and probe["status"] not in MANUAL_STATUSES: + raise MetricsError(f"{mid}: manual status must be one of {', '.join(MANUAL_STATUSES)}") + if kind in ("grep_at_ref", "pr_body"): + if kind == "grep_at_ref" and probe.get("each") and "{each}" not in probe["pattern"]: + raise MetricsError(f"{mid}: pattern must contain {{each}} when each is set") + for fragment in _each_items(probe) or [("", "")]: + try: + re.compile(probe["pattern"].replace("{each}", fragment[1])) + except re.error as e: + raise MetricsError(f"{mid}: bad pattern: {e}") from e + if probe.get("max_status") not in (None, "partial"): + raise MetricsError(f"{mid}: max_status can only be partial") + if probe.get("max_status") and not probe.get("cap_reason"): + raise MetricsError(f"{mid}: max_status needs a cap_reason") + + +def _each_items(probe: dict) -> list[tuple[str, str]]: + each = probe.get("each") + if not each: + return [] + if isinstance(each, dict): + return [(str(k), str(v)) for k, v in each.items()] + return [(str(v), re.escape(str(v))) for v in each] + + +# --- CI rollup ------------------------------------------------------------------------------- + + +@dataclass +class CISummary: + counts: dict[str, int] = field(default_factory=lambda: dict.fromkeys(CI_BUCKETS, 0)) + failing: list[str] = field(default_factory=list) + + def text(self) -> str: + parts = [f"{n} {b}" for b, n in self.counts.items() if n] + if not parts: + return "no checks" + text = ", ".join(parts) + return f"{text}; failing: {', '.join(self.failing)}" if self.failing else text + + +def classify_check(entry: dict) -> tuple[str, str] | None: + """(name, bucket) for one rollup entry, or None for entries with no name or outcome. + + A CheckRun that is not COMPLETED is pending; a completed one with no conclusion is + skipped entirely, as is any entry without a name. An unknown conclusion counts as a + failure so it shows up rather than vanishing. + """ + name = entry.get("name") or entry.get("context") + if not name: + return None + if entry.get("__typename") == "StatusContext" or "state" in entry: + state = entry.get("state") + if not state: + return None + return name, CONTEXT_STATES.get(state, "failure") + status = entry.get("status") + if status and status != "COMPLETED": + return name, "pending" + conclusion = entry.get("conclusion") + if not conclusion: + return None + return name, CHECK_CONCLUSIONS.get(conclusion, "failure") + + +def summarise_rollup(entries: list[dict] | None) -> CISummary: + summary = CISummary() + for entry in entries or []: + classified = classify_check(entry) + if classified is None: + continue + name, bucket = classified + summary.counts[bucket] += 1 + if bucket == "failure": + summary.failing.append(name) + return summary + + +# --- probes ---------------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Outcome: + status: str + evidence: str + links: tuple[tuple[str, str], ...] = () + + +class Tree(Protocol): + def files(self, sha: str) -> list[str]: ... + def show(self, sha: str, path: str) -> str | None: ... + + +class GitTree: + """Files at a commit, read through gitref without touching any working tree.""" + + def __init__(self) -> None: + self._files: dict[str, list[str]] = {} + + def files(self, sha: str) -> list[str]: + if sha not in self._files: + self._files[sha] = gitref.git("ls-tree", "-r", "--name-only", sha).splitlines() + return self._files[sha] + + def show(self, sha: str, path: str) -> str | None: + return gitref.show(sha, path) + + +@dataclass +class Context: + tree: Tree + prs: dict[str, dict] | None # branch -> PR; None when the PR list could not be read + slow: bool = False + slow_only: frozenset[str] = frozenset() # with slow: run just these ids; empty runs all + cache: dict[str, dict] = field(default_factory=dict) + worktree: Callable[[str], contextlib.AbstractContextManager[Path]] = gitref.detached_worktree + now: Callable[[], dt.datetime] = lambda: dt.datetime.now().astimezone() + + +def blob_link(sha: str, path: str) -> tuple[str, str]: + return path, f"{BLOB_URL}/{sha}/{path}" + + +def short_home(path: str | Path) -> str: + text = str(path) + home = str(Path.home()) + return "~" + text[len(home) :] if text.startswith(home + os.sep) else text + + +def probe_file(probe: dict, sha: str, ctx: Context) -> Outcome: + paths = probe["paths"] or [probe["path"]] + present = set(ctx.tree.files(sha)) + found = [p for p in paths if p in present] + missing = [p for p in paths if p not in present] + links = tuple(blob_link(sha, p) for p in found) + if missing: + status = "partial" if found else "gap" + return Outcome(status, f"missing at {sha[:8]}: {', '.join(missing)}", links) + return Outcome("met", f"present at {sha[:8]}", links) + + +def _matching_files(sha: str, globs: list[str], ctx: Context) -> list[str]: + return [f for f in ctx.tree.files(sha) if any(fnmatch.fnmatchcase(f, g) for g in globs)] + + +def probe_grep(probe: dict, sha: str, ctx: Context) -> Outcome: + files = _matching_files(sha, probe["paths"], ctx) + texts = {f: ctx.tree.show(sha, f) or "" for f in files} + need = probe["min_matches"] + + def search(pattern: str) -> tuple[int, list[str]]: + rx = re.compile(pattern, re.MULTILINE) + hits = {f: len(rx.findall(t)) for f, t in texts.items()} + return sum(hits.values()), [f for f, n in hits.items() if n] + + items = _each_items(probe) + if not items: + total, where = search(probe["pattern"]) + links = tuple(blob_link(sha, f) for f in where[:3]) + if total >= need: + return Outcome("met", f"{total} matches in {len(where)} files", links) + scope = f"{len(files)} files" if files else "no files match " + ", ".join(probe["paths"]) + return Outcome("gap", f"{total} of {need} matches needed, searched {scope}", links) + + found: list[str] = [] + missing: list[str] = [] + where_all: list[str] = [] + for label, fragment in items: + total, where = search(probe["pattern"].replace("{each}", fragment)) + (found if total >= need else missing).append(label) + where_all += [f for f in where if f not in where_all] + links = tuple(blob_link(sha, f) for f in where_all[:3]) + if not missing: + return Outcome("met", f"all {len(found)} found", links) + if not files: + return Outcome("gap", "no files match " + ", ".join(probe["paths"])) + evidence = f"missing {', '.join(missing)}" + if found: + return Outcome("partial", f"found {', '.join(found)}; {evidence}", links) + return Outcome("gap", f"none found in {len(files)} files; {evidence}", links) + + +def _tail(output: str, n: int = 3) -> str: + lines = [" ".join(line.split()) for line in output.splitlines() if line.strip()] + return " / ".join(lines[-n:]) or "(no output)" + + +def run_shell(run: str, cwd: Path, timeout_s: float) -> tuple[int | None, str]: + """(exit code, combined output); exit code None on timeout. Kills the whole process group, + since a shell's children (uv, pytest) would otherwise outlive it and hold the pipe open.""" + env = {k: v for k, v in os.environ.items() if k != "VIRTUAL_ENV"} + proc = subprocess.Popen( + run, + shell=True, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) + try: + out, _ = proc.communicate(timeout=timeout_s) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + out, _ = proc.communicate() + return None, out + return proc.returncode, out + + +def probe_command(metric: Metric, sha: str, ctx: Context) -> Outcome: + probe = metric.probe + if probe["slow"] and not (ctx.slow and (not ctx.slow_only or metric.id in ctx.slow_only)): + cached = ctx.cache.get(metric.id) + if cached and cached.get("sha") == sha: + return Outcome( + "not run", + f"slow; last run {cached['ran_at']}: {cached['status']}, {cached['evidence']}", + ) + return Outcome("not run", "slow; run with --slow") + with ctx.worktree(sha) as root: + cwd = root / probe["cwd"] + if not cwd.is_dir(): + return Outcome("gap", f"no {probe['cwd']}/ at {sha[:8]}") + code, out = run_shell(probe["run"], cwd, probe["timeout_s"]) + if code is None: + outcome = Outcome("gap", f"timed out after {probe['timeout_s']} s: {_tail(out)}") + else: + outcome = Outcome("met" if code == 0 else "gap", f"exit {code}: {_tail(out)}") + ctx.cache[metric.id] = { + "sha": sha, + "ran_at": ctx.now().strftime("%Y-%m-%d %H:%M %Z"), + "status": outcome.status, + "evidence": outcome.evidence, + } + return outcome + + +def lookup(data: Any, dotted: str) -> Any: + """Value at a dotted path; list elements by integer index. Raises KeyError when absent.""" + for part in dotted.split("."): + if isinstance(data, dict) and part in data: + data = data[part] + elif isinstance(data, list) and re.fullmatch(r"-?\d+", part): + try: + data = data[int(part)] + except IndexError as e: + raise KeyError(dotted) from e + else: + raise KeyError(dotted) + return data + + +def compare(value: Any, probe: dict) -> tuple[bool, str]: + """(passed, description of the value) for the report's single comparator.""" + if probe.get("equals") is not None: + return value == probe["equals"], f"{json.dumps(value)} (want {probe['equals']!r})" + if probe.get("less_than") is not None: + ok = isinstance(value, int | float) and not isinstance(value, bool) + return ok and value < probe["less_than"], f"{value} (want < {probe['less_than']})" + if not isinstance(value, list | dict | str): + return False, f"{json.dumps(value)} has no length" + if probe.get("length_equals") is not None: + return len(value) == probe["length_equals"], ( + f"length {len(value)} (want {probe['length_equals']})" + ) + return len(value) >= probe["length_at_least"], ( + f"length {len(value)} (want at least {probe['length_at_least']})" + ) + + +def _read_report(path: Path) -> dict | None: + try: + data = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return None + return data if isinstance(data, dict) else None + + +def _value_or_none(data: dict, key: str): + try: + return lookup(data, key) + except KeyError: + return None + + +def probe_report(probe: dict, sha: str | None) -> Outcome: + paths = sorted( + (Path(p) for p in globlib.glob(os.path.expanduser(probe["glob"]))), + key=lambda p: (p.stat().st_mtime, str(p)), + reverse=True, + ) + if not paths: + return Outcome("no evidence yet", f"no report matches {probe['glob']}") + reports = [(p, _read_report(p)) for p in paths] + if probe["require_current_sha"]: + if sha is None: + raise ValueError("require_current_sha needs the ref's SHA") + current = [(p, d) for p, d in reports if d is not None and d.get("sha") == sha] + if not current: + newest, data = reports[0] + got = str((data or {}).get("sha") or "no sha")[:8] + return Outcome("stale", f"report for {got}, ref at {sha[:8]}: {short_home(newest)}") + reports = current + # A run that did not exercise this metric leaves its key null; use the newest that did. + with_value = [ + (p, d) for p, d in reports if d is not None and _value_or_none(d, probe["key"]) is not None + ] + if not with_value: + path, data = reports[0] + if data is None: + return Outcome("gap", f"unreadable report {short_home(path)}") + return Outcome("no evidence yet", f"no {probe['key']} in {short_home(path)}") + path, data = with_value[0] + value = lookup(data, probe["key"]) + ok, described = compare(value, probe) + return Outcome("met" if ok else "gap", f"{probe['key']} = {described}: {short_home(path)}") + + +def _pr_for(ref: str, ctx: Context) -> tuple[dict | None, str]: + branch = ref.removeprefix("origin/") + if ctx.prs is None: + return None, "PR list unavailable" + pr = ctx.prs.get(branch) + return pr, "" if pr else f"no open PR for {branch}" + + +def probe_pr_check(probe: dict, ref: str, sha: str, ctx: Context) -> Outcome: + pr, why = _pr_for(ref, ctx) + if pr is None: + return Outcome("no evidence yet", why) + link = (f"#{pr['number']}", f"{PR_URL}/{pr['number']}") + buckets = [ + c[1] + for c in map(classify_check, pr.get("statusCheckRollup") or []) + if c and c[0] == probe["name"] + ] + if not buckets: + return Outcome("no evidence yet", f"no check named {probe['name']!r}", (link,)) + # A re-run leaves several entries under one name; any failure or pending one decides. + bucket = next((b for b in ("failure", "pending") if b in buckets), buckets[0]) + head = pr["headRefOid"] + if bucket == "success" and head != sha: + return Outcome("stale", f"{probe['name']} passed for {head[:8]}, ref at {sha[:8]}", (link,)) + status = {"success": "met", "failure": "gap", "pending": "not run"}.get(bucket, "gap") + return Outcome(status, f"{probe['name']}: {bucket} at {head[:8]}", (link,)) + + +def probe_pr_body(probe: dict, ref: str, ctx: Context) -> Outcome: + pr, why = _pr_for(ref, ctx) + if pr is None: + return Outcome("no evidence yet", why) + link = (f"#{pr['number']}", f"{PR_URL}/{pr['number']}") + n = len(re.findall(probe["pattern"], pr.get("body") or "", re.MULTILINE)) + need = probe["min_matches"] + return Outcome("met" if n >= need else "gap", f"{n} of {need} matches in the PR body", (link,)) + + +def needs_sha(metric: Metric) -> bool: + if metric.kind == "manual": + return False + if metric.kind == "report": + return bool(metric.probe["require_current_sha"]) + return True + + +def evaluate(metric: Metric, sha: str | None, ctx: Context) -> Outcome: + if sha is None and needs_sha(metric): + return Outcome("ref missing", f"{metric.ref} does not exist") + p = metric.probe + match metric.kind: + case "manual": + outcome = Outcome(p["status"], p["evidence"]) + case "file_at_ref": + outcome = probe_file(p, sha, ctx) + case "grep_at_ref": + outcome = probe_grep(p, sha, ctx) + case "command": + outcome = probe_command(metric, sha, ctx) + case "report": + outcome = probe_report(p, sha) + case "pr_check": + outcome = probe_pr_check(p, metric.ref, sha, ctx) + case "pr_body": + outcome = probe_pr_body(p, metric.ref, ctx) + case _: + raise MetricsError(f"{metric.id}: unknown probe {metric.kind}") + if p.get("max_status") == "partial" and outcome.status == "met": + return Outcome("partial", f"{outcome.evidence} ({p['cap_reason']})", outcome.links) + return outcome + + +# --- board ----------------------------------------------------------------------------------- + + +@dataclass +class Row: + id: str + text: str + ref: str + sha: str | None + status: str + evidence: str + links: list[list[str]] + + +@dataclass +class Board: + generated_at: str + command: str + slow: bool | list[str] # True, False, or the ids of the only slow probes run + prs: list[dict] | None + pr_error: str | None + workstreams: list[dict] + + +def pr_rows(prs: list[dict]) -> list[dict]: + rows = [] + for pr in sorted(prs, key=lambda p: -p["number"]): + ci = summarise_rollup(pr.get("statusCheckRollup")) + rows.append( + { + "number": pr["number"], + "title": pr["title"], + "branch": pr["headRefName"], + "head": pr["headRefOid"], + "draft": bool(pr["isDraft"]), + "updated": pr["updatedAt"], + "ci": {"counts": ci.counts, "failing": ci.failing, "text": ci.text()}, + } + ) + return rows + + +def build_board( + titles: dict[str, str], + metrics: list[Metric], + shas: dict[str, str | None], + prs: list[dict] | None, + ctx: Context, + pr_error: str | None = None, +) -> Board: + streams = [] + for ws, title in titles.items(): + rows = [] + for m in (m for m in metrics if m.workstream == ws): + sha = shas[m.ref] + o = evaluate(m, sha, ctx) + links = [list(link) for link in o.links] + rows.append(Row(m.id, m.text, m.ref, sha, o.status, o.evidence, links)) + refs = list(dict.fromkeys(r.ref for r in rows)) + streams.append( + { + "id": ws, + "title": title, + "refs": [{"ref": r, "sha": shas[r]} for r in refs], + "metrics": [asdict(r) for r in rows], + } + ) + return Board( + generated_at=ctx.now().isoformat(timespec="seconds"), + command=COMMAND, + slow=sorted(ctx.slow_only) if ctx.slow and ctx.slow_only else ctx.slow, + prs=pr_rows(prs) if prs is not None else None, + pr_error=pr_error, + workstreams=streams, + ) + + +def _cell(text: object) -> str: + return str(text).replace("|", "\\|").replace("\n", " ") + + +def _local(iso: str) -> str: + stamp = dt.datetime.fromisoformat(iso.replace("Z", "+00:00")) + return stamp.astimezone().strftime("%Y-%m-%d %H:%M %Z") + + +def render_markdown(board: Board) -> str: + if isinstance(board.slow, list): + slow = f"slow probes ran for {', '.join(board.slow)} only" + else: + slow = "slow probes ran" if board.slow else "slow probes not run (add ARGS=--slow)" + lines = [ + "# Scoreboard", + "", + f"Generated {_local(board.generated_at)} by `make scoreboard` (`{board.command}`) in " + f"`verification/`; " + f"{slow}. Probes are defined in `verification/scoreboard/metrics.yaml`.", + "", + "## Open pull requests", + "", + ] + if board.prs is None: + lines.append(f"Could not list PRs: {board.pr_error}") + elif not board.prs: + lines.append("No open pull requests.") + else: + lines += [ + "| # | Branch | Head | Draft | CI | Updated |", + "| --- | --- | --- | --- | --- | --- |", + ] + lines += [ + f"| [#{p['number']}]({PR_URL}/{p['number']}) | `{p['branch']}` | " + f"`{p['head'][:8]}` | {'yes' if p['draft'] else 'no'} | {_cell(p['ci']['text'])} | " + f"{_local(p['updated'])} |" + for p in board.prs + ] + for ws in board.workstreams: + refs = ", ".join( + f"`{r['ref']}` at [`{r['sha'][:8]}`]({TREE_URL}/{r['sha']})" + if r["sha"] + else f"`{r['ref']}` (missing)" + for r in ws["refs"] + ) + counts = {s: 0 for s in STATUSES} + for m in ws["metrics"]: + counts[m["status"]] += 1 + tally = ", ".join(f"{n} {s}" for s, n in counts.items() if n) + lines += ["", f"## {ws['id']} {ws['title']}", "", f"{refs}. {tally}.", ""] + lines += ["| Metric | Text | Status | Evidence |", "| --- | --- | --- | --- |"] + for m in ws["metrics"]: + evidence = _cell(m["evidence"]) + if m["links"]: + evidence += " " + ", ".join(f"[{_cell(label)}]({url})" for label, url in m["links"]) + lines.append(f"| {m['id']} | {_cell(m['text'])} | {m['status']} | {evidence} |") + return "\n".join(lines) + "\n" + + +# --- I/O ------------------------------------------------------------------------------------- + + +def list_prs() -> list[dict]: + out = subprocess.run( + ["gh", "pr", "list", "--repo", GITHUB_REPO, "--state", "open", "--json", PR_FIELDS], + check=True, + capture_output=True, + text=True, + ).stdout + return json.loads(out) + + +def resolve_refs(refs: list[str]) -> dict[str, str | None]: + return {r: gitref.resolve(r) if gitref.ref_exists(r) else None for r in refs} + + +@contextlib.contextmanager +def cached(path: Path) -> Iterator[dict]: + data = json.loads(path.read_text()) if path.exists() else {} + yield data + path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--slow", + nargs="*", + metavar="ID", + help="also run slow command probes; with ids, only those", + ) + parser.add_argument("--no-fetch", action="store_true", help="skip `git fetch origin`") + args = parser.parse_args(argv) + + titles, metrics = load_metrics() + if not args.no_fetch: + gitref.fetch() + shas = resolve_refs(list(dict.fromkeys(m.ref for m in metrics))) + prs: list[dict] | None + pr_error = None + try: + prs = list_prs() + except (OSError, subprocess.CalledProcessError, json.JSONDecodeError) as e: + prs = None + pr_error = (getattr(e, "stderr", "") or str(e)).strip() + print(f"warning: could not list PRs: {pr_error}", file=sys.stderr) + with cached(CACHE_PATH) as cache: + ctx = Context( + tree=GitTree(), + prs={p["headRefName"]: p for p in prs} if prs is not None else None, + slow=args.slow is not None, + slow_only=frozenset(args.slow or ()), + cache=cache, + ) + board = build_board(titles, metrics, shas, prs, ctx, pr_error) + markdown = render_markdown(board) + MARKDOWN_PATH.write_text(markdown) + JSON_PATH.write_text(json.dumps(asdict(board), indent=2) + "\n") + print(markdown, end="") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/verification/scoreboard/.gitignore b/verification/scoreboard/.gitignore new file mode 100644 index 000000000..b1a62cafa --- /dev/null +++ b/verification/scoreboard/.gitignore @@ -0,0 +1 @@ +.cache.json diff --git a/verification/scoreboard/metrics.yaml b/verification/scoreboard/metrics.yaml new file mode 100644 index 000000000..3de33721b --- /dev/null +++ b/verification/scoreboard/metrics.yaml @@ -0,0 +1,274 @@ +# Overnight-plan metrics for `make scoreboard`. hsverify/scoreboard.py documents each probe kind. +# A probe that only shows something exists is capped at partial with a cap_reason. +# Where only a person can judge, the probe is `manual`; the lead edits status and evidence. +# +# e2e report keys (hsverify/e2e.py): `sha` is the server SHA; `latency_ms` is the slowest real +# scene's round trip; `app_export_scene_valid` and `app_export_passed` cover scenes the iOS app +# exported from a replay. A key is null when the run did not exercise it; the probe then uses +# the newest report where it is set. + +workstreams: + S1: {title: Evals on real data} + S2: {title: Server and solver} + S3: {title: iOS capture app} + S4: {title: Verification} + M: {title: Maintenance} + +metrics: + # --- S1 -------------------------------------------------------------------------------------- + - id: S1-1 + workstream: S1 + ref: origin/t3/evals + text: One command regenerates every number from a clean checkout + manual: {status: gap, evidence: not yet reviewed} + + - id: S1-2 + workstream: S1 + ref: origin/t3/evals + text: Dataset URLs and sha256 recorded + # A README table row naming the dataset, with a URL and a 64-hex digest on the same row. + grep_at_ref: + paths: [experiments/evals/README.md] + pattern: '(?im)^\|[^\n]*{each}[^\n]*https?://[^\n]*\b[0-9a-f]{64}\b' + each: {ADVIO: ADVIO, ETH3D: ETH3D} + + - id: S1-3 + workstream: S1 + ref: origin/t3/evals + text: Metric code tested on hand-computed cases + command: + run: uv run pytest -q + cwd: experiments/evals + timeout_s: 600 + slow: true + max_status: partial + cap_reason: tests pass; whether the cases are hand-computed needs a read + + - id: S1-4 + workstream: S1 + ref: origin/t3/evals + text: ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft + # A Markdown table under experiments/evals with a row per distance and a p90 column. + grep_at_ref: + paths: ['experiments/evals/*.md'] + pattern: '(?im)^\|[^\n]*(?- + ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) + with a verdict per question + grep_at_ref: + paths: ['experiments/evals/*.md'] + pattern: '(?im)^\|[^\n]*(?:{each})' + each: + 1-3 m row: '1\s*(?:-|–|to)\s*3\s*m' + 3-10 m row: '3\s*(?:-|–|to)\s*10\s*m' + scale error: 'scale error' + verdict: 'verdict' + max_status: partial + cap_reason: the table exists; per-model numbers and verdicts need a read + + # --- S2 -------------------------------------------------------------------------------------- + - id: S2-1 + workstream: S2 + ref: origin/t3/server + text: pytest green + command: {run: uv run pytest -q, cwd: server, timeout_s: 600, slow: true} + + - id: S2-2 + workstream: S2 + ref: origin/t3/server + text: Every public golden test 01-14 of docs/research/t3-lane-c-review.md present + # Test names such as test_01_..., test_golden_01_... or an id like "golden 07". + grep_at_ref: + paths: ['server/tests/*.py'] + pattern: '(?i)def test_(?:golden_?)?{each}(?!\d)|golden[^0-9\n]{0,12}{each}(?!\d)' + each: ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14'] + + - id: S2-3 + workstream: S2 + ref: origin/t3/server + text: >- + Property tests: missing coverage never passes, equality is unsure, left/right mirror, + larger clearance never turns pass into fail + # A Hypothesis test: @given(...) (two levels of nested parentheses), any further + # decorators, then a test whose name says which property it checks. + grep_at_ref: + paths: ['server/tests/*.py'] + pattern: '@given\((?:[^()]|\((?:[^()]|\([^()]*\))*\))*\)\s*(?:@[^\n]*\n\s*)*def test_\w*(?i:{each})' + each: + missing coverage: 'coverage|unobserved|unseen' + equality unsure: 'equal' + mirror: 'mirror|symmetr' + larger clearance: 'monoton|larger_clearance|clearance' + + - id: S2-4 + workstream: S2 + ref: origin/t3/server + text: Schemas published + file_at_ref: {paths: [server/schemas/scene.schema.json, server/schemas/result.schema.json]} + + - id: S2-5 + workstream: S2 + ref: origin/t3/server + text: Schemas validated in tests + grep_at_ref: + paths: ['server/tests/*.py'] + pattern: '{each}' + each: {scene schema: 'scene\.schema\.json', result schema: 'result\.schema\.json'} + + - id: S2-6 + workstream: S2 + ref: origin/t3/server + text: A real-derived scene returns a result in under 1 s + report: + glob: ~/house-scanning-data/reports/e2e/*/report.json + key: latency_ms + less_than: 1000 + require_current_sha: true + + - id: S2-7 + workstream: S2 + ref: origin/t3/server + text: ruff clean + command: + run: uv run ruff check . && uv run ruff format --check . + cwd: server + timeout_s: 300 + slow: true + + # --- S3 -------------------------------------------------------------------------------------- + - id: S3-1 + workstream: S3 + ref: origin/t3/ios-mvf + text: CI build green + pr_check: {name: iOS build} + + - id: S3-2 + workstream: S3 + ref: origin/t3/ios-mvf + text: Built as Swift 6 with strict concurrency + grep_at_ref: + paths: [ios/project.yml] + pattern: '{each}' + each: + Swift 6: 'SWIFT_VERSION:\s*"?6' + strict concurrency: 'SWIFT_STRICT_CONCURRENCY:\s*complete' + + - id: S3-3 + workstream: S3 + ref: origin/t3/ios-mvf + text: Zero build warnings + report: + glob: ~/house-scanning-data/reports/sim/*/report.json + key: build.warnings + length_equals: 0 + require_current_sha: true + + - id: S3-4 + workstream: S3 + ref: origin/t3/ios-mvf + text: Unit tests for coverage, auto-capture and guidance + grep_at_ref: + paths: ['ios/HouseScanKit/Tests/*.swift', 'ios/HouseScanTests/*.swift'] + pattern: '(?i)(?:func test|@Test)[^\n]*(?:{each})' + each: {coverage: coverage, auto-capture: 'auto_?capture', guidance: guidance} + + - id: S3-5 + workstream: S3 + ref: origin/t3/ios-mvf + text: >- + UI tests run the whole flow from a real replay with no manual input, including one + gap-instruction-recapture loop + grep_at_ref: + paths: ['ios/HouseScanUITests/*.swift'] + pattern: '{each}' + each: {replay argument: '"-replay"', autopilot argument: '"-autopilot"', gap loop: gapRequest} + max_status: partial + cap_reason: the tests exist; passing is S3-1 + + - id: S3-6 + workstream: S3 + ref: origin/t3/ios-mvf + text: Accessibility audit on every screen + grep_at_ref: + paths: ['ios/HouseScanUITests/*.swift'] + pattern: 'performAccessibilityAudit' + max_status: partial + cap_reason: an audit exists; which screens it covers needs a read + + - id: S3-7 + workstream: S3 + ref: origin/t3/ios-mvf + text: Screenshots of each state in the PR + # Nine states: ScanContract.swift lists ten, and `unsupported` needs a device without LiDAR. + pr_body: {pattern: '!\[[^\]]*\]\([^)]+\)|- + a=$(wc -l < AGENTS.md | tr -d ' '); b=$(git show origin/main:AGENTS.md | wc -l | tr -d ' '); + echo "AGENTS.md $a lines, main $b"; test "$a" -lt "$b" + timeout_s: 60 + slow: false + + - id: M-2 + workstream: M + ref: origin/t3/docs-cleanup + text: AGENTS.md accurate, every path and command real + manual: {status: gap, evidence: not yet reviewed} diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json new file mode 100644 index 000000000..bf3b0029b --- /dev/null +++ b/verification/scoreboard/scoreboard.json @@ -0,0 +1,545 @@ +{ + "generated_at": "2026-09-26T03:21:08-05:00", + "command": "uv run python -m hsverify.scoreboard", + "slow": false, + "prs": [ + { + "number": 14, + "title": "docs: trim AGENTS.md and fix stale decisions in the team docs", + "branch": "t3/docs-cleanup", + "head": "9c9ceb2a608e9762bd002f20f1b979694bb0c92f", + "draft": false, + "updated": "2026-09-26T08:15:28Z", + "ci": { + "counts": { + "success": 4, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "4 success" + } + }, + { + "number": 13, + "title": "verification: independent end-to-end checks (S4)", + "branch": "t3/verification", + "head": "328373d81f280496380c0a9737f1dd0cc3ac50bb", + "draft": true, + "updated": "2026-09-26T08:19:36Z", + "ci": { + "counts": { + "success": 4, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "4 success" + } + }, + { + "number": 12, + "title": "feat(evals): real-data accuracy evals, starting with an ADVIO replay session", + "branch": "t3/evals", + "head": "b0b8a2745e3ea67cb8f70df3943177e146f591b9", + "draft": true, + "updated": "2026-09-26T08:11:18Z", + "ci": { + "counts": { + "success": 4, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "4 success" + } + }, + { + "number": 11, + "title": "server: placement solver, rules and API (S2)", + "branch": "t3/server", + "head": "fcd457411a68171509d53b87b11c115f55a2a765", + "draft": true, + "updated": "2026-09-26T08:20:33Z", + "ci": { + "counts": { + "success": 4, + "failure": 0, + "pending": 1, + "skipped": 0 + }, + "failing": [], + "text": "4 success, 1 pending" + } + }, + { + "number": 10, + "title": "ios: guided wall scan, from meter to battery spot", + "branch": "t3/ios-mvf", + "head": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", + "draft": true, + "updated": "2026-09-26T08:07:57Z", + "ci": { + "counts": { + "success": 5, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "5 success" + } + }, + { + "number": 9, + "title": "experiments: validate camera metadata for reconstruction", + "branch": "codex/capture-contract-probe", + "head": "4b6c430956bb35edff546f140e3152ed75c56246", + "draft": true, + "updated": "2026-09-26T05:08:13Z", + "ci": { + "counts": { + "success": 4, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "4 success" + } + }, + { + "number": 8, + "title": "ci: add manual TestFlight upload workflow", + "branch": "t3/testflight", + "head": "b896d59fb564321df5fcdea1a5b50b45d27bc7a1", + "draft": false, + "updated": "2026-09-26T08:18:47Z", + "ci": { + "counts": { + "success": 4, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "4 success" + } + }, + { + "number": 7, + "title": "experiments: add measure-lab, a no-LiDAR measurement rig", + "branch": "t3/measure-lab", + "head": "45770a3b1bd2ab90967e82a974fdfbf4d7bfbecb", + "draft": false, + "updated": "2026-09-26T08:19:30Z", + "ci": { + "counts": { + "success": 6, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "6 success" + } + }, + { + "number": 6, + "title": "docs: add research notes on capture, lane reviews and scoring", + "branch": "t3/research", + "head": "a7d91f162c43e5e364e68d8ade0ad1002dd6b203", + "draft": false, + "updated": "2026-09-26T07:11:53Z", + "ci": { + "counts": { + "success": 4, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "4 success" + } + }, + { + "number": 4, + "title": "Add a scoring harness that compares capture pipelines with a tape survey", + "branch": "t3/scoring-harness", + "head": "77ce830dd08acf0c7982edcec91cd3240e0565ea", + "draft": false, + "updated": "2026-09-26T08:17:23Z", + "ci": { + "counts": { + "success": 5, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "5 success" + } + } + ], + "pr_error": null, + "workstreams": [ + { + "id": "S1", + "title": "Evals on real data", + "refs": [ + { + "ref": "origin/t3/evals", + "sha": "b0b8a2745e3ea67cb8f70df3943177e146f591b9" + } + ], + "metrics": [ + { + "id": "S1-1", + "text": "One command regenerates every number from a clean checkout", + "ref": "origin/t3/evals", + "sha": "b0b8a2745e3ea67cb8f70df3943177e146f591b9", + "status": "gap", + "evidence": "not yet reviewed", + "links": [] + }, + { + "id": "S1-2", + "text": "Dataset URLs and sha256 recorded", + "ref": "origin/t3/evals", + "sha": "b0b8a2745e3ea67cb8f70df3943177e146f591b9", + "status": "partial", + "evidence": "found ADVIO; missing ETH3D", + "links": [ + [ + "experiments/evals/README.md", + "https://github.com/SamGu-NRX/house-scanning/blob/b0b8a2745e3ea67cb8f70df3943177e146f591b9/experiments/evals/README.md" + ] + ] + }, + { + "id": "S1-3", + "text": "Metric code tested on hand-computed cases", + "ref": "origin/t3/evals", + "sha": "b0b8a2745e3ea67cb8f70df3943177e146f591b9", + "status": "not run", + "evidence": "slow; run with --slow", + "links": [] + }, + { + "id": "S1-4", + "text": "ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft", + "ref": "origin/t3/evals", + "sha": "b0b8a2745e3ea67cb8f70df3943177e146f591b9", + "status": "met", + "evidence": "all 5 found", + "links": [ + [ + "experiments/evals/results/advio_drift.md", + "https://github.com/SamGu-NRX/house-scanning/blob/b0b8a2745e3ea67cb8f70df3943177e146f591b9/experiments/evals/results/advio_drift.md" + ] + ] + }, + { + "id": "S1-5", + "text": "ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question", + "ref": "origin/t3/evals", + "sha": "b0b8a2745e3ea67cb8f70df3943177e146f591b9", + "status": "gap", + "evidence": "none found in 2 files; missing 1-3 m row, 3-10 m row, scale error, verdict", + "links": [] + } + ] + }, + { + "id": "S2", + "title": "Server and solver", + "refs": [ + { + "ref": "origin/t3/server", + "sha": "fcd457411a68171509d53b87b11c115f55a2a765" + } + ], + "metrics": [ + { + "id": "S2-1", + "text": "pytest green", + "ref": "origin/t3/server", + "sha": "fcd457411a68171509d53b87b11c115f55a2a765", + "status": "not run", + "evidence": "slow; run with --slow", + "links": [] + }, + { + "id": "S2-2", + "text": "Every public golden test 01-14 of docs/research/t3-lane-c-review.md present", + "ref": "origin/t3/server", + "sha": "fcd457411a68171509d53b87b11c115f55a2a765", + "status": "met", + "evidence": "all 14 found", + "links": [ + [ + "server/tests/test_golden_public.py", + "https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/tests/test_golden_public.py" + ] + ] + }, + { + "id": "S2-3", + "text": "Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail", + "ref": "origin/t3/server", + "sha": "fcd457411a68171509d53b87b11c115f55a2a765", + "status": "gap", + "evidence": "none found in 5 files; missing missing coverage, equality unsure, mirror, larger clearance", + "links": [] + }, + { + "id": "S2-4", + "text": "Schemas published", + "ref": "origin/t3/server", + "sha": "fcd457411a68171509d53b87b11c115f55a2a765", + "status": "met", + "evidence": "present at fcd45741", + "links": [ + [ + "server/schemas/scene.schema.json", + "https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/schemas/scene.schema.json" + ], + [ + "server/schemas/result.schema.json", + "https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/schemas/result.schema.json" + ] + ] + }, + { + "id": "S2-5", + "text": "Schemas validated in tests", + "ref": "origin/t3/server", + "sha": "fcd457411a68171509d53b87b11c115f55a2a765", + "status": "met", + "evidence": "all 2 found", + "links": [ + [ + "server/tests/test_golden_public.py", + "https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/tests/test_golden_public.py" + ], + [ + "server/tests/test_schemas.py", + "https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/tests/test_schemas.py" + ] + ] + }, + { + "id": "S2-6", + "text": "A real-derived scene returns a result in under 1 s", + "ref": "origin/t3/server", + "sha": "fcd457411a68171509d53b87b11c115f55a2a765", + "status": "no evidence yet", + "evidence": "no report matches ~/house-scanning-data/reports/e2e/*/report.json", + "links": [] + }, + { + "id": "S2-7", + "text": "ruff clean", + "ref": "origin/t3/server", + "sha": "fcd457411a68171509d53b87b11c115f55a2a765", + "status": "not run", + "evidence": "slow; run with --slow", + "links": [] + } + ] + }, + { + "id": "S3", + "title": "iOS capture app", + "refs": [ + { + "ref": "origin/t3/ios-mvf", + "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d" + } + ], + "metrics": [ + { + "id": "S3-1", + "text": "CI build green", + "ref": "origin/t3/ios-mvf", + "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", + "status": "met", + "evidence": "iOS build: success at 6885b7b5", + "links": [ + [ + "#10", + "https://github.com/SamGu-NRX/house-scanning/pull/10" + ] + ] + }, + { + "id": "S3-2", + "text": "Built as Swift 6 with strict concurrency", + "ref": "origin/t3/ios-mvf", + "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", + "status": "met", + "evidence": "all 2 found", + "links": [ + [ + "ios/project.yml", + "https://github.com/SamGu-NRX/house-scanning/blob/6885b7b5ad872df7f73a6b915e5837cd38a2034d/ios/project.yml" + ] + ] + }, + { + "id": "S3-3", + "text": "Zero build warnings", + "ref": "origin/t3/ios-mvf", + "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", + "status": "met", + "evidence": "build.warnings = length 0 (want 0): ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json", + "links": [] + }, + { + "id": "S3-4", + "text": "Unit tests for coverage, auto-capture and guidance", + "ref": "origin/t3/ios-mvf", + "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", + "status": "gap", + "evidence": "none found in 1 files; missing coverage, auto-capture, guidance", + "links": [] + }, + { + "id": "S3-5", + "text": "UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop", + "ref": "origin/t3/ios-mvf", + "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", + "status": "gap", + "evidence": "none found in 1 files; missing replay argument, autopilot argument, gap loop", + "links": [] + }, + { + "id": "S3-6", + "text": "Accessibility audit on every screen", + "ref": "origin/t3/ios-mvf", + "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", + "status": "gap", + "evidence": "0 of 1 matches needed, searched 1 files", + "links": [] + }, + { + "id": "S3-7", + "text": "Screenshots of each state in the PR", + "ref": "origin/t3/ios-mvf", + "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", + "status": "gap", + "evidence": "0 of 9 matches in the PR body", + "links": [ + [ + "#10", + "https://github.com/SamGu-NRX/house-scanning/pull/10" + ] + ] + }, + { + "id": "S3-8", + "text": "Exported scene.json validates against C1", + "ref": "origin/t3/ios-mvf", + "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", + "status": "no evidence yet", + "evidence": "no report matches ~/house-scanning-data/reports/e2e/*/report.json", + "links": [] + }, + { + "id": "S3-9", + "text": "Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design", + "ref": "origin/t3/ios-mvf", + "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", + "status": "gap", + "evidence": "not yet reviewed", + "links": [] + } + ] + }, + { + "id": "S4", + "title": "Verification", + "refs": [ + { + "ref": "origin/t3/verification", + "sha": "328373d81f280496380c0a9737f1dd0cc3ac50bb" + }, + { + "ref": "origin/t3/ios-mvf", + "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d" + } + ], + "metrics": [ + { + "id": "S4-1", + "text": "End to end from a real replay to a placement result passes", + "ref": "origin/t3/verification", + "sha": "328373d81f280496380c0a9737f1dd0cc3ac50bb", + "status": "no evidence yet", + "evidence": "no report matches ~/house-scanning-data/reports/e2e/*/report.json", + "links": [] + }, + { + "id": "S4-2", + "text": "Screenshot report per app state", + "ref": "origin/t3/ios-mvf", + "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", + "status": "gap", + "evidence": "states = length 0 (want at least 9): ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json", + "links": [] + }, + { + "id": "S4-3", + "text": "Product description written from the code and checked against screenshots", + "ref": "origin/t3/verification", + "sha": "328373d81f280496380c0a9737f1dd0cc3ac50bb", + "status": "gap", + "evidence": "not yet reviewed", + "links": [] + }, + { + "id": "S4-4", + "text": "Scoreboard of every PR", + "ref": "origin/t3/verification", + "sha": "328373d81f280496380c0a9737f1dd0cc3ac50bb", + "status": "gap", + "evidence": "missing at 328373d8: verification/SCOREBOARD.md", + "links": [] + } + ] + }, + { + "id": "M", + "title": "Maintenance", + "refs": [ + { + "ref": "origin/t3/docs-cleanup", + "sha": "9c9ceb2a608e9762bd002f20f1b979694bb0c92f" + } + ], + "metrics": [ + { + "id": "M-1", + "text": "AGENTS.md shorter than on main", + "ref": "origin/t3/docs-cleanup", + "sha": "9c9ceb2a608e9762bd002f20f1b979694bb0c92f", + "status": "met", + "evidence": "exit 0: AGENTS.md 39 lines, main 45", + "links": [] + }, + { + "id": "M-2", + "text": "AGENTS.md accurate, every path and command real", + "ref": "origin/t3/docs-cleanup", + "sha": "9c9ceb2a608e9762bd002f20f1b979694bb0c92f", + "status": "gap", + "evidence": "not yet reviewed", + "links": [] + } + ] + } + ] +} diff --git a/verification/tests/test_scoreboard.py b/verification/tests/test_scoreboard.py new file mode 100644 index 000000000..87fd2f07d --- /dev/null +++ b/verification/tests/test_scoreboard.py @@ -0,0 +1,593 @@ +import contextlib +import datetime as dt +import json +import os +import textwrap +import time + +import pytest + +from hsverify import scoreboard as sb +from hsverify.scoreboard import ( + Context, + MetricsError, + Outcome, + build_board, + evaluate, + load_metrics, + parse_metric, + render_markdown, + summarise_rollup, +) + +SHA = "a" * 40 +OLD = "b" * 40 +WS = {"S1": {"title": "Evals"}} +FIXED_NOW = dt.datetime(2026, 9, 26, 7, 30, tzinfo=dt.UTC) + + +class FakeTree: + """Files per SHA; show returns None for a missing file, like gitref.show.""" + + def __init__(self, files: dict[str, dict[str, str]]): + self._files = files + + def files(self, sha): + return list(self._files.get(sha, {})) + + def show(self, sha, path): + return self._files.get(sha, {}).get(path) + + +def ctx(files=None, prs=None, **kw) -> Context: + return Context(tree=FakeTree(files or {}), prs=prs, now=lambda: FIXED_NOW, **kw) + + +def metric(kind: str, ref: str = "origin/t3/evals", **probe): + return parse_metric( + {"id": "S1-1", "workstream": "S1", "ref": ref, "text": "t", kind: probe}, WS + ) + + +# --- CI rollup ------------------------------------------------------------------------------- + + +def run(name, conclusion, status="COMPLETED"): + return {"__typename": "CheckRun", "name": name, "status": status, "conclusion": conclusion} + + +def context_status(name, state): + return {"__typename": "StatusContext", "context": name, "state": state} + + +def test_rollup_counts_each_bucket_and_names_failures(): + s = summarise_rollup( + [ + run("iOS build", "SUCCESS"), + run("Lint", "FAILURE"), + run("Server", None, status="IN_PROGRESS"), + run("Optional", "SKIPPED"), + context_status("CodeRabbit", "SUCCESS"), + context_status("Vercel", "ERROR"), + context_status("Queue", "PENDING"), + ] + ) + assert s.counts == {"success": 2, "failure": 2, "pending": 2, "skipped": 1} + assert s.failing == ["Lint", "Vercel"] + assert s.text() == "2 success, 2 failure, 2 pending, 1 skipped; failing: Lint, Vercel" + + +def test_rollup_skips_entries_without_name_or_outcome(): + s = summarise_rollup( + [ + {"__typename": "CheckRun", "name": None, "status": "COMPLETED", "conclusion": None}, + run("Done without conclusion", None), + context_status(None, "SUCCESS"), + context_status("No state", None), + run("ok", "SUCCESS"), + ] + ) + assert s.counts == {"success": 1, "failure": 0, "pending": 0, "skipped": 0} + assert s.text() == "1 success" + + +def test_rollup_unknown_conclusion_counts_as_failure(): + s = summarise_rollup([run("Weird", "SOMETHING_NEW")]) + assert s.failing == ["Weird"] + + +def test_rollup_empty(): + assert summarise_rollup(None).text() == "no checks" + + +# --- metrics file ---------------------------------------------------------------------------- + + +def test_real_metrics_file_loads(): + titles, metrics = load_metrics() + assert set(titles) == {"S1", "S2", "S3", "S4", "M"} + assert {m.workstream for m in metrics} == set(titles) + + +@pytest.mark.parametrize( + ("raw", "message"), + [ + ({"id": "X", "workstream": "S1", "ref": "r"}, "missing text"), + ({"id": "X", "workstream": "S9", "ref": "r", "text": "t", "manual": {}}, "unknown work"), + ({"id": "X", "workstream": "S1", "ref": "r", "text": "t"}, "exactly one probe"), + ( + {"id": "X", "workstream": "S1", "ref": "r", "text": "t", "manual": {}, "pr_body": {}}, + "exactly one probe", + ), + ( + {"id": "X", "workstream": "S1", "ref": "r", "text": "t", "manual": {"status": "ok"}}, + "manual needs evidence", + ), + ( + { + "id": "X", + "workstream": "S1", + "ref": "r", + "text": "t", + "manual": {"status": "done", "evidence": "e"}, + }, + "manual status must be", + ), + ( + {"id": "X", "workstream": "S1", "ref": "r", "text": "t", "report": {"glob": "g"}}, + "report needs key", + ), + ( + { + "id": "X", + "workstream": "S1", + "ref": "r", + "text": "t", + "report": {"glob": "g", "key": "k", "equals": 1, "less_than": 2}, + }, + "exactly one of equals", + ), + ( + {"id": "X", "workstream": "S1", "ref": "r", "text": "t", "file_at_ref": {}}, + "exactly one of path, paths", + ), + ( + { + "id": "X", + "workstream": "S1", + "ref": "r", + "text": "t", + "grep_at_ref": {"paths": ["a"], "pattern": "x", "each": ["1"]}, + }, + "must contain {each}", + ), + ( + { + "id": "X", + "workstream": "S1", + "ref": "r", + "text": "t", + "grep_at_ref": {"paths": ["a"], "pattern": "("}, + }, + "bad pattern", + ), + ( + {"id": "X", "workstream": "S1", "ref": "r", "text": "t", "pr_check": {"nme": "x"}}, + "does not take nme", + ), + ( + { + "id": "X", + "workstream": "S1", + "ref": "r", + "text": "t", + "pr_check": {"name": "x", "max_status": "partial"}, + }, + "needs a cap_reason", + ), + ], +) +def test_bad_metrics_fail_loudly(raw, message): + with pytest.raises(MetricsError, match=message): + parse_metric(raw, WS) + + +def test_duplicate_ids_rejected(tmp_path): + path = tmp_path / "m.yaml" + path.write_text( + textwrap.dedent( + """ + workstreams: {S1: {title: E}} + metrics: + - {id: A, workstream: S1, ref: r, text: t, manual: {status: gap, evidence: e}} + - {id: A, workstream: S1, ref: r, text: t, manual: {status: gap, evidence: e}} + """ + ) + ) + with pytest.raises(MetricsError, match="duplicate metric ids: A"): + load_metrics(path) + + +# --- ref missing ----------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("kind", "probe"), + [ + ("file_at_ref", {"path": "a"}), + ("grep_at_ref", {"paths": ["a"], "pattern": "x"}), + ("command", {"run": "true"}), + ("pr_check", {"name": "x"}), + ("pr_body", {"pattern": "x"}), + ( + "report", + {"glob": "/nonexistent/*.json", "key": "k", "equals": 1, "require_current_sha": True}, + ), + ], +) +def test_missing_ref_is_reported_not_raised(kind, probe): + out = evaluate(metric(kind, **probe), None, ctx(slow=True)) + assert out == Outcome("ref missing", "origin/t3/evals does not exist") + + +def test_manual_and_sha_free_report_ignore_missing_ref(tmp_path): + out = evaluate(metric("manual", status="partial", evidence="half"), None, ctx()) + assert out == Outcome("partial", "half") + probe = {"glob": str(tmp_path / "*.json"), "key": "k", "equals": 1} + assert evaluate(metric("report", **probe), None, ctx()).status == "no evidence yet" + + +# --- file_at_ref and grep_at_ref ------------------------------------------------------------- + + +def test_file_at_ref(): + c = ctx({SHA: {"server/a.json": "{}"}}) + met = evaluate(metric("file_at_ref", path="server/a.json"), SHA, c) + assert met.status == "met" + assert met.links == (("server/a.json", f"{sb.BLOB_URL}/{SHA}/server/a.json"),) + part = evaluate(metric("file_at_ref", paths=["server/a.json", "server/b.json"]), SHA, c) + assert part == Outcome("partial", "missing at aaaaaaaa: server/b.json", met.links) + assert evaluate(metric("file_at_ref", path="nope"), SHA, c).status == "gap" + + +GREP_FILES = { + SHA: { + "server/tests/test_golden.py": "def test_golden_01_x(): ...\ndef test_golden_02_y(): ...\n", + "server/tests/test_other.py": "def test_golden_11(): ...\n", + "server/solver.py": "golden_03 is not a test\n", + } +} + + +def test_grep_without_each_counts_matches(): + c = ctx(GREP_FILES) + m = metric("grep_at_ref", paths=["server/tests/*.py"], pattern=r"def test_", min_matches=3) + out = evaluate(m, SHA, c) + assert out.status == "met" + assert out.evidence == "3 matches in 2 files" + m = metric("grep_at_ref", paths=["server/tests/*.py"], pattern=r"def test_", min_matches=4) + assert evaluate(m, SHA, c).evidence == "3 of 4 matches needed, searched 2 files" + + +def test_grep_each_is_partial_and_names_what_is_missing(): + m = metric( + "grep_at_ref", + paths=["server/tests/*.py"], + pattern=r"golden[^0-9\n]{0,12}{each}(?!\d)", + each=["01", "02", "03", "1"], + ) + out = evaluate(m, SHA, ctx(GREP_FILES)) + # 03 exists only outside the globbed paths; "1" must not match inside "11" or "01". + assert out.status == "partial" + assert out.evidence == "found 01, 02; missing 03, 1" + assert [label for label, _ in out.links] == ["server/tests/test_golden.py"] + + +def test_grep_each_mapping_uses_labels_and_all_found_is_met(): + m = metric( + "grep_at_ref", + paths=["server/tests/*.py"], + pattern="{each}", + each={"first": r"golden_0[12]", "eleven": r"golden_11"}, + ) + assert evaluate(m, SHA, ctx(GREP_FILES)).evidence == "all 2 found" + + +def test_grep_with_no_matching_files_is_a_gap(): + m = metric("grep_at_ref", paths=["ios/*.swift"], pattern="{each}", each=["a"]) + assert evaluate(m, SHA, ctx(GREP_FILES)) == Outcome("gap", "no files match ios/*.swift") + + +def test_max_status_caps_met_at_partial(): + m = metric( + "grep_at_ref", + paths=["server/tests/*.py"], + pattern="def", + max_status="partial", + cap_reason="exists only", + ) + out = evaluate(m, SHA, ctx(GREP_FILES)) + assert out.status == "partial" + assert out.evidence.endswith("(exists only)") + + +# --- command --------------------------------------------------------------------------------- + + +def fake_worktree(root): + @contextlib.contextmanager + def wt(sha): + yield root + + return wt + + +def test_command_runs_in_worktree_keeps_last_lines_and_caches(tmp_path): + (tmp_path / "server").mkdir() + (tmp_path / "server" / "marker").write_text("x") + cache: dict = {} + c = ctx(slow=True, cache=cache, worktree=fake_worktree(tmp_path)) + run = "ls; echo one; echo two; echo three; echo four" + out = evaluate(metric("command", run=run, cwd="server"), SHA, c) + assert out == Outcome("met", "exit 0: two / three / four") + assert cache["S1-1"]["sha"] == SHA + assert cache["S1-1"]["status"] == "met" + + out = evaluate(metric("command", run="echo bad; exit 3", cwd="server"), SHA, c) + assert out == Outcome("gap", "exit 3: bad") + + +def test_command_missing_cwd_and_timeout(tmp_path): + c = ctx(slow=True, worktree=fake_worktree(tmp_path)) + assert evaluate(metric("command", run="true", cwd="server"), SHA, c) == Outcome( + "gap", "no server/ at aaaaaaaa" + ) + out = evaluate(metric("command", run="echo started; sleep 5", timeout_s=0.5), SHA, c) + assert out == Outcome("gap", "timed out after 0.5 s: started") + + +def test_slow_command_not_run_shows_cache_only_for_same_sha(tmp_path): + cache = { + "S1-1": {"sha": SHA, "ran_at": "2026-09-26 01:00 PDT", "status": "met", "evidence": "ok"} + } + + def no_worktree(sha): + raise AssertionError("slow probe must not run") + + c = ctx(cache=cache, worktree=no_worktree) + m = metric("command", run="true") + assert evaluate(m, SHA, c) == Outcome("not run", "slow; last run 2026-09-26 01:00 PDT: met, ok") + assert evaluate(m, OLD, c) == Outcome("not run", "slow; run with --slow") + + +# --- report ---------------------------------------------------------------------------------- + + +def write_report(path, data, mtime): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data)) + os.utime(path, (mtime, mtime)) + return path + + +def report_metric(tmp_path, **probe): + return metric("report", glob=str(tmp_path / "*" / "report.json"), **probe) + + +def test_report_uses_newest_and_compares(tmp_path): + write_report(tmp_path / "old" / "report.json", {"latency_ms": 2000}, 100) + new = write_report(tmp_path / "new" / "report.json", {"latency_ms": 420}, 200) + out = evaluate(report_metric(tmp_path, key="latency_ms", less_than=1000), SHA, ctx()) + assert out == Outcome("met", f"latency_ms = 420 (want < 1000): {new}") + + +@pytest.mark.parametrize( + ("data", "probe", "status"), + [ + ({"build": {"warnings": []}}, {"key": "build.warnings", "length_equals": 0}, "met"), + ({"build": {"warnings": ["w"]}}, {"key": "build.warnings", "length_equals": 0}, "gap"), + ({"states": [1, 2, 3]}, {"key": "states", "length_at_least": 3}, "met"), + ({"states": [1, 2]}, {"key": "states", "length_at_least": 3}, "gap"), + ({"ok": True}, {"key": "ok", "equals": True}, "met"), + ({"ok": 1}, {"key": "ok", "equals": "yes"}, "gap"), + ({"latency_ms": True}, {"key": "latency_ms", "less_than": 1000}, "gap"), + ({"states": [{"state": "result"}]}, {"key": "states.0.state", "equals": "result"}, "met"), + ({"n": 3}, {"key": "n", "length_equals": 0}, "gap"), + ], +) +def test_report_comparators(tmp_path, data, probe, status): + write_report(tmp_path / "r" / "report.json", data, 100) + assert evaluate(report_metric(tmp_path, **probe), SHA, ctx()).status == status + + +def test_report_missing_key_and_no_reports(tmp_path): + m = report_metric(tmp_path, key="build.ok", equals=True) + assert evaluate(m, SHA, ctx()).status == "no evidence yet" + path = write_report(tmp_path / "r" / "report.json", {"build": {}}, 100) + assert evaluate(m, SHA, ctx()) == Outcome("no evidence yet", f"no build.ok in {path}") + + +def test_report_skips_newer_runs_that_did_not_set_the_key(tmp_path): + m = report_metric(tmp_path, key="latency_ms", less_than=1000) + write_report(tmp_path / "old" / "report.json", {"latency_ms": 420.0}, 100) + write_report(tmp_path / "new" / "report.json", {"latency_ms": None}, 200) + outcome = evaluate(m, SHA, ctx()) + assert outcome.status == "met" + assert "old" in outcome.evidence + + +def test_report_requiring_current_sha_marks_stale_or_picks_matching(tmp_path): + older = write_report(tmp_path / "a" / "report.json", {"sha": SHA, "states": [1]}, 100) + newest = write_report(tmp_path / "b" / "report.json", {"sha": OLD, "states": []}, 200) + m = report_metric(tmp_path, key="states", length_at_least=1, require_current_sha=True) + out = evaluate(m, SHA, ctx()) + assert out == Outcome("met", f"states = length 1 (want at least 1): {older}") + + out = evaluate(m, "c" * 40, ctx()) + assert out == Outcome("stale", f"report for bbbbbbbb, ref at cccccccc: {newest}") + + +def test_report_path_shortens_home(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + assert sb.short_home(tmp_path / "x" / "report.json") == "~/x/report.json" + assert sb.short_home("/elsewhere/report.json") == "/elsewhere/report.json" + + +# --- PR probes ------------------------------------------------------------------------------- + + +def pr(branch="t3/evals", head=SHA, checks=(), body=""): + return { + "number": 12, + "title": "Evals", + "headRefName": branch, + "headRefOid": head, + "isDraft": True, + "updatedAt": "2026-09-26T07:47:51Z", + "statusCheckRollup": list(checks), + "body": body, + } + + +def test_pr_check_states(): + m = metric("pr_check", name="iOS build") + ok = {"t3/evals": pr(checks=[run("iOS build", "SUCCESS")])} + assert evaluate(m, SHA, ctx(prs=ok)).status == "met" + assert evaluate(m, OLD, ctx(prs=ok)) == Outcome( + "stale", + "iOS build passed for aaaaaaaa, ref at bbbbbbbb", + (("#12", f"{sb.PR_URL}/12"),), + ) + rerun = {"t3/evals": pr(checks=[run("iOS build", "SUCCESS"), run("iOS build", "FAILURE")])} + assert evaluate(m, SHA, ctx(prs=rerun)).status == "gap" + pending = {"t3/evals": pr(checks=[run("iOS build", None, status="QUEUED")])} + assert evaluate(m, SHA, ctx(prs=pending)).status == "not run" + other = {"t3/evals": pr(checks=[run("Lint", "SUCCESS")])} + assert evaluate(m, SHA, ctx(prs=other)).evidence == "no check named 'iOS build'" + assert evaluate(m, SHA, ctx(prs={})) == Outcome("no evidence yet", "no open PR for t3/evals") + assert evaluate(m, SHA, ctx(prs=None)) == Outcome("no evidence yet", "PR list unavailable") + + +def test_pr_body_counts_images(): + m = metric("pr_body", pattern=r"!\[[^\]]*\]\([^)]+\)| None: ... +def test_10_margins_cover_equality(e, offset, expected) -> None: ... +def test_11a_unseen_wall_past_a_tapped_corner() -> None: ... +def test_golden_12_last_start() -> None: ... + +@given(st.lists(st.floats(0, 1), min_size=1), st.booleans()) +@settings(max_examples=50) +def test_missing_coverage_never_passes(xs, flag) -> None: ... + +@given(st.floats()) +def test_mirror_gives_the_same_decision(x) -> None: ... +""" + + +def real_metric(mid): + return next(m for m in load_metrics()[1] if m.id == mid) + + +def test_golden_pattern_accepts_numbered_test_names_only(): + out = evaluate(real_metric("S2-2"), SHA, ctx({SHA: {"server/tests/test_g.py": S2_TESTS}})) + assert out.status == "partial" + assert out.evidence.startswith("found 07, 10, 11, 12; missing 01, 02, 03, 04, 05, 06, 08") + + +def test_property_pattern_needs_a_given_decorator(): + # test_07 and test_10 name coverage and equality but are not Hypothesis tests. + out = evaluate(real_metric("S2-3"), SHA, ctx({SHA: {"server/tests/test_p.py": S2_TESTS}})) + assert out.status == "partial" + assert out.evidence.startswith( + "found missing coverage, mirror; missing equality unsure, larger clearance" + ) + + +def test_slow_only_runs_the_named_probe(tmp_path): + c = ctx(slow=True, slow_only=frozenset({"S2-1"}), worktree=fake_worktree(tmp_path)) + assert evaluate(metric("command", run="true"), SHA, c).status == "not run" + c.slow_only = frozenset({"S1-1"}) + assert evaluate(metric("command", run="true"), SHA, c).status == "met" + + +def test_tail_keeps_last_three_lines_with_spaces_collapsed(): + assert ( + sb._tail("a\n\n... [ 82%]\nb\n 87 passed in 1s \n") + == "... [ 82%] / b / 87 passed in 1s" + ) diff --git a/verification/ux/checklist.md b/verification/ux/checklist.md index f0a6e8800..71abe7d19 100644 --- a/verification/ux/checklist.md +++ b/verification/ux/checklist.md @@ -104,24 +104,17 @@ Numeric limits marked *(reviewer threshold)* are review conventions. No user tes ## Per-state table -The state names below are placeholders. Match them to the app's real `STATE=` names once C4 logging lands. As of 2026-09-26 the local `t3/ios-mvf` worktree has none and isn't pushed. Its tracking cases (`normal`, `limited` with `excessiveMotion`, `insufficientFeatures`, `initializing` or `relocalizing`, `notAvailable`, `cameraAccessDenied`) should map onto the rows below. - -| State (placeholder) | Must show | Key items | -|---|---|---| -| onboarding | What the scan does and roughly how long it takes, in one or two sentences, and one start button | I1, I3, A1, A4, V2 | -| camera_permission / denied | Why the camera is needed. When denied, "Open Settings" | R9, I3, A1 | -| ar_unavailable | A plain explanation and what to do instead | R10, I3 | -| go_to_meter | "Go to your electric meter" and **I'm here** | I1, V8, A4 | -| meter_closeup | A reticle, no shutter, a named fix on failure, "Can't get a clear shot" after two tries | I5, T5, T6, R11, A8 | -| tap_meter_anchor | "Tap the meter". With no plane, "step closer" | I8, I2, A6 | -| step_back | A standing cue distinct from the aiming cue | I2, I1 | -| wall_walk (strip, fog) | One instruction, the strip with non-colour cell marks, upload status kept separate, "I can't access this area" | R1, R2, R3, R7, I6, I7, T4, A2, A9, V1, V4 | -| tracking_lost | A cause-specific prompt, no modal, coverage and arrows frozen | T1, T2, T3, R3, A3 | -| relocalizing | The saved meter view and "Point at the meter like this.", with a fallback to tapping the meter again | T7, R4, R5 | -| gap_recapture | One line with the reason and target, a skip option | I10, T4, R1 | -| panel | Safety wording and Skip, auto-capture with no shutter | I9, I5, T5 | -| feature_marking | The saved frame with one instruction for marking a missed object and one confirm button | I1, A1, A6, H1 | -| homeowner_question | One question and two large answer buttons | I4, A4, A6 | -| upload_processing | Upload count and pending state, no approval wording | R6, R7, H3 | -| result_reveal | The box on the meter anchor, numbers with error on tap, unsure checks as installer review | H1 to H7, A7 | -| offline_error | Saved on the phone and not sent, local guidance still running, retry | R6, R8, I3 | +State names are the raw values of `ScanPhase` in `ios/HouseScan/Contract/ScanContract.swift` on `t3/ios-mvf` (at `6885b7b`), which the app logs as `STATE=`. Several screens the first-try note describes are conditions inside a phase rather than phases of their own; the second column says where to look for them. + +| State | Also covers | Must show | Key items | +|---|---|---|---| +| `onboarding` | camera permission, denied permission | What the scan does and roughly how long it takes, in one or two sentences, and one start button. When camera access is denied, why it is needed and "Open Settings" | I1, I3, A1, A4, V2, R9 | +| `unsupported` | | A plain explanation and what to do instead | R10, I3 | +| `findMeter` | tap the meter, step back | "Go to your electric meter" and **I'm here**; then "Tap the meter", with "step closer" when there is no wall plane; then a standing cue distinct from the aiming cue | I1, I2, I8, V8, A4, A6 | +| `meterCloseUp` | panel close-ups | A reticle, no shutter, a named fix on failure, "Can't get a clear shot" after two tries. Panel shots add the safety wording and Skip | I5, I9, T5, T6, R11, A8 | +| `wallWalk` | tracking lost, relocalizing | One instruction, the strip with non-colour cell marks, upload status kept separate, "I can't access this area". When tracking is limited: a cause-specific prompt, no modal, coverage and arrows frozen. After an interruption: the saved meter view | R1, R2, R3, R4, R5, R7, I6, I7, T1, T2, T3, T4, T7, A2, A3, A9, V1, V4 | +| `gapRequest` | | One line with the reason and target, and a skip option | I10, T4, R1 | +| `markFeatures` | homeowner questions | The saved frame with one instruction for marking a missed object and one confirm button. Questions: one at a time, two large answer buttons | I1, I4, A1, A4, A6, H1 | +| `uploading` | offline | Upload count and pending state, no approval wording. Offline: saved on the phone and not sent, retry | R6, R7, R8, H3 | +| `result` | | Decision in plain words, numbers with error on tap, unsure checks as installer review, the unseen side disclosed | H1, H2, H3, H5, H6, H7 | +| `resultAR` | | The box on the meter anchor and the cable line, hidden with an explanation when tracking is not normal | H4, A7, T3 | From ede8be1a772fc293225a3f467682aac518a08972 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 03:21:41 -0500 Subject: [PATCH 09/59] verification: regenerate scoreboard Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/SCOREBOARD.md | 8 +++--- verification/scoreboard/scoreboard.json | 35 ++++++++++++++----------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index aefb762ce..2c48fa461 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -7,9 +7,9 @@ Generated 2026-09-26 03:21 CDT by `make scoreboard` (`uv run python -m hsverify. | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | | [#14](https://github.com/SamGu-NRX/house-scanning/pull/14) | `t3/docs-cleanup` | `9c9ceb2a` | no | 4 success | 2026-09-26 03:15 CDT | -| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `328373d8` | yes | 4 success | 2026-09-26 03:19 CDT | +| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `2ae5a802` | yes | no checks | 2026-09-26 03:21 CDT | | [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `b0b8a274` | yes | 4 success | 2026-09-26 03:11 CDT | -| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `fcd45741` | yes | 4 success, 1 pending | 2026-09-26 03:20 CDT | +| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `fcd45741` | yes | 5 success | 2026-09-26 03:20 CDT | | [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `6885b7b5` | yes | 5 success | 2026-09-26 03:07 CDT | | [#9](https://github.com/SamGu-NRX/house-scanning/pull/9) | `codex/capture-contract-probe` | `4b6c4309` | yes | 4 success | 2026-09-26 00:08 CDT | | [#8](https://github.com/SamGu-NRX/house-scanning/pull/8) | `t3/testflight` | `b896d59f` | no | 4 success | 2026-09-26 03:18 CDT | @@ -61,14 +61,14 @@ Generated 2026-09-26 03:21 CDT by `make scoreboard` (`uv run python -m hsverify. ## S4 Verification -`origin/t3/verification` at [`328373d8`](https://github.com/SamGu-NRX/house-scanning/tree/328373d81f280496380c0a9737f1dd0cc3ac50bb), `origin/t3/ios-mvf` at [`6885b7b5`](https://github.com/SamGu-NRX/house-scanning/tree/6885b7b5ad872df7f73a6b915e5837cd38a2034d). 3 gap, 1 no evidence yet. +`origin/t3/verification` at [`2ae5a802`](https://github.com/SamGu-NRX/house-scanning/tree/2ae5a802b41793f6135e56d6c22f33e3109e7761), `origin/t3/ios-mvf` at [`6885b7b5`](https://github.com/SamGu-NRX/house-scanning/tree/6885b7b5ad872df7f73a6b915e5837cd38a2034d). 1 met, 2 gap, 1 no evidence yet. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S4-1 | End to end from a real replay to a placement result passes | no evidence yet | no report matches ~/house-scanning-data/reports/e2e/*/report.json | | S4-2 | Screenshot report per app state | gap | states = length 0 (want at least 9): ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json | | S4-3 | Product description written from the code and checked against screenshots | gap | not yet reviewed | -| S4-4 | Scoreboard of every PR | gap | missing at 328373d8: verification/SCOREBOARD.md | +| S4-4 | Scoreboard of every PR | met | present at 2ae5a802 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/2ae5a802b41793f6135e56d6c22f33e3109e7761/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index bf3b0029b..b88bbdf7f 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-09-26T03:21:08-05:00", + "generated_at": "2026-09-26T03:21:41-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": false, "prs": [ @@ -25,18 +25,18 @@ "number": 13, "title": "verification: independent end-to-end checks (S4)", "branch": "t3/verification", - "head": "328373d81f280496380c0a9737f1dd0cc3ac50bb", + "head": "2ae5a802b41793f6135e56d6c22f33e3109e7761", "draft": true, - "updated": "2026-09-26T08:19:36Z", + "updated": "2026-09-26T08:21:37Z", "ci": { "counts": { - "success": 4, + "success": 0, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "4 success" + "text": "no checks" } }, { @@ -66,13 +66,13 @@ "updated": "2026-09-26T08:20:33Z", "ci": { "counts": { - "success": 4, + "success": 5, "failure": 0, - "pending": 1, + "pending": 0, "skipped": 0 }, "failing": [], - "text": "4 success, 1 pending" + "text": "5 success" } }, { @@ -465,7 +465,7 @@ "refs": [ { "ref": "origin/t3/verification", - "sha": "328373d81f280496380c0a9737f1dd0cc3ac50bb" + "sha": "2ae5a802b41793f6135e56d6c22f33e3109e7761" }, { "ref": "origin/t3/ios-mvf", @@ -477,7 +477,7 @@ "id": "S4-1", "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/verification", - "sha": "328373d81f280496380c0a9737f1dd0cc3ac50bb", + "sha": "2ae5a802b41793f6135e56d6c22f33e3109e7761", "status": "no evidence yet", "evidence": "no report matches ~/house-scanning-data/reports/e2e/*/report.json", "links": [] @@ -495,7 +495,7 @@ "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "328373d81f280496380c0a9737f1dd0cc3ac50bb", + "sha": "2ae5a802b41793f6135e56d6c22f33e3109e7761", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -504,10 +504,15 @@ "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "328373d81f280496380c0a9737f1dd0cc3ac50bb", - "status": "gap", - "evidence": "missing at 328373d8: verification/SCOREBOARD.md", - "links": [] + "sha": "2ae5a802b41793f6135e56d6c22f33e3109e7761", + "status": "met", + "evidence": "present at 2ae5a802", + "links": [ + [ + "verification/SCOREBOARD.md", + "https://github.com/SamGu-NRX/house-scanning/blob/2ae5a802b41793f6135e56d6c22f33e3109e7761/verification/SCOREBOARD.md" + ] + ] } ] }, From c47c70ef9b96391dccb9bb0ede71c65c52be48c3 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 03:26:45 -0500 Subject: [PATCH 10/59] verification: independent placement cases from the public goldens 27 scenes written from docs/research/t3-lane-c-review.md and contract C5, not from the server's fixtures, each with the outcomes its geometry forces and the thresholds it assumes. generate.py writes them; the README shows the arithmetic. Expectations can now pin a check's measured value and error. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/e2e/cases/README.md | 111 ++++ verification/e2e/cases/c5-no-coverage.json | 86 +++ .../e2e/cases/g01-unseen-beyond-garage.json | 141 +++++ .../e2e/cases/g03-negative-s-near-edge.json | 227 +++++++ .../e2e/cases/g05-inside-corner-gas.json | 169 +++++ .../e2e/cases/g06-regulator-footprint.json | 170 +++++ .../e2e/cases/g08a-garage-blocks-route.json | 147 +++++ .../cases/g08b-missing-wall-blocks-route.json | 152 +++++ .../e2e/cases/g10-derived-compound-error.json | 158 +++++ verification/e2e/cases/g10-e03-fail.json | 169 +++++ .../e2e/cases/g10-e03-lower-edge.json | 158 +++++ verification/e2e/cases/g10-e03-pass.json | 154 +++++ .../e2e/cases/g10-e03-upper-edge.json | 158 +++++ verification/e2e/cases/g10-e05-fail.json | 169 +++++ .../e2e/cases/g10-e05-lower-edge.json | 158 +++++ verification/e2e/cases/g10-e05-pass.json | 154 +++++ .../e2e/cases/g10-e05-upper-edge.json | 158 +++++ verification/e2e/cases/g10-e15-fail.json | 169 +++++ .../e2e/cases/g10-e15-lower-edge.json | 158 +++++ verification/e2e/cases/g10-e15-pass.json | 154 +++++ .../e2e/cases/g10-e15-upper-edge.json | 158 +++++ .../e2e/cases/g11a-unexplored-corner.json | 160 +++++ .../cases/g11b-both-ends-real-all-fail.json | 229 +++++++ .../e2e/cases/g11c-unobserved-ground.json | 174 ++++++ .../e2e/cases/g12a-subgrid-start.json | 173 ++++++ .../e2e/cases/g12b-exact-fit-last-start.json | 181 ++++++ .../g13a-unsure-never-outranks-pass.json | 168 +++++ verification/e2e/cases/g13b-unsure-alone.json | 174 ++++++ verification/e2e/cases/generate.py | 588 ++++++++++++++++++ verification/hsverify/resultcheck.py | 8 + verification/tests/test_resultcheck.py | 7 + 31 files changed, 5140 insertions(+) create mode 100644 verification/e2e/cases/README.md create mode 100644 verification/e2e/cases/c5-no-coverage.json create mode 100644 verification/e2e/cases/g01-unseen-beyond-garage.json create mode 100644 verification/e2e/cases/g03-negative-s-near-edge.json create mode 100644 verification/e2e/cases/g05-inside-corner-gas.json create mode 100644 verification/e2e/cases/g06-regulator-footprint.json create mode 100644 verification/e2e/cases/g08a-garage-blocks-route.json create mode 100644 verification/e2e/cases/g08b-missing-wall-blocks-route.json create mode 100644 verification/e2e/cases/g10-derived-compound-error.json create mode 100644 verification/e2e/cases/g10-e03-fail.json create mode 100644 verification/e2e/cases/g10-e03-lower-edge.json create mode 100644 verification/e2e/cases/g10-e03-pass.json create mode 100644 verification/e2e/cases/g10-e03-upper-edge.json create mode 100644 verification/e2e/cases/g10-e05-fail.json create mode 100644 verification/e2e/cases/g10-e05-lower-edge.json create mode 100644 verification/e2e/cases/g10-e05-pass.json create mode 100644 verification/e2e/cases/g10-e05-upper-edge.json create mode 100644 verification/e2e/cases/g10-e15-fail.json create mode 100644 verification/e2e/cases/g10-e15-lower-edge.json create mode 100644 verification/e2e/cases/g10-e15-pass.json create mode 100644 verification/e2e/cases/g10-e15-upper-edge.json create mode 100644 verification/e2e/cases/g11a-unexplored-corner.json create mode 100644 verification/e2e/cases/g11b-both-ends-real-all-fail.json create mode 100644 verification/e2e/cases/g11c-unobserved-ground.json create mode 100644 verification/e2e/cases/g12a-subgrid-start.json create mode 100644 verification/e2e/cases/g12b-exact-fit-last-start.json create mode 100644 verification/e2e/cases/g13a-unsure-never-outranks-pass.json create mode 100644 verification/e2e/cases/g13b-unsure-alone.json create mode 100644 verification/e2e/cases/generate.py diff --git a/verification/e2e/cases/README.md b/verification/e2e/cases/README.md new file mode 100644 index 000000000..5c373673e --- /dev/null +++ b/verification/e2e/cases/README.md @@ -0,0 +1,111 @@ +# Placement cases for the server's HTTP API + +Black-box cases for the placement server. Each `.json` holds a scene and what the result must +show. The scenes were written from `server/schemas/scene.schema.json` and `result.schema.json` at +origin/t3/server 69c6c3c and the golden tests in `docs/research/t3-lane-c-review.md` at +origin/t3/research a7d91f1. Nobody read the server's code or fixtures to write them, so a passing +server has been checked against the spec, not against its own tests. + +`generate.py` writes every JSON file in this folder. Edit it, not the JSON, then run from +`verification/`: + +```sh +uv run python e2e/cases/generate.py +``` + +## Why the assertions avoid the decision + +`server/rules.yaml` was not on origin/t3/server when these cases were written, so the numbers come +from the review's parameter table and the rules table in `docs/01-feature-map.md`. While the rules +hold placeholder values, `policy.auto_approve` is false and any would-be pass or reject becomes +`manual_review`. The cases therefore assert on the sweep, on single checks, on `spot` and on +`missing_evidence`. They assert a decision only where the scene alone forces it: + +- An unsure spot or an unobserved area rules out `pass`. +- An unexplored end rules out `reject`. So does a missing `coverage`, since its ends then default + to unexplored. + +## Expectation keys + +The keys follow the lead's case format. The cases use two additions and one convention: + +- `checks[].measured_ft` and `checks[].plus_minus_ft` (added): the gap and error the check must + report, within 1e-6. The g10 and g13b outcomes mean something only if the server measured the + gap the case built. With a different battery depth it measures a different gap, and these keys + catch that before the outcome is compared. +- `missing_evidence_empty: false` means the list must be non-empty. +- `rules_assumed` maps a check-id substring to the `threshold_ft` the case depends on. `gas` and + `opening` are both 3 ft (Base help page; IRC R328). + +## Fixture conventions + +- Units are feet. Walls are exact (`plus_minus_ft: 0`) unless a case says otherwise. So are the + meter and every object: `source: "tape"` with an explicit error, so no default error applies. +- Battery: 31/12 ft wide (W ≈ 2.5833) and 11/6 ft deep (D ≈ 1.8333), from the review's table. The + schema can't carry the battery size, so the checker can't tell which one the server used. + Sweep intervals are chosen to hold for any width from 30.68 in to 3 ft. The cases that depend on + the exact size are named in the table. +- A straight wall along +x has outward +z (the schema's clockwise rule). s = x − meter x. +- Cases g10, g11a, g11c and g13 place the wall at z = −11/6. A flush battery's front edge then + sits at z = 0.0 exactly in floating point, so a gas strip at z = d measures exactly d. The + equality rows need this: 3.3 − 0.3 and 2.7 + 0.3 both evaluate to exactly 3.0, so a server + that forgets equality lands on the wrong side and fails the case. +- "Fully observed" means wall, overhead, facing and ground (10 ft out) bands over the whole chain, + both ends `limit`, facing depth 9, no overheads, and concrete ground over the whole area. +- A gas "strip" is a long, thin gas footprint parallel to the wall. It spans past both ends of + the stretch it covers, so every battery beside it has the same perpendicular gap. It stands in + for the review's "every start has the same gas gap". + +## Cases + +| Case | Golden | Asserts | Arithmetic | +| --- | --- | --- | --- | +| g01-unseen-beyond-garage | 01 | `missing_evidence` empty; starts in (−13.9, −9.7) fail with `route` | Wall s = [−14, 10], garage [−7, −3], nothing observed left of −7. A start left of the garage has its battery at s ≤ −7, so its cable must cross the garage. The last such start is −7 − W = −9.583. | +| g03-negative-s-near-edge | 03 | spot on w1 within [−9, −6]; starts in (−8.4, 6.9) fail; no photo request | Pads are straight 3 ft segments at s = [−9, −6] and [7, 10]. Between them sit 7 segments of 13/7 ≈ 1.857 ft, each shorter than any battery. Headings run 32°, 24°, …, −24°, −32°, so every corner is an outside corner. Left route = 6 (near edge), right = 7, so left wins. Measuring to the far edge would give 103/12 ≈ 8.583 > 7 and pick the right pad. Pad starts are (−9, −8.583] and [7, 7.417], and every other start crosses a corner. | +| g05-inside-corner-gas | 05 | starts on w1 in (6, 7.4) fail with `gas`; never pass | w1 is z = 0 from x = −7 to 4 (w1 extended 1 ft left of the golden, so the meter at x = −6 sits inside the wall rather than on its end); s = x + 6. w2 is x = 4 going +z; gas point [4, 4] at s = 14. For start s the battery is x ∈ [s − 6, s − 6 + W], z ∈ [0, 11/6]. At s = 6 the nearest corner is (31/12, 11/6): √((17/12)² + (13/6)²) = √965/12 ≈ 2.589 < 3. Fail needs the right edge past 4 − √(9 − (13/6)²) = 4 − √155/6 ≈ 1.925, so s > 5.342. The last start on w1 is 89/12 ≈ 7.417 (2.167). Unrolled gap 14 − 103/12 = 65/12 ≈ 5.417 would wrongly pass. | +| g06-regulator-footprint | 06 | starts in (3, 9) fail with `gas`; never pass | Gas footprint: body x ∈ [6.5, 7.5], z ∈ [9.5, 10.5], with a spike reaching [7, 23/6]. Battery front at z = 11/6, so the gap to the tip is 23/6 − 11/6 = 2 whenever the battery covers x = 7, i.e. starts in [7 − W, 7] ≈ [4.417, 7]. Outside that the gap is √(dx² + 4) < 3 while dx < √5 ≈ 2.236, so starts in (2.18, 9.24) fail. The body alone gives 10 − 11/6 ≈ 8.17 and would pass. | +| g08a-garage-blocks-route | 08 | spot null; starts in (5, 9.4) fail with `route`; no photo request; never pass | Wall s = [−0.5, 12], garage [1, 5]. The earliest start is −0.5, so every battery ends at ≥ −0.5 + W ≈ 2.08 > 1 and overlaps the garage (opening gap 0 < 3), or starts at ≥ 5 and needs a cable across the garage. The golden's garage [3, 7] and pad [10, 13] were shifted so the meter side holds no start. | +| g08b-missing-wall-blocks-route | 08 | spot null; starts on w2 in (2, 9.4) fail with `route`; no photo request; never pass | w1 s = [−0.5, 1] (1.5 ft, shorter than the battery), no wall for s = (1, 2), w2 s = [2, 12]. The schema says no battery backs onto and no cable crosses a no-wall stretch, so every start fails by geometry. | +| g10-e03-{pass, upper-edge, lower-edge, fail} | 10 | gas check pass / unsure (margin) / unsure (margin) / fail, with measured d and ±0.3; no photo request; the fail row also needs spot null and every start in (−5, 12.4) failing with `gas` | T = 3, e = 0.3. d = 3.31: 3.31 − 0.3 = 3.01 > 3, pass. d = 3.3: 3.3 − 0.3 = 3.0, not > 3; 3.6 not < 3, so unsure. d = 2.7: 2.7 + 0.3 = 3.0, not < 3, so unsure. d = 2.69: 2.99 < 3, fail. | +| g10-e05-* | 10 | as above with ±0.5 | d = 3.51, 3.5, 2.5, 2.49 → 3.01 > 3; 3.0; 3.0; 2.99 < 3. | +| g10-e15-* | 10 | as above with ±1.5 | d = 4.51, 4.5, 1.5, 1.49 → 3.01 > 3; 3.0; 3.0; 2.99 < 3. | +| g10-derived-compound-error | 10 | gas unsure (margin), measured 3.5, ±0.6; never pass | Wall ±0.3 plus gas ±0.3 = ±0.6 (linear, as the review specifies). 3.5 − 0.6 = 2.9 is not > 3, and 3.5 + 0.6 = 4.1 is not < 3. Root-sum-square would give ±0.424 and 3.076 > 3, a wrong pass. Every g10 gas object also carries `conf: 0.6`; treating 1 − conf as an error would shift every row. | +| g11a-unexplored-corner | 11 (a) | decision manual_review; spot null; `missing_evidence` non-empty; starts in (−3, 5.4) fail with `gas` | Strip at gap 1: 1 + 0 < 3 fails every start. The right end at s = 8 is `unexplored`, so reject is ruled out and the wall past the corner must be requested. The end sits 8 ft out so it is within any plausible reach. | +| g11b-both-ends-real-all-fail | 11 (b) | decision manual_review or reject; spot null; no photo request; w1 starts in (−3, 5.4) and w2 starts in (8, 11.4) fail with `gas` | w1 s = [−3, 8], outside corner, w2 runs x = 8 toward −z, s = [8, 14], outward +x. Strips sit 1 ft beyond each wall's battery front: z = 11/6 + 1 and x = 8 + 11/6 + 1. | +| g11c-unobserved-ground | 11 (c) | decision manual_review; spot within [2.328, 10]; `missing_evidence` non-empty; starts in (−9.9, 2.2) fail with `gas`; starts in (2.5, 7.3) unsure | Strip x ∈ [−12, −0.5] at gap 1. A start a > −0.5 has gap √((a + 0.5)² + 1), under 3 while a < 2√2 − 0.5 ≈ 2.328. Ground was observed only for s ≤ 1, so starts past 2.328 pass gas on unseen ground: unsure. The last start is 10 − W ≈ 7.417. | +| g12a-subgrid-start | 12 | spot within [7, 7.1 + W]; starts in (−1, 6.95) and (7.2, 13.4) fail with `opening` | Windows [3, 4] and [10.1 + W, 11.1 + W] ≈ [12.683, 13.683]. Legal start: a − 4 > 3 and (10.1 + W) − (a + W) > 3, so 7 < a < 7.1. Both ends give a gap of exactly 3, which is unsure, not pass. The 2 in grid from 0 samples 7 and 43/6 ≈ 7.167, both outside. The wall starts at −1, so any battery left of window 1 would need its right edge < 0 but has it ≥ −1 + W > 0. Past window 2 a start needs a > 16.683, beyond the wall end at 16. Depends on W: with 30.68 in the legal interval is (7, 7.127), still inside the assertion. | +| g12b-exact-fit-last-start | 12 | spot within [6, 6 + W]; no photo request | Five 1.8 ft segments (headings 40° to 8°) for s = [−3, 6], then one segment of exactly W at heading 0° ending the wall. Its only start, 6, is also its last. Measured in floating point, the segment is 4.4e-16 longer than W. | +| g13a-unsure-never-outranks-pass | 13 | spot within [−8, −0.506]; gas check at the spot passes; starts in (0.5, 5) unsure; no photo request | Strip x ∈ [0.3, 12] at gap 3.2, ±0.3: every battery overlapping x ≥ 0.3 gets 3.2 − 0.3 = 2.9, unsure. A battery with right edge b < 0.3 gets √((0.3 − b)² + 3.2²). That passes when > 3.3, so (0.3 − b)² > 0.65 and b < 0.3 − √0.65 ≈ −0.506. Unsure spots have routes near 0 and passing ones ≥ 0.506, and the passing spot must still win. | +| g13b-unsure-alone | 13 | decision manual_review; spot on w1; gas unsure (margin), measured 3.2, ±0.3; starts in (−0.4, 5.3) unsure; no photo request | The wall starts at −0.5, so every battery ends at ≥ 2.08 and overlaps the strip: gap 3.2 for all. The spot is fully observed and unsure only by margin, so no photos. | +| c5-no-coverage | C5, scene schema | decision manual_review; `missing_evidence` non-empty; starts in (−6, 7.4) unsure | Clean wall s = [−6, 10] with ground and facing data but no `coverage`. "Absent means nothing is known to be observed" makes every start unsure. Absent ends default to unexplored, which rules out reject. | + +## Assumptions a failure should be checked against + +- A battery must sit flush on one straight segment (review item 4, golden 04; the result's + `spot.segment`). g03 and g12b use short segments to rule out starts, and g08b uses a 1.5 ft wall. +- Concrete ground, a 9 ft facing gap and no overheads pass. Only g03, g11c (spot), g12a, g12b, + g13a, g13b and the unsure sweep in c5 need this. A server whose placeholder rules fail concrete or + a 9 ft gap fails those cases without having broken the rule under test. +- A cable may pass under a window (golden 12 places a window between meter and spot and still + expects pass). g12a depends on it. +- A garage door is an opening under the 3 ft rule, and the route may not cross it (golden 08). + g01 and g08a depend on it. The crossing effect is policy (review item 7); if the placeholder rules + make it `review`, g01 and g08a fail and the reason is that rule. +- Gas objects with a footprint are measured by footprint. Their `span_ft` is the footprint's + stretch along the wall and does not block the cable route. In g10, g11a and g11b the span covers + the meter. A server that routes around spans still passes those, because they assert only gas + and photo requests. g13a keeps the span off the left routes. +- s counts the length of a no-wall stretch (g08b: w2 starts at s = 2, after a 1 ft gap). + +## Goldens not encoded + +- 02: its only assertion is the policy flag itself. Checking `policy.auto_approve` is false and no + result is `pass` across all runs covers it. +- 04: the golden gives w1 outward [0, −1] for a baseline running +x, which contradicts the + schema's clockwise rule. Its pad also relies on a ground check whose passing surfaces are not + published. The flush rule it tests is exercised by g03 and g12b. +- 07: needs `headroom_ft`, which is not published. +- 09: needs `confident_reach_ft` and `max_route_ft`, which are not published. +- 11 (d): its expectation is a pass, which placeholder rules can't produce. Its photo-request half + duplicates g01. +- 14: needs a pass. The result schema already requires every field it lists. diff --git a/verification/e2e/cases/c5-no-coverage.json b/verification/e2e/cases/c5-no-coverage.json new file mode 100644 index 000000000..8b88b16ba --- /dev/null +++ b/verification/e2e/cases/c5-no-coverage.json @@ -0,0 +1,86 @@ +{ + "id": "c5-no-coverage", + "source": "scene.schema.json coverage and end descriptions at origin/t3/server 69c6c3c; C5", + "description": "A clean straight wall with ground and facing measurements but no coverage field. Nothing is known to be observed, so no start may pass, the result must ask for views, and it cannot reject because both ends default to unexplored.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -6, + 0.0 + ], + [ + 10, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -6, + 0.0 + ], + [ + 10, + 0.0 + ], + [ + 10, + 10.0 + ], + [ + -6, + 10.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -6.0, + 10.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ] + }, + "rules_assumed": {}, + "expect": { + "decision_in": [ + "manual_review" + ], + "missing_evidence_empty": false, + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + -6, + 7.4 + ], + "outcome": "unsure" + } + ] + } +} diff --git a/verification/e2e/cases/g01-unseen-beyond-garage.json b/verification/e2e/cases/g01-unseen-beyond-garage.json new file mode 100644 index 000000000..136a684d5 --- /dev/null +++ b/verification/e2e/cases/g01-unseen-beyond-garage.json @@ -0,0 +1,141 @@ +{ + "id": "g01-unseen-beyond-garage", + "source": "t3-lane-c-review.md golden 01 at origin/t3/research a7d91f1", + "description": "The wall beyond a garage door left of the meter was never observed. No spot there can be reached by a cable, so the result asks for no photos, and every start beyond the garage fails its route.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -14, + 0.0 + ], + [ + 10, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "garage_door", + "wall_id": "w1", + "span_ft": [ + -7, + -3 + ], + "bottom_ft": 0.0, + "top_ft": 7.0, + "attrs": { + "operable": true + }, + "source": "tape", + "plus_minus_ft": 0.0 + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -14, + 0.0 + ], + [ + 10, + 0.0 + ], + [ + 10, + 10.0 + ], + [ + -14, + 10.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -14.0, + 10.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -7, + 10 + ] + }, + { + "band": "overhead", + "span_ft": [ + -7, + 10 + ] + }, + { + "band": "facing", + "span_ft": [ + -7, + 10 + ] + }, + { + "band": "ground", + "span_ft": [ + -7, + 10 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": {}, + "expect": { + "missing_evidence_empty": true, + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + -13.9, + -9.7 + ], + "outcome": "fail", + "failing_match": "route" + } + ] + } +} diff --git a/verification/e2e/cases/g03-negative-s-near-edge.json b/verification/e2e/cases/g03-negative-s-near-edge.json new file mode 100644 index 000000000..646d19ebc --- /dev/null +++ b/verification/e2e/cases/g03-negative-s-near-edge.json @@ -0,0 +1,227 @@ +{ + "id": "g03-negative-s-near-edge", + "source": "t3-lane-c-review.md golden 03 at origin/t3/research a7d91f1", + "description": "Two straight 3 ft pads, s = [-9, -6] and [7, 10], joined by segments 13/7 ft long that no battery fits on. The left pad's route runs to its near edge (6 ft), so it beats the right pad (7 ft); measuring to the far edge (103/12 ft) would pick the right pad.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -8.293569701353903, + -3.1154881209977927 + ], + [ + -5.749425412884625, + -1.525730328298178 + ], + [ + -4.052840991548366, + -0.7703622768716918 + ], + [ + -2.267640699091488, + -0.2584643303544074 + ], + [ + -0.4285714285714288, + 0.0 + ], + [ + 1.4285714285714288, + 0.0 + ], + [ + 3.267640699091489, + -0.2584643303544074 + ], + [ + 5.052840991548367, + -0.7703622768716918 + ], + [ + 6.7494254128846265, + -1.5257303282981778 + ], + [ + 9.293569701353903, + -3.1154881209977923 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -8.293569701353903, + -3.1154881209977927 + ], + [ + -5.749425412884625, + -1.525730328298178 + ], + [ + -4.052840991548366, + -0.7703622768716918 + ], + [ + -2.267640699091488, + -0.2584643303544074 + ], + [ + -0.4285714285714288, + 0.0 + ], + [ + 1.4285714285714288, + 0.0 + ], + [ + 3.267640699091489, + -0.2584643303544074 + ], + [ + 5.052840991548367, + -0.7703622768716918 + ], + [ + 6.7494254128846265, + -1.5257303282981778 + ], + [ + 9.293569701353903, + -3.1154881209977923 + ], + [ + 15.652600872152364, + 7.06108903287932 + ], + [ + 12.383084166315315, + 9.069640786008945 + ], + [ + 9.15708271145639, + 10.505949172559209 + ], + [ + 5.762580988904602, + 11.479306878451261 + ], + [ + 2.2656491135009325, + 11.970768603117891 + ], + [ + -1.2656491135009331, + 11.970768603117891 + ], + [ + -4.762580988904602, + 11.479306878451261 + ], + [ + -8.15708271145639, + 10.505949172559209 + ], + [ + -11.383084166315314, + 9.069640786008947 + ], + [ + -14.652600872152362, + 7.06108903287932 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -9, + 10 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -9, + 10 + ] + }, + { + "band": "overhead", + "span_ft": [ + -9, + 10 + ] + }, + { + "band": "facing", + "span_ft": [ + -9, + 10 + ] + }, + { + "band": "ground", + "span_ft": [ + -9, + 10 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": {}, + "expect": { + "spot": { + "wall_id": "w1", + "span_within": [ + -9, + -6 + ] + }, + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + -8.4, + 6.9 + ], + "outcome": "fail" + } + ], + "missing_evidence_empty": true + } +} diff --git a/verification/e2e/cases/g05-inside-corner-gas.json b/verification/e2e/cases/g05-inside-corner-gas.json new file mode 100644 index 000000000..fb8429c76 --- /dev/null +++ b/verification/e2e/cases/g05-inside-corner-gas.json @@ -0,0 +1,169 @@ +{ + "id": "g05-inside-corner-gas", + "source": "t3-lane-c-review.md golden 05 at origin/t3/research a7d91f1", + "description": "A gas meter on the return wall of an inside corner is 65/12 ft away along the unrolled wall but about 2.589 ft away in plan from a battery at s = [6, 6 + 31/12]. Gas clearance is measured in plan, so those starts fail.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + -6.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -7, + 0.0 + ], + [ + 4, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + }, + { + "id": "w2", + "baseline": [ + [ + 4, + 0 + ], + [ + 4, + 12 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w2", + "span_ft": [ + 14, + 14 + ], + "source": "tape", + "plus_minus_ft": 0.0, + "footprint": [ + [ + 4, + 4 + ] + ] + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -7, + 0 + ], + [ + 4, + 0 + ], + [ + 4, + 12 + ], + [ + -7, + 12 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -1, + 10 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + }, + { + "wall_id": "w2", + "span_ft": [ + 10, + 22 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -1, + 22 + ] + }, + { + "band": "overhead", + "span_ft": [ + -1, + 22 + ] + }, + { + "band": "facing", + "span_ft": [ + -1, + 22 + ] + }, + { + "band": "ground", + "span_ft": [ + -1, + 22 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "decision_not": [ + "pass" + ], + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + 6, + 7.4 + ], + "outcome": "fail", + "failing_match": "gas" + } + ] + } +} diff --git a/verification/e2e/cases/g06-regulator-footprint.json b/verification/e2e/cases/g06-regulator-footprint.json new file mode 100644 index 000000000..f8e5ef864 --- /dev/null +++ b/verification/e2e/cases/g06-regulator-footprint.json @@ -0,0 +1,170 @@ +{ + "id": "g06-regulator-footprint", + "source": "t3-lane-c-review.md golden 06 at origin/t3/research a7d91f1", + "description": "The gas meter body stands 10 ft from the wall but its regulator reaches [7, 23/6], 2 ft from the front edge of any battery covering x = 7. Clearance is measured to every part of the footprint.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -3, + 0.0 + ], + [ + 16, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + 6.5, + 7.5 + ], + "source": "tape", + "plus_minus_ft": 0.0, + "footprint": [ + [ + 6.5, + 10.5 + ], + [ + 6.5, + 9.5 + ], + [ + 6.9, + 9.5 + ], + [ + 7.0, + 3.8333333333333335 + ], + [ + 7.1, + 9.5 + ], + [ + 7.5, + 9.5 + ], + [ + 7.5, + 10.5 + ] + ] + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -3, + 0.0 + ], + [ + 16, + 0.0 + ], + [ + 16, + 10.0 + ], + [ + -3, + 10.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -3, + 16 + ], + "depth_ft": 12.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -3.0, + 16.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -3.0, + 16.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -3.0, + 16.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -3.0, + 16.0 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "decision_not": [ + "pass" + ], + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + 3, + 9 + ], + "outcome": "fail", + "failing_match": "gas" + } + ] + } +} diff --git a/verification/e2e/cases/g08a-garage-blocks-route.json b/verification/e2e/cases/g08a-garage-blocks-route.json new file mode 100644 index 000000000..d615532ef --- /dev/null +++ b/verification/e2e/cases/g08a-garage-blocks-route.json @@ -0,0 +1,147 @@ +{ + "id": "g08a-garage-blocks-route", + "source": "t3-lane-c-review.md golden 08 at origin/t3/research a7d91f1", + "description": "A garage door spans s = [1, 5] and the wall ends at s = -0.5, so every start either overlaps the door or lies beyond it, where the cable would have to cross the door.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -0.5, + 0.0 + ], + [ + 12, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "garage_door", + "wall_id": "w1", + "span_ft": [ + 1, + 5 + ], + "bottom_ft": 0.0, + "top_ft": 7.0, + "attrs": { + "operable": true + }, + "source": "tape", + "plus_minus_ft": 0.0 + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -0.5, + 0.0 + ], + [ + 12, + 0.0 + ], + [ + 12, + 10.0 + ], + [ + -0.5, + 10.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -0.5, + 12.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -0.5, + 12.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -0.5, + 12.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -0.5, + 12.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -0.5, + 12.0 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "opening": 3.0 + }, + "expect": { + "decision_not": [ + "pass" + ], + "spot": null, + "missing_evidence_empty": true, + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + 5, + 9.4 + ], + "outcome": "fail", + "failing_match": "route" + } + ] + } +} diff --git a/verification/e2e/cases/g08b-missing-wall-blocks-route.json b/verification/e2e/cases/g08b-missing-wall-blocks-route.json new file mode 100644 index 000000000..aa2a95010 --- /dev/null +++ b/verification/e2e/cases/g08b-missing-wall-blocks-route.json @@ -0,0 +1,152 @@ +{ + "id": "g08b-missing-wall-blocks-route", + "source": "t3-lane-c-review.md golden 08 at origin/t3/research a7d91f1", + "description": "The meter's wall w1 is 1.5 ft long, too short for a battery; after a 1 ft stretch with no wall, w2 runs from s = 2 to 12. No cable can cross the gap, so every start on w2 fails its route and no spot exists.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -0.5, + 0.0 + ], + [ + 1, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + }, + { + "id": "w2", + "baseline": [ + [ + 2, + 0.0 + ], + [ + 12, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -0.5, + 0 + ], + [ + 12, + 0 + ], + [ + 12, + 10 + ], + [ + -0.5, + 10 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -0.5, + 1 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + }, + { + "wall_id": "w2", + "span_ft": [ + 2, + 12 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -0.5, + 12 + ] + }, + { + "band": "overhead", + "span_ft": [ + -0.5, + 12 + ] + }, + { + "band": "facing", + "span_ft": [ + -0.5, + 12 + ] + }, + { + "band": "ground", + "span_ft": [ + -0.5, + 12 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": {}, + "expect": { + "decision_not": [ + "pass" + ], + "spot": null, + "missing_evidence_empty": true, + "sweep_runs": [ + { + "wall_id": "w2", + "start_ft": [ + 2, + 9.4 + ], + "outcome": "fail", + "failing_match": "route" + } + ] + } +} diff --git a/verification/e2e/cases/g10-derived-compound-error.json b/verification/e2e/cases/g10-derived-compound-error.json new file mode 100644 index 000000000..4b5b9e9ef --- /dev/null +++ b/verification/e2e/cases/g10-derived-compound-error.json @@ -0,0 +1,158 @@ +{ + "id": "g10-derived-compound-error", + "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", + "description": "Gas gap 3.5 ft between a gas meter at +/- 0.3 and a battery on a wall at +/- 0.3. The gap carries +/- 0.6 (linear sum), so 3.5 - 0.6 = 2.9 is not above 3: unsure by margin. A root-sum-square error of 0.424 would wrongly pass it.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + -1.8333333333333333 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ] + ], + "plus_minus_ft": 0.3 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + -5, + 15 + ], + "source": "tape", + "plus_minus_ft": 0.3, + "footprint": [ + [ + -10, + 3.5 + ], + [ + 20, + 3.5 + ], + [ + 20, + 4.0 + ], + [ + -10, + 4.0 + ] + ], + "conf": 0.6 + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ], + [ + 15, + 8.166666666666666 + ], + [ + -5, + 8.166666666666666 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -5.0, + 15.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -5.0, + 15.0 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "decision_not": [ + "pass" + ], + "checks": [ + { + "match": "gas", + "outcome": "unsure", + "unsure_cause": "margin", + "measured_ft": 3.5, + "plus_minus_ft": 0.6 + } + ], + "missing_evidence_empty": true + } +} diff --git a/verification/e2e/cases/g10-e03-fail.json b/verification/e2e/cases/g10-e03-fail.json new file mode 100644 index 000000000..16a80bb2e --- /dev/null +++ b/verification/e2e/cases/g10-e03-fail.json @@ -0,0 +1,169 @@ +{ + "id": "g10-e03-fail", + "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", + "description": "Every start has gas gap d = 2.69 ft with error 0.3 ft (gas meter +/- 0.3, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + -1.8333333333333333 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + -5, + 15 + ], + "source": "tape", + "plus_minus_ft": 0.3, + "footprint": [ + [ + -10, + 2.69 + ], + [ + 20, + 2.69 + ], + [ + 20, + 3.19 + ], + [ + -10, + 3.19 + ] + ], + "conf": 0.6 + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ], + [ + 15, + 8.166666666666666 + ], + [ + -5, + 8.166666666666666 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -5.0, + 15.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -5.0, + 15.0 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "checks": [ + { + "match": "gas", + "outcome": "fail", + "measured_ft": 2.69, + "plus_minus_ft": 0.3 + } + ], + "missing_evidence_empty": true, + "decision_not": [ + "pass" + ], + "spot": null, + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + -5, + 12.4 + ], + "outcome": "fail", + "failing_match": "gas" + } + ] + } +} diff --git a/verification/e2e/cases/g10-e03-lower-edge.json b/verification/e2e/cases/g10-e03-lower-edge.json new file mode 100644 index 000000000..be75ffcaf --- /dev/null +++ b/verification/e2e/cases/g10-e03-lower-edge.json @@ -0,0 +1,158 @@ +{ + "id": "g10-e03-lower-edge", + "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", + "description": "Every start has gas gap d = 2.7 ft with error 0.3 ft (gas meter +/- 0.3, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + -1.8333333333333333 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + -5, + 15 + ], + "source": "tape", + "plus_minus_ft": 0.3, + "footprint": [ + [ + -10, + 2.7 + ], + [ + 20, + 2.7 + ], + [ + 20, + 3.2 + ], + [ + -10, + 3.2 + ] + ], + "conf": 0.6 + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ], + [ + 15, + 8.166666666666666 + ], + [ + -5, + 8.166666666666666 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -5.0, + 15.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -5.0, + 15.0 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "checks": [ + { + "match": "gas", + "outcome": "unsure", + "measured_ft": 2.7, + "plus_minus_ft": 0.3, + "unsure_cause": "margin" + } + ], + "missing_evidence_empty": true, + "decision_not": [ + "pass" + ] + } +} diff --git a/verification/e2e/cases/g10-e03-pass.json b/verification/e2e/cases/g10-e03-pass.json new file mode 100644 index 000000000..3c0965753 --- /dev/null +++ b/verification/e2e/cases/g10-e03-pass.json @@ -0,0 +1,154 @@ +{ + "id": "g10-e03-pass", + "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", + "description": "Every start has gas gap d = 3.31 ft with error 0.3 ft (gas meter +/- 0.3, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + -1.8333333333333333 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + -5, + 15 + ], + "source": "tape", + "plus_minus_ft": 0.3, + "footprint": [ + [ + -10, + 3.31 + ], + [ + 20, + 3.31 + ], + [ + 20, + 3.81 + ], + [ + -10, + 3.81 + ] + ], + "conf": 0.6 + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ], + [ + 15, + 8.166666666666666 + ], + [ + -5, + 8.166666666666666 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -5.0, + 15.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -5.0, + 15.0 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "checks": [ + { + "match": "gas", + "outcome": "pass", + "measured_ft": 3.31, + "plus_minus_ft": 0.3 + } + ], + "missing_evidence_empty": true + } +} diff --git a/verification/e2e/cases/g10-e03-upper-edge.json b/verification/e2e/cases/g10-e03-upper-edge.json new file mode 100644 index 000000000..98ba5f9e8 --- /dev/null +++ b/verification/e2e/cases/g10-e03-upper-edge.json @@ -0,0 +1,158 @@ +{ + "id": "g10-e03-upper-edge", + "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", + "description": "Every start has gas gap d = 3.3 ft with error 0.3 ft (gas meter +/- 0.3, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + -1.8333333333333333 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + -5, + 15 + ], + "source": "tape", + "plus_minus_ft": 0.3, + "footprint": [ + [ + -10, + 3.3 + ], + [ + 20, + 3.3 + ], + [ + 20, + 3.8 + ], + [ + -10, + 3.8 + ] + ], + "conf": 0.6 + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ], + [ + 15, + 8.166666666666666 + ], + [ + -5, + 8.166666666666666 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -5.0, + 15.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -5.0, + 15.0 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "checks": [ + { + "match": "gas", + "outcome": "unsure", + "measured_ft": 3.3, + "plus_minus_ft": 0.3, + "unsure_cause": "margin" + } + ], + "missing_evidence_empty": true, + "decision_not": [ + "pass" + ] + } +} diff --git a/verification/e2e/cases/g10-e05-fail.json b/verification/e2e/cases/g10-e05-fail.json new file mode 100644 index 000000000..8be51050f --- /dev/null +++ b/verification/e2e/cases/g10-e05-fail.json @@ -0,0 +1,169 @@ +{ + "id": "g10-e05-fail", + "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", + "description": "Every start has gas gap d = 2.49 ft with error 0.5 ft (gas meter +/- 0.5, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + -1.8333333333333333 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + -5, + 15 + ], + "source": "tape", + "plus_minus_ft": 0.5, + "footprint": [ + [ + -10, + 2.49 + ], + [ + 20, + 2.49 + ], + [ + 20, + 2.99 + ], + [ + -10, + 2.99 + ] + ], + "conf": 0.6 + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ], + [ + 15, + 8.166666666666666 + ], + [ + -5, + 8.166666666666666 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -5.0, + 15.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -5.0, + 15.0 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "checks": [ + { + "match": "gas", + "outcome": "fail", + "measured_ft": 2.49, + "plus_minus_ft": 0.5 + } + ], + "missing_evidence_empty": true, + "decision_not": [ + "pass" + ], + "spot": null, + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + -5, + 12.4 + ], + "outcome": "fail", + "failing_match": "gas" + } + ] + } +} diff --git a/verification/e2e/cases/g10-e05-lower-edge.json b/verification/e2e/cases/g10-e05-lower-edge.json new file mode 100644 index 000000000..f66e93366 --- /dev/null +++ b/verification/e2e/cases/g10-e05-lower-edge.json @@ -0,0 +1,158 @@ +{ + "id": "g10-e05-lower-edge", + "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", + "description": "Every start has gas gap d = 2.5 ft with error 0.5 ft (gas meter +/- 0.5, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + -1.8333333333333333 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + -5, + 15 + ], + "source": "tape", + "plus_minus_ft": 0.5, + "footprint": [ + [ + -10, + 2.5 + ], + [ + 20, + 2.5 + ], + [ + 20, + 3.0 + ], + [ + -10, + 3.0 + ] + ], + "conf": 0.6 + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ], + [ + 15, + 8.166666666666666 + ], + [ + -5, + 8.166666666666666 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -5.0, + 15.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -5.0, + 15.0 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "checks": [ + { + "match": "gas", + "outcome": "unsure", + "measured_ft": 2.5, + "plus_minus_ft": 0.5, + "unsure_cause": "margin" + } + ], + "missing_evidence_empty": true, + "decision_not": [ + "pass" + ] + } +} diff --git a/verification/e2e/cases/g10-e05-pass.json b/verification/e2e/cases/g10-e05-pass.json new file mode 100644 index 000000000..1f3e047ab --- /dev/null +++ b/verification/e2e/cases/g10-e05-pass.json @@ -0,0 +1,154 @@ +{ + "id": "g10-e05-pass", + "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", + "description": "Every start has gas gap d = 3.51 ft with error 0.5 ft (gas meter +/- 0.5, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + -1.8333333333333333 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + -5, + 15 + ], + "source": "tape", + "plus_minus_ft": 0.5, + "footprint": [ + [ + -10, + 3.51 + ], + [ + 20, + 3.51 + ], + [ + 20, + 4.01 + ], + [ + -10, + 4.01 + ] + ], + "conf": 0.6 + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ], + [ + 15, + 8.166666666666666 + ], + [ + -5, + 8.166666666666666 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -5.0, + 15.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -5.0, + 15.0 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "checks": [ + { + "match": "gas", + "outcome": "pass", + "measured_ft": 3.51, + "plus_minus_ft": 0.5 + } + ], + "missing_evidence_empty": true + } +} diff --git a/verification/e2e/cases/g10-e05-upper-edge.json b/verification/e2e/cases/g10-e05-upper-edge.json new file mode 100644 index 000000000..2c4a02470 --- /dev/null +++ b/verification/e2e/cases/g10-e05-upper-edge.json @@ -0,0 +1,158 @@ +{ + "id": "g10-e05-upper-edge", + "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", + "description": "Every start has gas gap d = 3.5 ft with error 0.5 ft (gas meter +/- 0.5, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + -1.8333333333333333 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + -5, + 15 + ], + "source": "tape", + "plus_minus_ft": 0.5, + "footprint": [ + [ + -10, + 3.5 + ], + [ + 20, + 3.5 + ], + [ + 20, + 4.0 + ], + [ + -10, + 4.0 + ] + ], + "conf": 0.6 + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ], + [ + 15, + 8.166666666666666 + ], + [ + -5, + 8.166666666666666 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -5.0, + 15.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -5.0, + 15.0 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "checks": [ + { + "match": "gas", + "outcome": "unsure", + "measured_ft": 3.5, + "plus_minus_ft": 0.5, + "unsure_cause": "margin" + } + ], + "missing_evidence_empty": true, + "decision_not": [ + "pass" + ] + } +} diff --git a/verification/e2e/cases/g10-e15-fail.json b/verification/e2e/cases/g10-e15-fail.json new file mode 100644 index 000000000..0edfc98ca --- /dev/null +++ b/verification/e2e/cases/g10-e15-fail.json @@ -0,0 +1,169 @@ +{ + "id": "g10-e15-fail", + "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", + "description": "Every start has gas gap d = 1.49 ft with error 1.5 ft (gas meter +/- 1.5, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + -1.8333333333333333 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + -5, + 15 + ], + "source": "tape", + "plus_minus_ft": 1.5, + "footprint": [ + [ + -10, + 1.49 + ], + [ + 20, + 1.49 + ], + [ + 20, + 1.99 + ], + [ + -10, + 1.99 + ] + ], + "conf": 0.6 + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ], + [ + 15, + 8.166666666666666 + ], + [ + -5, + 8.166666666666666 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -5.0, + 15.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -5.0, + 15.0 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "checks": [ + { + "match": "gas", + "outcome": "fail", + "measured_ft": 1.49, + "plus_minus_ft": 1.5 + } + ], + "missing_evidence_empty": true, + "decision_not": [ + "pass" + ], + "spot": null, + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + -5, + 12.4 + ], + "outcome": "fail", + "failing_match": "gas" + } + ] + } +} diff --git a/verification/e2e/cases/g10-e15-lower-edge.json b/verification/e2e/cases/g10-e15-lower-edge.json new file mode 100644 index 000000000..76c68b443 --- /dev/null +++ b/verification/e2e/cases/g10-e15-lower-edge.json @@ -0,0 +1,158 @@ +{ + "id": "g10-e15-lower-edge", + "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", + "description": "Every start has gas gap d = 1.5 ft with error 1.5 ft (gas meter +/- 1.5, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + -1.8333333333333333 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + -5, + 15 + ], + "source": "tape", + "plus_minus_ft": 1.5, + "footprint": [ + [ + -10, + 1.5 + ], + [ + 20, + 1.5 + ], + [ + 20, + 2.0 + ], + [ + -10, + 2.0 + ] + ], + "conf": 0.6 + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ], + [ + 15, + 8.166666666666666 + ], + [ + -5, + 8.166666666666666 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -5.0, + 15.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -5.0, + 15.0 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "checks": [ + { + "match": "gas", + "outcome": "unsure", + "measured_ft": 1.5, + "plus_minus_ft": 1.5, + "unsure_cause": "margin" + } + ], + "missing_evidence_empty": true, + "decision_not": [ + "pass" + ] + } +} diff --git a/verification/e2e/cases/g10-e15-pass.json b/verification/e2e/cases/g10-e15-pass.json new file mode 100644 index 000000000..faebe0533 --- /dev/null +++ b/verification/e2e/cases/g10-e15-pass.json @@ -0,0 +1,154 @@ +{ + "id": "g10-e15-pass", + "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", + "description": "Every start has gas gap d = 4.51 ft with error 1.5 ft (gas meter +/- 1.5, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + -1.8333333333333333 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + -5, + 15 + ], + "source": "tape", + "plus_minus_ft": 1.5, + "footprint": [ + [ + -10, + 4.51 + ], + [ + 20, + 4.51 + ], + [ + 20, + 5.01 + ], + [ + -10, + 5.01 + ] + ], + "conf": 0.6 + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ], + [ + 15, + 8.166666666666666 + ], + [ + -5, + 8.166666666666666 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -5.0, + 15.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -5.0, + 15.0 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "checks": [ + { + "match": "gas", + "outcome": "pass", + "measured_ft": 4.51, + "plus_minus_ft": 1.5 + } + ], + "missing_evidence_empty": true + } +} diff --git a/verification/e2e/cases/g10-e15-upper-edge.json b/verification/e2e/cases/g10-e15-upper-edge.json new file mode 100644 index 000000000..bdf8cb87e --- /dev/null +++ b/verification/e2e/cases/g10-e15-upper-edge.json @@ -0,0 +1,158 @@ +{ + "id": "g10-e15-upper-edge", + "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", + "description": "Every start has gas gap d = 4.5 ft with error 1.5 ft (gas meter +/- 1.5, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + -1.8333333333333333 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + -5, + 15 + ], + "source": "tape", + "plus_minus_ft": 1.5, + "footprint": [ + [ + -10, + 4.5 + ], + [ + 20, + 4.5 + ], + [ + 20, + 5.0 + ], + [ + -10, + 5.0 + ] + ], + "conf": 0.6 + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -5, + -1.8333333333333333 + ], + [ + 15, + -1.8333333333333333 + ], + [ + 15, + 8.166666666666666 + ], + [ + -5, + 8.166666666666666 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -5.0, + 15.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -5.0, + 15.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -5.0, + 15.0 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "checks": [ + { + "match": "gas", + "outcome": "unsure", + "measured_ft": 4.5, + "plus_minus_ft": 1.5, + "unsure_cause": "margin" + } + ], + "missing_evidence_empty": true, + "decision_not": [ + "pass" + ] + } +} diff --git a/verification/e2e/cases/g11a-unexplored-corner.json b/verification/e2e/cases/g11a-unexplored-corner.json new file mode 100644 index 000000000..660922c32 --- /dev/null +++ b/verification/e2e/cases/g11a-unexplored-corner.json @@ -0,0 +1,160 @@ +{ + "id": "g11a-unexplored-corner", + "source": "t3-lane-c-review.md golden 11 (a) at origin/t3/research a7d91f1", + "description": "Every start on w1 fails gas by 2 ft (gap 1 ft). The walk stopped at the corner at s = 8 without seeing the next wall, so the result cannot reject and must ask for the wall past the corner.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + -1.8333333333333333 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -3, + -1.8333333333333333 + ], + [ + 8, + -1.8333333333333333 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + -3, + 8 + ], + "source": "tape", + "plus_minus_ft": 0.0, + "footprint": [ + [ + -10, + 1.0 + ], + [ + 20, + 1.0 + ], + [ + 20, + 1.5 + ], + [ + -10, + 1.5 + ] + ] + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -3, + -1.8333333333333333 + ], + [ + 8, + -1.8333333333333333 + ], + [ + 8, + 8.166666666666666 + ], + [ + -3, + 8.166666666666666 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -3.0, + 8.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "unexplored" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -3, + 8 + ] + }, + { + "band": "overhead", + "span_ft": [ + -3, + 8 + ] + }, + { + "band": "facing", + "span_ft": [ + -3, + 8 + ] + }, + { + "band": "ground", + "span_ft": [ + -3, + 8 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "decision_in": [ + "manual_review" + ], + "spot": null, + "missing_evidence_empty": false, + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + -3, + 5.4 + ], + "outcome": "fail", + "failing_match": "gas" + } + ] + } +} diff --git a/verification/e2e/cases/g11b-both-ends-real-all-fail.json b/verification/e2e/cases/g11b-both-ends-real-all-fail.json new file mode 100644 index 000000000..b8648d3d8 --- /dev/null +++ b/verification/e2e/cases/g11b-both-ends-real-all-fail.json @@ -0,0 +1,229 @@ +{ + "id": "g11b-both-ends-real-all-fail", + "source": "t3-lane-c-review.md golden 11 (b) at origin/t3/research a7d91f1", + "description": "w1 turns an outside corner at s = 8 onto w2, walked to its real end at s = 14. Gas strips 1 ft in front of each wall fail every start. Nothing is unobserved, so there is no spot and no photo request.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -3, + 0.0 + ], + [ + 8, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + }, + { + "id": "w2", + "baseline": [ + [ + 8, + 0 + ], + [ + 8, + -6 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + -3, + 8 + ], + "source": "tape", + "plus_minus_ft": 0.0, + "footprint": [ + [ + -10, + 2.833333333333333 + ], + [ + 12, + 2.833333333333333 + ], + [ + 12, + 3.333333333333333 + ], + [ + -10, + 3.333333333333333 + ] + ] + }, + { + "type": "gas_meter", + "wall_id": "w2", + "span_ft": [ + 8, + 14 + ], + "source": "tape", + "plus_minus_ft": 0.0, + "footprint": [ + [ + 10.833333333333332, + -10 + ], + [ + 11.333333333333332, + -10 + ], + [ + 11.333333333333332, + 3 + ], + [ + 10.833333333333332, + 3 + ] + ] + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -3, + 0 + ], + [ + 8, + 0 + ], + [ + 8, + -6 + ], + [ + 18, + -6 + ], + [ + 18, + 10 + ], + [ + -3, + 10 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -3, + 8 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + }, + { + "wall_id": "w2", + "span_ft": [ + 8, + 14 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -3, + 14 + ] + }, + { + "band": "overhead", + "span_ft": [ + -3, + 14 + ] + }, + { + "band": "facing", + "span_ft": [ + -3, + 14 + ] + }, + { + "band": "ground", + "span_ft": [ + -3, + 14 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "decision_in": [ + "manual_review", + "reject" + ], + "spot": null, + "missing_evidence_empty": true, + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + -3, + 5.4 + ], + "outcome": "fail", + "failing_match": "gas" + }, + { + "wall_id": "w2", + "start_ft": [ + 8, + 11.4 + ], + "outcome": "fail", + "failing_match": "gas" + } + ] + } +} diff --git a/verification/e2e/cases/g11c-unobserved-ground.json b/verification/e2e/cases/g11c-unobserved-ground.json new file mode 100644 index 000000000..6163c3319 --- /dev/null +++ b/verification/e2e/cases/g11c-unobserved-ground.json @@ -0,0 +1,174 @@ +{ + "id": "g11c-unobserved-ground", + "source": "t3-lane-c-review.md golden 11 (c) at origin/t3/research a7d91f1", + "description": "Starts below s = 2*sqrt(2) - 0.5 fail gas. Starts beyond it pass gas but stand on ground nobody saw (observed only up to s = 1), so they are unsure, never pass, and the result asks for that ground.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + -1.8333333333333333 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -10, + -1.8333333333333333 + ], + [ + 10, + -1.8333333333333333 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + -10, + -0.5 + ], + "source": "tape", + "plus_minus_ft": 0.0, + "footprint": [ + [ + -12, + 1.0 + ], + [ + -0.5, + 1.0 + ], + [ + -0.5, + 1.5 + ], + [ + -12, + 1.5 + ] + ] + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -10, + -1.8333333333333333 + ], + [ + 1, + -1.8333333333333333 + ], + [ + 1, + 8.166666666666666 + ], + [ + -10, + 8.166666666666666 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -10.0, + 10.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -10, + 10 + ] + }, + { + "band": "overhead", + "span_ft": [ + -10, + 10 + ] + }, + { + "band": "facing", + "span_ft": [ + -10, + 10 + ] + }, + { + "band": "ground", + "span_ft": [ + -10, + 1 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "decision_in": [ + "manual_review" + ], + "spot": { + "wall_id": "w1", + "span_within": [ + 2.3284271247461903, + 10 + ] + }, + "missing_evidence_empty": false, + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + -9.9, + 2.2 + ], + "outcome": "fail", + "failing_match": "gas" + }, + { + "wall_id": "w1", + "start_ft": [ + 2.5, + 7.3 + ], + "outcome": "unsure" + } + ] + } +} diff --git a/verification/e2e/cases/g12a-subgrid-start.json b/verification/e2e/cases/g12a-subgrid-start.json new file mode 100644 index 000000000..d42ea95c1 --- /dev/null +++ b/verification/e2e/cases/g12a-subgrid-start.json @@ -0,0 +1,173 @@ +{ + "id": "g12a-subgrid-start", + "source": "t3-lane-c-review.md golden 12 at origin/t3/research a7d91f1", + "description": "Windows at s = [3, 4] and [10.1 + 31/12, 11.1 + 31/12] with 3 ft opening clearance leave legal starts only in (7, 7.1). A 2 in grid from s = 0 samples 7 and 43/6 and misses them.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -1, + 0.0 + ], + [ + 16, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "window", + "wall_id": "w1", + "span_ft": [ + 3, + 4 + ], + "bottom_ft": 4.0, + "top_ft": 7.0, + "attrs": { + "operable": true + }, + "source": "tape", + "plus_minus_ft": 0.0 + }, + { + "type": "window", + "wall_id": "w1", + "span_ft": [ + 12.683333333333334, + 13.683333333333334 + ], + "bottom_ft": 4.0, + "top_ft": 7.0, + "attrs": { + "operable": true + }, + "source": "tape", + "plus_minus_ft": 0.0 + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -1, + 0.0 + ], + [ + 16, + 0.0 + ], + [ + 16, + 10.0 + ], + [ + -1, + 10.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -1.0, + 16.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -1.0, + 16.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -1.0, + 16.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -1.0, + 16.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -1.0, + 16.0 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "opening": 3.0 + }, + "expect": { + "spot": { + "wall_id": "w1", + "span_within": [ + 7, + 9.683333333333334 + ] + }, + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + -1, + 6.95 + ], + "outcome": "fail", + "failing_match": "opening" + }, + { + "wall_id": "w1", + "start_ft": [ + 7.2, + 13.4 + ], + "outcome": "fail", + "failing_match": "opening" + } + ] + } +} diff --git a/verification/e2e/cases/g12b-exact-fit-last-start.json b/verification/e2e/cases/g12b-exact-fit-last-start.json new file mode 100644 index 000000000..d7548161c --- /dev/null +++ b/verification/e2e/cases/g12b-exact-fit-last-start.json @@ -0,0 +1,181 @@ +{ + "id": "g12b-exact-fit-last-start", + "source": "t3-lane-c-review.md golden 12 at origin/t3/research a7d91f1", + "description": "Five 1.8 ft segments, then one straight segment exactly 31/12 ft long at s = [6, 6 + 31/12] where the wall ends. Its single start, s = 6, is also the last start and must be enumerated.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -2.3965377130018712, + -1.7929208145156164 + ], + [ + -1.0176577153877109, + -0.6359031170798457 + ], + [ + 0.5088288576938558, + 0.3179515585399231 + ], + [ + 2.1532106814505374, + 1.0500775160763633 + ], + [ + 3.8834817341395116, + 1.546224756546962 + ], + [ + 5.665964257874338, + 1.7967363382750796 + ], + [ + 8.249297591207672, + 1.7967363382750796 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -2.3965377130018712, + -1.7929208145156164 + ], + [ + -1.0176577153877109, + -0.6359031170798457 + ], + [ + 0.5088288576938558, + 0.3179515585399231 + ], + [ + 2.1532106814505374, + 1.0500775160763633 + ], + [ + 3.8834817341395116, + 1.546224756546962 + ], + [ + 5.665964257874338, + 1.7967363382750796 + ], + [ + 8.249297591207672, + 1.7967363382750796 + ], + [ + 8.249297591207672, + 13.79673633827508 + ], + [ + 4.8288865729448345, + 13.767504941392971 + ], + [ + 1.3885414443264001, + 13.28399596535263 + ], + [ + -1.9510310384574874, + 12.326388965507265 + ], + [ + -5.124829895736833, + 10.913322672847046 + ], + [ + -8.071080742897388, + 9.072300815419522 + ], + [ + -10.109989029240344, + 7.39961250291212 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -3, + 8.583333333333334 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -3, + 8.583333333333334 + ] + }, + { + "band": "overhead", + "span_ft": [ + -3, + 8.583333333333334 + ] + }, + { + "band": "facing", + "span_ft": [ + -3, + 8.583333333333334 + ] + }, + { + "band": "ground", + "span_ft": [ + -3, + 8.583333333333334 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": {}, + "expect": { + "spot": { + "wall_id": "w1", + "span_within": [ + 6, + 8.583333333333334 + ] + }, + "missing_evidence_empty": true + } +} diff --git a/verification/e2e/cases/g13a-unsure-never-outranks-pass.json b/verification/e2e/cases/g13a-unsure-never-outranks-pass.json new file mode 100644 index 000000000..2a50f0734 --- /dev/null +++ b/verification/e2e/cases/g13a-unsure-never-outranks-pass.json @@ -0,0 +1,168 @@ +{ + "id": "g13a-unsure-never-outranks-pass", + "source": "t3-lane-c-review.md golden 13 at origin/t3/research a7d91f1", + "description": "Starts right of the meter have gas gap 3.2 +/- 0.3 (unsure) and the shortest routes. Batteries whose right edge is left of s = 0.3 - sqrt(0.65) clear gas and pass. The chosen spot must be a passing one on the left.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + -1.8333333333333333 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -8, + -1.8333333333333333 + ], + [ + 8, + -1.8333333333333333 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + 0.3, + 8 + ], + "source": "tape", + "plus_minus_ft": 0.3, + "footprint": [ + [ + 0.3, + 3.2 + ], + [ + 12, + 3.2 + ], + [ + 12, + 3.7 + ], + [ + 0.3, + 3.7 + ] + ] + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -8, + -1.8333333333333333 + ], + [ + 8, + -1.8333333333333333 + ], + [ + 8, + 8.166666666666666 + ], + [ + -8, + 8.166666666666666 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -8.0, + 8.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -8.0, + 8.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -8.0, + 8.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -8.0, + 8.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -8.0, + 8.0 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "spot": { + "wall_id": "w1", + "span_within": [ + -8, + -0.5062257748298531 + ] + }, + "checks": [ + { + "match": "gas", + "outcome": "pass" + } + ], + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + 0.5, + 5 + ], + "outcome": "unsure" + } + ], + "missing_evidence_empty": true + } +} diff --git a/verification/e2e/cases/g13b-unsure-alone.json b/verification/e2e/cases/g13b-unsure-alone.json new file mode 100644 index 000000000..07c88235c --- /dev/null +++ b/verification/e2e/cases/g13b-unsure-alone.json @@ -0,0 +1,174 @@ +{ + "id": "g13b-unsure-alone", + "source": "t3-lane-c-review.md golden 13 at origin/t3/research a7d91f1", + "description": "The wall ends at s = -0.5, so every start has gas gap 3.2 +/- 0.3. The best spot is unsure by margin and fully observed: manual review with no photo request.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + -1.8333333333333333 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -0.5, + -1.8333333333333333 + ], + [ + 8, + -1.8333333333333333 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + 0.3, + 8 + ], + "source": "tape", + "plus_minus_ft": 0.3, + "footprint": [ + [ + 0.3, + 3.2 + ], + [ + 12, + 3.2 + ], + [ + 12, + 3.7 + ], + [ + 0.3, + 3.7 + ] + ] + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -0.5, + -1.8333333333333333 + ], + [ + 8, + -1.8333333333333333 + ], + [ + 8, + 8.166666666666666 + ], + [ + -0.5, + 8.166666666666666 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -0.5, + 8.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -0.5, + 8.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -0.5, + 8.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -0.5, + 8.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -0.5, + 8.0 + ], + "out_ft": 10.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0 + }, + "expect": { + "decision_in": [ + "manual_review" + ], + "spot": { + "wall_id": "w1", + "span_within": [ + -0.5, + 8 + ] + }, + "checks": [ + { + "match": "gas", + "outcome": "unsure", + "unsure_cause": "margin", + "measured_ft": 3.2, + "plus_minus_ft": 0.3 + } + ], + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + -0.4, + 5.3 + ], + "outcome": "unsure" + } + ], + "missing_evidence_empty": true + } +} diff --git a/verification/e2e/cases/generate.py b/verification/e2e/cases/generate.py new file mode 100644 index 000000000..c0885529d --- /dev/null +++ b/verification/e2e/cases/generate.py @@ -0,0 +1,588 @@ +"""Write the black-box placement cases in this folder, one JSON file per case. + +Run from verification/: `uv run python e2e/cases/generate.py`. README.md gives the arithmetic +behind every expected number. The scenes follow server/schemas/scene.schema.json at +origin/t3/server; they were written from that schema and the lane C review, not from the +server's code or fixtures. +""" + +import json +import math +from pathlib import Path + +HERE = Path(__file__).parent +RESEARCH_SHA = "a7d91f1" +SERVER_SHA = "69c6c3c" + +# Battery footprint from the lane C review's parameter table: 31 in wide, 22 in deep. +W = 31 / 12 +D = 11 / 6 +METER_Y = 4.0 +# Walls placed at z = -D put a flush battery's front edge at z = 0 exactly in floating point, +# so a gas strip at z = d is measured as exactly d. The margin cases need that: 3.3 - 0.3 and +# 2.7 + 0.3 both evaluate to exactly 3.0, while 11/6 + 3.3 - 11/6 does not return 3.3. +FRONT0 = -D + + +def golden(n): + return f"t3-lane-c-review.md golden {n} at origin/t3/research {RESEARCH_SHA}" + + +def straight(wall_id, x0, x1, z=0.0, pm=0.0): + return {"id": wall_id, "baseline": [[x0, z], [x1, z]], "plus_minus_ft": pm} + + +def rect(x0, x1, z0, z1): + return [[x0, z0], [x1, z0], [x1, z1], [x0, z1]] + + +def concrete(polygon): + return {"type": "concrete", "polygon": polygon, "plus_minus_ft": 0.0} + + +def facing(wall_id, span, depth=9.0): + return {"wall_id": wall_id, "span_ft": list(span), "depth_ft": depth, "plus_minus_ft": 0.0} + + +def coverage(span, ground=None, left="limit", right="limit", bands=("wall", "overhead", "facing")): + """Observed bands over `span`; the ground band over `ground` (default: `span`), 10 ft out.""" + observed = [{"band": b, "span_ft": list(span)} for b in bands] + for g in [span] if ground is None else ground: + observed.append({"band": "ground", "span_ft": list(g), "out_ft": 10.0}) + return {"ends": {"left": {"kind": left}, "right": {"kind": right}}, "observed": observed} + + +def gas(wall_id, span, footprint, pm=0.0, conf=None): + obj = { + "type": "gas_meter", + "wall_id": wall_id, + "span_ft": list(span), + "source": "tape", + "plus_minus_ft": pm, + "footprint": footprint, + } + if conf is not None: + obj["conf"] = conf + return obj + + +def opening(kind, wall_id, span, bottom, top): + return { + "type": kind, + "wall_id": wall_id, + "span_ft": list(span), + "bottom_ft": bottom, + "top_ft": top, + "attrs": {"operable": True}, + "source": "tape", + "plus_minus_ft": 0.0, + } + + +def meter(x, z, wall_id="w1"): + return {"pos": [x, METER_Y, z], "wall_id": wall_id, "plus_minus_ft": 0.0} + + +def polyline_wall(wall_id, s_start, segments): + """A wall of straight segments given as (length, heading in degrees), heading 0 = +x. + + The first point sits at s = s_start. The baseline is shifted so the point at s = 0 (the + meter) is the origin. Headings fall left to right, so every corner is an outside corner. + """ + pts = [(0.0, 0.0)] + s_marks = [s_start] + for length, heading in segments: + a = math.radians(heading) + x, z = pts[-1] + pts.append((x + length * math.cos(a), z + length * math.sin(a))) + s_marks.append(s_marks[-1] + length) + # Find the point at s = 0 and shift it to the origin. + for i in range(len(segments)): + if s_marks[i] <= 0 <= s_marks[i + 1]: + t = (0 - s_marks[i]) / (s_marks[i + 1] - s_marks[i]) + ox = pts[i][0] + t * (pts[i + 1][0] - pts[i][0]) + oz = pts[i][1] + t * (pts[i + 1][1] - pts[i][1]) + break + shifted = [[x - ox, z - oz] for x, z in pts] + return {"id": wall_id, "baseline": shifted, "plus_minus_ft": 0.0} + + +def offset_ground(baseline, out=12.0): + """Ground polygon in front of a convex wall: the baseline, then the baseline pushed out.""" + front = [] + n = len(baseline) + for i, (x, z) in enumerate(baseline): + normals = [] + for j in (i - 1, i): + if 0 <= j < n - 1: + dx = baseline[j + 1][0] - baseline[j][0] + dz = baseline[j + 1][1] - baseline[j][1] + length = math.hypot(dx, dz) + normals.append((-dz / length, dx / length)) + nx = sum(v[0] for v in normals) / len(normals) + nz = sum(v[1] for v in normals) / len(normals) + norm = math.hypot(nx, nz) + front.append([x + out * nx / norm, z + out * nz / norm]) + return concrete([list(p) for p in baseline] + front[::-1]) + + +CASES = [] + + +def case(cid, source, description, scene, expect, rules_assumed=None): + CASES.append( + { + "id": cid, + "source": source, + "description": description, + "scene": scene, + "rules_assumed": rules_assumed or {}, + "expect": expect, + } + ) + + +def straight_scene(x0, x1, z=0.0, meter_x=0.0, objects=(), ground=None, cov=None, wall_pm=0.0): + scene = { + "schema_version": "1.0", + "meter": meter(meter_x, z), + "walls": [straight("w1", x0, x1, z, wall_pm)], + "objects": list(objects), + "ground": [concrete(rect(x0, x1, z, z + 10))] if ground is None else ground, + "facing": [facing("w1", (x0 - meter_x, x1 - meter_x))], + } + scene["coverage"] = coverage((x0 - meter_x, x1 - meter_x)) if cov is None else cov + return scene + + +# g01: an unseen stretch behind a garage door needs no photo, since no cable can reach it. +case( + "g01-unseen-beyond-garage", + golden("01"), + "The wall beyond a garage door left of the meter was never observed. No spot there can be " + "reached by a cable, so the result asks for no photos, and every start beyond the garage " + "fails its route.", + straight_scene( + -14, + 10, + objects=[opening("garage_door", "w1", (-7, -3), 0.0, 7.0)], + cov=coverage((-7, 10)), + ), + { + "missing_evidence_empty": True, + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [-13.9, -9.7], + "outcome": "fail", + "failing_match": "route", + } + ], + }, +) + +# g03: pads at s = [-9, -6] and [7, 10]; the stretch between is short segments. +seg = 13 / 7 +g03_wall = polyline_wall("w1", -9, [(3, 32)] + [(seg, 24 - 8 * k) for k in range(7)] + [(3, -32)]) +case( + "g03-negative-s-near-edge", + golden("03"), + "Two straight 3 ft pads, s = [-9, -6] and [7, 10], joined by segments 13/7 ft long that no " + "battery fits on. The left pad's route runs to its near edge (6 ft), so it beats the right " + "pad (7 ft); measuring to the far edge (103/12 ft) would pick the right pad.", + { + "schema_version": "1.0", + "meter": meter(0.0, 0.0), + "walls": [g03_wall], + "objects": [], + "ground": [offset_ground(g03_wall["baseline"])], + "facing": [facing("w1", (-9, 10))], + "coverage": coverage((-9, 10)), + }, + { + "spot": {"wall_id": "w1", "span_within": [-9, -6]}, + "sweep_runs": [{"wall_id": "w1", "start_ft": [-8.4, 6.9], "outcome": "fail"}], + "missing_evidence_empty": True, + }, +) + +# g05: inside corner, gas meter on the return wall at [4, 4]. +g05 = { + "schema_version": "1.0", + "meter": meter(-6.0, 0.0), + "walls": [ + straight("w1", -7, 4), + {"id": "w2", "baseline": [[4, 0], [4, 12]], "plus_minus_ft": 0.0}, + ], + "objects": [gas("w2", (14, 14), [[4, 4]])], + "ground": [concrete(rect(-7, 4, 0, 12))], + "facing": [facing("w1", (-1, 10)), facing("w2", (10, 22))], + "coverage": coverage((-1, 22)), +} +case( + "g05-inside-corner-gas", + golden("05"), + "A gas meter on the return wall of an inside corner is 65/12 ft away along the unrolled " + "wall but about 2.589 ft away in plan from a battery at s = [6, 6 + 31/12]. Gas clearance " + "is measured in plan, so those starts fail.", + g05, + { + "decision_not": ["pass"], + "sweep_runs": [ + {"wall_id": "w1", "start_ft": [6, 7.4], "outcome": "fail", "failing_match": "gas"} + ], + }, + {"gas": 3.0}, +) + +# g06: gas meter body 10 ft out, regulator reaching [7, 23/6]. +spike = [[6.5, 10.5], [6.5, 9.5], [6.9, 9.5], [7.0, 23 / 6], [7.1, 9.5], [7.5, 9.5], [7.5, 10.5]] +case( + "g06-regulator-footprint", + golden("06"), + "The gas meter body stands 10 ft from the wall but its regulator reaches [7, 23/6], 2 ft " + "from the front edge of any battery covering x = 7. Clearance is measured to every part of " + "the footprint.", + straight_scene(-3, 16, objects=[gas("w1", (6.5, 7.5), spike)]) + | {"facing": [facing("w1", (-3, 16), depth=12.0)]}, + { + "decision_not": ["pass"], + "sweep_runs": [ + {"wall_id": "w1", "start_ft": [3, 9], "outcome": "fail", "failing_match": "gas"} + ], + }, + {"gas": 3.0}, +) + +# g08a: garage door right of the meter, nothing usable before it. +case( + "g08a-garage-blocks-route", + golden("08"), + "A garage door spans s = [1, 5] and the wall ends at s = -0.5, so every start either " + "overlaps the door or lies beyond it, where the cable would have to cross the door.", + straight_scene(-0.5, 12, objects=[opening("garage_door", "w1", (1, 5), 0.0, 7.0)]), + { + "decision_not": ["pass"], + "spot": None, + "missing_evidence_empty": True, + "sweep_runs": [ + {"wall_id": "w1", "start_ft": [5, 9.4], "outcome": "fail", "failing_match": "route"} + ], + }, + {"opening": 3.0}, +) + +# g08b: a 1 ft stretch with no wall between the meter's short wall and the rest. +g08b = { + "schema_version": "1.0", + "meter": meter(0.0, 0.0), + "walls": [straight("w1", -0.5, 1), straight("w2", 2, 12)], + "objects": [], + "ground": [concrete(rect(-0.5, 12, 0, 10))], + "facing": [facing("w1", (-0.5, 1)), facing("w2", (2, 12))], + "coverage": coverage((-0.5, 12)), +} +case( + "g08b-missing-wall-blocks-route", + golden("08"), + "The meter's wall w1 is 1.5 ft long, too short for a battery; after a 1 ft stretch with no " + "wall, w2 runs from s = 2 to 12. No cable can cross the gap, so every start on w2 fails " + "its route and no spot exists.", + g08b, + { + "decision_not": ["pass"], + "spot": None, + "missing_evidence_empty": True, + "sweep_runs": [ + {"wall_id": "w2", "start_ft": [2, 9.4], "outcome": "fail", "failing_match": "route"} + ], + }, +) + +# g10: uniform gas gap d with error e. The strip spans past both wall ends, so every start +# on the wall sees the same perpendicular gap. +T = 3.0 + + +def g10_scene(d, e, wall_pm=0.0): + strip = gas("w1", (-5, 15), rect(-10, 20, d, d + 0.5), pm=e, conf=0.6) + return straight_scene(-5, 15, z=FRONT0, objects=[strip], wall_pm=wall_pm) | { + "ground": [concrete(rect(-5, 15, FRONT0, FRONT0 + 10))] + } + + +ROWS = [ + ("pass", lambda e: round(T + e + 0.01, 2), "pass", None), + ("upper-edge", lambda e: round(T + e, 2), "unsure", "margin"), + ("lower-edge", lambda e: round(T - e, 2), "unsure", "margin"), + ("fail", lambda e: round(T - e - 0.01, 2), "fail", None), +] +for e in (0.3, 0.5, 1.5): + for name, dfun, outcome, cause in ROWS: + d = dfun(e) + check = {"match": "gas", "outcome": outcome, "measured_ft": d, "plus_minus_ft": e} + if cause: + check["unsure_cause"] = cause + expect = {"checks": [check], "missing_evidence_empty": True} + if outcome != "pass": + expect["decision_not"] = ["pass"] + if outcome == "fail": + expect["spot"] = None + expect["sweep_runs"] = [ + {"wall_id": "w1", "start_ft": [-5, 12.4], "outcome": "fail", "failing_match": "gas"} + ] + case( + f"g10-e{round(e * 10):02d}-{name}", + golden("10"), + f"Every start has gas gap d = {d} ft with error {e} ft (gas meter +/- {e}, wall " + f"exact), against a {T} ft threshold. The gas object's recognition conf of 0.6 " + "must not change the error.", + g10_scene(d, e), + expect, + {"gas": T}, + ) + +case( + "g10-derived-compound-error", + golden("10"), + "Gas gap 3.5 ft between a gas meter at +/- 0.3 and a battery on a wall at +/- 0.3. The gap " + "carries +/- 0.6 (linear sum), so 3.5 - 0.6 = 2.9 is not above 3: unsure by margin. A " + "root-sum-square error of 0.424 would wrongly pass it.", + g10_scene(3.5, 0.3, wall_pm=0.3), + { + "decision_not": ["pass"], + "checks": [ + { + "match": "gas", + "outcome": "unsure", + "unsure_cause": "margin", + "measured_ft": 3.5, + "plus_minus_ft": 0.6, + } + ], + "missing_evidence_empty": True, + }, + {"gas": T}, +) + +# g11a: every start fails gas, but the right end is an untapped corner. +case( + "g11a-unexplored-corner", + golden("11 (a)"), + "Every start on w1 fails gas by 2 ft (gap 1 ft). The walk stopped at the corner at s = 8 " + "without seeing the next wall, so the result cannot reject and must ask for the wall past " + "the corner.", + straight_scene( + -3, + 8, + z=FRONT0, + objects=[gas("w1", (-3, 8), rect(-10, 20, 1.0, 1.5))], + ground=[concrete(rect(-3, 8, FRONT0, FRONT0 + 10))], + cov=coverage((-3, 8), right="unexplored"), + ), + { + "decision_in": ["manual_review"], + "spot": None, + "missing_evidence_empty": False, + "sweep_runs": [ + {"wall_id": "w1", "start_ft": [-3, 5.4], "outcome": "fail", "failing_match": "gas"} + ], + }, + {"gas": 3.0}, +) + +# g11b: both walls walked to real ends, every start fails gas. +z1 = D + 1.0 +g11b = { + "schema_version": "1.0", + "meter": meter(0.0, 0.0), + "walls": [ + straight("w1", -3, 8), + {"id": "w2", "baseline": [[8, 0], [8, -6]], "plus_minus_ft": 0.0}, + ], + "objects": [ + gas("w1", (-3, 8), rect(-10, 12, z1, z1 + 0.5)), + gas("w2", (8, 14), [[8 + z1, -10], [8 + z1 + 0.5, -10], [8 + z1 + 0.5, 3], [8 + z1, 3]]), + ], + "ground": [concrete([[-3, 0], [8, 0], [8, -6], [18, -6], [18, 10], [-3, 10]])], + "facing": [facing("w1", (-3, 8)), facing("w2", (8, 14))], + "coverage": coverage((-3, 14)), +} +case( + "g11b-both-ends-real-all-fail", + golden("11 (b)"), + "w1 turns an outside corner at s = 8 onto w2, walked to its real end at s = 14. Gas strips " + "1 ft in front of each wall fail every start. Nothing is unobserved, so there is no spot " + "and no photo request.", + g11b, + { + "decision_in": ["manual_review", "reject"], + "spot": None, + "missing_evidence_empty": True, + "sweep_runs": [ + {"wall_id": "w1", "start_ft": [-3, 5.4], "outcome": "fail", "failing_match": "gas"}, + {"wall_id": "w2", "start_ft": [8, 11.4], "outcome": "fail", "failing_match": "gas"}, + ], + }, + {"gas": 3.0}, +) + +# g11c: observed starts fail gas; the ground right of s = 1 was never seen (closed gate). +case( + "g11c-unobserved-ground", + golden("11 (c)"), + "Starts below s = 2*sqrt(2) - 0.5 fail gas. Starts beyond it pass gas but stand on ground " + "nobody saw (observed only up to s = 1), so they are unsure, never pass, and the result " + "asks for that ground.", + straight_scene( + -10, + 10, + z=FRONT0, + objects=[gas("w1", (-10, -0.5), rect(-12, -0.5, 1.0, 1.5))], + ground=[concrete(rect(-10, 1, FRONT0, FRONT0 + 10))], + cov=coverage((-10, 10), ground=[(-10, 1)]), + ), + { + "decision_in": ["manual_review"], + "spot": {"wall_id": "w1", "span_within": [2 * math.sqrt(2) - 0.5, 10]}, + "missing_evidence_empty": False, + "sweep_runs": [ + {"wall_id": "w1", "start_ft": [-9.9, 2.2], "outcome": "fail", "failing_match": "gas"}, + {"wall_id": "w1", "start_ft": [2.5, 7.3], "outcome": "unsure"}, + ], + }, + {"gas": 3.0}, +) + +# g12a: legal starts only in the open interval (7, 7.1), narrower than a 2 in grid step. +win2 = 10.1 + W +case( + "g12a-subgrid-start", + golden("12"), + "Windows at s = [3, 4] and [10.1 + 31/12, 11.1 + 31/12] with 3 ft opening clearance leave " + "legal starts only in (7, 7.1). A 2 in grid from s = 0 samples 7 and 43/6 and misses them.", + straight_scene( + -1, + 16, + objects=[ + opening("window", "w1", (3, 4), 4.0, 7.0), + opening("window", "w1", (win2, win2 + 1), 4.0, 7.0), + ], + ), + { + "spot": {"wall_id": "w1", "span_within": [7, 7.1 + W]}, + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [-1, 6.95], + "outcome": "fail", + "failing_match": "opening", + }, + { + "wall_id": "w1", + "start_ft": [7.2, 13.4], + "outcome": "fail", + "failing_match": "opening", + }, + ], + }, + {"opening": 3.0}, +) + +# g12b: one straight segment exactly one battery wide; its only start is its last start. +g12b_wall = polyline_wall("w1", -3, [(1.8, 40 - 8 * k) for k in range(5)] + [(W, 0)]) +case( + "g12b-exact-fit-last-start", + golden("12"), + "Five 1.8 ft segments, then one straight segment exactly 31/12 ft long at s = [6, 6 + 31/12] " + "where the wall ends. Its single start, s = 6, is also the last start and must be " + "enumerated.", + { + "schema_version": "1.0", + "meter": meter(0.0, 0.0), + "walls": [g12b_wall], + "objects": [], + "ground": [offset_ground(g12b_wall["baseline"])], + "facing": [facing("w1", (-3, 6 + W))], + "coverage": coverage((-3, 6 + W)), + }, + {"spot": {"wall_id": "w1", "span_within": [6, 6 + W]}, "missing_evidence_empty": True}, +) + + +# g13: F = right of the meter, gas gap 3.2 +/- 0.3 (unsure); S = left, passes. +def g13_scene(x0): + strip = gas("w1", (0.3, 8), rect(0.3, 12, 3.2, 3.7), pm=0.3) + return straight_scene(x0, 8, z=FRONT0, objects=[strip]) | { + "ground": [concrete(rect(x0, 8, FRONT0, FRONT0 + 10))] + } + + +s_edge = 0.3 - math.sqrt(3.3**2 - 3.2**2) +case( + "g13a-unsure-never-outranks-pass", + golden("13"), + "Starts right of the meter have gas gap 3.2 +/- 0.3 (unsure) and the shortest routes. " + "Batteries whose right edge is left of s = 0.3 - sqrt(0.65) clear gas and pass. The chosen " + "spot must be a passing one on the left.", + g13_scene(-8), + { + "spot": {"wall_id": "w1", "span_within": [-8, s_edge]}, + "checks": [{"match": "gas", "outcome": "pass"}], + "sweep_runs": [{"wall_id": "w1", "start_ft": [0.5, 5], "outcome": "unsure"}], + "missing_evidence_empty": True, + }, + {"gas": 3.0}, +) +case( + "g13b-unsure-alone", + golden("13"), + "The wall ends at s = -0.5, so every start has gas gap 3.2 +/- 0.3. The best spot is " + "unsure by margin and fully observed: manual review with no photo request.", + g13_scene(-0.5), + { + "decision_in": ["manual_review"], + "spot": {"wall_id": "w1", "span_within": [-0.5, 8]}, + "checks": [ + { + "match": "gas", + "outcome": "unsure", + "unsure_cause": "margin", + "measured_ft": 3.2, + "plus_minus_ft": 0.3, + } + ], + "sweep_runs": [{"wall_id": "w1", "start_ft": [-0.4, 5.3], "outcome": "unsure"}], + "missing_evidence_empty": True, + }, + {"gas": 3.0}, +) + +# No coverage field: nothing counts as observed and both ends default to unexplored. +nocov = straight_scene(-6, 10) +del nocov["coverage"] +case( + "c5-no-coverage", + f"scene.schema.json coverage and end descriptions at origin/t3/server {SERVER_SHA}; C5", + "A clean straight wall with ground and facing measurements but no coverage field. Nothing is " + "known to be observed, so no start may pass, the result must ask for views, and it cannot " + "reject because both ends default to unexplored.", + nocov, + { + "decision_in": ["manual_review"], + "missing_evidence_empty": False, + "sweep_runs": [{"wall_id": "w1", "start_ft": [-6, 7.4], "outcome": "unsure"}], + }, +) + + +def main(): + for old in HERE.glob("*.json"): + old.unlink() + for c in CASES: + (HERE / f"{c['id']}.json").write_text(json.dumps(c, indent=2) + "\n") + print(f"wrote {len(CASES)} cases") + + +if __name__ == "__main__": + main() diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 2c73cf49a..db6d85bf9 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -26,6 +26,7 @@ from jsonschema import Draft202012Validator EPS = 1e-6 +NUMBER_TOL_FT = 1e-5 # Spot centres and offsets are rounded by the server; a hundredth of a foot is 1/8 inch. OFFSET_TOL_FT = 0.01 # Sweep start ranges are sampled; a pass run may reach one 2 in step past a covered edge. @@ -265,6 +266,13 @@ def expectation_problems(expect: dict, result: dict) -> list[str]: problems.append( f"check {check['id']}: {field} {check.get(field)}, expected {rule[field]}" ) + # Pinned numbers catch a server that measures from somewhere else (for example + # another battery depth) and lands on the right outcome by accident. The server + # rounds to 6 decimals. + for field in ("measured_ft", "plus_minus_ft"): + got = check.get(field) + if field in rule and (got is None or abs(got - rule[field]) > NUMBER_TOL_FT): + problems.append(f"check {check['id']}: {field} {got}, expected {rule[field]}") if "missing_evidence_empty" in expect: empty = not result.get("missing_evidence") diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 299734058..259234b04 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -250,6 +250,13 @@ def test_expectations(): assert len(msgs) == 4, msgs +def test_pinned_check_numbers(): + r = result() + assert expectation_problems({"checks": [{"match": "gas", "measured_ft": 5.0000004}]}, r) == [] + msgs = expectation_problems({"checks": [{"match": "gas", "measured_ft": 4.5}]}, r) + assert msgs == ["check gas_clearance: measured_ft 5.0, expected 4.5"] + + def test_assumption_mismatch(): assert assumption_mismatches({"gas": 3.0}, result()) == [] assert assumption_mismatches({"gas": 4.0}, result()) From 5ed6fdaeab466384e924a338e5824b03d0836b2d Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 03:28:14 -0500 Subject: [PATCH 11/59] verification: count contract problems apart from case expectations A case can be wrong itself, so the report now counts broken invariants and properties separately; the scoreboard's new S2-8 reads that count for the server's current SHA. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/hsverify/e2e.py | 23 ++++++++++++++++------- verification/scoreboard/metrics.yaml | 13 +++++++++++++ verification/tests/test_e2e.py | 7 +++++++ 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index 5fac903d7..8d3467508 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -282,25 +282,29 @@ def judge(url: str, endpoint: Endpoint, item: SceneInput, schemas: dict) -> dict status, payload, ms = post(url, endpoint, item) record |= {"http_status": status, "latency_ms": round(ms, 1)} if status != 200: - return record | {"status": "fail", "problems": [f"HTTP {status}: {payload[:500]!r}"]} + failure = [f"HTTP {status} for a scene that validates: {payload[:500]!r}"] + return record | {"status": "fail", "problems": failure, "contract_problems": failure} result = json.loads(payload) record["result"] = result problems = [f"result schema: {e}" for e in schema_errors(result, schemas["result"])] if problems: - return record | {"status": "fail", "problems": problems[:20]} + return record | {"status": "fail", "problems": problems[:20], "contract_problems": problems} - problems += invariant_problems(item.scene, result, sent=item.raw) + # Contract problems hold for any scene; expectation problems depend on one case's geometry. + contract = problems + invariant_problems(item.scene, result, sent=item.raw) + contract += property_problems(url, endpoint, item, result) if item.real and ms > LATENCY_BUDGET_MS: - problems.append(f"real-derived scene took {ms:.0f} ms, budget {LATENCY_BUDGET_MS:.0f}") + contract.append(f"real-derived scene took {ms:.0f} ms, budget {LATENCY_BUDGET_MS:.0f}") mismatches = assumption_mismatches(item.rules_assumed, result) - if item.expect and not mismatches: - problems += expectation_problems(item.expect, result) - problems += property_problems(url, endpoint, item, result) + expected = expectation_problems(item.expect, result) if item.expect and not mismatches else [] + problems = contract + expected record["decision"] = result["decision"] record["outcome_lengths_ft"] = outcome_lengths(result) if mismatches: record["assumption_mismatches"] = mismatches + record["contract_problems"] = contract + record["expectation_problems"] = expected record["problems"] = problems record["status"] = "fail" if problems else "assumption mismatch" if mismatches else "pass" return record @@ -358,6 +362,11 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: "counts": counts, "all_passed": all(r["status"] in ("pass", "skipped") for r in records) and bool(records), "latency_ms": max(real) if real else None, + "scenes_answered": sum(1 for r in records if "decision" in r), + # Invariants and properties broken, summed over every answered scene. Case + # expectations are counted separately because a case can be wrong itself. + "contract_problem_count": sum(len(r.get("contract_problems", [])) for r in records), + "expectation_problem_count": sum(len(r.get("expectation_problems", [])) for r in records), "real_scene_passed": any(r.get("real") and r["status"] == "pass" for r in records), # Null when the run had no app export, so the scoreboard looks for an older run. "app_export_scene_valid": all(r["status"] != "bad input" for r in exported) diff --git a/verification/scoreboard/metrics.yaml b/verification/scoreboard/metrics.yaml index 3de33721b..b772430e8 100644 --- a/verification/scoreboard/metrics.yaml +++ b/verification/scoreboard/metrics.yaml @@ -130,6 +130,19 @@ metrics: less_than: 1000 require_current_sha: true + - id: S2-8 + workstream: S2 + ref: origin/t3/server + text: >- + Black-box (verification e2e): every answer valid under C2, consistent with its decision + and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric + when mirrored, and never passing without coverage + report: + glob: ~/house-scanning-data/reports/e2e/*/report.json + key: contract_problem_count + equals: 0 + require_current_sha: true + - id: S2-7 workstream: S2 ref: origin/t3/server diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index 54c44f9e9..abad80a4e 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -183,6 +183,13 @@ def test_server_breaking_coverage_rule_fails(fake_url): assert re.search(r"ground under .* was not observed", joined) +def test_a_result_breaking_its_schema_counts_against_the_contract(fake_url): + strict = {"scene": {}, "result": {"required": ["nothing_has_this"]}} + record = judge(fake_url, Endpoint("/place", "application/json"), item(), strict) + assert record["status"] == "fail" + assert record["contract_problems"] and record["contract_problems"] == record["problems"] + + def test_bad_input_is_not_blamed_on_the_server(fake_url): strict = {"scene": {"required": ["nothing_has_this"]}, "result": {}} record = judge(fake_url, Endpoint("/place", "application/json"), item(), strict) From 8610d63603a135e38b12d6c7b9f3a560337e0216 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 03:31:05 -0500 Subject: [PATCH 12/59] verification: accessibility audit harness; probe gains a negative control A UI test bundle launches an installed app by bundle id with the C4 arguments and runs performAccessibilityAudit on every distinct screen, printing each issue as a line the runner can collect. The probe's middle screen now carries an unlabelled 12 pt button so a clean audit can be told apart from a blind one. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../A11yAudit.xcodeproj/project.pbxproj | 388 ++++++++++++++++++ .../contents.xcworkspacedata | 7 + .../xcschemes/AccessibilityAudit.xcscheme | 116 ++++++ .../AccessibilityAuditTests.swift | 94 +++++ .../a11y-audit/AuditHost/AuditHostApp.swift | 8 + verification/fixtures/a11y-audit/project.yml | 40 ++ .../StateProbe/StateProbeApp.swift | 6 + 7 files changed, 659 insertions(+) create mode 100644 verification/fixtures/a11y-audit/A11yAudit.xcodeproj/project.pbxproj create mode 100644 verification/fixtures/a11y-audit/A11yAudit.xcodeproj/project.xcworkspace/contents.xcworkspacedata create mode 100644 verification/fixtures/a11y-audit/A11yAudit.xcodeproj/xcshareddata/xcschemes/AccessibilityAudit.xcscheme create mode 100644 verification/fixtures/a11y-audit/AccessibilityAudit/AccessibilityAuditTests.swift create mode 100644 verification/fixtures/a11y-audit/AuditHost/AuditHostApp.swift create mode 100644 verification/fixtures/a11y-audit/project.yml diff --git a/verification/fixtures/a11y-audit/A11yAudit.xcodeproj/project.pbxproj b/verification/fixtures/a11y-audit/A11yAudit.xcodeproj/project.pbxproj new file mode 100644 index 000000000..2b83f88b3 --- /dev/null +++ b/verification/fixtures/a11y-audit/A11yAudit.xcodeproj/project.pbxproj @@ -0,0 +1,388 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 77; + objects = { + +/* Begin PBXBuildFile section */ + 427F7989BACF1CEC6DA2D9C5 /* AccessibilityAuditTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C29531C38DAF3788E36F3030 /* AccessibilityAuditTests.swift */; }; + 7155F226D9F0FACC47C3489D /* AuditHostApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 143718565DE5764127A93948 /* AuditHostApp.swift */; }; +/* End PBXBuildFile section */ + +/* Begin PBXContainerItemProxy section */ + B1FE4BA30BEA3FD8D3508D07 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 58DE58FA588828120ADE9232 /* Project object */; + proxyType = 1; + remoteGlobalIDString = 3247AD21C2DC26C0631148F6; + remoteInfo = AuditHost; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXFileReference section */ + 143718565DE5764127A93948 /* AuditHostApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AuditHostApp.swift; sourceTree = ""; }; + B56910E308F1D85FFF1B2292 /* AccessibilityAudit.xctest */ = {isa = PBXFileReference; includeInIndex = 0; lastKnownFileType = wrapper.cfbundle; path = AccessibilityAudit.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; + C29531C38DAF3788E36F3030 /* AccessibilityAuditTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccessibilityAuditTests.swift; sourceTree = ""; }; + E21BA944B212ED015C3642FE /* AuditHost.app */ = {isa = PBXFileReference; includeInIndex = 0; lastKnownFileType = wrapper.application; path = AuditHost.app; sourceTree = BUILT_PRODUCTS_DIR; }; +/* End PBXFileReference section */ + +/* Begin PBXGroup section */ + 7EC2B27B2CFA88AA1E34F6C2 = { + isa = PBXGroup; + children = ( + BB4FF326C88E53CFE7056923 /* AccessibilityAudit */, + 9ABA268B579D85C69FB70292 /* AuditHost */, + C17A46CCDC0398A153016DE6 /* Products */, + ); + sourceTree = ""; + }; + 9ABA268B579D85C69FB70292 /* AuditHost */ = { + isa = PBXGroup; + children = ( + 143718565DE5764127A93948 /* AuditHostApp.swift */, + ); + path = AuditHost; + sourceTree = ""; + }; + BB4FF326C88E53CFE7056923 /* AccessibilityAudit */ = { + isa = PBXGroup; + children = ( + C29531C38DAF3788E36F3030 /* AccessibilityAuditTests.swift */, + ); + path = AccessibilityAudit; + sourceTree = ""; + }; + C17A46CCDC0398A153016DE6 /* Products */ = { + isa = PBXGroup; + children = ( + B56910E308F1D85FFF1B2292 /* AccessibilityAudit.xctest */, + E21BA944B212ED015C3642FE /* AuditHost.app */, + ); + name = Products; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 3247AD21C2DC26C0631148F6 /* AuditHost */ = { + isa = PBXNativeTarget; + buildConfigurationList = 6505F0C3EE204A6145D7C7E5 /* Build configuration list for PBXNativeTarget "AuditHost" */; + buildPhases = ( + D0D4BE765A5BE20AE2232C7E /* Sources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = AuditHost; + packageProductDependencies = ( + ); + productName = AuditHost; + productReference = E21BA944B212ED015C3642FE /* AuditHost.app */; + productType = "com.apple.product-type.application"; + }; + 8CCD665844B0D8DF2B1EE105 /* AccessibilityAudit */ = { + isa = PBXNativeTarget; + buildConfigurationList = D0152AB46C653989BEA3FA0D /* Build configuration list for PBXNativeTarget "AccessibilityAudit" */; + buildPhases = ( + B11EDB5C8C3342DCB0A4A327 /* Sources */, + ); + buildRules = ( + ); + dependencies = ( + 00EA29F202EF6AECBCD5C1FC /* PBXTargetDependency */, + ); + name = AccessibilityAudit; + packageProductDependencies = ( + ); + productName = AccessibilityAudit; + productReference = B56910E308F1D85FFF1B2292 /* AccessibilityAudit.xctest */; + productType = "com.apple.product-type.bundle.ui-testing"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 58DE58FA588828120ADE9232 /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = YES; + LastUpgradeCheck = 1430; + TargetAttributes = { + 8CCD665844B0D8DF2B1EE105 = { + TestTargetID = 3247AD21C2DC26C0631148F6; + }; + }; + }; + buildConfigurationList = 2042BAAFD5134746EA8592DF /* Build configuration list for PBXProject "A11yAudit" */; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + Base, + en, + ); + mainGroup = 7EC2B27B2CFA88AA1E34F6C2; + minimizedProjectReferenceProxies = 1; + preferredProjectObjectVersion = 77; + productRefGroup = C17A46CCDC0398A153016DE6 /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 8CCD665844B0D8DF2B1EE105 /* AccessibilityAudit */, + 3247AD21C2DC26C0631148F6 /* AuditHost */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXSourcesBuildPhase section */ + B11EDB5C8C3342DCB0A4A327 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 427F7989BACF1CEC6DA2D9C5 /* AccessibilityAuditTests.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + D0D4BE765A5BE20AE2232C7E /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 7155F226D9F0FACC47C3489D /* AuditHostApp.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXTargetDependency section */ + 00EA29F202EF6AECBCD5C1FC /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 3247AD21C2DC26C0631148F6 /* AuditHost */; + targetProxy = B1FE4BA30BEA3FD8D3508D07 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + +/* Begin XCBuildConfiguration section */ + 3C9E1A85CDCE6151C6199A4E /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + CODE_SIGNING_ALLOWED = NO; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 26.0; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = iphoneos; + SWIFT_COMPILATION_MODE = wholemodule; + SWIFT_OPTIMIZATION_LEVEL = "-O"; + SWIFT_VERSION = 6.0; + }; + name = Release; + }; + 708EB10FCF910F1ACB74A421 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + GENERATE_INFOPLIST_FILE = YES; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + "@loader_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = dev.housescanning.accessibilityaudit; + SDKROOT = iphoneos; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_TARGET_NAME = AuditHost; + }; + name = Debug; + }; + 841E56A50F19DED71F190C76 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + GENERATE_INFOPLIST_FILE = YES; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + "@loader_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = dev.housescanning.accessibilityaudit; + SDKROOT = iphoneos; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_TARGET_NAME = AuditHost; + }; + name = Release; + }; + BE3EA69AAFB233D09E1F40CA /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CODE_SIGN_IDENTITY = "iPhone Developer"; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_UILaunchScreen_Generation = YES; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = dev.housescanning.audithost; + SDKROOT = iphoneos; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + D55271EAC8D6256B4B11B08F /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CODE_SIGN_IDENTITY = "iPhone Developer"; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_UILaunchScreen_Generation = YES; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = dev.housescanning.audithost; + SDKROOT = iphoneos; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; + EC24429FA6E4DFBA10CCD5C8 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + CODE_SIGNING_ALLOWED = NO; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "$(inherited)", + "DEBUG=1", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 26.0; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = iphoneos; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_VERSION = 6.0; + }; + name = Debug; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 2042BAAFD5134746EA8592DF /* Build configuration list for PBXProject "A11yAudit" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + EC24429FA6E4DFBA10CCD5C8 /* Debug */, + 3C9E1A85CDCE6151C6199A4E /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Debug; + }; + 6505F0C3EE204A6145D7C7E5 /* Build configuration list for PBXNativeTarget "AuditHost" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + BE3EA69AAFB233D09E1F40CA /* Debug */, + D55271EAC8D6256B4B11B08F /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Debug; + }; + D0152AB46C653989BEA3FA0D /* Build configuration list for PBXNativeTarget "AccessibilityAudit" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 708EB10FCF910F1ACB74A421 /* Debug */, + 841E56A50F19DED71F190C76 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Debug; + }; +/* End XCConfigurationList section */ + }; + rootObject = 58DE58FA588828120ADE9232 /* Project object */; +} diff --git a/verification/fixtures/a11y-audit/A11yAudit.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/verification/fixtures/a11y-audit/A11yAudit.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 000000000..919434a62 --- /dev/null +++ b/verification/fixtures/a11y-audit/A11yAudit.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/verification/fixtures/a11y-audit/A11yAudit.xcodeproj/xcshareddata/xcschemes/AccessibilityAudit.xcscheme b/verification/fixtures/a11y-audit/A11yAudit.xcodeproj/xcshareddata/xcschemes/AccessibilityAudit.xcscheme new file mode 100644 index 000000000..c5ee0e95f --- /dev/null +++ b/verification/fixtures/a11y-audit/A11yAudit.xcodeproj/xcshareddata/xcschemes/AccessibilityAudit.xcscheme @@ -0,0 +1,116 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/verification/fixtures/a11y-audit/AccessibilityAudit/AccessibilityAuditTests.swift b/verification/fixtures/a11y-audit/AccessibilityAudit/AccessibilityAuditTests.swift new file mode 100644 index 000000000..c6e833f6e --- /dev/null +++ b/verification/fixtures/a11y-audit/AccessibilityAudit/AccessibilityAuditTests.swift @@ -0,0 +1,94 @@ +import XCTest + +/// Launches the app under test by bundle id with the launch arguments it is given, then runs +/// `performAccessibilityAudit` once per distinct screen until the app stops changing. +/// +/// A screen is identified by the labels of its static texts and buttons, so a new instruction +/// counts as a new screen. Each issue is printed as one `A11Y_ISSUE=` line and each +/// screen as `A11Y_SCREEN=`, which hsverify.a11yaudit reads from the test output. +/// Screenshots go to AUDIT_OUT on the host. +/// +/// Inputs arrive as environment variables (set on the xcodebuild command line with the +/// TEST_RUNNER_ prefix): AUDIT_BUNDLE, AUDIT_ARGS (arguments separated by U+001F), +/// AUDIT_SECONDS (overall limit), AUDIT_IDLE (stop after this long without a new screen), +/// AUDIT_OUT (folder for screenshots). +final class AccessibilityAuditTests: XCTestCase { + @MainActor + func testAuditEveryScreen() throws { + let env = ProcessInfo.processInfo.environment + let bundle = try XCTUnwrap(env["AUDIT_BUNDLE"], "AUDIT_BUNDLE is required") + let args = (env["AUDIT_ARGS"] ?? "").split(separator: "\u{1F}").map(String.init) + let limit = Double(env["AUDIT_SECONDS"] ?? "") ?? 120 + let idle = Double(env["AUDIT_IDLE"] ?? "") ?? 15 + let out = env["AUDIT_OUT"].map { URL(fileURLWithPath: $0) } + + let app = XCUIApplication(bundleIdentifier: bundle) + app.launchArguments = args + app.launch() + + let start = Date() + var lastChange = Date() + var seen: [String] = [] + while Date().timeIntervalSince(start) < limit, Date().timeIntervalSince(lastChange) < idle { + guard app.state == .runningForeground else { + emit("A11Y_SCREEN", ["index": seen.count + 1, "error": "app not running"]) + break + } + let labels = screenLabels(app) + let key = labels.joined(separator: "\n") + if !seen.contains(key) { + seen.append(key) + lastChange = Date() + audit(app, index: seen.count, labels: labels, out: out, + at: Date().timeIntervalSince(start)) + } + RunLoop.current.run(until: Date().addingTimeInterval(0.5)) + } + } + + @MainActor + private func screenLabels(_ app: XCUIApplication) -> [String] { + let texts = app.staticTexts.allElementsBoundByIndex.map { "text:" + $0.label } + let buttons = app.buttons.allElementsBoundByIndex.map { "button:" + $0.label } + return texts + buttons + } + + @MainActor + private func audit( + _ app: XCUIApplication, index: Int, labels: [String], out: URL?, at seconds: Double + ) { + var shot: String? = nil + if let out { + let name = String(format: "a11y-%02d.png", index) + try? XCUIScreen.main.screenshot().pngRepresentation + .write(to: out.appendingPathComponent(name)) + shot = name + } + var issues = 0 + do { + try app.performAccessibilityAudit(for: .all) { issue in + issues += 1 + self.emit("A11Y_ISSUE", [ + "screen": index, + "type": String(describing: issue.auditType), + "description": issue.compactDescription, + "detail": issue.detailedDescription, + "element": issue.element.map { "\($0.elementType.rawValue) \($0.label)" } ?? "", + ]) + return true // record every issue; never stop at the first + } + } catch { + emit("A11Y_SCREEN", ["index": index, "error": "\(error)"]) + } + emit("A11Y_SCREEN", [ + "index": index, "seconds": seconds, "labels": labels, "issues": issues, + "screenshot": shot ?? "", + ]) + } + + private func emit(_ tag: String, _ object: [String: Any]) { + let data = (try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys])) + ?? Data("{}".utf8) + print("\(tag)=\(String(decoding: data, as: UTF8.self))") + } +} diff --git a/verification/fixtures/a11y-audit/AuditHost/AuditHostApp.swift b/verification/fixtures/a11y-audit/AuditHost/AuditHostApp.swift new file mode 100644 index 000000000..bed353394 --- /dev/null +++ b/verification/fixtures/a11y-audit/AuditHost/AuditHostApp.swift @@ -0,0 +1,8 @@ +import SwiftUI + +@main +struct AuditHostApp: App { + var body: some Scene { + WindowGroup { Text("Accessibility audit host") } + } +} diff --git a/verification/fixtures/a11y-audit/project.yml b/verification/fixtures/a11y-audit/project.yml new file mode 100644 index 000000000..b668951b9 --- /dev/null +++ b/verification/fixtures/a11y-audit/project.yml @@ -0,0 +1,40 @@ +# Accessibility audit harness for hsverify.a11yaudit. The UI test launches an app that is already +# installed on the Simulator, by bundle id, and runs Apple's accessibility audit on every distinct +# screen it shows. AuditHost exists only because a UI test target needs a host app. +name: A11yAudit +options: + deploymentTarget: + iOS: "26.0" +settings: + base: + SWIFT_VERSION: "6.0" + CODE_SIGNING_ALLOWED: NO +targets: + AuditHost: + type: application + platform: iOS + sources: [AuditHost] + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: dev.housescanning.audithost + GENERATE_INFOPLIST_FILE: YES + INFOPLIST_KEY_UILaunchScreen_Generation: YES + AccessibilityAudit: + type: bundle.ui-testing + platform: iOS + sources: [AccessibilityAudit] + dependencies: + - target: AuditHost + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: dev.housescanning.accessibilityaudit + GENERATE_INFOPLIST_FILE: YES + TEST_TARGET_NAME: AuditHost +schemes: + AccessibilityAudit: + build: + targets: + AuditHost: all + AccessibilityAudit: [test] + test: + targets: [AccessibilityAudit] diff --git a/verification/fixtures/state-probe/StateProbe/StateProbeApp.swift b/verification/fixtures/state-probe/StateProbe/StateProbeApp.swift index 7daf0a53e..d7fd894ee 100644 --- a/verification/fixtures/state-probe/StateProbe/StateProbeApp.swift +++ b/verification/fixtures/state-probe/StateProbe/StateProbeApp.swift @@ -32,6 +32,12 @@ struct ProbeView: View { Text(current).font(.largeTitle.bold()) Text(CommandLine.arguments.dropFirst().joined(separator: " ")) .font(.footnote.monospaced()) + if current == "probe_middle" { + // Negative control for the accessibility audit: an icon-only button with no + // label and a 12 pt target. The audit must report it. + Button {} label: { Image(systemName: "gearshape").font(.system(size: 9)) } + .frame(width: 12, height: 12) + } } .padding() .task { From 0d4f1d64b2225177749946f8e41d88feca5b6e98 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 03:35:47 -0500 Subject: [PATCH 13/59] verification: fix cases that missed public rules; accept a named raw-body endpoint The first run showed the cases, not the server, were wrong in 16 places: they ignored NEC 110.26 working space at the meter and left ground unobserved within the placeholder pool and driveway clearances. g03 stays: it catches a real server disagreement. The server's POST /v1/placements reads the raw body, so a named endpoint without a body schema is taken as JSON. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/e2e/cases/README.md | 117 ++++++++++++------ verification/e2e/cases/c5-no-coverage.json | 17 +++ .../e2e/cases/g01-unseen-beyond-garage.json | 34 ++--- .../e2e/cases/g03-negative-s-near-edge.json | 18 +-- .../e2e/cases/g06-regulator-footprint.json | 18 +-- .../e2e/cases/g08a-garage-blocks-route.json | 22 ++-- .../e2e/cases/g10-derived-compound-error.json | 18 +-- verification/e2e/cases/g10-e03-fail.json | 18 +-- .../e2e/cases/g10-e03-lower-edge.json | 18 +-- verification/e2e/cases/g10-e03-pass.json | 18 +-- .../e2e/cases/g10-e03-upper-edge.json | 18 +-- verification/e2e/cases/g10-e05-fail.json | 18 +-- .../e2e/cases/g10-e05-lower-edge.json | 18 +-- verification/e2e/cases/g10-e05-pass.json | 18 +-- .../e2e/cases/g10-e05-upper-edge.json | 18 +-- verification/e2e/cases/g10-e15-fail.json | 18 +-- .../e2e/cases/g10-e15-lower-edge.json | 18 +-- verification/e2e/cases/g10-e15-pass.json | 18 +-- .../e2e/cases/g10-e15-upper-edge.json | 18 +-- .../e2e/cases/g12a-subgrid-start.json | 18 +-- .../e2e/cases/g12b-exact-fit-last-start.json | 18 +-- .../g13a-unsure-never-outranks-pass.json | 31 +++-- verification/e2e/cases/g13b-unsure-alone.json | 27 ++-- verification/e2e/cases/generate.py | 72 ++++++++--- verification/hsverify/e2e.py | 11 +- verification/tests/test_e2e.py | 8 ++ 26 files changed, 384 insertions(+), 261 deletions(-) diff --git a/verification/e2e/cases/README.md b/verification/e2e/cases/README.md index 5c373673e..f64b8be33 100644 --- a/verification/e2e/cases/README.md +++ b/verification/e2e/cases/README.md @@ -3,8 +3,10 @@ Black-box cases for the placement server. Each `.json` holds a scene and what the result must show. The scenes were written from `server/schemas/scene.schema.json` and `result.schema.json` at origin/t3/server 69c6c3c and the golden tests in `docs/research/t3-lane-c-review.md` at -origin/t3/research a7d91f1. Nobody read the server's code or fixtures to write them, so a passing -server has been checked against the spec, not against its own tests. +origin/t3/research a7d91f1. After the first run they were corrected against `server/rules.yaml` +at origin/t3/server 2c9348f, which is public configuration. Nobody read the server's code, tests +or fixtures to write them, so a passing server has been checked against the spec, not against its +own tests. `generate.py` writes every JSON file in this folder. Edit it, not the JSON, then run from `verification/`: @@ -15,9 +17,7 @@ uv run python e2e/cases/generate.py ## Why the assertions avoid the decision -`server/rules.yaml` was not on origin/t3/server when these cases were written, so the numbers come -from the review's parameter table and the rules table in `docs/01-feature-map.md`. While the rules -hold placeholder values, `policy.auto_approve` is false and any would-be pass or reject becomes +`rules.yaml` sets `policy.auto_approve: false`, so any would-be pass or reject becomes `manual_review`. The cases therefore assert on the sweep, on single checks, on `spot` and on `missing_evidence`. They assert a decision only where the scene alone forces it: @@ -34,24 +34,41 @@ The keys follow the lead's case format. The cases use two additions and one conv gap the case built. With a different battery depth it measures a different gap, and these keys catch that before the outcome is compared. - `missing_evidence_empty: false` means the list must be non-empty. -- `rules_assumed` maps a check-id substring to the `threshold_ft` the case depends on. `gas` and - `opening` are both 3 ft (Base help page; IRC R328). +- `rules_assumed` maps a check-id substring to the `threshold_ft` the case depends on: `gas` 3 ft + (`clearances.gas_ft`) and `opening` 3 ft (`clearances.opening_ft`). + +## The rules these cases depend on + +From `rules.yaml` at 2c9348f: + +- Battery 31/12 × 11/6 ft (W ≈ 2.5833, D ≈ 1.8333), matching the review's table. +- `meter_working_space` 2.5 ft wide, centred on the meter. A battery overlapping s = (−1.25, 1.25) + fails, so starts in (−1.25 − W, 1.25) = (−3.833, 1.25) fail. The starts −3.833 and 1.25 only + touch the space: gap 0 against threshold 0, which is unsure. +- `pool_ft` 10 and `drive_ft` 5, both placeholders. With no pool or driveway in the scene these + checks pass only if the ground was seen 10 ft around the footprint. Otherwise they are unsure + and ask for a photo. +- `openings.types` is `[door, window]`. A garage door is not an opening, so it has no 3 ft + clearance. It does fail the route (`route.crossing.garage_door: fail`). +- `route.crossing.window: detour` and `gas_meter: detour`. `route.corner_allowance_ft` adds 0.5 ft + per corner the cable turns. +- `facing.measured_from: battery_front`, `min_ft` 3: a 9 ft facing gap gives 9 − 11/6 ≈ 7.17. +- `ground.allowed` includes concrete. ## Fixture conventions - Units are feet. Walls are exact (`plus_minus_ft: 0`) unless a case says otherwise. So are the meter and every object: `source: "tape"` with an explicit error, so no default error applies. -- Battery: 31/12 ft wide (W ≈ 2.5833) and 11/6 ft deep (D ≈ 1.8333), from the review's table. The - schema can't carry the battery size, so the checker can't tell which one the server used. - Sweep intervals are chosen to hold for any width from 30.68 in to 3 ft. The cases that depend on - the exact size are named in the table. - A straight wall along +x has outward +z (the schema's clockwise rule). s = x − meter x. - Cases g10, g11a, g11c and g13 place the wall at z = −11/6. A flush battery's front edge then sits at z = 0.0 exactly in floating point, so a gas strip at z = d measures exactly d. The equality rows need this: 3.3 − 0.3 and 2.7 + 0.3 both evaluate to exactly 3.0, so a server that forgets equality lands on the wrong side and fails the case. -- "Fully observed" means wall, overhead, facing and ground (10 ft out) bands over the whole chain, - both ends `limit`, facing depth 9, no overheads, and concrete ground over the whole area. +- "Fully observed" means wall, overhead, facing and ground bands from 15 ft past the left end to + 15 ft past the right end, ground seen 15 ft out, both ends `limit`, facing depth 9, no overheads, + and concrete ground over the chain. 15 ft clears the 10 ft pool distance measured from a + footprint whose front is 11/6 ft out. g05, g08b and g11a–c keep coverage ending at the chain, + since they passed with it and g11 depends on exact gaps. - A gas "strip" is a long, thin gas footprint parallel to the wall. It spans past both ends of the stretch it covers, so every battery beside it has the same perpendicular gap. It stands in for the review's "every start has the same gas gap". @@ -60,43 +77,63 @@ The keys follow the lead's case format. The cases use two additions and one conv | Case | Golden | Asserts | Arithmetic | | --- | --- | --- | --- | -| g01-unseen-beyond-garage | 01 | `missing_evidence` empty; starts in (−13.9, −9.7) fail with `route` | Wall s = [−14, 10], garage [−7, −3], nothing observed left of −7. A start left of the garage has its battery at s ≤ −7, so its cable must cross the garage. The last such start is −7 − W = −9.583. | -| g03-negative-s-near-edge | 03 | spot on w1 within [−9, −6]; starts in (−8.4, 6.9) fail; no photo request | Pads are straight 3 ft segments at s = [−9, −6] and [7, 10]. Between them sit 7 segments of 13/7 ≈ 1.857 ft, each shorter than any battery. Headings run 32°, 24°, …, −24°, −32°, so every corner is an outside corner. Left route = 6 (near edge), right = 7, so left wins. Measuring to the far edge would give 103/12 ≈ 8.583 > 7 and pick the right pad. Pad starts are (−9, −8.583] and [7, 7.417], and every other start crosses a corner. | +| g01-unseen-beyond-garage | 01 | `missing_evidence` empty; starts in (−19.9, −8.2) fail with `route` | Wall s = [−20, 10], garage [−5.5, −1], nothing observed left of s = −9. A battery with right edge b < −1 needs a cable across the garage (b ≤ −5.5 means a ≤ −8.083). A battery with b > −1 overlaps the working space. So every start left of the meter fails. The first start right of the meter that clears the working space is 1.25, and pool needs ground from 1.25 − 10 = −8.75 ≥ −9. | +| g03-negative-s-near-edge | 03 | spot on w1 within [−9, −6]; starts in (−8.4, 6.9) fail; no photo request | Pads are straight 3 ft segments at s = [−9, −6] and [7, 10]. Between them sit 7 segments of 13/7 ≈ 1.857 ft, each shorter than the battery, with corners at s = −4.143, −2.286, −0.429, 1.429, 3.286, 5.143. Headings run 32°, 24°, …, −24°, −32°, so every corner is an outside corner. Left route = 6 + 3 corners × 0.5 = 7.5; right = 7 + 3 × 0.5 = 8.5; left wins. Measuring to the far edge gives 103/12 + 1.5 ≈ 10.08 > 8.5 and would pick the right pad. Pad starts are (−9, −8.583] and [7, 7.417]; every other start crosses a corner. | | g05-inside-corner-gas | 05 | starts on w1 in (6, 7.4) fail with `gas`; never pass | w1 is z = 0 from x = −7 to 4 (w1 extended 1 ft left of the golden, so the meter at x = −6 sits inside the wall rather than on its end); s = x + 6. w2 is x = 4 going +z; gas point [4, 4] at s = 14. For start s the battery is x ∈ [s − 6, s − 6 + W], z ∈ [0, 11/6]. At s = 6 the nearest corner is (31/12, 11/6): √((17/12)² + (13/6)²) = √965/12 ≈ 2.589 < 3. Fail needs the right edge past 4 − √(9 − (13/6)²) = 4 − √155/6 ≈ 1.925, so s > 5.342. The last start on w1 is 89/12 ≈ 7.417 (2.167). Unrolled gap 14 − 103/12 = 65/12 ≈ 5.417 would wrongly pass. | -| g06-regulator-footprint | 06 | starts in (3, 9) fail with `gas`; never pass | Gas footprint: body x ∈ [6.5, 7.5], z ∈ [9.5, 10.5], with a spike reaching [7, 23/6]. Battery front at z = 11/6, so the gap to the tip is 23/6 − 11/6 = 2 whenever the battery covers x = 7, i.e. starts in [7 − W, 7] ≈ [4.417, 7]. Outside that the gap is √(dx² + 4) < 3 while dx < √5 ≈ 2.236, so starts in (2.18, 9.24) fail. The body alone gives 10 − 11/6 ≈ 8.17 and would pass. | -| g08a-garage-blocks-route | 08 | spot null; starts in (5, 9.4) fail with `route`; no photo request; never pass | Wall s = [−0.5, 12], garage [1, 5]. The earliest start is −0.5, so every battery ends at ≥ −0.5 + W ≈ 2.08 > 1 and overlaps the garage (opening gap 0 < 3), or starts at ≥ 5 and needs a cable across the garage. The golden's garage [3, 7] and pad [10, 13] were shifted so the meter side holds no start. | +| g06-regulator-footprint | 06 | starts in (3, 9) fail with `gas`; never pass | Gas footprint: body x ∈ [6.5, 7.5], z ∈ [9.5, 10.5], with a spike reaching [7, 23/6]. Battery front at z = 11/6, so the gap to the tip is 23/6 − 11/6 = 2 whenever the battery covers x = 7, i.e. starts in [7 − W, 7] ≈ [4.417, 7]. Outside that the gap is √(dx² + 4) < 3 while dx < √5 ≈ 2.236, so starts in (2.18, 9.24) fail. The body alone gives 10 − 11/6 ≈ 8.17 and would pass. Facing depth is 12 here so the body stands inside it. | +| g08a-garage-blocks-route | 08 | spot null; starts in (5, 9.4) fail with `route`; no photo request; never pass | Wall s = [−0.5, 12], garage [1, 5]. Starts in [−0.5, 1.25) overlap the working space. From 1.25 on, the route to the near edge covers [0, a] and crosses the garage, which starts at 1. The golden's garage [3, 7] and pad [10, 13] were shifted so the meter side holds no start. | | g08b-missing-wall-blocks-route | 08 | spot null; starts on w2 in (2, 9.4) fail with `route`; no photo request; never pass | w1 s = [−0.5, 1] (1.5 ft, shorter than the battery), no wall for s = (1, 2), w2 s = [2, 12]. The schema says no battery backs onto and no cable crosses a no-wall stretch, so every start fails by geometry. | | g10-e03-{pass, upper-edge, lower-edge, fail} | 10 | gas check pass / unsure (margin) / unsure (margin) / fail, with measured d and ±0.3; no photo request; the fail row also needs spot null and every start in (−5, 12.4) failing with `gas` | T = 3, e = 0.3. d = 3.31: 3.31 − 0.3 = 3.01 > 3, pass. d = 3.3: 3.3 − 0.3 = 3.0, not > 3; 3.6 not < 3, so unsure. d = 2.7: 2.7 + 0.3 = 3.0, not < 3, so unsure. d = 2.69: 2.99 < 3, fail. | | g10-e05-* | 10 | as above with ±0.5 | d = 3.51, 3.5, 2.5, 2.49 → 3.01 > 3; 3.0; 3.0; 2.99 < 3. | | g10-e15-* | 10 | as above with ±1.5 | d = 4.51, 4.5, 1.5, 1.49 → 3.01 > 3; 3.0; 3.0; 2.99 < 3. | | g10-derived-compound-error | 10 | gas unsure (margin), measured 3.5, ±0.6; never pass | Wall ±0.3 plus gas ±0.3 = ±0.6 (linear, as the review specifies). 3.5 − 0.6 = 2.9 is not > 3, and 3.5 + 0.6 = 4.1 is not < 3. Root-sum-square would give ±0.424 and 3.076 > 3, a wrong pass. Every g10 gas object also carries `conf: 0.6`; treating 1 − conf as an error would shift every row. | -| g11a-unexplored-corner | 11 (a) | decision manual_review; spot null; `missing_evidence` non-empty; starts in (−3, 5.4) fail with `gas` | Strip at gap 1: 1 + 0 < 3 fails every start. The right end at s = 8 is `unexplored`, so reject is ruled out and the wall past the corner must be requested. The end sits 8 ft out so it is within any plausible reach. | +| g11a-unexplored-corner | 11 (a) | decision manual_review; spot null; `missing_evidence` non-empty; starts in (−3, 5.4) fail with `gas` | Strip at gap 1: 1 + 0 < 3 fails every start. The right end at s = 8 is `unexplored`, so reject is ruled out and the wall past the corner must be requested. The end sits 8 ft out, inside `route.max_ft` 20. | | g11b-both-ends-real-all-fail | 11 (b) | decision manual_review or reject; spot null; no photo request; w1 starts in (−3, 5.4) and w2 starts in (8, 11.4) fail with `gas` | w1 s = [−3, 8], outside corner, w2 runs x = 8 toward −z, s = [8, 14], outward +x. Strips sit 1 ft beyond each wall's battery front: z = 11/6 + 1 and x = 8 + 11/6 + 1. | | g11c-unobserved-ground | 11 (c) | decision manual_review; spot within [2.328, 10]; `missing_evidence` non-empty; starts in (−9.9, 2.2) fail with `gas`; starts in (2.5, 7.3) unsure | Strip x ∈ [−12, −0.5] at gap 1. A start a > −0.5 has gap √((a + 0.5)² + 1), under 3 while a < 2√2 − 0.5 ≈ 2.328. Ground was observed only for s ≤ 1, so starts past 2.328 pass gas on unseen ground: unsure. The last start is 10 − W ≈ 7.417. | -| g12a-subgrid-start | 12 | spot within [7, 7.1 + W]; starts in (−1, 6.95) and (7.2, 13.4) fail with `opening` | Windows [3, 4] and [10.1 + W, 11.1 + W] ≈ [12.683, 13.683]. Legal start: a − 4 > 3 and (10.1 + W) − (a + W) > 3, so 7 < a < 7.1. Both ends give a gap of exactly 3, which is unsure, not pass. The 2 in grid from 0 samples 7 and 43/6 ≈ 7.167, both outside. The wall starts at −1, so any battery left of window 1 would need its right edge < 0 but has it ≥ −1 + W > 0. Past window 2 a start needs a > 16.683, beyond the wall end at 16. Depends on W: with 30.68 in the legal interval is (7, 7.127), still inside the assertion. | -| g12b-exact-fit-last-start | 12 | spot within [6, 6 + W]; no photo request | Five 1.8 ft segments (headings 40° to 8°) for s = [−3, 6], then one segment of exactly W at heading 0° ending the wall. Its only start, 6, is also its last. Measured in floating point, the segment is 4.4e-16 longer than W. | -| g13a-unsure-never-outranks-pass | 13 | spot within [−8, −0.506]; gas check at the spot passes; starts in (0.5, 5) unsure; no photo request | Strip x ∈ [0.3, 12] at gap 3.2, ±0.3: every battery overlapping x ≥ 0.3 gets 3.2 − 0.3 = 2.9, unsure. A battery with right edge b < 0.3 gets √((0.3 − b)² + 3.2²). That passes when > 3.3, so (0.3 − b)² > 0.65 and b < 0.3 − √0.65 ≈ −0.506. Unsure spots have routes near 0 and passing ones ≥ 0.506, and the passing spot must still win. | -| g13b-unsure-alone | 13 | decision manual_review; spot on w1; gas unsure (margin), measured 3.2, ±0.3; starts in (−0.4, 5.3) unsure; no photo request | The wall starts at −0.5, so every battery ends at ≥ 2.08 and overlaps the strip: gap 3.2 for all. The spot is fully observed and unsure only by margin, so no photos. | -| c5-no-coverage | C5, scene schema | decision manual_review; `missing_evidence` non-empty; starts in (−6, 7.4) unsure | Clean wall s = [−6, 10] with ground and facing data but no `coverage`. "Absent means nothing is known to be observed" makes every start unsure. Absent ends default to unexplored, which rules out reject. | +| g12a-subgrid-start | 12 | spot within [7, 7.1 + W]; starts in (−1, 6.95) and (7.2, 13.4) fail with `opening` | Windows [3, 4] and [10.1 + W, 11.1 + W] ≈ [12.683, 13.683]. Legal start: a − 4 > 3 and (10.1 + W) − (a + W) > 3, so 7 < a < 7.1. Both ends give a gap of exactly 3, which is unsure, not pass. The 2 in grid from 0 samples 7 and 43/6 ≈ 7.167, both outside. The wall starts at −1, so any battery left of window 1 would need its right edge < 0 but has it ≥ −1 + W > 0. Past window 2 a start needs a > 16.683, beyond the wall end at 16. The cable detours under window 1. | +| g12b-exact-fit-last-start | 12 | spot within [6, 6 + W]; no photo request | Five 1.8 ft segments (headings 40° to 8°) for s = [−3, 6], then one segment of exactly W at heading 0° ending the wall. Its only start, 6, is also its last. Measured in floating point, the segment is 4.4e-16 longer than W. Route 6 + 3 corners × 0.5 = 7.5. | +| g13a-unsure-never-outranks-pass | 13 | spot within [−8, −1.25]; gas check at the spot passes; starts in (−3.8, 1.2) fail with `meter_working_space`; starts in (1.3, 5) unsure; no photo request | Strip x ∈ [0.3, 12] at gap 3.2, ±0.3: every battery overlapping x ≥ 0.3 gets 3.2 − 0.3 = 2.9, unsure. A battery with right edge b < 0.3 gets √((0.3 − b)² + 3.2²), which passes when > 3.3: (0.3 − b)² > 0.65, b < 0.3 − √0.65 ≈ −0.506. The working space also needs b ≤ −1.25, so passing spots have b ≤ −1.25 and route ≥ 1.25. Unsure spots start at 1.25 (route 1.25), and the passing spot must still win. | +| g13b-unsure-alone | 13 | decision manual_review; spot on w1; gas unsure (margin), measured 3.2, ±0.3; starts in (−0.4, 1.2) fail with `meter_working_space`; starts in (1.3, 5.3) unsure; no photo request | The wall starts at −0.5, so every battery ends at ≥ 2.08 and overlaps the strip: gap 3.2 for all. The spot is fully observed and unsure only by margin, so no photos. | +| c5-no-coverage | C5, scene schema | decision manual_review; `missing_evidence` non-empty; starts in (−6, −3.9) and (1.3, 7.4) unsure, starts in (−3.8, 1.2) fail with `meter_working_space` | Clean wall s = [−6, 10] with ground and facing data but no `coverage`. "Absent means nothing is known to be observed" makes every start unsure except where the working space, which needs no observation, fails it. Absent ends default to unexplored, which rules out reject. | ## Assumptions a failure should be checked against - A battery must sit flush on one straight segment (review item 4, golden 04; the result's `spot.segment`). g03 and g12b use short segments to rule out starts, and g08b uses a 1.5 ft wall. -- Concrete ground, a 9 ft facing gap and no overheads pass. Only g03, g11c (spot), g12a, g12b, - g13a, g13b and the unsure sweep in c5 need this. A server whose placeholder rules fail concrete or - a 9 ft gap fails those cases without having broken the rule under test. -- A cable may pass under a window (golden 12 places a window between meter and spot and still - expects pass). g12a depends on it. -- A garage door is an opening under the 3 ft rule, and the route may not cross it (golden 08). - g01 and g08a depend on it. The crossing effect is policy (review item 7); if the placeholder rules - make it `review`, g01 and g08a fail and the reason is that rule. -- Gas objects with a footprint are measured by footprint. Their `span_ft` is the footprint's - stretch along the wall and does not block the cable route. In g10, g11a and g11b the span covers - the meter. A server that routes around spans still passes those, because they assert only gas - and photo requests. g13a keeps the span off the left routes. +- Gas objects with a footprint are measured by footprint. In g10, g11a and g11b the strip's span + covers the meter, which makes the cable detour; those cases assert only gas and photo requests. + g13a keeps the span off the left routes. - s counts the length of a no-wall stretch (g08b: w2 starts at s = 2, after a 1 ft gap). +## Changes after the first run + +First run: origin/t3/server 66d7ec8 through the solver shim, 10 of 27 passing +(`~/house-scanning-data/reports/e2e/20260926-032427-66d7ec89/report.json`). These were case +errors, corrected once `rules.yaml` was published: + +- **Meter working space.** c5, g13a and g13b expected unsure over starts in (−3.833, 1.25), which + `meter_working_space` fails. Their sweep assertions now expect that fail and assert unsure only + outside it. The g13a spot bound moved from −0.506 to −1.25. +- **Ground coverage for pool and driveway.** Every "no photo request" case got + `missing_evidence` asking for ground up to 10 ft beyond the spot, for `pool_clearance` (and + `drive_clearance`, gas, AC and opening in g03 and g12b, where coverage stopped at the chain's + ends). "Fully observed" now reaches 15 ft past each end and 15 ft out. The straight-wall helper + applies this too, so g06, g08a, g10 and g12a gained coverage as well; their assertions are + unchanged. +- **g01 layout.** The old spot right of the meter needed ground 10 ft to its left, which reached + into the unseen stretch, so the photo request was correct. The garage moved to [−5.5, −1] and the + unseen stretch to s < −9: no start between garage and meter survives the working space, and the + spot's 10 ft pool area is observed. +- **Garage doors are not openings.** `openings.types` is `[door, window]`. g08a still holds through + the working space and the route. Its `rules_assumed` no longer lists `opening`. + +## Known server disagreement at 66d7ec8 + +- **g03, corner-crossing starts.** The sweep is one run, `{"start_ft": [-9.0, 7.416667], + "outcome": "unsure"}`, and `stats.candidates` is 14. Starts in (−8.583, 7) cross a corner. They + are not evaluated, yet the run covering them claims they are unsure. Golden 04 says they fail + backing. The chosen spot itself is right: `[-8.583, -6.0]`, route 7.5. + ## Goldens not encoded - 02: its only assertion is the policy flag itself. Checking `policy.auto_approve` is false and no @@ -104,8 +141,10 @@ The keys follow the lead's case format. The cases use two additions and one conv - 04: the golden gives w1 outward [0, −1] for a baseline running +x, which contradicts the schema's clockwise rule. Its pad also relies on a ground check whose passing surfaces are not published. The flush rule it tests is exercised by g03 and g12b. -- 07: needs `headroom_ft`, which is not published. -- 09: needs `confident_reach_ft` and `max_route_ft`, which are not published. -- 11 (d): its expectation is a pass, which placeholder rules can't produce. Its photo-request half - duplicates g01. +- 07: needs headroom around `headroom.min_ft` 6.5, a placeholder. Now that the value is + published this could be encoded; it isn't yet. +- 09: needs `confident_reach_ft` 15 and `max_route_ft` 20. `confident_reach_ft` is a placeholder + but published, so this could also be encoded now. +- 11 (d): its expectation is a pass, which `auto_approve: false` can't produce. Its photo-request + half duplicates g01. - 14: needs a pass. The result schema already requires every field it lists. diff --git a/verification/e2e/cases/c5-no-coverage.json b/verification/e2e/cases/c5-no-coverage.json index 8b88b16ba..28058b40a 100644 --- a/verification/e2e/cases/c5-no-coverage.json +++ b/verification/e2e/cases/c5-no-coverage.json @@ -77,6 +77,23 @@ "wall_id": "w1", "start_ft": [ -6, + -3.9 + ], + "outcome": "unsure" + }, + { + "wall_id": "w1", + "start_ft": [ + -3.8, + 1.2 + ], + "outcome": "fail", + "failing_match": "meter_working_space" + }, + { + "wall_id": "w1", + "start_ft": [ + 1.3, 7.4 ], "outcome": "unsure" diff --git a/verification/e2e/cases/g01-unseen-beyond-garage.json b/verification/e2e/cases/g01-unseen-beyond-garage.json index 136a684d5..241755970 100644 --- a/verification/e2e/cases/g01-unseen-beyond-garage.json +++ b/verification/e2e/cases/g01-unseen-beyond-garage.json @@ -18,7 +18,7 @@ "id": "w1", "baseline": [ [ - -14, + -20, 0.0 ], [ @@ -34,8 +34,8 @@ "type": "garage_door", "wall_id": "w1", "span_ft": [ - -7, - -3 + -5.5, + -1 ], "bottom_ft": 0.0, "top_ft": 7.0, @@ -51,7 +51,7 @@ "type": "concrete", "polygon": [ [ - -14, + -20, 0.0 ], [ @@ -63,7 +63,7 @@ 10.0 ], [ - -14, + -20, 10.0 ] ], @@ -74,7 +74,7 @@ { "wall_id": "w1", "span_ft": [ - -14.0, + -20.0, 10.0 ], "depth_ft": 9.0, @@ -94,31 +94,31 @@ { "band": "wall", "span_ft": [ - -7, - 10 + -9.0, + 25.0 ] }, { "band": "overhead", "span_ft": [ - -7, - 10 + -9.0, + 25.0 ] }, { "band": "facing", "span_ft": [ - -7, - 10 + -9.0, + 25.0 ] }, { "band": "ground", "span_ft": [ - -7, - 10 + -9.0, + 25.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } @@ -130,8 +130,8 @@ { "wall_id": "w1", "start_ft": [ - -13.9, - -9.7 + -19.9, + -8.2 ], "outcome": "fail", "failing_match": "route" diff --git a/verification/e2e/cases/g03-negative-s-near-edge.json b/verification/e2e/cases/g03-negative-s-near-edge.json index 646d19ebc..1fc150ec0 100644 --- a/verification/e2e/cases/g03-negative-s-near-edge.json +++ b/verification/e2e/cases/g03-negative-s-near-edge.json @@ -174,31 +174,31 @@ { "band": "wall", "span_ft": [ - -9, - 10 + -24.0, + 25.0 ] }, { "band": "overhead", "span_ft": [ - -9, - 10 + -24.0, + 25.0 ] }, { "band": "facing", "span_ft": [ - -9, - 10 + -24.0, + 25.0 ] }, { "band": "ground", "span_ft": [ - -9, - 10 + -24.0, + 25.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } diff --git a/verification/e2e/cases/g06-regulator-footprint.json b/verification/e2e/cases/g06-regulator-footprint.json index f8e5ef864..6c7ca5fa0 100644 --- a/verification/e2e/cases/g06-regulator-footprint.json +++ b/verification/e2e/cases/g06-regulator-footprint.json @@ -119,31 +119,31 @@ { "band": "wall", "span_ft": [ - -3.0, - 16.0 + -18.0, + 31.0 ] }, { "band": "overhead", "span_ft": [ - -3.0, - 16.0 + -18.0, + 31.0 ] }, { "band": "facing", "span_ft": [ - -3.0, - 16.0 + -18.0, + 31.0 ] }, { "band": "ground", "span_ft": [ - -3.0, - 16.0 + -18.0, + 31.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } diff --git a/verification/e2e/cases/g08a-garage-blocks-route.json b/verification/e2e/cases/g08a-garage-blocks-route.json index d615532ef..7c66472fa 100644 --- a/verification/e2e/cases/g08a-garage-blocks-route.json +++ b/verification/e2e/cases/g08a-garage-blocks-route.json @@ -94,38 +94,36 @@ { "band": "wall", "span_ft": [ - -0.5, - 12.0 + -15.5, + 27.0 ] }, { "band": "overhead", "span_ft": [ - -0.5, - 12.0 + -15.5, + 27.0 ] }, { "band": "facing", "span_ft": [ - -0.5, - 12.0 + -15.5, + 27.0 ] }, { "band": "ground", "span_ft": [ - -0.5, - 12.0 + -15.5, + 27.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } }, - "rules_assumed": { - "opening": 3.0 - }, + "rules_assumed": {}, "expect": { "decision_not": [ "pass" diff --git a/verification/e2e/cases/g10-derived-compound-error.json b/verification/e2e/cases/g10-derived-compound-error.json index 4b5b9e9ef..9439c3c6a 100644 --- a/verification/e2e/cases/g10-derived-compound-error.json +++ b/verification/e2e/cases/g10-derived-compound-error.json @@ -108,31 +108,31 @@ { "band": "wall", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "overhead", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "facing", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "ground", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } diff --git a/verification/e2e/cases/g10-e03-fail.json b/verification/e2e/cases/g10-e03-fail.json index 16a80bb2e..ced2093ad 100644 --- a/verification/e2e/cases/g10-e03-fail.json +++ b/verification/e2e/cases/g10-e03-fail.json @@ -108,31 +108,31 @@ { "band": "wall", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "overhead", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "facing", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "ground", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } diff --git a/verification/e2e/cases/g10-e03-lower-edge.json b/verification/e2e/cases/g10-e03-lower-edge.json index be75ffcaf..c27e9d47e 100644 --- a/verification/e2e/cases/g10-e03-lower-edge.json +++ b/verification/e2e/cases/g10-e03-lower-edge.json @@ -108,31 +108,31 @@ { "band": "wall", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "overhead", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "facing", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "ground", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } diff --git a/verification/e2e/cases/g10-e03-pass.json b/verification/e2e/cases/g10-e03-pass.json index 3c0965753..669aa74f3 100644 --- a/verification/e2e/cases/g10-e03-pass.json +++ b/verification/e2e/cases/g10-e03-pass.json @@ -108,31 +108,31 @@ { "band": "wall", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "overhead", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "facing", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "ground", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } diff --git a/verification/e2e/cases/g10-e03-upper-edge.json b/verification/e2e/cases/g10-e03-upper-edge.json index 98ba5f9e8..ef5225b4c 100644 --- a/verification/e2e/cases/g10-e03-upper-edge.json +++ b/verification/e2e/cases/g10-e03-upper-edge.json @@ -108,31 +108,31 @@ { "band": "wall", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "overhead", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "facing", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "ground", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } diff --git a/verification/e2e/cases/g10-e05-fail.json b/verification/e2e/cases/g10-e05-fail.json index 8be51050f..948fb024d 100644 --- a/verification/e2e/cases/g10-e05-fail.json +++ b/verification/e2e/cases/g10-e05-fail.json @@ -108,31 +108,31 @@ { "band": "wall", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "overhead", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "facing", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "ground", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } diff --git a/verification/e2e/cases/g10-e05-lower-edge.json b/verification/e2e/cases/g10-e05-lower-edge.json index f66e93366..1f063283c 100644 --- a/verification/e2e/cases/g10-e05-lower-edge.json +++ b/verification/e2e/cases/g10-e05-lower-edge.json @@ -108,31 +108,31 @@ { "band": "wall", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "overhead", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "facing", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "ground", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } diff --git a/verification/e2e/cases/g10-e05-pass.json b/verification/e2e/cases/g10-e05-pass.json index 1f3e047ab..27bf2452d 100644 --- a/verification/e2e/cases/g10-e05-pass.json +++ b/verification/e2e/cases/g10-e05-pass.json @@ -108,31 +108,31 @@ { "band": "wall", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "overhead", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "facing", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "ground", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } diff --git a/verification/e2e/cases/g10-e05-upper-edge.json b/verification/e2e/cases/g10-e05-upper-edge.json index 2c4a02470..9c5f6b391 100644 --- a/verification/e2e/cases/g10-e05-upper-edge.json +++ b/verification/e2e/cases/g10-e05-upper-edge.json @@ -108,31 +108,31 @@ { "band": "wall", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "overhead", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "facing", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "ground", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } diff --git a/verification/e2e/cases/g10-e15-fail.json b/verification/e2e/cases/g10-e15-fail.json index 0edfc98ca..ed1f8bcfd 100644 --- a/verification/e2e/cases/g10-e15-fail.json +++ b/verification/e2e/cases/g10-e15-fail.json @@ -108,31 +108,31 @@ { "band": "wall", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "overhead", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "facing", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "ground", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } diff --git a/verification/e2e/cases/g10-e15-lower-edge.json b/verification/e2e/cases/g10-e15-lower-edge.json index 76c68b443..e0aa9b7d6 100644 --- a/verification/e2e/cases/g10-e15-lower-edge.json +++ b/verification/e2e/cases/g10-e15-lower-edge.json @@ -108,31 +108,31 @@ { "band": "wall", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "overhead", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "facing", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "ground", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } diff --git a/verification/e2e/cases/g10-e15-pass.json b/verification/e2e/cases/g10-e15-pass.json index faebe0533..794827e40 100644 --- a/verification/e2e/cases/g10-e15-pass.json +++ b/verification/e2e/cases/g10-e15-pass.json @@ -108,31 +108,31 @@ { "band": "wall", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "overhead", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "facing", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "ground", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } diff --git a/verification/e2e/cases/g10-e15-upper-edge.json b/verification/e2e/cases/g10-e15-upper-edge.json index bdf8cb87e..00e86c52b 100644 --- a/verification/e2e/cases/g10-e15-upper-edge.json +++ b/verification/e2e/cases/g10-e15-upper-edge.json @@ -108,31 +108,31 @@ { "band": "wall", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "overhead", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "facing", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ] }, { "band": "ground", "span_ft": [ - -5.0, - 15.0 + -20.0, + 30.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } diff --git a/verification/e2e/cases/g12a-subgrid-start.json b/verification/e2e/cases/g12a-subgrid-start.json index d42ea95c1..35d6d893e 100644 --- a/verification/e2e/cases/g12a-subgrid-start.json +++ b/verification/e2e/cases/g12a-subgrid-start.json @@ -109,31 +109,31 @@ { "band": "wall", "span_ft": [ - -1.0, - 16.0 + -16.0, + 31.0 ] }, { "band": "overhead", "span_ft": [ - -1.0, - 16.0 + -16.0, + 31.0 ] }, { "band": "facing", "span_ft": [ - -1.0, - 16.0 + -16.0, + 31.0 ] }, { "band": "ground", "span_ft": [ - -1.0, - 16.0 + -16.0, + 31.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } diff --git a/verification/e2e/cases/g12b-exact-fit-last-start.json b/verification/e2e/cases/g12b-exact-fit-last-start.json index d7548161c..646b4c77d 100644 --- a/verification/e2e/cases/g12b-exact-fit-last-start.json +++ b/verification/e2e/cases/g12b-exact-fit-last-start.json @@ -138,31 +138,31 @@ { "band": "wall", "span_ft": [ - -3, - 8.583333333333334 + -18.0, + 23.583333333333336 ] }, { "band": "overhead", "span_ft": [ - -3, - 8.583333333333334 + -18.0, + 23.583333333333336 ] }, { "band": "facing", "span_ft": [ - -3, - 8.583333333333334 + -18.0, + 23.583333333333336 ] }, { "band": "ground", "span_ft": [ - -3, - 8.583333333333334 + -18.0, + 23.583333333333336 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } diff --git a/verification/e2e/cases/g13a-unsure-never-outranks-pass.json b/verification/e2e/cases/g13a-unsure-never-outranks-pass.json index 2a50f0734..ee5dd1ba6 100644 --- a/verification/e2e/cases/g13a-unsure-never-outranks-pass.json +++ b/verification/e2e/cases/g13a-unsure-never-outranks-pass.json @@ -107,31 +107,31 @@ { "band": "wall", "span_ft": [ - -8.0, - 8.0 + -23.0, + 23.0 ] }, { "band": "overhead", "span_ft": [ - -8.0, - 8.0 + -23.0, + 23.0 ] }, { "band": "facing", "span_ft": [ - -8.0, - 8.0 + -23.0, + 23.0 ] }, { "band": "ground", "span_ft": [ - -8.0, - 8.0 + -23.0, + 23.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } @@ -144,7 +144,7 @@ "wall_id": "w1", "span_within": [ -8, - -0.5062257748298531 + -1.25 ] }, "checks": [ @@ -157,7 +157,16 @@ { "wall_id": "w1", "start_ft": [ - 0.5, + -3.8, + 1.2 + ], + "outcome": "fail", + "failing_match": "meter_working_space" + }, + { + "wall_id": "w1", + "start_ft": [ + 1.3, 5 ], "outcome": "unsure" diff --git a/verification/e2e/cases/g13b-unsure-alone.json b/verification/e2e/cases/g13b-unsure-alone.json index 07c88235c..d4e97a416 100644 --- a/verification/e2e/cases/g13b-unsure-alone.json +++ b/verification/e2e/cases/g13b-unsure-alone.json @@ -107,31 +107,31 @@ { "band": "wall", "span_ft": [ - -0.5, - 8.0 + -15.5, + 23.0 ] }, { "band": "overhead", "span_ft": [ - -0.5, - 8.0 + -15.5, + 23.0 ] }, { "band": "facing", "span_ft": [ - -0.5, - 8.0 + -15.5, + 23.0 ] }, { "band": "ground", "span_ft": [ - -0.5, - 8.0 + -15.5, + 23.0 ], - "out_ft": 10.0 + "out_ft": 15.0 } ] } @@ -164,6 +164,15 @@ "wall_id": "w1", "start_ft": [ -0.4, + 1.2 + ], + "outcome": "fail", + "failing_match": "meter_working_space" + }, + { + "wall_id": "w1", + "start_ft": [ + 1.3, 5.3 ], "outcome": "unsure" diff --git a/verification/e2e/cases/generate.py b/verification/e2e/cases/generate.py index c0885529d..e38073719 100644 --- a/verification/e2e/cases/generate.py +++ b/verification/e2e/cases/generate.py @@ -22,6 +22,9 @@ # so a gas strip at z = d is measured as exactly d. The margin cases need that: 3.3 - 0.3 and # 2.7 + 0.3 both evaluate to exactly 3.0, while 11/6 + 3.3 - 11/6 does not return 3.3. FRONT0 = -D +# rules.yaml meter_working_space: 2.5 ft wide centred on the meter (NEC 110.26). A battery +# overlapping s = (-1.25, 1.25) fails, i.e. starts in (-1.25 - W, 1.25) = (-3.833, 1.25). +WS = "meter_working_space" def golden(n): @@ -44,14 +47,35 @@ def facing(wall_id, span, depth=9.0): return {"wall_id": wall_id, "span_ft": list(span), "depth_ft": depth, "plus_minus_ft": 0.0} -def coverage(span, ground=None, left="limit", right="limit", bands=("wall", "overhead", "facing")): - """Observed bands over `span`; the ground band over `ground` (default: `span`), 10 ft out.""" - observed = [{"band": b, "span_ft": list(span)} for b in bands] - for g in [span] if ground is None else ground: - observed.append({"band": "ground", "span_ft": list(g), "out_ft": 10.0}) +# How far "fully observed" coverage reaches past the chain's ends and out from the wall. +# rules.yaml (origin/t3/server) has pool_ft 10 and drive_ft 5, and an unseen pool or driveway +# within that distance of the footprint makes the check UNSURE and asks for a photo. The +# footprint's front is 11/6 ft out, so the ground must be seen to 10 + 11/6 ft out and 10 ft +# along the wall past any footprint. 15 ft clears both. +PAST = 15.0 + + +def coverage( + span, + ground=None, + left="limit", + right="limit", + bands=("wall", "overhead", "facing"), + past=(0.0, 0.0), + out=10.0, +): + """Observed bands over `span` widened by `past` (left, right); ground over `ground`.""" + wide = [span[0] - past[0], span[1] + past[1]] if any(past) else list(span) + observed = [{"band": b, "span_ft": wide} for b in bands] + for g in [wide] if ground is None else ground: + observed.append({"band": "ground", "span_ft": list(g), "out_ft": out}) return {"ends": {"left": {"kind": left}, "right": {"kind": right}}, "observed": observed} +def full_coverage(span): + return coverage(span, past=(PAST, PAST), out=PAST) + + def gas(wall_id, span, footprint, pm=0.0, conf=None): obj = { "type": "gas_meter", @@ -151,11 +175,14 @@ def straight_scene(x0, x1, z=0.0, meter_x=0.0, objects=(), ground=None, cov=None "ground": [concrete(rect(x0, x1, z, z + 10))] if ground is None else ground, "facing": [facing("w1", (x0 - meter_x, x1 - meter_x))], } - scene["coverage"] = coverage((x0 - meter_x, x1 - meter_x)) if cov is None else cov + scene["coverage"] = full_coverage((x0 - meter_x, x1 - meter_x)) if cov is None else cov return scene # g01: an unseen stretch behind a garage door needs no photo, since no cable can reach it. +# The garage ends at s = -1, inside the meter's working space, so no battery can sit between +# garage and meter; the unseen stretch starts at s = -9, more than pool_ft (10) left of the +# first start right of the meter (1.25). case( "g01-unseen-beyond-garage", golden("01"), @@ -163,17 +190,17 @@ def straight_scene(x0, x1, z=0.0, meter_x=0.0, objects=(), ground=None, cov=None "reached by a cable, so the result asks for no photos, and every start beyond the garage " "fails its route.", straight_scene( - -14, + -20, 10, - objects=[opening("garage_door", "w1", (-7, -3), 0.0, 7.0)], - cov=coverage((-7, 10)), + objects=[opening("garage_door", "w1", (-5.5, -1), 0.0, 7.0)], + cov=coverage((-9, 10), past=(0.0, PAST), out=PAST), ), { "missing_evidence_empty": True, "sweep_runs": [ { "wall_id": "w1", - "start_ft": [-13.9, -9.7], + "start_ft": [-19.9, -8.2], "outcome": "fail", "failing_match": "route", } @@ -197,7 +224,7 @@ def straight_scene(x0, x1, z=0.0, meter_x=0.0, objects=(), ground=None, cov=None "objects": [], "ground": [offset_ground(g03_wall["baseline"])], "facing": [facing("w1", (-9, 10))], - "coverage": coverage((-9, 10)), + "coverage": full_coverage((-9, 10)), }, { "spot": {"wall_id": "w1", "span_within": [-9, -6]}, @@ -269,7 +296,6 @@ def straight_scene(x0, x1, z=0.0, meter_x=0.0, objects=(), ground=None, cov=None {"wall_id": "w1", "start_ft": [5, 9.4], "outcome": "fail", "failing_match": "route"} ], }, - {"opening": 3.0}, ) # g08b: a 1 ft stretch with no wall between the meter's short wall and the rest. @@ -504,7 +530,7 @@ def g10_scene(d, e, wall_pm=0.0): "objects": [], "ground": [offset_ground(g12b_wall["baseline"])], "facing": [facing("w1", (-3, 6 + W))], - "coverage": coverage((-3, 6 + W)), + "coverage": full_coverage((-3, 6 + W)), }, {"spot": {"wall_id": "w1", "span_within": [6, 6 + W]}, "missing_evidence_empty": True}, ) @@ -518,7 +544,9 @@ def g13_scene(x0): } -s_edge = 0.3 - math.sqrt(3.3**2 - 3.2**2) +# Passing spots need their right edge left of both the gas edge 0.3 - sqrt(0.65) ~ -0.506 and +# the meter working space's left edge, -1.25 (rules.yaml meter_working_space width 2.5). +s_edge = min(0.3 - math.sqrt(3.3**2 - 3.2**2), -1.25) case( "g13a-unsure-never-outranks-pass", golden("13"), @@ -529,7 +557,10 @@ def g13_scene(x0): { "spot": {"wall_id": "w1", "span_within": [-8, s_edge]}, "checks": [{"match": "gas", "outcome": "pass"}], - "sweep_runs": [{"wall_id": "w1", "start_ft": [0.5, 5], "outcome": "unsure"}], + "sweep_runs": [ + {"wall_id": "w1", "start_ft": [-3.8, 1.2], "outcome": "fail", "failing_match": WS}, + {"wall_id": "w1", "start_ft": [1.3, 5], "outcome": "unsure"}, + ], "missing_evidence_empty": True, }, {"gas": 3.0}, @@ -552,7 +583,10 @@ def g13_scene(x0): "plus_minus_ft": 0.3, } ], - "sweep_runs": [{"wall_id": "w1", "start_ft": [-0.4, 5.3], "outcome": "unsure"}], + "sweep_runs": [ + {"wall_id": "w1", "start_ft": [-0.4, 1.2], "outcome": "fail", "failing_match": WS}, + {"wall_id": "w1", "start_ft": [1.3, 5.3], "outcome": "unsure"}, + ], "missing_evidence_empty": True, }, {"gas": 3.0}, @@ -571,7 +605,11 @@ def g13_scene(x0): { "decision_in": ["manual_review"], "missing_evidence_empty": False, - "sweep_runs": [{"wall_id": "w1", "start_ft": [-6, 7.4], "outcome": "unsure"}], + "sweep_runs": [ + {"wall_id": "w1", "start_ft": [-6, -3.9], "outcome": "unsure"}, + {"wall_id": "w1", "start_ft": [-3.8, 1.2], "outcome": "fail", "failing_match": WS}, + {"wall_id": "w1", "start_ft": [1.3, 7.4], "outcome": "unsure"}, + ], }, ) diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index 8d3467508..a9df22362 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -188,9 +188,14 @@ def discover_endpoint(openapi: dict, override: str | None = None) -> Endpoint: if kind in body: candidates.append((path, kind, body[kind].get("schema", {}))) if override: - candidates = [c for c in candidates if c[0] == override] - if not candidates: - raise SystemExit(f"--endpoint {override} is not a POST with a JSON or multipart body") + declared = [c for c in candidates if c[0] == override] + # An endpoint that reads the raw request body declares no body schema; the caller has + # named it, so it takes the scene as JSON. + if not declared and "post" in openapi.get("paths", {}).get(override, {}): + declared = [(override, "application/json", {})] + if not declared: + raise SystemExit(f"--endpoint {override} is not a POST in the server's OpenAPI") + candidates = declared elif len(candidates) > 1: named = [c for c in candidates if re.search(r"scene|place|solve|placement", c[0])] candidates = named if len(named) == 1 else candidates diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index abad80a4e..f80cfb691 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -64,6 +64,14 @@ def test_ambiguity_is_loud(): assert discover_endpoint(openapi({"/a": json_op(), "/b": json_op()}), "/b").path == "/b" +def test_named_post_without_a_body_schema_takes_json(): + raw_body = {"post": {"responses": {}}} + found = discover_endpoint(openapi({"/v1/placements": raw_body}), "/v1/placements") + assert found == Endpoint("/v1/placements", "application/json") + with pytest.raises(SystemExit, match="not a POST"): + discover_endpoint(openapi({"/v1/placements": raw_body}), "/v1/other") + + def test_case_file_is_loaded_with_its_expectations(tmp_path): path = tmp_path / "c.json" path.write_text( From 69592e600f35a11f236364e2884231aed014a478 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 03:38:29 -0500 Subject: [PATCH 14/59] verification: independent accessibility audit per screen Wraps the audit harness: builds the app from a ref, runs Apple's audit on each distinct screen with the C4 arguments, and also flags buttons labelled with an SF Symbol name, which the audit accepts. Checked against the probe's negative control: both defects found. Scoreboard S3-6b reads the report for the app's current SHA. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/README.md | 11 ++ verification/hsverify/a11yaudit.py | 230 +++++++++++++++++++++++++++ verification/hsverify/simrun.py | 8 +- verification/scoreboard/metrics.yaml | 13 ++ verification/tests/test_a11yaudit.py | 34 ++++ 5 files changed, 294 insertions(+), 2 deletions(-) create mode 100644 verification/hsverify/a11yaudit.py create mode 100644 verification/tests/test_a11yaudit.py diff --git a/verification/README.md b/verification/README.md index 71e1fb33e..1fd15abf7 100644 --- a/verification/README.md +++ b/verification/README.md @@ -9,6 +9,7 @@ workstreams push. Nothing here fixes their code; findings go back to the owning | App in the Simulator, a screenshot per `STATE` | `make sim REF=origin/t3/ios-mvf ARGS="--replay --autopilot --server-url "` | `~/house-scanning-data/reports/sim//index.html` | | Runner self-test | `make sim-probe` | same, for `fixtures/state-probe` | | A replay session (C3) is well formed, and how far its poses are from ground truth | `uv run python -m hsverify.replaycheck ` | printed | +| Apple's accessibility audit on every screen the app reaches | `uv run python -m hsverify.a11yaudit --ref origin/t3/ios-mvf --replay --autopilot` | `~/house-scanning-data/reports/a11y//index.html` | | Scenes through the server's HTTP API, judged against C1, C2 and C5 | `uv run python -m hsverify.e2e --server-ref origin/t3/server [--real scene.zip]` | `~/house-scanning-data/reports/e2e//report.md` | Requirements: uv, Xcode 26 or newer with an iOS Simulator runtime. @@ -70,3 +71,13 @@ coverage (no pass anywhere). Case files in `e2e/cases/` add the outcomes their g forces. Each one names the thresholds it assumes, and a case whose assumption differs from the server's rules is reported as an assumption mismatch, not as a pass or fail. Scenes passed with `--real` must answer within 1 s. + +## Accessibility audit + +`hsverify/a11yaudit.py` builds and installs the app as the Simulator runner does, then runs the +UI test bundle in `fixtures/a11y-audit`. The bundle launches the installed app by bundle id with +the C4 arguments and calls `performAccessibilityAudit(for: .all)` on each distinct screen (a +screen is new when the labels of its texts and buttons change). Apple's audit does not object +to a button whose label SwiftUI took from an SF Symbol name, which VoiceOver reads aloud, so the +wrapper flags those too. The probe app's middle screen has an unlabelled 12 pt button as a +negative control: an audit of the probe must report it, and does (hit area, symbol-name label). diff --git a/verification/hsverify/a11yaudit.py b/verification/hsverify/a11yaudit.py new file mode 100644 index 000000000..cb4e8ede6 --- /dev/null +++ b/verification/hsverify/a11yaudit.py @@ -0,0 +1,230 @@ +"""Run Apple's accessibility audit on every screen an app shows, from a git ref. + + uv run python -m hsverify.a11yaudit --ref origin/t3/ios-mvf \ + --replay ~/house-scanning-data/replays/ --autopilot + +The app is built and installed exactly as `hsverify.simrun` does. Then the UI test bundle in +`fixtures/a11y-audit` launches it by bundle id with the same launch arguments, and runs +`performAccessibilityAudit(for: .all)` on each distinct screen: contrast, Dynamic Type, hit +areas, clipped text, element descriptions and traits. A screen is new when the labels of its +texts and buttons change. + +Apple's audit accepts a button whose label SwiftUI took from an SF Symbol name, which +VoiceOver then reads aloud ("gearshape"). Those labels are flagged here as well. + +Reports go to ~/house-scanning-data/reports/a11y// with a screenshot per audited screen. +The probe's middle screen carries a deliberately undersized, unlabelled button, so +`--ref HEAD --project verification/fixtures/state-probe/StateProbe.xcodeproj --scheme +StateProbe` must report at least one issue; a clean result there means the audit is blind. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import html +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +from hsverify import gitref, simrun + +HARNESS = Path(__file__).resolve().parents[1] / "fixtures" / "a11y-audit" / "A11yAudit.xcodeproj" +DEFAULT_REPORTS = Path.home() / "house-scanning-data" / "reports" / "a11y" +HARNESS_DERIVED = Path("/tmp/hs-verify-a11y-derived-data") +# SF Symbol names are lowercase words joined by dots: "gearshape", "arrow.left.circle.fill". +SYMBOL_NAME = re.compile(r"^[a-z0-9]+(\.[a-z0-9]+)*$") +ARG_SEPARATOR = "\x1f" # the harness splits AUDIT_ARGS on U+001F + + +def parse_output(text: str) -> tuple[list[dict], list[dict]]: + """Screens and issues from the harness's `A11Y_SCREEN=` and `A11Y_ISSUE=` lines.""" + screens, issues = [], [] + for line in text.splitlines(): + for tag, sink in (("A11Y_SCREEN=", screens), ("A11Y_ISSUE=", issues)): + if tag in line: + try: + sink.append(json.loads(line.split(tag, 1)[1])) + except json.JSONDecodeError: + continue + return screens, issues + + +# One-word symbol names that are not ordinary English words. Dotted names ("arrow.left") are +# always symbol names; a plain word such as "camera" could be a deliberate label. +SYMBOL_WORDS = {"gearshape", "xmark", "checkmark", "ellipsis", "chevron", "info", "plus"} + + +def symbol_name_labels(screens: list[dict]) -> list[dict]: + """Buttons whose accessible label looks like an SF Symbol name.""" + found = [] + for screen in screens: + for label in screen.get("labels", []): + kind, _, text = label.partition(":") + looks_like_symbol = SYMBOL_NAME.match(text) and ("." in text or text in SYMBOL_WORDS) + if kind == "button" and looks_like_symbol: + found.append({"screen": screen["index"], "label": text}) + return found + + +def run_harness( + udid: str, bundle_id: str, args: argparse.Namespace, launch: list[str], out: Path +) -> str: + env = os.environ | { + "TEST_RUNNER_AUDIT_BUNDLE": bundle_id, + "TEST_RUNNER_AUDIT_ARGS": ARG_SEPARATOR.join(launch), + "TEST_RUNNER_AUDIT_SECONDS": str(args.seconds), + "TEST_RUNNER_AUDIT_IDLE": str(args.idle), + "TEST_RUNNER_AUDIT_OUT": str(out), + } + cmd = [ + "xcodebuild", + "test", + "-project", + str(HARNESS), + "-scheme", + "AccessibilityAudit", + "-destination", + f"id={udid}", + "-derivedDataPath", + str(HARNESS_DERIVED), + "CODE_SIGNING_ALLOWED=NO", + ] + simrun.wait_for_other_builds(args.build_wait) + result = subprocess.run(cmd, env=env, capture_output=True, text=True) + (out / "xcodebuild-test.log").write_text(result.stdout + result.stderr) + return result.stdout + + +def write_report(out: Path, data: dict) -> None: + (out / "report.json").write_text(json.dumps(data, indent=2) + "\n") + rows = [] + for screen in data["screens"]: + issues = [i for i in data["issues"] if i.get("screen") == screen["index"]] + symbols = [s for s in data["symbol_name_labels"] if s["screen"] == screen["index"]] + items = "".join( + f"
  • {html.escape(i['description'])} {html.escape(i.get('element', ''))}" + f"
    {html.escape(i.get('detail', ''))}
    • " + for i in issues + ) + "".join( + f"
  • Button label is a symbol name {html.escape(s['label'])}
    • " + for s in symbols + ) + texts = [lab.split(":", 1)[1] for lab in screen.get("labels", []) if lab.split(":", 1)[1]] + rows.append( + f"
    " + f"

    Screen {screen['index']} +{screen.get('seconds', 0):.1f} s" + f"

    {html.escape(' · '.join(texts[:8]))}

    " + f"
      {items or '
    • No issues
      • '}
    " + ) + page = ( + "Accessibility audit" + f"

    Accessibility audit: {html.escape(data['ref'])} at " + f"{data['sha'][:12]}

    {len(data['screens'])} screens, " + f"{data['issue_count']} issues. Text size {html.escape(str(data['content_size']))}, " + f"{html.escape(data['appearance'])}.

    " + "".join(rows) + ) + (out / "index.html").write_text(page) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--ref", default="origin/t3/ios-mvf") + parser.add_argument("--project", default="ios/HouseScan.xcodeproj") + parser.add_argument("--scheme", default="HouseScan") + parser.add_argument("--replay", type=Path) + parser.add_argument("--autopilot", action="store_true") + parser.add_argument("--server-url") + parser.add_argument("--extra-arg", action="append", default=[]) + parser.add_argument("--seconds", type=float, default=180) + parser.add_argument("--idle", type=float, default=20) + parser.add_argument("--appearance", choices=["light", "dark"], default="light") + parser.add_argument("--content-size") + parser.add_argument("--label", default="") + parser.add_argument("--build-wait", type=float, default=1200) + args = parser.parse_args(argv) + + gitref.fetch() + sha = gitref.resolve(args.ref) + stamp = dt.datetime.now().strftime("%Y%m%d-%H%M%S") + name = "-".join( + filter( + None, + [ + stamp, + simrun.slug(args.ref.removeprefix("origin/")), + sha[:8], + simrun.slug(args.label), + ], + ) + ) + out = DEFAULT_REPORTS / name + out.mkdir(parents=True, exist_ok=True) + + launch: list[str] = [] + if args.replay: + launch += ["-replay", str(args.replay.expanduser().resolve())] + if args.autopilot: + launch.append("-autopilot") + if args.server_url: + launch += ["-serverURL", args.server_url] + launch += args.extra_arg + + with gitref.detached_worktree(sha) as tree: + build, app = simrun.build_app(tree, out, args.build_wait, args.project, args.scheme) + if app is None: + print(f"Build failed; see {out / 'build.log'}") + return 1 + device = simrun.ensure_device() + udid = device["udid"] + try: + bundle_id = simrun.bundle_id_of(app) + simrun.prepare_display(udid, args.appearance, args.content_size) + subprocess.run(["xcrun", "simctl", "uninstall", udid, bundle_id], capture_output=True) + simrun.simctl("install", udid, str(app)) + subprocess.run( + ["xcrun", "simctl", "privacy", udid, "grant", "camera", bundle_id], capture_output=True + ) + print(f"Auditing {bundle_id} at {sha[:12]}...", flush=True) + output = run_harness(udid, bundle_id, args, launch, out) + finally: + subprocess.run(["xcrun", "simctl", "shutdown", udid], capture_output=True) + + screens, issues = parse_output(output) + symbols = symbol_name_labels(screens) + data = { + "ref": args.ref, + "build": build, + "sha": sha, + "launch_arguments": launch, + "appearance": args.appearance, + "content_size": args.content_size or "large", + "screens": screens, + "issues": issues, + "symbol_name_labels": symbols, + "issue_count": len(issues) + len(symbols), + "harness_ran": "** TEST SUCCEEDED **" in output or "** TEST FAILED **" in output, + } + write_report(out, data) + print(f"{len(screens)} screens, {data['issue_count']} issues. Open {out / 'index.html'}") + for issue in issues[:20]: + print( + f" screen {issue.get('screen')}: {issue.get('description')} {issue.get('element', '')}" + ) + for s in symbols: + print(f" screen {s['screen']}: button label is a symbol name: {s['label']}") + if not data["harness_ran"]: + print(f"The audit harness did not run; see {out / 'xcodebuild-test.log'}") + return 1 + return 0 if data["issue_count"] == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/verification/hsverify/simrun.py b/verification/hsverify/simrun.py index 71720e2f3..7c7d3bdd9 100644 --- a/verification/hsverify/simrun.py +++ b/verification/hsverify/simrun.py @@ -195,6 +195,11 @@ def build_app( return result, (app if code == 0 and app.exists() else None) +def bundle_id_of(app: Path) -> str: + with (app / "Info.plist").open("rb") as f: + return plistlib.load(f)["CFBundleIdentifier"] + + def shorten(line: str, tree: Path) -> str: return line.replace(str(tree) + "/", "") @@ -468,8 +473,7 @@ def main(argv: list[str] | None = None) -> int: rep.end_reason = "build failed" rep.problems.append("The app did not build; see build.log.") else: - with (app / "Info.plist").open("rb") as f: - bundle_id = plistlib.load(f)["CFBundleIdentifier"] + bundle_id = bundle_id_of(app) rep.device["bundle_id"] = bundle_id udid = device["udid"] prepare_display(udid, args.appearance, args.content_size) diff --git a/verification/scoreboard/metrics.yaml b/verification/scoreboard/metrics.yaml index b772430e8..4fb9275b2 100644 --- a/verification/scoreboard/metrics.yaml +++ b/verification/scoreboard/metrics.yaml @@ -213,6 +213,19 @@ metrics: max_status: partial cap_reason: an audit exists; which screens it covers needs a read + - id: S3-6b + workstream: S3 + ref: origin/t3/ios-mvf + text: >- + Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the + run reached, plus symbol-name button labels) finds no issues; S4-2 shows how many screens + the app reached + report: + glob: ~/house-scanning-data/reports/a11y/*/report.json + key: issue_count + equals: 0 + require_current_sha: true + - id: S3-7 workstream: S3 ref: origin/t3/ios-mvf diff --git a/verification/tests/test_a11yaudit.py b/verification/tests/test_a11yaudit.py new file mode 100644 index 000000000..afdf911dc --- /dev/null +++ b/verification/tests/test_a11yaudit.py @@ -0,0 +1,34 @@ +from hsverify.a11yaudit import parse_output, symbol_name_labels + + +def test_parse_output_reads_screens_and_issues_and_skips_noise(): + text = "\n".join( + [ + "Test Case started.", + 'A11Y_ISSUE={"screen": 2, "description": "Hit area is too small"}', + 'A11Y_SCREEN={"index": 2, "labels": ["text:hi"], "issues": 1}', + "A11Y_SCREEN={broken", + ] + ) + screens, issues = parse_output(text) + assert screens == [{"index": 2, "labels": ["text:hi"], "issues": 1}] + assert issues == [{"screen": 2, "description": "Hit area is too small"}] + + +def test_symbol_name_labels(): + screens = [ + { + "index": 1, + "labels": [ + "button:gearshape", + "button:arrow.left.circle.fill", + "button:camera", + "button:I'm here", + "text:xmark", + ], + } + ] + assert symbol_name_labels(screens) == [ + {"screen": 1, "label": "gearshape"}, + {"screen": 1, "label": "arrow.left.circle.fill"}, + ] From 6cf8692ee26f3474e12a1f08fb6a8a6780e91652 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 03:39:58 -0500 Subject: [PATCH 15/59] verification: pick the JSON-answering upload endpoint; variants keep bundle images The server now declares a JSON body and an upload on /v1/placements, plus an SVG site-plan route. Discovery keeps only POSTs that answer with JSON and prefers the upload, which is how the app sends scene.json with its JPEGs. Variants of a bundle keep its images, or a scene listing keyframes is rightly refused. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/hsverify/e2e.py | 17 +++++++++++++++-- verification/tests/test_e2e.py | 23 +++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index a9df22362..28a2d27e6 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -183,7 +183,13 @@ def discover_endpoint(openapi: dict, override: str | None = None) -> Endpoint: """The POST operation that takes a scene: JSON body, or a multipart upload of the bundle.""" candidates = [] for path, ops in openapi.get("paths", {}).items(): - body = ops.get("post", {}).get("requestBody", {}).get("content", {}) + post = ops.get("post", {}) + # The result is JSON; a POST declared to answer with something else (an SVG site + # plan, say) is not the placement endpoint. + answers = post.get("responses", {}).get("200", {}).get("content") + if answers and "application/json" not in answers: + continue + body = post.get("requestBody", {}).get("content", {}) for kind in ("application/json", "multipart/form-data"): if kind in body: candidates.append((path, kind, body[kind].get("schema", {}))) @@ -199,6 +205,10 @@ def discover_endpoint(openapi: dict, override: str | None = None) -> Endpoint: elif len(candidates) > 1: named = [c for c in candidates if re.search(r"scene|place|solve|placement", c[0])] candidates = named if len(named) == 1 else candidates + # One path taking both a JSON body and an upload: use the upload, which is how the app sends + # scene.json with its keyframe JPEGs (C1). + if len({c[0] for c in candidates}) == 1 and len(candidates) > 1: + candidates = [c for c in candidates if c[1] == "multipart/form-data"] or candidates[:1] if len(candidates) != 1: listed = ", ".join(f"{p} ({k})" for p, k, _ in candidates) or "none" raise SystemExit(f"Cannot tell which endpoint takes a scene: {listed}. Pass --endpoint.") @@ -268,7 +278,10 @@ def post(url: str, endpoint: Endpoint, item: SceneInput) -> tuple[int, bytes, fl def variant(item: SceneInput, name: str, scene: dict) -> SceneInput: raw = json.dumps(scene, indent=1).encode() - return SceneInput(f"{item.name}~{name}", scene, raw, real=item.real, source=item.source) + # Keep the bundle's images: a variant that still lists keyframes must still carry them. + return SceneInput( + f"{item.name}~{name}", scene, raw, images=item.images, real=item.real, source=item.source + ) def judge(url: str, endpoint: Endpoint, item: SceneInput, schemas: dict) -> dict: diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index f80cfb691..ae998c1e0 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -51,6 +51,29 @@ def test_multipart_endpoint_names_its_file_field(): ) +def test_one_path_with_json_and_upload_uses_the_upload(): + both = { + "post": { + "requestBody": { + "content": { + "application/json": {"schema": {}}, + "multipart/form-data": {"schema": {"$ref": "#/components/schemas/Upload"}}, + } + } + } + } + assert discover_endpoint(openapi({"/v1/placements": both})) == Endpoint( + "/v1/placements", "multipart/form-data", "bundle" + ) + + +def test_a_post_answering_with_svg_is_not_the_placement_endpoint(): + svg = json_op() + svg["post"]["responses"] = {"200": {"content": {"image/svg+xml": {}}}} + found = discover_endpoint(openapi({"/v1/placements": json_op(), "/v1/plan.svg": svg})) + assert found.path == "/v1/placements" + + def test_a_scene_named_path_wins_among_several(): found = discover_endpoint(openapi({"/login": json_op(), "/placement": json_op()})) assert found.path == "/placement" From 0b2160843de953eac0e9d7dc18ff3383c7950f97 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 03:41:07 -0500 Subject: [PATCH 16/59] verification: scope app evidence to the app branch's runs; regenerate scoreboard Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/SCOREBOARD.md | 38 ++++---- verification/scoreboard/metrics.yaml | 26 +++--- verification/scoreboard/scoreboard.json | 119 +++++++++++++++--------- 3 files changed, 106 insertions(+), 77 deletions(-) diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index 2c48fa461..38613fe25 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,15 +1,15 @@ # Scoreboard -Generated 2026-09-26 03:21 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-26 03:40 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | | [#14](https://github.com/SamGu-NRX/house-scanning/pull/14) | `t3/docs-cleanup` | `9c9ceb2a` | no | 4 success | 2026-09-26 03:15 CDT | -| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `2ae5a802` | yes | no checks | 2026-09-26 03:21 CDT | -| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `b0b8a274` | yes | 4 success | 2026-09-26 03:11 CDT | -| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `fcd45741` | yes | 5 success | 2026-09-26 03:20 CDT | +| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `6cf8692e` | yes | 4 success | 2026-09-26 03:40 CDT | +| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `df4cce67` | yes | 5 success | 2026-09-26 03:32 CDT | +| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `44607e04` | yes | 5 success | 2026-09-26 03:37 CDT | | [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `6885b7b5` | yes | 5 success | 2026-09-26 03:07 CDT | | [#9](https://github.com/SamGu-NRX/house-scanning/pull/9) | `codex/capture-contract-probe` | `4b6c4309` | yes | 4 success | 2026-09-26 00:08 CDT | | [#8](https://github.com/SamGu-NRX/house-scanning/pull/8) | `t3/testflight` | `b896d59f` | no | 4 success | 2026-09-26 03:18 CDT | @@ -19,33 +19,34 @@ Generated 2026-09-26 03:21 CDT by `make scoreboard` (`uv run python -m hsverify. ## S1 Evals on real data -`origin/t3/evals` at [`b0b8a274`](https://github.com/SamGu-NRX/house-scanning/tree/b0b8a2745e3ea67cb8f70df3943177e146f591b9). 1 met, 1 partial, 2 gap, 1 not run. +`origin/t3/evals` at [`df4cce67`](https://github.com/SamGu-NRX/house-scanning/tree/df4cce678b3b133b56e34f6c0cb910142a08f662). 1 met, 1 partial, 2 gap, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S1-1 | One command regenerates every number from a clean checkout | gap | not yet reviewed | -| S1-2 | Dataset URLs and sha256 recorded | partial | found ADVIO; missing ETH3D [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/b0b8a2745e3ea67cb8f70df3943177e146f591b9/experiments/evals/README.md) | +| S1-2 | Dataset URLs and sha256 recorded | partial | found ADVIO; missing ETH3D [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/df4cce678b3b133b56e34f6c0cb910142a08f662/experiments/evals/README.md) | | S1-3 | Metric code tested on hand-computed cases | not run | slow; run with --slow | -| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/b0b8a2745e3ea67cb8f70df3943177e146f591b9/experiments/evals/results/advio_drift.md) | +| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/df4cce678b3b133b56e34f6c0cb910142a08f662/experiments/evals/results/advio_drift.md) | | S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | gap | none found in 2 files; missing 1-3 m row, 3-10 m row, scale error, verdict | ## S2 Server and solver -`origin/t3/server` at [`fcd45741`](https://github.com/SamGu-NRX/house-scanning/tree/fcd457411a68171509d53b87b11c115f55a2a765). 3 met, 1 gap, 2 not run, 1 no evidence yet. +`origin/t3/server` at [`44607e04`](https://github.com/SamGu-NRX/house-scanning/tree/44607e04e4f851944c47cb535c9c4b13bbad1c5b). 5 met, 1 partial, 2 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S2-1 | pytest green | not run | slow; run with --slow | -| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/tests/test_golden_public.py) | -| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | gap | none found in 5 files; missing missing coverage, equality unsure, mirror, larger clearance | -| S2-4 | Schemas published | met | present at fcd45741 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/schemas/result.schema.json) | -| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/tests/test_schemas.py) | -| S2-6 | A real-derived scene returns a result in under 1 s | no evidence yet | no report matches ~/house-scanning-data/reports/e2e/*/report.json | +| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_golden_public.py) | +| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_properties.py) | +| S2-4 | Schemas published | met | present at 44607e04 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/schemas/result.schema.json) | +| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_schemas.py), [server/tests/test_api.py](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_api.py) | +| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 733.5 (want < 1000): ~/house-scanning-data/reports/e2e/20260926-034015-44607e04/report.json | | S2-7 | ruff clean | not run | slow; run with --slow | +| S2-8 | Black-box (verification e2e): every answer valid under C2, consistent with its decision and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric when mirrored, and never passing without coverage | met | contract_problem_count = 0 (want 0): ~/house-scanning-data/reports/e2e/20260926-034015-44607e04/report.json | ## S3 iOS capture app -`origin/t3/ios-mvf` at [`6885b7b5`](https://github.com/SamGu-NRX/house-scanning/tree/6885b7b5ad872df7f73a6b915e5837cd38a2034d). 3 met, 5 gap, 1 no evidence yet. +`origin/t3/ios-mvf` at [`6885b7b5`](https://github.com/SamGu-NRX/house-scanning/tree/6885b7b5ad872df7f73a6b915e5837cd38a2034d). 3 met, 5 gap, 2 no evidence yet. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | @@ -55,20 +56,21 @@ Generated 2026-09-26 03:21 CDT by `make scoreboard` (`uv run python -m hsverify. | S3-4 | Unit tests for coverage, auto-capture and guidance | gap | none found in 1 files; missing coverage, auto-capture, guidance | | S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | gap | none found in 1 files; missing replay argument, autopilot argument, gap loop | | S3-6 | Accessibility audit on every screen | gap | 0 of 1 matches needed, searched 1 files | +| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, plus symbol-name button labels) finds no issues; S4-2 shows how many screens the app reached | no evidence yet | no report matches ~/house-scanning-data/reports/a11y/*t3-ios-mvf*/report.json | | S3-7 | Screenshots of each state in the PR | gap | 0 of 9 matches in the PR body [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-8 | Exported scene.json validates against C1 | no evidence yet | no report matches ~/house-scanning-data/reports/e2e/*/report.json | +| S3-8 | Exported scene.json validates against C1 | no evidence yet | no app_export_scene_valid in ~/house-scanning-data/reports/e2e/20260926-034015-44607e04/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/verification` at [`2ae5a802`](https://github.com/SamGu-NRX/house-scanning/tree/2ae5a802b41793f6135e56d6c22f33e3109e7761), `origin/t3/ios-mvf` at [`6885b7b5`](https://github.com/SamGu-NRX/house-scanning/tree/6885b7b5ad872df7f73a6b915e5837cd38a2034d). 1 met, 2 gap, 1 no evidence yet. +`origin/t3/verification` at [`6cf8692e`](https://github.com/SamGu-NRX/house-scanning/tree/6cf8692ee26f3474e12a1f08fb6a8a6780e91652), `origin/t3/ios-mvf` at [`6885b7b5`](https://github.com/SamGu-NRX/house-scanning/tree/6885b7b5ad872df7f73a6b915e5837cd38a2034d). 1 met, 2 gap, 1 no evidence yet. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S4-1 | End to end from a real replay to a placement result passes | no evidence yet | no report matches ~/house-scanning-data/reports/e2e/*/report.json | +| S4-1 | End to end from a real replay to a placement result passes | no evidence yet | no app_export_passed in ~/house-scanning-data/reports/e2e/20260926-034015-44607e04/report.json | | S4-2 | Screenshot report per app state | gap | states = length 0 (want at least 9): ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json | | S4-3 | Product description written from the code and checked against screenshots | gap | not yet reviewed | -| S4-4 | Scoreboard of every PR | met | present at 2ae5a802 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/2ae5a802b41793f6135e56d6c22f33e3109e7761/verification/SCOREBOARD.md) | +| S4-4 | Scoreboard of every PR | met | present at 6cf8692e [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/6cf8692ee26f3474e12a1f08fb6a8a6780e91652/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/scoreboard/metrics.yaml b/verification/scoreboard/metrics.yaml index 4fb9275b2..b2225d606 100644 --- a/verification/scoreboard/metrics.yaml +++ b/verification/scoreboard/metrics.yaml @@ -130,6 +130,16 @@ metrics: less_than: 1000 require_current_sha: true + - id: S2-7 + workstream: S2 + ref: origin/t3/server + text: ruff clean + command: + run: uv run ruff check . && uv run ruff format --check . + cwd: server + timeout_s: 300 + slow: true + - id: S2-8 workstream: S2 ref: origin/t3/server @@ -143,16 +153,6 @@ metrics: equals: 0 require_current_sha: true - - id: S2-7 - workstream: S2 - ref: origin/t3/server - text: ruff clean - command: - run: uv run ruff check . && uv run ruff format --check . - cwd: server - timeout_s: 300 - slow: true - # --- S3 -------------------------------------------------------------------------------------- - id: S3-1 workstream: S3 @@ -176,7 +176,7 @@ metrics: ref: origin/t3/ios-mvf text: Zero build warnings report: - glob: ~/house-scanning-data/reports/sim/*/report.json + glob: ~/house-scanning-data/reports/sim/*t3-ios-mvf*/report.json key: build.warnings length_equals: 0 require_current_sha: true @@ -221,7 +221,7 @@ metrics: run reached, plus symbol-name button labels) finds no issues; S4-2 shows how many screens the app reached report: - glob: ~/house-scanning-data/reports/a11y/*/report.json + glob: ~/house-scanning-data/reports/a11y/*t3-ios-mvf*/report.json key: issue_count equals: 0 require_current_sha: true @@ -264,7 +264,7 @@ metrics: ref: origin/t3/ios-mvf text: Screenshot report per app state report: - glob: ~/house-scanning-data/reports/sim/*/report.json + glob: ~/house-scanning-data/reports/sim/*t3-ios-mvf*/report.json key: states length_at_least: 9 require_current_sha: true diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index b88bbdf7f..635789089 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-09-26T03:21:41-05:00", + "generated_at": "2026-09-26T03:40:54-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": false, "prs": [ @@ -25,45 +25,45 @@ "number": 13, "title": "verification: independent end-to-end checks (S4)", "branch": "t3/verification", - "head": "2ae5a802b41793f6135e56d6c22f33e3109e7761", + "head": "6cf8692ee26f3474e12a1f08fb6a8a6780e91652", "draft": true, - "updated": "2026-09-26T08:21:37Z", + "updated": "2026-09-26T08:40:06Z", "ci": { "counts": { - "success": 0, + "success": 4, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "no checks" + "text": "4 success" } }, { "number": 12, "title": "feat(evals): real-data accuracy evals, starting with an ADVIO replay session", "branch": "t3/evals", - "head": "b0b8a2745e3ea67cb8f70df3943177e146f591b9", + "head": "df4cce678b3b133b56e34f6c0cb910142a08f662", "draft": true, - "updated": "2026-09-26T08:11:18Z", + "updated": "2026-09-26T08:32:28Z", "ci": { "counts": { - "success": 4, + "success": 5, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "4 success" + "text": "5 success" } }, { "number": 11, "title": "server: placement solver, rules and API (S2)", "branch": "t3/server", - "head": "fcd457411a68171509d53b87b11c115f55a2a765", + "head": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", "draft": true, - "updated": "2026-09-26T08:20:33Z", + "updated": "2026-09-26T08:37:02Z", "ci": { "counts": { "success": 5, @@ -192,7 +192,7 @@ "refs": [ { "ref": "origin/t3/evals", - "sha": "b0b8a2745e3ea67cb8f70df3943177e146f591b9" + "sha": "df4cce678b3b133b56e34f6c0cb910142a08f662" } ], "metrics": [ @@ -200,7 +200,7 @@ "id": "S1-1", "text": "One command regenerates every number from a clean checkout", "ref": "origin/t3/evals", - "sha": "b0b8a2745e3ea67cb8f70df3943177e146f591b9", + "sha": "df4cce678b3b133b56e34f6c0cb910142a08f662", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -209,13 +209,13 @@ "id": "S1-2", "text": "Dataset URLs and sha256 recorded", "ref": "origin/t3/evals", - "sha": "b0b8a2745e3ea67cb8f70df3943177e146f591b9", + "sha": "df4cce678b3b133b56e34f6c0cb910142a08f662", "status": "partial", "evidence": "found ADVIO; missing ETH3D", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/b0b8a2745e3ea67cb8f70df3943177e146f591b9/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/df4cce678b3b133b56e34f6c0cb910142a08f662/experiments/evals/README.md" ] ] }, @@ -223,7 +223,7 @@ "id": "S1-3", "text": "Metric code tested on hand-computed cases", "ref": "origin/t3/evals", - "sha": "b0b8a2745e3ea67cb8f70df3943177e146f591b9", + "sha": "df4cce678b3b133b56e34f6c0cb910142a08f662", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -232,13 +232,13 @@ "id": "S1-4", "text": "ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft", "ref": "origin/t3/evals", - "sha": "b0b8a2745e3ea67cb8f70df3943177e146f591b9", + "sha": "df4cce678b3b133b56e34f6c0cb910142a08f662", "status": "met", "evidence": "all 5 found", "links": [ [ "experiments/evals/results/advio_drift.md", - "https://github.com/SamGu-NRX/house-scanning/blob/b0b8a2745e3ea67cb8f70df3943177e146f591b9/experiments/evals/results/advio_drift.md" + "https://github.com/SamGu-NRX/house-scanning/blob/df4cce678b3b133b56e34f6c0cb910142a08f662/experiments/evals/results/advio_drift.md" ] ] }, @@ -246,7 +246,7 @@ "id": "S1-5", "text": "ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question", "ref": "origin/t3/evals", - "sha": "b0b8a2745e3ea67cb8f70df3943177e146f591b9", + "sha": "df4cce678b3b133b56e34f6c0cb910142a08f662", "status": "gap", "evidence": "none found in 2 files; missing 1-3 m row, 3-10 m row, scale error, verdict", "links": [] @@ -259,7 +259,7 @@ "refs": [ { "ref": "origin/t3/server", - "sha": "fcd457411a68171509d53b87b11c115f55a2a765" + "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b" } ], "metrics": [ @@ -267,7 +267,7 @@ "id": "S2-1", "text": "pytest green", "ref": "origin/t3/server", - "sha": "fcd457411a68171509d53b87b11c115f55a2a765", + "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -276,13 +276,13 @@ "id": "S2-2", "text": "Every public golden test 01-14 of docs/research/t3-lane-c-review.md present", "ref": "origin/t3/server", - "sha": "fcd457411a68171509d53b87b11c115f55a2a765", + "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", "status": "met", "evidence": "all 14 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_golden_public.py" ] ] }, @@ -290,26 +290,31 @@ "id": "S2-3", "text": "Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail", "ref": "origin/t3/server", - "sha": "fcd457411a68171509d53b87b11c115f55a2a765", - "status": "gap", - "evidence": "none found in 5 files; missing missing coverage, equality unsure, mirror, larger clearance", - "links": [] + "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", + "status": "partial", + "evidence": "found missing coverage, mirror; missing equality unsure, larger clearance", + "links": [ + [ + "server/tests/test_properties.py", + "https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_properties.py" + ] + ] }, { "id": "S2-4", "text": "Schemas published", "ref": "origin/t3/server", - "sha": "fcd457411a68171509d53b87b11c115f55a2a765", + "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", "status": "met", - "evidence": "present at fcd45741", + "evidence": "present at 44607e04", "links": [ [ "server/schemas/scene.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/schemas/scene.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/schemas/scene.schema.json" ], [ "server/schemas/result.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/schemas/result.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/schemas/result.schema.json" ] ] }, @@ -317,17 +322,21 @@ "id": "S2-5", "text": "Schemas validated in tests", "ref": "origin/t3/server", - "sha": "fcd457411a68171509d53b87b11c115f55a2a765", + "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", "status": "met", "evidence": "all 2 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_golden_public.py" ], [ "server/tests/test_schemas.py", - "https://github.com/SamGu-NRX/house-scanning/blob/fcd457411a68171509d53b87b11c115f55a2a765/server/tests/test_schemas.py" + "https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_schemas.py" + ], + [ + "server/tests/test_api.py", + "https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_api.py" ] ] }, @@ -335,19 +344,28 @@ "id": "S2-6", "text": "A real-derived scene returns a result in under 1 s", "ref": "origin/t3/server", - "sha": "fcd457411a68171509d53b87b11c115f55a2a765", - "status": "no evidence yet", - "evidence": "no report matches ~/house-scanning-data/reports/e2e/*/report.json", + "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", + "status": "met", + "evidence": "latency_ms = 733.5 (want < 1000): ~/house-scanning-data/reports/e2e/20260926-034015-44607e04/report.json", "links": [] }, { "id": "S2-7", "text": "ruff clean", "ref": "origin/t3/server", - "sha": "fcd457411a68171509d53b87b11c115f55a2a765", + "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", "status": "not run", "evidence": "slow; run with --slow", "links": [] + }, + { + "id": "S2-8", + "text": "Black-box (verification e2e): every answer valid under C2, consistent with its decision and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric when mirrored, and never passing without coverage", + "ref": "origin/t3/server", + "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", + "status": "met", + "evidence": "contract_problem_count = 0 (want 0): ~/house-scanning-data/reports/e2e/20260926-034015-44607e04/report.json", + "links": [] } ] }, @@ -425,6 +443,15 @@ "evidence": "0 of 1 matches needed, searched 1 files", "links": [] }, + { + "id": "S3-6b", + "text": "Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, plus symbol-name button labels) finds no issues; S4-2 shows how many screens the app reached", + "ref": "origin/t3/ios-mvf", + "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", + "status": "no evidence yet", + "evidence": "no report matches ~/house-scanning-data/reports/a11y/*t3-ios-mvf*/report.json", + "links": [] + }, { "id": "S3-7", "text": "Screenshots of each state in the PR", @@ -445,7 +472,7 @@ "ref": "origin/t3/ios-mvf", "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", "status": "no evidence yet", - "evidence": "no report matches ~/house-scanning-data/reports/e2e/*/report.json", + "evidence": "no app_export_scene_valid in ~/house-scanning-data/reports/e2e/20260926-034015-44607e04/report.json", "links": [] }, { @@ -465,7 +492,7 @@ "refs": [ { "ref": "origin/t3/verification", - "sha": "2ae5a802b41793f6135e56d6c22f33e3109e7761" + "sha": "6cf8692ee26f3474e12a1f08fb6a8a6780e91652" }, { "ref": "origin/t3/ios-mvf", @@ -477,9 +504,9 @@ "id": "S4-1", "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/verification", - "sha": "2ae5a802b41793f6135e56d6c22f33e3109e7761", + "sha": "6cf8692ee26f3474e12a1f08fb6a8a6780e91652", "status": "no evidence yet", - "evidence": "no report matches ~/house-scanning-data/reports/e2e/*/report.json", + "evidence": "no app_export_passed in ~/house-scanning-data/reports/e2e/20260926-034015-44607e04/report.json", "links": [] }, { @@ -495,7 +522,7 @@ "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "2ae5a802b41793f6135e56d6c22f33e3109e7761", + "sha": "6cf8692ee26f3474e12a1f08fb6a8a6780e91652", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -504,13 +531,13 @@ "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "2ae5a802b41793f6135e56d6c22f33e3109e7761", + "sha": "6cf8692ee26f3474e12a1f08fb6a8a6780e91652", "status": "met", - "evidence": "present at 2ae5a802", + "evidence": "present at 6cf8692e", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/2ae5a802b41793f6135e56d6c22f33e3109e7761/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/6cf8692ee26f3474e12a1f08fb6a8a6780e91652/verification/SCOREBOARD.md" ] ] } From c692a8d5e3675edaa90d908b64e645d3198768bd Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 03:47:36 -0500 Subject: [PATCH 17/59] verification: goldens 07 and 09; cases can rule out one outcome and point at scenes outside git The case suite gains headroom and reach cases now that their values are published. outcome_not lets a real-geometry case forbid a pass without claiming which of unsure or fail the server should give, and scene_path lets a committed case describe a dataset-derived scene that stays outside the repository. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/e2e/cases/README.md | 44 ++- verification/e2e/cases/g07-headroom-fail.json | 150 +++++++++++ .../e2e/cases/g07-headroom-margin.json | 157 +++++++++++ .../e2e/cases/g07-headroom-partial-span.json | 154 +++++++++++ verification/e2e/cases/g07-headroom-pass.json | 153 +++++++++++ .../e2e/cases/g07-headroom-unobserved.json | 137 ++++++++++ verification/e2e/cases/g09-reach-14.json | 163 +++++++++++ verification/e2e/cases/g09-reach-15.json | 166 ++++++++++++ verification/e2e/cases/g09-reach-16.json | 166 ++++++++++++ verification/e2e/cases/g09-reach-20.json | 166 ++++++++++++ .../e2e/cases/g09-reach-20p2-pm03.json | 166 ++++++++++++ .../e2e/cases/g09-reach-20p4-pm03.json | 153 +++++++++++ verification/e2e/cases/g09-reach-21.json | 153 +++++++++++ verification/e2e/cases/g09-vertical-run.json | 183 +++++++++++++ verification/e2e/cases/generate.py | 252 ++++++++++++++++++ verification/hsverify/e2e.py | 16 ++ verification/hsverify/resultcheck.py | 10 +- verification/tests/test_e2e.py | 20 ++ verification/tests/test_resultcheck.py | 36 +++ 19 files changed, 2435 insertions(+), 10 deletions(-) create mode 100644 verification/e2e/cases/g07-headroom-fail.json create mode 100644 verification/e2e/cases/g07-headroom-margin.json create mode 100644 verification/e2e/cases/g07-headroom-partial-span.json create mode 100644 verification/e2e/cases/g07-headroom-pass.json create mode 100644 verification/e2e/cases/g07-headroom-unobserved.json create mode 100644 verification/e2e/cases/g09-reach-14.json create mode 100644 verification/e2e/cases/g09-reach-15.json create mode 100644 verification/e2e/cases/g09-reach-16.json create mode 100644 verification/e2e/cases/g09-reach-20.json create mode 100644 verification/e2e/cases/g09-reach-20p2-pm03.json create mode 100644 verification/e2e/cases/g09-reach-20p4-pm03.json create mode 100644 verification/e2e/cases/g09-reach-21.json create mode 100644 verification/e2e/cases/g09-vertical-run.json diff --git a/verification/e2e/cases/README.md b/verification/e2e/cases/README.md index f64b8be33..06b6c9634 100644 --- a/verification/e2e/cases/README.md +++ b/verification/e2e/cases/README.md @@ -35,7 +35,10 @@ The keys follow the lead's case format. The cases use two additions and one conv catch that before the outcome is compared. - `missing_evidence_empty: false` means the list must be non-empty. - `rules_assumed` maps a check-id substring to the `threshold_ft` the case depends on: `gas` 3 ft - (`clearances.gas_ft`) and `opening` 3 ft (`clearances.opening_ft`). + (`clearances.gas_ft`), `opening` 3 ft (`clearances.opening_ft`), `headroom` 6.5 ft + (`headroom.min_ft`) and `route_length` 20 ft (`route.max_ft`). `route.confident_reach_ft` (15) + is not reported as any check's `threshold_ft`, so g09 can't declare it there. A change to it + shows up as g09-reach-14 or g09-reach-15 failing. ## The rules these cases depend on @@ -50,10 +53,22 @@ From `rules.yaml` at 2c9348f: and ask for a photo. - `openings.types` is `[door, window]`. A garage door is not an opening, so it has no 3 ft clearance. It does fail the route (`route.crossing.garage_door: fail`). -- `route.crossing.window: detour` and `gas_meter: detour`. `route.corner_allowance_ft` adds 0.5 ft - per corner the cable turns. +- `route.crossing.window: detour`, `gas_meter: detour` and `elec_box: detour`. + `route.corner_allowance_ft` adds 0.5 ft per corner the cable turns. `route.height_ft` 1: the + cable runs 1 ft above the ground. The route length has no vertical run from the meter down to + that height (at 2c9348f a spot at s = 1.583 reports route 1.583). A detour over an object adds + 2 × (top − 1): up to its top and back down. Going under is impossible when the object sits on + the ground. A window whose bottom is above 1 ft is crossed with no extra (g12a: route 7.042 for + start 7.042). +- `route.confident_reach_ft` 15: a longer route is unsure. `route.max_ft` 20 (`route_length`, + `at_most`): pass needs length + error < 20, fail needs length − error > 20. The route's error + is the meter's and the wall's errors summed (at 2c9348f a wall at ±0.3 with an exact meter gives + a route at ±0.3). +- `headroom.min_ft` 6.5 (`at_least`): pass needs clearance − error > 6.5, fail needs + clearance + error < 6.5. +- `ground.allowed` is concrete, gravel, lawn and mulch. A deck under the footprint fails + `ground_surface`. - `facing.measured_from: battery_front`, `min_ft` 3: a 9 ft facing gap gives 9 − 11/6 ≈ 7.17. -- `ground.allowed` includes concrete. ## Fixture conventions @@ -94,6 +109,15 @@ From `rules.yaml` at 2c9348f: | g12b-exact-fit-last-start | 12 | spot within [6, 6 + W]; no photo request | Five 1.8 ft segments (headings 40° to 8°) for s = [−3, 6], then one segment of exactly W at heading 0° ending the wall. Its only start, 6, is also its last. Measured in floating point, the segment is 4.4e-16 longer than W. Route 6 + 3 corners × 0.5 = 7.5. | | g13a-unsure-never-outranks-pass | 13 | spot within [−8, −1.25]; gas check at the spot passes; starts in (−3.8, 1.2) fail with `meter_working_space`; starts in (1.3, 5) unsure; no photo request | Strip x ∈ [0.3, 12] at gap 3.2, ±0.3: every battery overlapping x ≥ 0.3 gets 3.2 − 0.3 = 2.9, unsure. A battery with right edge b < 0.3 gets √((0.3 − b)² + 3.2²), which passes when > 3.3: (0.3 − b)² > 0.65, b < 0.3 − √0.65 ≈ −0.506. The working space also needs b ≤ −1.25, so passing spots have b ≤ −1.25 and route ≥ 1.25. Unsure spots start at 1.25 (route 1.25), and the passing spot must still win. | | g13b-unsure-alone | 13 | decision manual_review; spot on w1; gas unsure (margin), measured 3.2, ±0.3; starts in (−0.4, 1.2) fail with `meter_working_space`; starts in (1.3, 5.3) unsure; no photo request | The wall starts at −0.5, so every battery ends at ≥ 2.08 and overlaps the strip: gap 3.2 for all. The spot is fully observed and unsure only by margin, so no photos. | +| g07-headroom-fail | 07 | headroom fail, measured 5.5, ±0.5; spot null; every start in (−5, 12.4) fails with `headroom`; no photo request | Overhead over the whole wall at 6.5 − 1 = 5.5 ± 0.5: 5.5 + 0.5 = 6.0 < 6.5. | +| g07-headroom-pass | 07 | headroom pass, measured 7.2, ±0.5; starts in (−5, −3.9) and (1.3, 12.4) pass; no photo request | 6.5 + 0.7 = 7.2: 7.2 − 0.5 = 6.7 > 6.5. The gap between the two intervals is the meter working space. | +| g07-headroom-margin | 07 | headroom unsure (margin), measured 6.8, ±0.5; same intervals unsure; no photo request | 6.5 + 0.3 = 6.8: 6.8 − 0.5 = 6.3 is not > 6.5 and 7.3 is not < 6.5. | +| g07-headroom-unobserved | 07 | headroom unsure (unobserved); same intervals unsure; `missing_evidence` non-empty | No overheads, and the overhead band is left out of coverage. | +| g07-headroom-partial-span | 07 | starts in (5.5, 8.4) fail with `headroom`; starts in (1.3, 5.3) and (8.6, 12.4) pass | Overhead 5.5 ± 0.5 over s = [8, 8.5] only. A battery [a, a + W] overlaps it when 8 − W < a < 8.5, i.e. 5.417 < a < 8.5, even if it covers only a few inches of the battery. The schema's overheads have no depth out from the wall, so the golden's "front strip only" is tested along the wall instead. | +| g09-reach-14, -15, -16, -20, -21 | 09 | `route_length` measured L, ±0: pass at 14, unsure at 15, 16 and 20, fail at 21. For pass and unsure the spot is [L, L + W] and the sweep at start L has that outcome. For fail the spot is null and the sweep at L fails with `route_length`. No photo request | The wall runs s = [−1, L + W], all ground before L is deck and [L, L + W] is concrete. Every start before L fails `ground_surface`, starts left of the meter also fail the working space, and L is the last start. So the route is L with no corner. 14 < 15, pass. 15 is not inside 15. 16 is past 15 but below 20. For 20: 20 + 0 is not < 20 and 20 − 0 is not > 20, so unsure. For 21: 21 − 0 > 20, fail. | +| g09-reach-20p2-pm03 | 09 | `route_length` unsure, measured 20.2, ±0.3; spot at L | Meter at ±0.3, wall exact. 20.2 − 0.3 = 19.9 is not > 20, and 20.5 is not < 20. | +| g09-reach-20p4-pm03 | 09 | spot null; the sweep at L fails with `route_length` | 20.4 − 0.3 = 20.1 > 20. | +| g09-vertical-run | 09 | `route_length` unsure, measured 16, ±0; `route_path` pass; spot at [11, 11 + W] | Pad at 15 − 4 = 11. An `elec_box` at s = [4, 5], 0 to 3.5 ft tall, sits across the 1 ft cable run, and elec boxes detour. Over: 2 × (3.5 − 1) = 5. Routed 11 + 5 = 16 = 15 + 1, past the confident reach. Measuring along the wall alone (11) would pass. | | c5-no-coverage | C5, scene schema | decision manual_review; `missing_evidence` non-empty; starts in (−6, −3.9) and (1.3, 7.4) unsure, starts in (−3.8, 1.2) fail with `meter_working_space` | Clean wall s = [−6, 10] with ground and facing data but no `coverage`. "Absent means nothing is known to be observed" makes every start unsure except where the working space, which needs no observation, fails it. Absent ends default to unexplored, which rules out reject. | ## Assumptions a failure should be checked against @@ -104,6 +128,14 @@ From `rules.yaml` at 2c9348f: covers the meter, which makes the cable detour; those cases assert only gas and photo requests. g13a keeps the span off the left routes. - s counts the length of a no-wall stretch (g08b: w2 starts at s = 2, after a 1 ft gap). +- g09's last start is end − 2.583333333333 (the width as rules.yaml writes it) in floating point. + For L = 16, 20, 20.4, 21 and 11 the wall end is chosen so that start is exactly L. For 14, 15 and + 20.2 no such end exists, so the start lands 1.8e-15, 1.8e-15 and 3.6e-15 past L. No outcome + changes, but g09-reach-15 tests "past the confident reach", not the equality at exactly 15. +- The vertical detour is an inference. Neither rules.yaml nor the result schema gives the + formula; "go over or under it at route height" does not say how far. If g09-vertical-run fails + on `measured_ft` alone, check `route.detours[].extra_ft` against 5 before calling it a server + error. ## Changes after the first run @@ -141,10 +173,6 @@ errors, corrected once `rules.yaml` was published: - 04: the golden gives w1 outward [0, −1] for a baseline running +x, which contradicts the schema's clockwise rule. Its pad also relies on a ground check whose passing surfaces are not published. The flush rule it tests is exercised by g03 and g12b. -- 07: needs headroom around `headroom.min_ft` 6.5, a placeholder. Now that the value is - published this could be encoded; it isn't yet. -- 09: needs `confident_reach_ft` 15 and `max_route_ft` 20. `confident_reach_ft` is a placeholder - but published, so this could also be encoded now. - 11 (d): its expectation is a pass, which `auto_approve: false` can't produce. Its photo-request half duplicates g01. - 14: needs a pass. The result schema already requires every field it lists. diff --git a/verification/e2e/cases/g07-headroom-fail.json b/verification/e2e/cases/g07-headroom-fail.json new file mode 100644 index 000000000..9e758201d --- /dev/null +++ b/verification/e2e/cases/g07-headroom-fail.json @@ -0,0 +1,150 @@ +{ + "id": "g07-headroom-fail", + "source": "t3-lane-c-review.md golden 07 at origin/t3/research a7d91f1", + "description": "An overhead over the whole wall has clear height 5.5 +/- 0.5 ft against headroom 6.5 ft, so every start's headroom check is fail.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -5, + 0.0 + ], + [ + 15, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -5, + 0.0 + ], + [ + 15, + 0.0 + ], + [ + 15, + 10.0 + ], + [ + -5, + 10.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -5.0, + 15.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -20.0, + 30.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -20.0, + 30.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -20.0, + 30.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -20.0, + 30.0 + ], + "out_ft": 15.0 + } + ] + }, + "overheads": [ + { + "wall_id": "w1", + "span_ft": [ + -5, + 15 + ], + "clearance_ft": 5.5, + "plus_minus_ft": 0.5 + } + ] + }, + "rules_assumed": { + "headroom": 6.5 + }, + "expect": { + "checks": [ + { + "match": "headroom", + "outcome": "fail", + "measured_ft": 5.5, + "plus_minus_ft": 0.5 + } + ], + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + -5, + 12.4 + ], + "outcome": "fail", + "failing_match": "headroom" + } + ], + "missing_evidence_empty": true, + "decision_not": [ + "pass" + ], + "spot": null + } +} diff --git a/verification/e2e/cases/g07-headroom-margin.json b/verification/e2e/cases/g07-headroom-margin.json new file mode 100644 index 000000000..6b1fe62d5 --- /dev/null +++ b/verification/e2e/cases/g07-headroom-margin.json @@ -0,0 +1,157 @@ +{ + "id": "g07-headroom-margin", + "source": "t3-lane-c-review.md golden 07 at origin/t3/research a7d91f1", + "description": "An overhead over the whole wall has clear height 6.8 +/- 0.5 ft against headroom 6.5 ft, so every start's headroom check is unsure.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -5, + 0.0 + ], + [ + 15, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -5, + 0.0 + ], + [ + 15, + 0.0 + ], + [ + 15, + 10.0 + ], + [ + -5, + 10.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -5.0, + 15.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -20.0, + 30.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -20.0, + 30.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -20.0, + 30.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -20.0, + 30.0 + ], + "out_ft": 15.0 + } + ] + }, + "overheads": [ + { + "wall_id": "w1", + "span_ft": [ + -5, + 15 + ], + "clearance_ft": 6.8, + "plus_minus_ft": 0.5 + } + ] + }, + "rules_assumed": { + "headroom": 6.5 + }, + "expect": { + "checks": [ + { + "match": "headroom", + "outcome": "unsure", + "measured_ft": 6.8, + "plus_minus_ft": 0.5, + "unsure_cause": "margin" + } + ], + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + -5, + -3.9 + ], + "outcome": "unsure" + }, + { + "wall_id": "w1", + "start_ft": [ + 1.3, + 12.4 + ], + "outcome": "unsure" + } + ], + "missing_evidence_empty": true, + "decision_not": [ + "pass" + ] + } +} diff --git a/verification/e2e/cases/g07-headroom-partial-span.json b/verification/e2e/cases/g07-headroom-partial-span.json new file mode 100644 index 000000000..303bedc2e --- /dev/null +++ b/verification/e2e/cases/g07-headroom-partial-span.json @@ -0,0 +1,154 @@ +{ + "id": "g07-headroom-partial-span", + "source": "t3-lane-c-review.md golden 07 at origin/t3/research a7d91f1", + "description": "A low overhead (5.5 +/- 0.5 ft) covers only s = [8, 8.5]. Every battery whose 31 in stretch overlaps it fails headroom, even when the overhead covers a few inches of it.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -5, + 0.0 + ], + [ + 15, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -5, + 0.0 + ], + [ + 15, + 0.0 + ], + [ + 15, + 10.0 + ], + [ + -5, + 10.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -5.0, + 15.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -20.0, + 30.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -20.0, + 30.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -20.0, + 30.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -20.0, + 30.0 + ], + "out_ft": 15.0 + } + ] + }, + "overheads": [ + { + "wall_id": "w1", + "span_ft": [ + 8, + 8.5 + ], + "clearance_ft": 5.5, + "plus_minus_ft": 0.5 + } + ] + }, + "rules_assumed": { + "headroom": 6.5 + }, + "expect": { + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + 1.3, + 5.3 + ], + "outcome": "pass" + }, + { + "wall_id": "w1", + "start_ft": [ + 5.5, + 8.4 + ], + "outcome": "fail", + "failing_match": "headroom" + }, + { + "wall_id": "w1", + "start_ft": [ + 8.6, + 12.4 + ], + "outcome": "pass" + } + ], + "missing_evidence_empty": true + } +} diff --git a/verification/e2e/cases/g07-headroom-pass.json b/verification/e2e/cases/g07-headroom-pass.json new file mode 100644 index 000000000..0f80b48d0 --- /dev/null +++ b/verification/e2e/cases/g07-headroom-pass.json @@ -0,0 +1,153 @@ +{ + "id": "g07-headroom-pass", + "source": "t3-lane-c-review.md golden 07 at origin/t3/research a7d91f1", + "description": "An overhead over the whole wall has clear height 7.2 +/- 0.5 ft against headroom 6.5 ft, so every start's headroom check is pass.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -5, + 0.0 + ], + [ + 15, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -5, + 0.0 + ], + [ + 15, + 0.0 + ], + [ + 15, + 10.0 + ], + [ + -5, + 10.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -5.0, + 15.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -20.0, + 30.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -20.0, + 30.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -20.0, + 30.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -20.0, + 30.0 + ], + "out_ft": 15.0 + } + ] + }, + "overheads": [ + { + "wall_id": "w1", + "span_ft": [ + -5, + 15 + ], + "clearance_ft": 7.2, + "plus_minus_ft": 0.5 + } + ] + }, + "rules_assumed": { + "headroom": 6.5 + }, + "expect": { + "checks": [ + { + "match": "headroom", + "outcome": "pass", + "measured_ft": 7.2, + "plus_minus_ft": 0.5 + } + ], + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + -5, + -3.9 + ], + "outcome": "pass" + }, + { + "wall_id": "w1", + "start_ft": [ + 1.3, + 12.4 + ], + "outcome": "pass" + } + ], + "missing_evidence_empty": true + } +} diff --git a/verification/e2e/cases/g07-headroom-unobserved.json b/verification/e2e/cases/g07-headroom-unobserved.json new file mode 100644 index 000000000..8aea3a8e8 --- /dev/null +++ b/verification/e2e/cases/g07-headroom-unobserved.json @@ -0,0 +1,137 @@ +{ + "id": "g07-headroom-unobserved", + "source": "t3-lane-c-review.md golden 07 at origin/t3/research a7d91f1", + "description": "Nothing overhead was recorded and the overhead band was never observed, so every start's headroom check is unsure for lack of a view, and the result asks for that view.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -5, + 0.0 + ], + [ + 15, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -5, + 0.0 + ], + [ + 15, + 0.0 + ], + [ + 15, + 10.0 + ], + [ + -5, + 10.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -5.0, + 15.0 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -20.0, + 30.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -20.0, + 30.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -20.0, + 30.0 + ], + "out_ft": 15.0 + } + ] + } + }, + "rules_assumed": { + "headroom": 6.5 + }, + "expect": { + "decision_not": [ + "pass" + ], + "checks": [ + { + "match": "headroom", + "outcome": "unsure", + "unsure_cause": "unobserved" + } + ], + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + -5, + -3.9 + ], + "outcome": "unsure" + }, + { + "wall_id": "w1", + "start_ft": [ + 1.3, + 12.4 + ], + "outcome": "unsure" + } + ], + "missing_evidence_empty": false + } +} diff --git a/verification/e2e/cases/g09-reach-14.json b/verification/e2e/cases/g09-reach-14.json new file mode 100644 index 000000000..2b8d162b3 --- /dev/null +++ b/verification/e2e/cases/g09-reach-14.json @@ -0,0 +1,163 @@ +{ + "id": "g09-reach-14", + "source": "t3-lane-c-review.md golden 09 at origin/t3/research a7d91f1", + "description": "Route 14 ft, 1 ft inside the 15 ft confident reach: the route passes.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -1, + 0.0 + ], + [ + 16.583333333333, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [], + "ground": [ + { + "type": "deck", + "polygon": [ + [ + -1, + 0 + ], + [ + 14.0, + 0 + ], + [ + 14.0, + 10 + ], + [ + -1, + 10 + ] + ], + "plus_minus_ft": 0.0 + }, + { + "type": "concrete", + "polygon": [ + [ + 14.0, + 0 + ], + [ + 16.583333333333, + 0 + ], + [ + 16.583333333333, + 10 + ], + [ + 14.0, + 10 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -1.0, + 16.583333333333 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -16.0, + 31.583333333333 + ] + }, + { + "band": "overhead", + "span_ft": [ + -16.0, + 31.583333333333 + ] + }, + { + "band": "facing", + "span_ft": [ + -16.0, + 31.583333333333 + ] + }, + { + "band": "ground", + "span_ft": [ + -16.0, + 31.583333333333 + ], + "out_ft": 15.0 + } + ] + } + }, + "rules_assumed": { + "route_length": 20.0 + }, + "expect": { + "missing_evidence_empty": true, + "spot": { + "wall_id": "w1", + "span_within": [ + 14.0, + 16.583333333333332 + ] + }, + "checks": [ + { + "match": "route_length", + "outcome": "pass", + "measured_ft": 14.0, + "plus_minus_ft": 0.0 + } + ], + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + 13.999, + 14.001 + ], + "outcome": "pass" + } + ] + } +} diff --git a/verification/e2e/cases/g09-reach-15.json b/verification/e2e/cases/g09-reach-15.json new file mode 100644 index 000000000..1e2aba004 --- /dev/null +++ b/verification/e2e/cases/g09-reach-15.json @@ -0,0 +1,166 @@ +{ + "id": "g09-reach-15", + "source": "t3-lane-c-review.md golden 09 at origin/t3/research a7d91f1", + "description": "Route exactly at the 15 ft confident reach: not inside it, so unsure.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -1, + 0.0 + ], + [ + 17.583333333333, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [], + "ground": [ + { + "type": "deck", + "polygon": [ + [ + -1, + 0 + ], + [ + 15.0, + 0 + ], + [ + 15.0, + 10 + ], + [ + -1, + 10 + ] + ], + "plus_minus_ft": 0.0 + }, + { + "type": "concrete", + "polygon": [ + [ + 15.0, + 0 + ], + [ + 17.583333333333, + 0 + ], + [ + 17.583333333333, + 10 + ], + [ + 15.0, + 10 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -1.0, + 17.583333333333 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -16.0, + 32.583333333333 + ] + }, + { + "band": "overhead", + "span_ft": [ + -16.0, + 32.583333333333 + ] + }, + { + "band": "facing", + "span_ft": [ + -16.0, + 32.583333333333 + ] + }, + { + "band": "ground", + "span_ft": [ + -16.0, + 32.583333333333 + ], + "out_ft": 15.0 + } + ] + } + }, + "rules_assumed": { + "route_length": 20.0 + }, + "expect": { + "missing_evidence_empty": true, + "spot": { + "wall_id": "w1", + "span_within": [ + 15.0, + 17.583333333333332 + ] + }, + "checks": [ + { + "match": "route_length", + "outcome": "unsure", + "measured_ft": 15.0, + "plus_minus_ft": 0.0 + } + ], + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + 14.999, + 15.001 + ], + "outcome": "unsure" + } + ], + "decision_not": [ + "pass" + ] + } +} diff --git a/verification/e2e/cases/g09-reach-16.json b/verification/e2e/cases/g09-reach-16.json new file mode 100644 index 000000000..8302220e3 --- /dev/null +++ b/verification/e2e/cases/g09-reach-16.json @@ -0,0 +1,166 @@ +{ + "id": "g09-reach-16", + "source": "t3-lane-c-review.md golden 09 at origin/t3/research a7d91f1", + "description": "Route 16 ft, between the 15 ft confident reach and the 20 ft maximum: unsure.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -1, + 0.0 + ], + [ + 18.583333333333, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [], + "ground": [ + { + "type": "deck", + "polygon": [ + [ + -1, + 0 + ], + [ + 16.0, + 0 + ], + [ + 16.0, + 10 + ], + [ + -1, + 10 + ] + ], + "plus_minus_ft": 0.0 + }, + { + "type": "concrete", + "polygon": [ + [ + 16.0, + 0 + ], + [ + 18.583333333333, + 0 + ], + [ + 18.583333333333, + 10 + ], + [ + 16.0, + 10 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -1.0, + 18.583333333333 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -16.0, + 33.583333333333 + ] + }, + { + "band": "overhead", + "span_ft": [ + -16.0, + 33.583333333333 + ] + }, + { + "band": "facing", + "span_ft": [ + -16.0, + 33.583333333333 + ] + }, + { + "band": "ground", + "span_ft": [ + -16.0, + 33.583333333333 + ], + "out_ft": 15.0 + } + ] + } + }, + "rules_assumed": { + "route_length": 20.0 + }, + "expect": { + "missing_evidence_empty": true, + "spot": { + "wall_id": "w1", + "span_within": [ + 16.0, + 18.583333333333332 + ] + }, + "checks": [ + { + "match": "route_length", + "outcome": "unsure", + "measured_ft": 16.0, + "plus_minus_ft": 0.0 + } + ], + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + 15.999, + 16.001 + ], + "outcome": "unsure" + } + ], + "decision_not": [ + "pass" + ] + } +} diff --git a/verification/e2e/cases/g09-reach-20.json b/verification/e2e/cases/g09-reach-20.json new file mode 100644 index 000000000..ddd0eddb2 --- /dev/null +++ b/verification/e2e/cases/g09-reach-20.json @@ -0,0 +1,166 @@ +{ + "id": "g09-reach-20", + "source": "t3-lane-c-review.md golden 09 at origin/t3/research a7d91f1", + "description": "Route exactly at the 20 ft maximum with no error: 20 + 0 is not below 20 and 20 - 0 is not above it, so unsure.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -1, + 0.0 + ], + [ + 22.583333333333, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [], + "ground": [ + { + "type": "deck", + "polygon": [ + [ + -1, + 0 + ], + [ + 20.0, + 0 + ], + [ + 20.0, + 10 + ], + [ + -1, + 10 + ] + ], + "plus_minus_ft": 0.0 + }, + { + "type": "concrete", + "polygon": [ + [ + 20.0, + 0 + ], + [ + 22.583333333333, + 0 + ], + [ + 22.583333333333, + 10 + ], + [ + 20.0, + 10 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -1.0, + 22.583333333333 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -16.0, + 37.583333333333 + ] + }, + { + "band": "overhead", + "span_ft": [ + -16.0, + 37.583333333333 + ] + }, + { + "band": "facing", + "span_ft": [ + -16.0, + 37.583333333333 + ] + }, + { + "band": "ground", + "span_ft": [ + -16.0, + 37.583333333333 + ], + "out_ft": 15.0 + } + ] + } + }, + "rules_assumed": { + "route_length": 20.0 + }, + "expect": { + "missing_evidence_empty": true, + "spot": { + "wall_id": "w1", + "span_within": [ + 20.0, + 22.583333333333332 + ] + }, + "checks": [ + { + "match": "route_length", + "outcome": "unsure", + "measured_ft": 20.0, + "plus_minus_ft": 0.0 + } + ], + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + 19.999, + 20.001 + ], + "outcome": "unsure" + } + ], + "decision_not": [ + "pass" + ] + } +} diff --git a/verification/e2e/cases/g09-reach-20p2-pm03.json b/verification/e2e/cases/g09-reach-20p2-pm03.json new file mode 100644 index 000000000..8fb581d32 --- /dev/null +++ b/verification/e2e/cases/g09-reach-20p2-pm03.json @@ -0,0 +1,166 @@ +{ + "id": "g09-reach-20p2-pm03", + "source": "t3-lane-c-review.md golden 09 at origin/t3/research a7d91f1", + "description": "Route 20.2 +/- 0.3 ft (meter tap error): 20.2 - 0.3 = 19.9 is not above 20, so unsure.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.3 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -1, + 0.0 + ], + [ + 22.783333333333, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [], + "ground": [ + { + "type": "deck", + "polygon": [ + [ + -1, + 0 + ], + [ + 20.2, + 0 + ], + [ + 20.2, + 10 + ], + [ + -1, + 10 + ] + ], + "plus_minus_ft": 0.0 + }, + { + "type": "concrete", + "polygon": [ + [ + 20.2, + 0 + ], + [ + 22.783333333333, + 0 + ], + [ + 22.783333333333, + 10 + ], + [ + 20.2, + 10 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -1.0, + 22.783333333333 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -16.0, + 37.783333333333005 + ] + }, + { + "band": "overhead", + "span_ft": [ + -16.0, + 37.783333333333005 + ] + }, + { + "band": "facing", + "span_ft": [ + -16.0, + 37.783333333333005 + ] + }, + { + "band": "ground", + "span_ft": [ + -16.0, + 37.783333333333005 + ], + "out_ft": 15.0 + } + ] + } + }, + "rules_assumed": { + "route_length": 20.0 + }, + "expect": { + "missing_evidence_empty": true, + "spot": { + "wall_id": "w1", + "span_within": [ + 20.2, + 22.78333333333333 + ] + }, + "checks": [ + { + "match": "route_length", + "outcome": "unsure", + "measured_ft": 20.2, + "plus_minus_ft": 0.3 + } + ], + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + 20.199, + 20.201 + ], + "outcome": "unsure" + } + ], + "decision_not": [ + "pass" + ] + } +} diff --git a/verification/e2e/cases/g09-reach-20p4-pm03.json b/verification/e2e/cases/g09-reach-20p4-pm03.json new file mode 100644 index 000000000..635fe8c8e --- /dev/null +++ b/verification/e2e/cases/g09-reach-20p4-pm03.json @@ -0,0 +1,153 @@ +{ + "id": "g09-reach-20p4-pm03", + "source": "t3-lane-c-review.md golden 09 at origin/t3/research a7d91f1", + "description": "Route 20.4 +/- 0.3 ft (meter tap error): 20.4 - 0.3 = 20.1 is above 20, so fail.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.3 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -1, + 0.0 + ], + [ + 22.983333333333, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [], + "ground": [ + { + "type": "deck", + "polygon": [ + [ + -1, + 0 + ], + [ + 20.4, + 0 + ], + [ + 20.4, + 10 + ], + [ + -1, + 10 + ] + ], + "plus_minus_ft": 0.0 + }, + { + "type": "concrete", + "polygon": [ + [ + 20.4, + 0 + ], + [ + 22.983333333333, + 0 + ], + [ + 22.983333333333, + 10 + ], + [ + 20.4, + 10 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -1.0, + 22.983333333333 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -16.0, + 37.983333333333 + ] + }, + { + "band": "overhead", + "span_ft": [ + -16.0, + 37.983333333333 + ] + }, + { + "band": "facing", + "span_ft": [ + -16.0, + 37.983333333333 + ] + }, + { + "band": "ground", + "span_ft": [ + -16.0, + 37.983333333333 + ], + "out_ft": 15.0 + } + ] + } + }, + "rules_assumed": { + "route_length": 20.0 + }, + "expect": { + "missing_evidence_empty": true, + "spot": null, + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + 20.399, + 20.401 + ], + "outcome": "fail", + "failing_match": "route_length" + } + ], + "decision_not": [ + "pass" + ] + } +} diff --git a/verification/e2e/cases/g09-reach-21.json b/verification/e2e/cases/g09-reach-21.json new file mode 100644 index 000000000..266489b6b --- /dev/null +++ b/verification/e2e/cases/g09-reach-21.json @@ -0,0 +1,153 @@ +{ + "id": "g09-reach-21", + "source": "t3-lane-c-review.md golden 09 at origin/t3/research a7d91f1", + "description": "Route 21 ft, 1 ft past the 20 ft maximum: the only start on usable ground fails.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -1, + 0.0 + ], + [ + 23.583333333333, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [], + "ground": [ + { + "type": "deck", + "polygon": [ + [ + -1, + 0 + ], + [ + 21.0, + 0 + ], + [ + 21.0, + 10 + ], + [ + -1, + 10 + ] + ], + "plus_minus_ft": 0.0 + }, + { + "type": "concrete", + "polygon": [ + [ + 21.0, + 0 + ], + [ + 23.583333333333, + 0 + ], + [ + 23.583333333333, + 10 + ], + [ + 21.0, + 10 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -1.0, + 23.583333333333 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -16.0, + 38.583333333333 + ] + }, + { + "band": "overhead", + "span_ft": [ + -16.0, + 38.583333333333 + ] + }, + { + "band": "facing", + "span_ft": [ + -16.0, + 38.583333333333 + ] + }, + { + "band": "ground", + "span_ft": [ + -16.0, + 38.583333333333 + ], + "out_ft": 15.0 + } + ] + } + }, + "rules_assumed": { + "route_length": 20.0 + }, + "expect": { + "missing_evidence_empty": true, + "spot": null, + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + 20.999, + 21.001 + ], + "outcome": "fail", + "failing_match": "route_length" + } + ], + "decision_not": [ + "pass" + ] + } +} diff --git a/verification/e2e/cases/g09-vertical-run.json b/verification/e2e/cases/g09-vertical-run.json new file mode 100644 index 000000000..14ab6aa2d --- /dev/null +++ b/verification/e2e/cases/g09-vertical-run.json @@ -0,0 +1,183 @@ +{ + "id": "g09-vertical-run", + "source": "t3-lane-c-review.md golden 09 at origin/t3/research a7d91f1", + "description": "The pad starts 11 ft along the wall, but the cable must climb over a 3.5 ft box at the 1 ft run height: 11 + 2 x 2.5 = 16 ft routed, past the 15 ft confident reach.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1", + "plus_minus_ft": 0.0 + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -1, + 0.0 + ], + [ + 13.583333333333, + 0.0 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "objects": [ + { + "type": "elec_box", + "wall_id": "w1", + "span_ft": [ + 4, + 5 + ], + "bottom_ft": 0.0, + "top_ft": 3.5, + "source": "tape", + "plus_minus_ft": 0.0 + } + ], + "ground": [ + { + "type": "deck", + "polygon": [ + [ + -1, + 0 + ], + [ + 11.0, + 0 + ], + [ + 11.0, + 10 + ], + [ + -1, + 10 + ] + ], + "plus_minus_ft": 0.0 + }, + { + "type": "concrete", + "polygon": [ + [ + 11.0, + 0 + ], + [ + 13.583333333333, + 0 + ], + [ + 13.583333333333, + 10 + ], + [ + 11.0, + 10 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -1.0, + 13.583333333333 + ], + "depth_ft": 9.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -16.0, + 28.583333333333 + ] + }, + { + "band": "overhead", + "span_ft": [ + -16.0, + 28.583333333333 + ] + }, + { + "band": "facing", + "span_ft": [ + -16.0, + 28.583333333333 + ] + }, + { + "band": "ground", + "span_ft": [ + -16.0, + 28.583333333333 + ], + "out_ft": 15.0 + } + ] + } + }, + "rules_assumed": { + "route_length": 20.0 + }, + "expect": { + "missing_evidence_empty": true, + "spot": { + "wall_id": "w1", + "span_within": [ + 11.0, + 13.583333333333334 + ] + }, + "checks": [ + { + "match": "route_length", + "outcome": "unsure", + "measured_ft": 16.0, + "plus_minus_ft": 0.0 + }, + { + "match": "route_path", + "outcome": "pass" + } + ], + "sweep_runs": [ + { + "wall_id": "w1", + "start_ft": [ + 10.999, + 11.001 + ], + "outcome": "unsure" + } + ], + "decision_not": [ + "pass" + ] + } +} diff --git a/verification/e2e/cases/generate.py b/verification/e2e/cases/generate.py index e38073719..ed3b4ae48 100644 --- a/verification/e2e/cases/generate.py +++ b/verification/e2e/cases/generate.py @@ -613,6 +613,258 @@ def g13_scene(x0): }, ) +# g07: headroom. rules.yaml headroom.min_ft = 6.5 (at_least). The schema's overheads carry only +# a stretch of wall (span_ft) and a clear height, not a depth out from the wall, so "covers the +# whole footprint" is tested along the wall: a partial overhead over part of the battery's width +# must still limit it. +HEADROOM = 6.5 + + +def overhead(span, clearance, pm=0.5): + return {"wall_id": "w1", "span_ft": list(span), "clearance_ft": clearance, "plus_minus_ft": pm} + + +G07_ROWS = [ + # name, clearance, outcome, unsure cause + ("fail", HEADROOM - 1, "fail", None), + ("pass", HEADROOM + 0.7, "pass", None), + ("margin", HEADROOM + 0.3, "unsure", "margin"), +] +for name, clearance, outcome, cause in G07_ROWS: + check = { + "match": "headroom", + "outcome": outcome, + "measured_ft": clearance, + "plus_minus_ft": 0.5, + } + if cause: + check["unsure_cause"] = cause + if outcome == "fail": + runs = [ + { + "wall_id": "w1", + "start_ft": [-5, 12.4], + "outcome": "fail", + "failing_match": "headroom", + } + ] + else: + runs = [ + {"wall_id": "w1", "start_ft": [-5, -3.9], "outcome": outcome}, + {"wall_id": "w1", "start_ft": [1.3, 12.4], "outcome": outcome}, + ] + expect = {"checks": [check], "sweep_runs": runs, "missing_evidence_empty": True} + if outcome != "pass": + expect["decision_not"] = ["pass"] + if outcome == "fail": + expect["spot"] = None + case( + f"g07-headroom-{name}", + golden("07"), + f"An overhead over the whole wall has clear height {clearance} +/- 0.5 ft against " + f"headroom {HEADROOM} ft, so every start's headroom check is {outcome}.", + straight_scene(-5, 15) | {"overheads": [overhead((-5, 15), clearance)]}, + expect, + {"headroom": HEADROOM}, + ) + +g07u = straight_scene( + -5, 15, cov=coverage((-5, 15), bands=("wall", "facing"), past=(PAST, PAST), out=PAST) +) +case( + "g07-headroom-unobserved", + golden("07"), + "Nothing overhead was recorded and the overhead band was never observed, so every start's " + "headroom check is unsure for lack of a view, and the result asks for that view.", + g07u, + { + "decision_not": ["pass"], + "checks": [{"match": "headroom", "outcome": "unsure", "unsure_cause": "unobserved"}], + "sweep_runs": [ + {"wall_id": "w1", "start_ft": [-5, -3.9], "outcome": "unsure"}, + {"wall_id": "w1", "start_ft": [1.3, 12.4], "outcome": "unsure"}, + ], + "missing_evidence_empty": False, + }, + {"headroom": HEADROOM}, +) +case( + "g07-headroom-partial-span", + golden("07"), + "A low overhead (5.5 +/- 0.5 ft) covers only s = [8, 8.5]. Every battery whose 31 in " + "stretch overlaps it fails headroom, even when the overhead covers a few inches of it.", + straight_scene(-5, 15) | {"overheads": [overhead((8, 8.5), HEADROOM - 1)]}, + { + "sweep_runs": [ + {"wall_id": "w1", "start_ft": [1.3, 5.3], "outcome": "pass"}, + { + "wall_id": "w1", + "start_ft": [5.5, 8.4], + "outcome": "fail", + "failing_match": "headroom", + }, + {"wall_id": "w1", "start_ft": [8.6, 12.4], "outcome": "pass"}, + ], + "missing_evidence_empty": True, + }, + {"headroom": HEADROOM}, +) + +# g09: reach uses the routed length. rules.yaml route.confident_reach_ft = 15 (beyond it the +# route is unsure), route.max_ft = 20 (route_length check, at_most; beyond it the route fails). +# Every start before L sits on deck, which rules.yaml ground.allowed excludes, so it fails +# ground_surface; the wall ends at L + W, so L is the last start and the only one on concrete. +# Starts left of the meter overlap its working space (the wall begins at -1). +CONFIDENT, MAX_ROUTE = 15.0, 20.0 +# The battery width as rules.yaml writes it. The last start is end - W_RULES in floating point. +# The wall's end is the nearest float to L + W_RULES for which that start is exactly L. For +# L = 14, 15 and 20.2 no such float exists (end has a coarser ulp than L), so the end is the +# smallest float whose last start is at or past L: 14 and 15 land 1.8e-15 past, 20.2 lands 3.6e-15 +# past. None of those flips an outcome except that L = 15 tests "past the confident reach", not +# the equality at 15 itself. +W_RULES = 2.583333333333 + + +def exact_end(L): + base = L + W_RULES + near = [base] + for direction in (-math.inf, math.inf): + x = base + for _ in range(4): + x = math.nextafter(x, direction) + near.append(x) + exact = [x for x in near if x - W_RULES == L] + end = ( + min(exact, key=lambda x: abs(x - base)) + if exact + else min(x for x in near if x - W_RULES >= L) + ) + assert (end + 1) - 1 == end + return end + + +def reach_scene(L, meter_pm=0.0, objects=()): + end = exact_end(L) + scene = straight_scene(-1, end, objects=objects) + scene["meter"]["plus_minus_ft"] = meter_pm + scene["ground"] = [ + {"type": "deck", "polygon": rect(-1, L, 0, 10), "plus_minus_ft": 0.0}, + concrete(rect(L, end, 0, 10)), + ] + return scene + + +def reach_case(cid, L, route_ft, e, outcome, description, objects=(), extra_checks=()): + at_L = [round(L - 0.001, 3), round(L + 0.001, 3)] + check = { + "match": "route_length", + "outcome": outcome, + "measured_ft": route_ft, + "plus_minus_ft": e, + } + expect = {"missing_evidence_empty": True} + if outcome == "fail": + expect["spot"] = None + expect["sweep_runs"] = [ + {"wall_id": "w1", "start_ft": at_L, "outcome": "fail", "failing_match": "route_length"} + ] + else: + expect["spot"] = {"wall_id": "w1", "span_within": [L, L + W]} + expect["checks"] = [check, *extra_checks] + expect["sweep_runs"] = [{"wall_id": "w1", "start_ft": at_L, "outcome": outcome}] + if outcome != "pass": + expect["decision_not"] = ["pass"] + case( + cid, + golden("09"), + description, + reach_scene(L, e, objects), + expect, + {"route_length": MAX_ROUTE}, + ) + + +reach_case( + "g09-reach-14", + 14.0, + 14.0, + 0.0, + "pass", + "Route 14 ft, 1 ft inside the 15 ft confident reach: the route passes.", +) +reach_case( + "g09-reach-15", + 15.0, + 15.0, + 0.0, + "unsure", + "Route exactly at the 15 ft confident reach: not inside it, so unsure.", +) +reach_case( + "g09-reach-16", + 16.0, + 16.0, + 0.0, + "unsure", + "Route 16 ft, between the 15 ft confident reach and the 20 ft maximum: unsure.", +) +reach_case( + "g09-reach-20", + 20.0, + 20.0, + 0.0, + "unsure", + "Route exactly at the 20 ft maximum with no error: 20 + 0 is not below 20 and 20 - 0 " + "is not above it, so unsure.", +) +reach_case( + "g09-reach-21", + 21.0, + 21.0, + 0.0, + "fail", + "Route 21 ft, 1 ft past the 20 ft maximum: the only start on usable ground fails.", +) +reach_case( + "g09-reach-20p2-pm03", + 20.2, + 20.2, + 0.3, + "unsure", + "Route 20.2 +/- 0.3 ft (meter tap error): 20.2 - 0.3 = 19.9 is not above 20, so unsure.", +) +reach_case( + "g09-reach-20p4-pm03", + 20.4, + 20.4, + 0.3, + "fail", + "Route 20.4 +/- 0.3 ft (meter tap error): 20.4 - 0.3 = 20.1 is above 20, so fail.", +) +# Vertical run: an electrical box standing on the ground at s = [4, 5], 3.5 ft tall, across the +# 1 ft cable run (rules.yaml route.height_ft). route.crossing.elec_box is detour; going under is +# impossible (bottom 0), so the cable climbs 3.5 - 1 = 2.5 ft and comes back down: 5 ft extra. +ebox = { + "type": "elec_box", + "wall_id": "w1", + "span_ft": [4, 5], + "bottom_ft": 0.0, + "top_ft": 3.5, + "source": "tape", + "plus_minus_ft": 0.0, +} +reach_case( + "g09-vertical-run", + CONFIDENT - 4, + CONFIDENT + 1, + 0.0, + "unsure", + "The pad starts 11 ft along the wall, but the cable must climb over a 3.5 ft box at " + "the 1 ft run height: 11 + 2 x 2.5 = 16 ft routed, past the 15 ft confident reach.", + objects=[ebox], + extra_checks=[{"match": "route_path", "outcome": "pass"}], +) + def main(): for old in HERE.glob("*.json"): diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index 28a2d27e6..2747efd00 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -91,6 +91,22 @@ def load_input(path: Path, real: bool = False) -> SceneInput: images = {Path(n).name: bundle.read(n) for n in names if n.lower().endswith(".jpg")} return SceneInput(path.stem, json.loads(raw), raw, images, real=real, source=str(path)) data = json.loads(path.read_text()) + if "scene_path" in data and "expect" in data: + # A committed case about a scene that stays outside git (dataset-derived). + target = Path(data["scene_path"]).expanduser() + if not target.exists(): + return SceneInput( + data["id"], + {}, + b"", + source=str(path), + skip_reason=f"{target} not built here; see the case's README", + ) + item = load_input(target, real=bool(data.get("real", real))) + item.name, item.expect = data["id"], data["expect"] + item.rules_assumed = data.get("rules_assumed", {}) + item.source = data.get("source", str(path)) + return item if "scene" in data and "expect" in data: raw = json.dumps(data["scene"], indent=1).encode() return SceneInput( diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index db6d85bf9..4703cd090 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -241,14 +241,20 @@ def expectation_problems(expect: dict, result: dict) -> list[str]: for r in result.get("sweep", []) if r["wall_id"] == rule["wall_id"] and r["start_ft"][0] < b and r["start_ft"][1] > a ] - if not runs: + if not runs and "outcome" in rule: problems.append(f"no sweep run overlaps {rule['wall_id']} starts ({a}, {b})") for run in runs: - if run["outcome"] != rule["outcome"]: + if "outcome" in rule and run["outcome"] != rule["outcome"]: problems.append( f"sweep {run['wall_id']} starts {run['start_ft']} is {run['outcome']}, " f"expected {rule['outcome']}" ) + if "outcome_not" in rule and run["outcome"] == rule["outcome_not"]: + why = f" ({rule['reason']})" if "reason" in rule else "" + problems.append( + f"sweep {run['wall_id']} starts {run['start_ft']} is {run['outcome']}, " + f"which this case rules out{why}" + ) for field, key in (("failing", "failing_match"), ("unsure", "unsure_match")): if key in rule and not any(rule[key] in c for c in run[field]): problems.append( diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index ae998c1e0..fdf0c3340 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -113,6 +113,26 @@ def test_case_file_is_loaded_with_its_expectations(tmp_path): assert json.loads(item.raw) == SCENE +def test_case_pointing_at_a_scene_outside_git(tmp_path): + scene = tmp_path / "scene.json" + scene.write_text(json.dumps(SCENE)) + case = tmp_path / "case.json" + case.write_text( + json.dumps( + { + "id": "real-1", + "real": True, + "scene_path": str(scene), + "expect": {"decision_not": ["pass"]}, + } + ) + ) + item = load_input(case) + assert (item.name, item.real, item.expect) == ("real-1", True, {"decision_not": ["pass"]}) + scene.unlink() + assert load_input(case).skip_reason.endswith("not built here; see the case's README") + + class FakeServer(BaseHTTPRequestHandler): """Answers every scene with a consistent manual_review; `decide_pass` breaks C5.""" diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 259234b04..d99d0c8bc 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -250,6 +250,42 @@ def test_expectations(): assert len(msgs) == 4, msgs +def test_outcome_not_rules_out_one_outcome_and_tolerates_no_runs(): + r = result( + sweep=[ + { + "wall_id": "w1", + "start_ft": [0.0, 2.0], + "outcome": "unsure", + "failing": [], + "unsure": [], + }, + { + "wall_id": "w1", + "start_ft": [2.1, 3.0], + "outcome": "pass", + "failing": [], + "unsure": [], + }, + ] + ) + ok = { + "sweep_runs": [ + {"wall_id": "w1", "start_ft": [0.5, 1.5], "outcome_not": "pass"}, + {"wall_id": "w9", "start_ft": [0, 9], "outcome_not": "pass"}, + ] + } + assert expectation_problems(ok, r) == [] + bad = { + "sweep_runs": [ + {"wall_id": "w1", "start_ft": [0.5, 2.5], "outcome_not": "pass", "reason": "window"} + ] + } + assert expectation_problems(bad, r) == [ + "sweep w1 starts [2.1, 3.0] is pass, which this case rules out (window)" + ] + + def test_pinned_check_numbers(): r = result() assert expectation_problems({"checks": [{"match": "gas", "measured_ft": 5.0000004}]}, r) == [] From 820ac912ef0b9f15d0b07991d41dd2683c7d75cb Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 03:50:27 -0500 Subject: [PATCH 18/59] verification: real-geometry scene from ETH3D facade A three-wall courtyard chain built from the ETH3D laser scan and COLMAP poses, with five windows and a door marked by hand in two or three photos each and checked against gaps in the scan. The meter is invented and labelled so. The case says no footprint may pass anywhere and the decision must be manual_review, with the arithmetic in the README. Only code, annotations and the case are committed; the dataset and the built scene stay outside git. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/pyproject.toml | 1 + verification/scenes/eth3d-facade/README.md | 135 +++ .../scenes/eth3d-facade/annotations.json | 415 ++++++++ .../scenes/eth3d-facade/build_scene.py | 941 ++++++++++++++++++ verification/scenes/eth3d-facade/case.json | 22 + verification/uv.lock | 73 ++ 6 files changed, 1587 insertions(+) create mode 100644 verification/scenes/eth3d-facade/README.md create mode 100644 verification/scenes/eth3d-facade/annotations.json create mode 100644 verification/scenes/eth3d-facade/build_scene.py create mode 100644 verification/scenes/eth3d-facade/case.json diff --git a/verification/pyproject.toml b/verification/pyproject.toml index e8669fc2a..6175f1785 100644 --- a/verification/pyproject.toml +++ b/verification/pyproject.toml @@ -6,6 +6,7 @@ requires-python = ">=3.12" dependencies = [ "jsonschema>=4.23", "numpy>=2.5.3", + "pillow>=12.3.0", "pyyaml>=6.0", ] diff --git a/verification/scenes/eth3d-facade/README.md b/verification/scenes/eth3d-facade/README.md new file mode 100644 index 000000000..26fe14937 --- /dev/null +++ b/verification/scenes/eth3d-facade/README.md @@ -0,0 +1,135 @@ +# ETH3D facade: a real-geometry test scene + +`scene.json` for the placement server, built from a real building: the south courtyard wing of the +ETH3D "facade" dataset (an ETH Zurich building, DSLR photos plus a metric laser scan). Doors, +windows and walls are real and measured. The electric meter is invented, because the building has +none where we need it. `case.json` says what a correct server must and must not answer, in the +format the e2e checker reads. + +## License + +ETH3D data is for research use only. Nothing from it is in this repo: no images, points or crops. +The script reads the dataset from outside the repo and writes its output there too. + +## Build + +Data: `facade_dslr_undistorted.7z` and `facade_dslr_scan_eval.7z` from eth3d.net, extracted to +`~/house-scanning-data/verify/eth3d/` (it creates `facade/`). Then, from `verification/`: + +``` +uv run python scenes/eth3d-facade/build_scene.py [--schema path/to/scene.schema.json] [--overlays] +``` + +This writes `~/house-scanning-data/verify/scenes/eth3d-facade/`: `scene.json`, `scene.zip` +(scene.json plus 7 keyframes at the top level), `bundle/`, and `build_report.json` with every fit +statistic. `--overlays` also draws the wall chain and the annotations on each keyframe. The run +takes about 10 s and is deterministic (fixed RANSAC seed). The output validates against +`server/schemas/scene.schema.json` from `origin/t3/server` (44607e0). + +## How each number is made + +- **Frame.** The laser scan and the COLMAP poses share one metric frame. I checked this by + projecting scan points into DSC_0419, where walls, window edges and the door line up. The ground + is a RANSAC plane fitted to scan points in the courtyard: 1.86 M inliers, 0.97 cm RMS. Its normal + becomes +y, so heights are above the local ground. The normal is 1.30 degrees off the scanner's + vertical, which is about the gravel's drainage slope, so this frame is not exactly gravity. + Scene x is scan x made horizontal, z = x cross y. All lengths are feet. The origin is the ground + point under the invented meter. +- **Walls.** Each wall's ground line is a RANSAC line fitted in plan to laser points 0.3-8 ft + high near a rough segment that I read off a top view (`annotations.json` `walls`). + Neighbouring lines meet at their intersection. The free ends are the last scanned points. I used + the laser scan rather than the COLMAP points because it is denser (millions of points on this + wing against 85 k for the whole SfM model) and it is the dataset's ground truth. Each wall's `plus_minus_ft` + is max(0.1, 3 x the fit's RMS). +- **Chain.** Left to right as seen from the courtyard: w1, a 52.0 ft (15.85 m) wall with five + ground-floor windows. Then w2, the 11.6 ft (3.54 m) east side of the entrance tower, which + projects 3.5 m into the courtyard. Then w3, the tower front with the entrance door, 16.2 ft + (4.95 m). Past w3 the wall steps back into w1's plane, but the scan and SfM points stop there, + so the chain ends. Both ends are `unexplored`. +- **Poses.** COLMAP gives world-to-camera `R, t` for an OpenCV camera (+y down, +z forward). + Camera-to-world is `R^T` and `C = -R^T t`. ARKit's camera has +y up and looks along -z, so + `R_c2w = A . R^T . diag(1, -1, -1)` and `C_scene = A (C - origin) / 0.3048`, where `A` rotates + scan axes into scene axes. The 16 numbers are that 4x4, column-major. Keyframes are the + undistorted images resized to 1/4, and fx, fy, cx, cy are scaled by the same factor. +- **Features.** For each image of a feature I clicked four points: one on the left jamb, one on + the right jamb, one on the sill (or door threshold) and the arch apex. Each point's ray + `((u-cx)/fx, -(v-cy)/fy, -1)` goes through the pose onto the wall's brick face. The brick face + sits 0.33 ft behind the ground line on w1, because the stone plinth stands proud. The hits give + `span_ft` (jamb positions along the chain) and `bottom_ft` / `top_ft` (heights). Every feature + uses 2 or 3 images, and the value is their mean. + `plus_minus_ft = max(0.1, largest disagreement between the images over the four values, + documented floor)`. Only the door has a floor (0.7 ft). It sits in a recess, and both photos + see it from the east, so their 0.19 ft agreement hides a shared bias that the scan exposes. +- **Coverage.** A stretch counts as observed for a band when one keyframe sees all of that band's + probe points. A point counts as seen when it is inside the image, on the camera side of its + wall, not behind another wall of the chain, and within 75 degrees of the wall's normal. The + 75-degree limit is my judgement, not a measured value. The probes are: wall, ground line and + battery height (3.29 ft); ground, out to 6 ft; overhead, 6.5 ft up at 0 and 3 ft out; facing, + 3 ft up, straight out. No camera sees w2 at better than grazing, so w2 is in no band. +- **Overheads and facing** come from the scan. An overhead is anything within 3 ft of the wall, + lower than 12 ft, whose underside is above 4 ft. With a 2 ft floor, the tops of the two bins by + the door read as overheads, because the scanner saw their tops but not the parts below. Facing is + the nearest point more than 1 ft out and 1-6 ft high. The 1 ft skips pilasters and sills. It + reports 60 ft, a lower bound, where the scan saw open ground that far out and nothing in between. + +## What is real and what is invented + +| Item | Real? | wall | span_ft (s) | bottom-top ft | +/- ft | laser-scan check | +|---|---|---|---|---|---|---| +| electric meter | **invented** | w1 | 0 (5 ft up) | | 0.3 | placed midway between windows 4 and 5 | +| window 1 | real | w1 | -40.08 to -34.97 | 3.00-11.12 | 1.29 | opening -40.20 to -34.60 | +| window 2 | real | w1 | -29.26 to -24.04 | 2.87-11.25 | 0.36 | -29.45 to -23.90 | +| window 3 | real | w1 | -18.59 to -13.26 | 3.14-11.58 | 0.33 | -18.80 to -13.20 | +| window 4 | real | w1 | -8.13 to -2.57 | 3.32-11.73 | 0.27 | -8.15 to -2.55 | +| window 5 | real | w1 | 2.54 to 7.92 | 3.52-11.98 | 0.40 | 2.55 to 8.05 | +| entrance door | real | w3 | 23.83 to 31.01 | 0.69-13.37 | 0.70 | 24.0 to 31.65 | + +The scan check column comes from gaps in the laser points on the wall face 4.5-6 ft up. Every +window edge from the photos lies within its stated +/- of the scan's edge. Scan sill heights come +out 0.1-0.5 ft lower than `bottom_ft`, because I clicked the top edge of the stone sill and the +scan measures the brick below it. There is no gas meter; any stretch it could constrain already +fails on a window. + +Present but not annotated: + +- A narrow window on w2, visible only edge-on in DSC_0419. w2 is unobserved, which is the point. +- A downspout in the w1/w2 corner. It shows up only as the 1.03 ft facing depth there and a 6.1 ft + overhead. +- Two bins beside the door. They show up as the 2.3-2.5 ft facing depths; `objects` has no type + for them. + +## Expected placement outcomes + +Rule values come from `server/rules.yaml` at 44607e0: `battery.width_ft` 2.5833 (31 in), +`battery.depth_ft` 1.8333, `clearances.opening_ft` 3.0 for `openings.types` [door, window] (no +height exemption: `exempt_bottom_above_ft` is null), `route.max_ft` 20.0 and +`route.corner_allowance_ft` 0.5 per corner. The solver measures opening clearance as the plan +distance from the footprint to the opening's stretch of wall line, so sill height does not enter. +`case.json` lists ranges of the battery's start position (its left edge in s, as the server's +`sweep` reports it), where start = centre - 1.2917. + +- **A window or door blocks every start within 3 ft of it.** For a span [a, b] the blocked starts + are (a - 2.58 - 3, b + 3). Window 4 gives (-8.13 - 5.58, -2.57 + 3) = (-13.71, 0.43). Window 5 + gives (-3.04, 10.92), clipped at the w1/w2 corner (9.9). The door gives (18.25, 34.01), of which + w3 holds [21.52, 34.01]. +- **Nothing on w1 can pass.** The gaps between windows are 5.71, 5.45, 5.13 and 5.11 ft. A battery + needs 2.58 + 3 + 3 = 8.58 ft, so each gap falls short by at least 2.87 ft. That is more than + twice the largest window error (1.29 ft), so the error bars can't rescue it. The window bands + overlap end to end and cover every start on w1, [-42.1, 9.9]. +- **Beyond reach.** Left of the meter the route runs to the battery's right edge, so it is + -(start + 2.58) > 20 when start < -22.58. On w2 the route is start + 0.5 (one corner), over 20 + past start 19.5. Every start on w3 has route >= 21.52 + 2 x 0.5 = 22.52. Measured in a straight + line, w3's start is only 15.1 ft from the meter, but the door blocks starts up to 34.01, and a + battery starting there is 25.0 ft away in a straight line. So w3 fails under either reading of + "within 20 ft". +- **w2 must never pass**, because no band observes it. +- **The decision must be `manual_review`.** It can't be `pass`, because no start passes. It can't + be `reject` either. w2 is in reach and unseen, and a battery on it starting at 12.93 or later + clears window 5 on the nominal numbers. Window 5's edge is 1.98 ft from the corner and the + battery stands 1.83 ft proud, so the gap is sqrt(d^2 + 0.15^2), where d = start - 9.9; that + reaches 3 at d = 3.0. An area that could hold a valid spot was not seen. Both chain ends lie + beyond reach, so only w2 blocks a reject. +- The server at 44607e0 answered `manual_review` (unsure_checks, unobserved_area) with its spot on + w2 starting at 12.405. There the gap to window 5 is 2.47 ft. The check allows window 5's ± plus + w2's ± (0.40 + 0.13 = 0.53), and 2.47 + 0.53 reaches 3.0, so the check is UNSURE, not FAIL. That + agrees with the numbers above. diff --git a/verification/scenes/eth3d-facade/annotations.json b/verification/scenes/eth3d-facade/annotations.json new file mode 100644 index 000000000..b85a23899 --- /dev/null +++ b/verification/scenes/eth3d-facade/annotations.json @@ -0,0 +1,415 @@ +{ + "about": "Hand annotations for the ETH3D facade test scene. Read README.md first. Pixel coordinates are in the saved keyframes (the dataset images downscaled by `keyframes.scale`), origin at the image's top-left corner.", + "roi_scan_xy_m": [ + [ + -6.0, + -34.0 + ], + [ + 26.0, + -5.0 + ] + ], + "ground_fit_scan_xy_m": [ + [ + -2.0, + -27.0 + ], + [ + 20.0, + -12.0 + ] + ], + "walls": [ + { + "id": "w1", + "rough_scan_xy_m": [ + [ + 20.3, + -24.84 + ], + [ + 4.9, + -28.0 + ] + ] + }, + { + "id": "w2", + "rough_scan_xy_m": [ + [ + 4.9, + -28.0 + ], + [ + 4.16, + -24.43 + ] + ] + }, + { + "id": "w3", + "rough_scan_xy_m": [ + [ + 4.16, + -24.43 + ], + [ + -0.05, + -25.3 + ] + ] + } + ], + "chain_ends": { + "left": { + "kind": "unexplored", + "note": "w1 meets the main building in an inside corner and the wall continues along that building; not modelled" + }, + "right": { + "kind": "unexplored", + "note": "past the tower's west pilaster the wall steps back; the laser scan and the SfM points stop here, so it is not modelled" + } + }, + "keyframes": { + "scale": 0.25, + "images": [ + "DSC_0359", + "DSC_0360", + "DSC_0361", + "DSC_0362", + "DSC_0363", + "DSC_0419", + "DSC_0420" + ] + }, + "meter": { + "wall_id": "w1", + "from_wall_left_end_ft": 42.1, + "height_ft": 5.0, + "plus_minus_ft": 0.3, + "note": "INVENTED: this building has no electric meter here. Placed midway in the gap between w1 windows 4 and 5." + }, + "min_plus_minus_ft": 0.1, + "single_view_plus_minus_ft": 0.5, + "features": [ + { + "label": "w1 window 1", + "type": "window", + "wall_id": "w1", + "left_is_bottom_left_corner_in": [ + "DSC_0359", + "DSC_0360" + ], + "points_px": { + "DSC_0359": { + "left": [ + 1104, + 911 + ], + "right": [ + 1180, + 914 + ], + "bottom": [ + 1142, + 912 + ], + "top": [ + 1135, + 776 + ] + }, + "DSC_0360": { + "left": [ + 660, + 905 + ], + "right": [ + 718, + 905 + ], + "bottom": [ + 690, + 905 + ], + "top": [ + 690, + 765 + ] + } + } + }, + { + "label": "w1 window 2", + "type": "window", + "wall_id": "w1", + "left_is_bottom_left_corner_in": [ + "DSC_0360", + "DSC_0361" + ], + "points_px": { + "DSC_0360": { + "left": [ + 789, + 907 + ], + "right": [ + 869, + 909 + ], + "bottom": [ + 830, + 908 + ], + "top": [ + 835, + 755 + ] + }, + "DSC_0361": { + "left": [ + 190, + 818 + ], + "right": [ + 292, + 818 + ], + "bottom": [ + 240, + 818 + ], + "top": [ + 264, + 645 + ] + } + } + }, + { + "label": "w1 window 3", + "type": "window", + "wall_id": "w1", + "left_is_bottom_left_corner_in": [ + "DSC_0361", + "DSC_0419" + ], + "points_px": { + "DSC_0361": { + "left": [ + 402, + 822 + ], + "right": [ + 508, + 822 + ], + "bottom": [ + 455, + 822 + ], + "top": [ + 465, + 645 + ] + }, + "DSC_0419": { + "left": [ + 234, + 848 + ], + "right": [ + 365, + 848 + ], + "bottom": [ + 300, + 848 + ], + "top": [ + 315, + 654 + ] + } + } + }, + { + "label": "w1 window 4", + "type": "window", + "wall_id": "w1", + "left_is_bottom_left_corner_in": [ + "DSC_0419", + "DSC_0420" + ], + "points_px": { + "DSC_0419": { + "left": [ + 487, + 848 + ], + "right": [ + 616, + 848 + ], + "bottom": [ + 550, + 848 + ], + "top": [ + 556, + 653 + ] + }, + "DSC_0420": { + "left": [ + 43, + 825 + ], + "right": [ + 218, + 824 + ], + "bottom": [ + 130, + 824 + ], + "top": [ + 140, + 605 + ] + }, + "DSC_0361": { + "left": [ + 614, + 823 + ], + "right": [ + 730, + 823 + ], + "bottom": [ + 672, + 823 + ], + "top": [ + 680, + 647 + ] + } + } + }, + { + "label": "w1 window 5", + "type": "window", + "wall_id": "w1", + "left_is_bottom_left_corner_in": [ + "DSC_0419", + "DSC_0420" + ], + "points_px": { + "DSC_0419": { + "left": [ + 733, + 842 + ], + "right": [ + 848, + 842 + ], + "bottom": [ + 790, + 842 + ], + "top": [ + 791, + 653 + ] + }, + "DSC_0420": { + "left": [ + 368, + 822 + ], + "right": [ + 505, + 818 + ], + "bottom": [ + 436, + 820 + ], + "top": [ + 440, + 622 + ] + }, + "DSC_0361": { + "left": [ + 833, + 826 + ], + "right": [ + 952, + 827 + ], + "bottom": [ + 892, + 826 + ], + "top": [ + 895, + 645 + ] + } + } + }, + { + "label": "w3 entrance door", + "type": "door", + "wall_id": "w3", + "points_px": { + "DSC_0419": { + "left": [ + 1020, + 700 + ], + "right": [ + 1226, + 700 + ], + "bottom": [ + 1150, + 958 + ], + "top": [ + 1122, + 567 + ] + }, + "DSC_0420": { + "left": [ + 666, + 720 + ], + "right": [ + 855, + 720 + ], + "bottom": [ + 760, + 919 + ], + "top": [ + 760, + 547 + ] + } + }, + "plus_minus_floor_ft": { + "value": 0.7, + "why": "The door sits 1.2 ft behind the tower face, with its brick jambs 0.5 ft back, and both photos view it from the east. They agree to 0.19 ft, but the laser scan puts the door's right edge 0.64 ft further right (s = 31.65 vs 31.01), so their agreement does not bound the error." + } + } + ] +} diff --git a/verification/scenes/eth3d-facade/build_scene.py b/verification/scenes/eth3d-facade/build_scene.py new file mode 100644 index 000000000..5d3d0b94b --- /dev/null +++ b/verification/scenes/eth3d-facade/build_scene.py @@ -0,0 +1,941 @@ +"""Build a real-geometry test scene.json from the ETH3D "facade" dataset. + +Reads the laser scans, the COLMAP poses of the undistorted DSLR images and the hand annotations in +annotations.json next to this file, then writes scene.json, the keyframe JPEGs it references and +scene.zip into the output folder. See README.md for what every number means. + + uv run python scenes/eth3d-facade/build_scene.py # from verification/ + uv run python scenes/eth3d-facade/build_scene.py --schema path/to/scene.schema.json + +Frames. The scan and the COLMAP poses share one metric frame (scan z is up to within a degree). +The scene frame is ARKit-like: y is the fitted ground normal, so the ground is y = 0, x and z are +horizontal, right-handed, and every length is in feet. The origin is the ground point under the +hypothetical meter. +""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import sys +import zipfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +from PIL import Image, ImageDraw + +HERE = Path(__file__).resolve().parent +DEFAULT_DATA = Path.home() / "house-scanning-data/verify/eth3d/facade" +DEFAULT_OUT = Path.home() / "house-scanning-data/verify/scenes/eth3d-facade" + +M_PER_FT = 0.3048 # exact by definition +RNG_SEED = 7 # fixed so RANSAC, and therefore every number in README.md, is reproducible + +# Coverage: a surface point counts as seen by a keyframe only if it projects inside the image, is +# on the camera side of its wall, is not hidden behind another wall of the chain, and is viewed +# within this angle of the wall normal. 75 degrees is a judgement call, not a calibrated value: +# at 75 degrees one pixel spans about 4x its head-on width on the wall, and a box drawn there +# moves by feet for a few pixels of error. +MAX_VIEW_ANGLE_DEG = 75.0 +COVERAGE_STEP_FT = 0.25 +BATTERY_HEIGHT_FT = 39.5 / 12 # Base Core height, docs/01 rules table; the wall band's top probe +HEADROOM_FT = 6.5 # NEC 110.26 headroom, docs/01; the overhead band's top probe +GROUND_OUT_FT = 6.0 # how far out the ground band claims to have been seen +OVERHEAD_OUT_FT = 3.0 # overhead search depth in front of the wall +OVERHEAD_MAX_FT = 12.0 # anything higher than this is not reported as an overhead +OVERHEAD_MIN_FT = 4.0 # see scan_profiles +FACING_MAX_FT = 60.0 + + +# -------------------------------------------------------------------------------------------- +# Reading the dataset + + +def read_alignment(path: Path) -> dict[str, np.ndarray]: + txt = path.read_text() + out = { + m.group(1): np.array(m.group(2).split(), float).reshape(4, 4) + for m in re.finditer(r'filename="([^"]+)">\s*([^<]+)<', txt) + } + if len(out) != 3: + raise SystemExit(f"expected 3 scans in {path}, found {sorted(out)}") + return out + + +def load_scan_crop(scan_dir: Path, roi_xy: list[list[float]]) -> np.ndarray: + """Laser points inside the plan box roi_xy = [[xmin, ymin], [xmax, ymax]], scan frame, m.""" + (x0, y0), (x1, y1) = roi_xy + out = [] + for name, T in read_alignment(scan_dir / "scan_alignment.mlp").items(): + with open(scan_dir / name, "rb") as f: + header = b"" + while not header.endswith(b"end_header\n"): + line = f.readline() + if not line: + raise SystemExit(f"{name}: no end_header") + header += line + if b"binary_little_endian" not in header or not re.search( + rb"element vertex \d+\nproperty float x\nproperty float y\nproperty float z\nelement", + header, + ): + raise SystemExit(f"{name}: expected little-endian float x,y,z vertices first") + n = int(re.search(rb"element vertex (\d+)", header).group(1)) + X = np.memmap(scan_dir / name, dtype=" x0) & (x[:, 0] < x1) & (x[:, 1] > y0) & (x[:, 1] < y1) + out.append(x[keep]) + return np.concatenate(out) + + +def qvec_to_rot(q: np.ndarray) -> np.ndarray: + w, x, y, z = q / np.linalg.norm(q) + return np.array( + [ + [1 - 2 * (y * y + z * z), 2 * (x * y - w * z), 2 * (x * z + w * y)], + [2 * (x * y + w * z), 1 - 2 * (x * x + z * z), 2 * (y * z - w * x)], + [2 * (x * z - w * y), 2 * (y * z + w * x), 1 - 2 * (x * x + y * y)], + ] + ) + + +@dataclass +class ColmapImage: + name: str # stem, e.g. DSC_0419 + R: np.ndarray # world-to-camera rotation, OpenCV camera (+x right, +y down, +z forward) + t: np.ndarray + cam: np.ndarray # fx, fy, cx, cy of the full-size undistorted image + w: int + h: int + uv: np.ndarray # observed keypoints, full-size pixels + pid: np.ndarray # their 3D point ids, -1 when untriangulated + + +def read_colmap(calib: Path) -> dict[str, ColmapImage]: + cams = {} + for ln in (calib / "cameras.txt").read_text().splitlines(): + if ln.startswith("#") or not ln.strip(): + continue + p = ln.split() + if p[1] != "PINHOLE": + raise SystemExit( + f"camera {p[0]} is {p[1]}; this script only handles undistorted PINHOLE" + ) + cams[int(p[0])] = (int(p[2]), int(p[3]), np.array(p[4:8], float)) + lines = [ln for ln in (calib / "images.txt").read_text().splitlines() if not ln.startswith("#")] + out = {} + for head, pts in zip(lines[0::2], lines[1::2], strict=True): + p = head.split() + w, h, k = cams[int(p[8])] + a = np.array(pts.split(), float).reshape(-1, 3) if pts.strip() else np.zeros((0, 3)) + name = Path(p[9]).stem + out[name] = ColmapImage( + name, + qvec_to_rot(np.array(p[1:5], float)), + np.array(p[5:8], float), + k, + w, + h, + a[:, :2], + a[:, 2].astype(int), + ) + return out + + +def read_points3d(calib: Path) -> dict[int, np.ndarray]: + pts = {} + for ln in (calib / "points3D.txt").read_text().splitlines(): + if ln.startswith("#") or not ln.strip(): + continue + p = ln.split() + pts[int(p[0])] = np.array(p[1:4], float) + return pts + + +# -------------------------------------------------------------------------------------------- +# Geometry + + +def ransac_plane(X, thr, rng, iters=600, accept=lambda n: True): + Xs = X[rng.integers(0, len(X), min(len(X), 40000))] + best_k, best = -1, None + for _ in range(iters): + s = Xs[rng.integers(0, len(Xs), 3)] + n = np.cross(s[1] - s[0], s[2] - s[0]) + if np.linalg.norm(n) < 1e-9: + continue + n /= np.linalg.norm(n) + if not accept(n): + continue + k = int((np.abs((Xs - s[0]) @ n) < thr).sum()) + if k > best_k: + best_k, best = k, (n, s[0]) + if best is None: + raise SystemExit("plane RANSAC found no acceptable plane") + inl = np.abs((X - best[1]) @ best[0]) < thr + c = X[inl].mean(0) + n = np.linalg.svd(X[inl] - c, full_matrices=False)[2][-1] + return n, c, inl + + +def ransac_line2d(Q, thr, rng, iters=400): + """Line through 2D points: returns (point, unit direction, inlier mask).""" + best_k, best = -1, None + for _ in range(iters): + a, b = Q[rng.integers(0, len(Q), 2)] + d = b - a + if np.linalg.norm(d) < 0.2: + continue + d /= np.linalg.norm(d) + nrm = np.array([-d[1], d[0]]) + k = int((np.abs((Q - a) @ nrm) < thr).sum()) + if k > best_k: + best_k, best = k, (a, nrm) + inl = np.abs((Q - best[0]) @ best[1]) < thr + c = Q[inl].mean(0) + d = np.linalg.svd(Q[inl] - c, full_matrices=False)[2][0] + return c, d, inl + + +def intersect_lines(p1, d1, p2, d2): + A = np.array([d1, -d2]).T + t = np.linalg.solve(A, p2 - p1) + return p1 + t[0] * d1 + + +@dataclass +class Frame: + """Scan frame (m, z up) -> scene frame (ft, y = ground normal).""" + + A: np.ndarray # rows are the scene x, y, z axes expressed in scan coordinates + origin: np.ndarray # scan-frame point that becomes the scene origin + + def pts(self, X): + return (np.asarray(X) - self.origin) @ self.A.T / M_PER_FT + + def dirs(self, V): + return np.asarray(V) @ self.A.T + + +def make_frame(ground_n: np.ndarray, ground_c: np.ndarray) -> Frame: + y = ground_n / np.linalg.norm(ground_n) + x = np.array([1.0, 0, 0]) - y[0] * y # scan x made horizontal + x /= np.linalg.norm(x) + z = np.cross(x, y) # right-handed: x cross y = z + return Frame(np.array([x, y, z]), ground_c.copy()) + + +@dataclass +class Wall: + id: str + p0: np.ndarray # plan [x, z] ft, left end as seen from outside + p1: np.ndarray + s0: float = 0.0 # unrolled position of p0 along the chain before the meter shift + height_ft: float = 0.0 + fit_rms_ft: float = 0.0 + # Signed distance (along `outward`) from the baseline to the wall face 4-8 ft up. The baseline + # is fitted over 0.3-8 ft and lands on the stone plinth, which stands proud of the brick; the + # windows sit in the brick, so clicks are cast onto this face, not the baseline. + face_offset_ft: float = 0.0 + + @property + def length(self): + return float(np.linalg.norm(self.p1 - self.p0)) + + @property + def along(self): + return (self.p1 - self.p0) / self.length + + @property + def outward(self): + # scene.schema.json: outward is the baseline direction turned 90 degrees clockwise seen + # from +y. With y up, x right on screen puts +z down-screen, so (dx, dz) -> (-dz, dx). + dx, dz = self.along + return np.array([-dz, dx]) + + def s_of(self, q): # q: plan points (..., 2) + return self.s0 + (np.asarray(q) - self.p0) @ self.along + + def plan_at(self, s): + return self.p0 + np.multiply.outer(np.asarray(s) - self.s0, self.along) + + +# -------------------------------------------------------------------------------------------- +# Cameras in the scene frame + + +@dataclass +class Keyframe: + id: str + name: str + R_c2w: np.ndarray # ARKit camera axes (+x right, +y up, looks along -z) in scene axes + C: np.ndarray # camera centre, scene ft + K: np.ndarray # fx, fy, cx, cy for the saved (downscaled) image + w: int + h: int + + def project(self, X): + """ARKit convention: a camera-space point (x, y, z) with z < 0 maps to + u = cx + fx * x / -z, v = cy - fy * y / -z (v grows downward in the image).""" + Xc = (np.asarray(X) - self.C) @ self.R_c2w # = R_c2w^T (X - C), row form + z = -Xc[..., 2] + fx, fy, cx, cy = self.K + with np.errstate(divide="ignore", invalid="ignore"): + u = cx + fx * Xc[..., 0] / z + v = cy - fy * Xc[..., 1] / z + return u, v, z + + def ray(self, u, v): + """World direction of pixel (u, v): camera ray ((u-cx)/fx, -(v-cy)/fy, -1).""" + fx, fy, cx, cy = self.K + d = np.array([(u - cx) / fx, -(v - cy) / fy, -1.0]) + return self.R_c2w @ d + + def pose16(self): + T = np.eye(4) + T[:3, :3] = self.R_c2w + T[:3, 3] = self.C + return [round(float(v), 6) for v in T.flatten(order="F")] # column-major + + +OPENCV_TO_ARKIT_CAM = np.diag([1.0, -1.0, -1.0]) # flip y (down -> up) and z (forward -> back) + + +def make_keyframe(img: ColmapImage, frame: Frame, scale: float, kid: str) -> Keyframe: + w, h = round(img.w * scale), round(img.h * scale) + sx, sy = w / img.w, h / img.h + fx, fy, cx, cy = img.cam + # Pixel coordinates are continuous with (0, 0) at the image's top-left corner, so a resize + # scales focal lengths and principal point alike. + K = np.array([fx * sx, fy * sy, cx * sx, cy * sy]) + R_c2w_scan = img.R.T @ OPENCV_TO_ARKIT_CAM + C_scan = -img.R.T @ img.t + return Keyframe(kid, img.name, frame.A @ R_c2w_scan, frame.pts(C_scan), K, w, h) + + +# -------------------------------------------------------------------------------------------- +# Scene pieces + + +def fit_ground(P, roi_ground, rng): + (x0, y0), (x1, y1) = roi_ground + zmed = np.median(P[:, 2]) + m = (P[:, 0] > x0) & (P[:, 0] < x1) & (P[:, 1] > y0) & (P[:, 1] < y1) & (P[:, 2] < zmed) + n, c, inl = ransac_plane(P[m], 0.03, rng, accept=lambda n: abs(n[2]) > np.cos(np.radians(10))) + n = n if n[2] > 0 else -n + rms = float(np.sqrt(np.mean(((P[m][inl] - c) @ n) ** 2))) + return ( + n, + c, + { + "inliers": int(inl.sum()), + "rms_m": rms, + "tilt_from_scan_z_deg": float(np.degrees(np.arccos(n[2]))), + }, + ) + + +def fit_walls(P_scene, ann_walls, frame, rng): + """Fit each wall's plan line to scan points near its rough segment, then join neighbours.""" + lines, stats = [], [] + h = P_scene[:, 1] + plan = P_scene[:, [0, 2]] + for w in ann_walls: + # rough ends are plan points read off a top view; their scan z does not matter beyond + # a few centimetres because the scene y axis is within 1.2 degrees of scan z + a, b = frame.pts(np.c_[np.array(w["rough_scan_xy_m"]), np.zeros(2)])[:, [0, 2]] + d = (b - a) / np.linalg.norm(b - a) + nrm = np.array([-d[1], d[0]]) + t = (plan - a) @ d + L = np.linalg.norm(b - a) + trim = 1.0 # ft kept away from the rough ends so neighbouring walls do not leak in + m = (np.abs((plan - a) @ nrm) < 1.0) & (t > trim) & (t < L - trim) & (h > 0.3) & (h < 8.0) + if m.sum() < 1000: + raise SystemExit(f"wall {w['id']}: only {m.sum()} scan points near its rough segment") + c, dd, inl = ransac_line2d(plan[m], 0.1, rng) + if dd @ d < 0: + dd = -dd + res = (plan[m][inl] - c) @ np.array([-dd[1], dd[0]]) + near = (np.abs((plan - c) @ np.array([-dd[1], dd[0]])) < 0.15) & (t > trim) & (t < L - trim) + lines.append((c, dd, a, b)) + stats.append( + { + "rms_ft": float(np.sqrt(np.mean(res**2))), + "inliers": int(inl.sum()), + "height_ft": float(np.percentile(h[near], 99.5)), + } + ) + walls = [] + for i, (w, (c, d, a, b), st) in enumerate(zip(ann_walls, lines, stats, strict=True)): + if i > 0: + p0 = intersect_lines(*lines[i - 1][:2], c, d) + else: + p0 = extent_end(plan, h, c, d, a, side=0) + if i < len(lines) - 1: + p1 = intersect_lines(c, d, *lines[i + 1][:2]) + else: + p1 = extent_end(plan, h, c, d, b, side=1) + walls.append( + Wall(w["id"], p0, p1, height_ft=round(st["height_ft"], 1), fit_rms_ft=st["rms_ft"]) + ) + for w, st in zip(walls, stats, strict=True): + lat = (plan - w.p0) @ w.along + out = (plan - w.p0) @ w.outward + k = (lat > 1.0) & (lat < w.length - 1.0) & (h > 4.0) & (h < 8.0) & (np.abs(out) < 1.0) + hist, edges = np.histogram(out[k], bins=np.arange(-1.0, 1.0001, 0.02)) + w.face_offset_ft = float(edges[hist.argmax()] + 0.01) + st["face_offset_ft"] = w.face_offset_ft + s = 0.0 + for w in walls: + w.s0 = s + s += w.length + return walls, stats + + +def extent_end(plan, h, c, d, rough_end, side): + """Free chain end: the last scanned wall point along the fitted line near the rough end.""" + nrm = np.array([-d[1], d[0]]) + t_rough = (rough_end - c) @ d + m = ( + (np.abs((plan - c) @ nrm) < 0.15) + & (h > 0.3) + & (h < 8.0) + & (np.abs((plan - c) @ d - t_rough) < 4) + ) + t = (plan[m] - c) @ d + te = np.percentile(t, 99.8) if side == 1 else np.percentile(t, 0.2) + return c + te * d + + +def chain_segments(walls): + return [(w.p0, w.p1, w.id) for w in walls] + + +def seg_cross(a, b, c, d): + def orient(p, q, r): + return np.sign((q[0] - p[0]) * (r[1] - p[1]) - (q[1] - p[1]) * (r[0] - p[0])) + + return orient(a, b, c) * orient(a, b, d) < 0 and orient(c, d, a) * orient(c, d, b) < 0 + + +def visible(kf: Keyframe, wall: Wall, X, walls) -> bool: + """X: scene point on or in front of `wall`. See MAX_VIEW_ANGLE_DEG for the criteria.""" + u, v, z = kf.project(X) + if not (z > 0.5 and 0 <= u < kf.w and 0 <= v < kf.h): + return False + to_cam = kf.C - X + n3 = np.array([wall.outward[0], 0.0, wall.outward[1]]) + cosang = (to_cam @ n3) / np.linalg.norm(to_cam) + if cosang < np.cos(np.radians(MAX_VIEW_ANGLE_DEG)): + return False + a, b = kf.C[[0, 2]], X[[0, 2]] + for w in walls: + if w is wall: + continue + # shrink the other wall slightly so a shared corner does not count as a crossing + e = 0.02 * (w.p1 - w.p0) + if seg_cross(a, b, w.p0 + e, w.p1 - e): + return False + return True + + +def band_probes(band: str, wall: Wall, s: float, facing_depth: float | None): + q = wall.plan_at(s) + o = wall.outward + + def at(out, y): + p = q + out * o + return np.array([p[0], y, p[1]]) + + if band == "wall": + return [at(0.0, 0.0), at(0.0, BATTERY_HEIGHT_FT)] + if band == "ground": + return [at(0.0, 0.0), at(GROUND_OUT_FT, 0.0)] + if band == "overhead": + return [at(0.0, HEADROOM_FT), at(OVERHEAD_OUT_FT, HEADROOM_FT), at(OVERHEAD_OUT_FT, 0.0)] + if band == "facing": + if facing_depth is None: + return None + return [at(0.0, 3.0), at(min(facing_depth, 20.0), 3.0)] + raise ValueError(band) + + +def intervals(ss, ok, step): + """Sample centres ss with flags ok -> [start, end] spans of consecutive True samples.""" + out = [] + for s, k in zip(ss, ok, strict=True): + if not k: + continue + if out and abs(out[-1][1] - (s - step / 2)) < 1e-9: + out[-1][1] = s + step / 2 + else: + out.append([s - step / 2, s + step / 2]) + return out + + +WALL_RELIEF_FT = 1.0 # pilasters, sills and downspouts stand out less than this; see scan_profiles +BIN_INSET_FT = 0.15 # keeps a neighbouring wall that meets this one at a corner out of the end bins + + +def scan_profiles(P_scene, walls): + """Per 1 ft of chain: the lowest thing overhead near the wall and the straight-out gap. + + Overhead: something with open space under it. Points within OVERHEAD_OUT_FT of the wall are + binned into 0.25 ft cells in plan; a cell whose lowest point is above OVERHEAD_MIN_FT does not + reach down, so its lowest point is a clearance. Cells that reach lower (a pilaster, the + neighbouring wall at a corner) are obstacles, not overheads. With a 2 ft floor the tops of + the two bins by the door and the downspout shoe read as 2.1-2.5 ft overheads, because the + scanner saw their tops but not their lower parts; the photos show nothing hanging over this + wall below the tower's bay window, so the floor is 4 ft. A real overhang lower than that + would be missed. + Facing: the nearest point more than WALL_RELIEF_FT out, between 1 and 6 ft high. The wall's + own relief is excluded so it does not read as a fence; the cost is that an obstacle within + 1 ft of the wall is not reported.""" + plan, h = P_scene[:, [0, 2]], P_scene[:, 1] + over, facing = [], [] + for w in walls: + lat = (plan - w.p0) @ w.along + out = (plan - w.p0) @ w.outward + near = ( + (out > 0.3) + & (out < OVERHEAD_OUT_FT) + & (h > 0.5) + & (h < OVERHEAD_MAX_FT) + & (lat > 0) + & (lat < w.length) + ) + cell = np.floor(np.c_[lat[near], out[near]] / 0.25).astype(int) + cell_min, cell_n = {}, {} + for c, hh in zip(map(tuple, cell), h[near], strict=True): + cell_min[c] = min(cell_min.get(c, np.inf), hh) + cell_n[c] = cell_n.get(c, 0) + 1 + for a in np.arange(0.0, w.length, 1.0): + b = min(a + 1.0, w.length) + k = (lat >= a + BIN_INSET_FT) & (lat < b - BIN_INSET_FT) + lows = [ + v + for c, v in cell_min.items() + if a <= (c[0] + 0.5) * 0.25 < b and v > OVERHEAD_MIN_FT and cell_n[c] >= 5 + ] + over.append((w.id, w.s0 + a, w.s0 + b, float(min(lows)) if lows else None)) + kf = k & (out > WALL_RELIEF_FT) & (out < FACING_MAX_FT) & (h > 1.0) & (h < 6.0) + if kf.sum() > 20: + depth = float(out[kf].min()) + elif ( + k & (np.abs(h) < 0.5) & (out > FACING_MAX_FT - 5) & (out < FACING_MAX_FT) + ).sum() > 20: + # the scan saw open ground that far out and nothing standing before it: report + # the cap, a lower bound, which is safe for an "at least" check + depth = FACING_MAX_FT + else: + depth = None + facing.append((w.id, w.s0 + a, w.s0 + b, depth)) + return over, facing + + +def merge_runs(rows, key): + """(wall_id, s_a, s_b, value) rows -> merged runs, grouped by key(value), keeping the min.""" + out = [] + for wid, a, b, v in rows: + if v is None: + continue + if out and out[-1][0] == wid and abs(out[-1][2] - a) < 1e-6 and key(out[-1][3]) == key(v): + out[-1] = (wid, out[-1][1], b, min(out[-1][3], v)) + else: + out.append((wid, a, b, v)) + return out + + +# -------------------------------------------------------------------------------------------- +# Annotations + + +def backproject(kf: Keyframe, wall: Wall, uv): + """Intersect pixel rays with the wall's vertical plane. Returns scene points (n, 3).""" + n3 = np.array([wall.outward[0], 0.0, wall.outward[1]]) + face = wall.p0 + wall.face_offset_ft * wall.outward + P0 = np.array([face[0], 0.0, face[1]]) + out = [] + for u, v in uv: + d = kf.ray(u, v) + den = d @ n3 + if abs(den) < 1e-6: + raise SystemExit(f"{kf.name}: ray parallel to wall {wall.id}") + t = ((P0 - kf.C) @ n3) / den + if t <= 0: + raise SystemExit(f"{kf.name}: wall {wall.id} is behind the camera at pixel {u, v}") + out.append(kf.C + t * d) + return np.array(out) + + +POINT_ROLES = ("left", "right", "bottom", "top") + + +def measure_feature(feat, kfs_by_name, walls_by_id): + """Each image gives [s_left, s_right, bottom, top]. Every value comes from one clicked point + cast onto the wall plane: a point anywhere on the left jamb gives s_left, on the right jamb + s_right, on the sill or threshold the bottom height, at the arch apex the top height. One point + per value means a hidden corner or a leaning vertical in a tilted photo does not matter.""" + wall = walls_by_id[feat["wall_id"]] + per_image = {} + for img, pts in feat["points_px"].items(): + if sorted(pts) != sorted(POINT_ROLES): + raise SystemExit( + f"{feat['label']} @ {img}: need exactly the points {POINT_ROLES}, got {sorted(pts)}" + ) + X = backproject(kfs_by_name[img], wall, [pts[r] for r in POINT_ROLES]) + s = wall.s_of(X[:, [0, 2]]) + per_image[img] = np.array([s[0], s[1], X[2, 1], X[3, 1]]) + if not (per_image[img][0] < per_image[img][1] and per_image[img][2] < per_image[img][3]): + raise SystemExit(f"{feat['label']} @ {img}: left/right or bottom/top came out swapped") + vals = np.array(list(per_image.values())) + mean = vals.mean(0) + spread = float(np.max(vals.max(0) - vals.min(0))) if len(vals) >= 2 else None + return wall, mean, spread, per_image + + +# -------------------------------------------------------------------------------------------- + + +def main(): + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("--data", type=Path, default=DEFAULT_DATA, help="extracted ETH3D facade folder") + ap.add_argument("--out", type=Path, default=DEFAULT_OUT) + ap.add_argument("--annotations", type=Path, default=HERE / "annotations.json") + ap.add_argument("--schema", type=Path, help="validate scene.json against this JSON Schema") + ap.add_argument( + "--overlays", + action="store_true", + help="also write keyframes with the annotations and the wall chain drawn on", + ) + args = ap.parse_args() + + ann = json.loads(args.annotations.read_text()) + rng = np.random.default_rng(RNG_SEED) + report = {} + + # 1. Laser scan, ground plane, scene frame --------------------------------------------- + P = load_scan_crop(args.data / "dslr_scan_eval", ann["roi_scan_xy_m"]) + g_n, g_c, g_stats = fit_ground(P, ann["ground_fit_scan_xy_m"], rng) + frame = make_frame(g_n, g_c) + report["ground"] = g_stats + + # 2. Wall chain (first pass with a provisional origin) --------------------------------- + P_scene = frame.pts(P) + walls, wall_stats = fit_walls(P_scene, ann["walls"], frame, rng) + + # 3. Meter: hypothetical, placed on a wall at a documented distance from that wall's left end + mt = ann["meter"] + wm = next(w for w in walls if w.id == mt["wall_id"]) + if not 0 <= mt["from_wall_left_end_ft"] <= wm.length: + raise SystemExit(f"meter is off wall {wm.id} (length {wm.length:.2f} ft)") + foot_plan = wm.p0 + mt["from_wall_left_end_ft"] * wm.along + # move the origin to the ground point under the meter and redo everything in that frame + foot_scan = frame.origin + frame.A.T @ (np.array([foot_plan[0], 0.0, foot_plan[1]]) * M_PER_FT) + frame = Frame(frame.A, foot_scan) + P_scene = frame.pts(P) + rng = np.random.default_rng(RNG_SEED) + walls, wall_stats = fit_walls(P_scene, ann["walls"], frame, rng) + wm = next(w for w in walls if w.id == mt["wall_id"]) + s_meter = wm.s0 + mt["from_wall_left_end_ft"] + for w in walls: + w.s0 -= s_meter # s = 0 at the meter from here on + walls_by_id = {w.id: w for w in walls} + meter_pos = np.array([*(wm.p0 + mt["from_wall_left_end_ft"] * wm.along), mt["height_ft"]])[ + [0, 2, 1] + ] + report["walls"] = [ + { + "id": w.id, + "length_ft": w.length, + "length_m": w.length * M_PER_FT, + "s_range_ft": [w.s0, w.s0 + w.length], + "fit": st, + } + for w, st in zip(walls, wall_stats, strict=True) + ] + + # 4. Keyframes ------------------------------------------------------------------------ + colmap = read_colmap(args.data / "dslr_calibration_undistorted") + scale = ann["keyframes"]["scale"] + kfs = [ + make_keyframe(colmap[n], frame, scale, f"k{i + 1}") + for i, n in enumerate(ann["keyframes"]["images"]) + ] + kfs_by_name = {k.name: k for k in kfs} + for w in walls: + cams_in_front = [k.name for k in kfs if (k.C[[0, 2]] - w.p0) @ w.outward > 0] + if not cams_in_front: + raise SystemExit( + f"wall {w.id}: no keyframe on its outward side; baseline order is wrong" + ) + + # Convention check: COLMAP's own triangulated points, moved into the scene frame and + # projected with the ARKit formula, must land on the keypoints COLMAP observed them at. + pts3d = read_points3d(args.data / "dslr_calibration_undistorted") + conv = {} + for k in kfs: + im = colmap[k.name] + sel = im.pid >= 0 + X = frame.pts(np.array([pts3d[i] for i in im.pid[sel]])) + u, v, z = k.project(X) + sx, sy = k.w / im.w, k.h / im.h + err = np.hypot(u - im.uv[sel, 0] * sx, v - im.uv[sel, 1] * sy) + conv[k.name] = { + "points": int(sel.sum()), + "all_in_front": bool((z > 0).all()), + "median_px": float(np.median(err)), + "p95_px": float(np.percentile(err, 95)), + } + report["arkit_projection_check"] = conv + if any(c["median_px"] > 1.0 or not c["all_in_front"] for c in conv.values()): + raise SystemExit(f"pose convention check failed: {conv}") + + # 5. Objects -------------------------------------------------------------------------- + objects, feature_rows = [], [] + for feat in ann["features"]: + wall, mean, spread, per_image = measure_feature(feat, kfs_by_name, walls_by_id) + # plus_minus_ft = max(0.1 ft, largest disagreement between images over the four values, + # the feature's documented floor). The floor exists for cases where the images share a + # bias that their agreement cannot show (see the door in annotations.json). + pm = max( + ann["min_plus_minus_ft"], + spread if spread is not None else ann["single_view_plus_minus_ft"], + feat.get("plus_minus_floor_ft", {}).get("value", 0.0), + ) + obj = { + "type": feat["type"], + "wall_id": wall.id, + "span_ft": [round(float(mean[0]), 2), round(float(mean[1]), 2)], + "bottom_ft": round(max(0.0, float(mean[2])), 2), + "top_ft": round(float(mean[3]), 2), + "source": "tap", + "plus_minus_ft": round(float(pm), 2), + } + if "attrs" in feat: + obj["attrs"] = feat["attrs"] + objects.append(obj) + feature_rows.append( + { + "label": feat["label"], + **obj, + "per_image": {k: [round(float(x), 2) for x in v] for k, v in per_image.items()}, + "spread_ft": spread, + } + ) + + # 6. Overheads and facing from the scan, coverage from the keyframes ----------------------- + over, facing = scan_profiles(P_scene, walls) # walls are already in meter-relative s + over_runs = merge_runs(over, key=lambda v: round(v)) + # facing depths are grouped in 2 ft steps (everything past 20 ft in one group) so the list + # stays short; each run reports the smallest depth inside it + face_runs = merge_runs(facing, key=lambda v: min(int(v // 2), 10)) + + observed = [] + per_band = {} + for band in ("wall", "ground", "overhead", "facing"): + spans = [] + for w in walls: + ss = np.arange(w.s0 + COVERAGE_STEP_FT / 2, w.s0 + w.length, COVERAGE_STEP_FT) + ok = [] + for s in ss: + fd = next((v for (wid, a, b, v) in facing if wid == w.id and a <= s < b), None) + probes = band_probes(band, w, s, fd) + ok.append( + probes is not None + and any(all(visible(k, w, X, walls) for X in probes) for k in kfs) + ) + spans += [(w.id, a, b) for a, b in intervals(ss, ok, COVERAGE_STEP_FT)] + # join spans that touch across a corner + merged = [] + for _, a, b in spans: + if merged and abs(merged[-1][1] - a) < 1e-6: + merged[-1][1] = b + else: + merged.append([a, b]) + per_band[band] = merged + for a, b in merged: + entry = {"band": band, "span_ft": [round(a, 2), round(b, 2)]} + if band == "ground": + entry["out_ft"] = GROUND_OUT_FT + observed.append(entry) + + # 7. Assemble ---------------------------------------------------------------------------- + r3 = lambda a: [round(float(x), 3) for x in a] # noqa: E731 + scene = { + "schema_version": "1.0", + "meter": {"pos": r3(meter_pos), "wall_id": wm.id, "plus_minus_ft": mt["plus_minus_ft"]}, + "walls": [ + { + "id": w.id, + "baseline": [r3(w.p0), r3(w.p1)], + "height_ft": w.height_ft, + "plus_minus_ft": round(max(0.1, 3 * w.fit_rms_ft), 2), + } + for w in walls + ], + "objects": objects, + "overheads": [ + { + "wall_id": wid, + "span_ft": [round(a, 2), round(b, 2)], + "clearance_ft": round(v, 2), + "plus_minus_ft": 0.1, + } + for wid, a, b, v in over_runs + ], + "facing": [ + { + "wall_id": wid, + "span_ft": [round(a, 2), round(b, 2)], + "depth_ft": round(v, 2), + "plus_minus_ft": 0.1, + } + for wid, a, b, v in face_runs + ], + "coverage": {"ends": ann["chain_ends"], "observed": observed}, + "keyframes": [ + { + "id": k.id, + "pose": k.pose16(), + "intrinsics": [round(float(x), 4) for x in k.K], + "w": k.w, + "h": k.h, + "img": f"{k.id}.jpg", + } + for k in kfs + ], + } + + # 8. Write ---------------------------------------------------------------------------------- + out = args.out + if out.exists(): + shutil.rmtree(out) + (out / "bundle").mkdir(parents=True) + img_dir = args.data / "images/dslr_images_undistorted" + for k in kfs: + im = Image.open(img_dir / f"{k.name}.JPG") + im.draft("RGB", (k.w, k.h)) + im.convert("RGB").resize((k.w, k.h), Image.Resampling.LANCZOS).save( + out / "bundle" / f"{k.id}.jpg", quality=90 + ) + (out / "bundle/scene.json").write_text(json.dumps(scene, indent=1) + "\n") + shutil.copy(out / "bundle/scene.json", out / "scene.json") + with zipfile.ZipFile(out / "scene.zip", "w", zipfile.ZIP_DEFLATED) as z: + for f in sorted((out / "bundle").iterdir()): + z.write(f, f.name) + + # Sanity check: a corner clicked in one image, cast onto the wall, projected into the other + # image, compared with where it was clicked there. Only features whose "left" point is the + # same physical corner (bottom-left, at the sill) in both images take part. + repro = [] + for feat in ann["features"]: + if feat.get("left_is_bottom_left_corner_in") is None: + continue + i1, i2 = feat["left_is_bottom_left_corner_in"] + wall = walls_by_id[feat["wall_id"]] + X = backproject(kfs_by_name[i1], wall, [feat["points_px"][i1]["left"]]) + u, v, _ = kfs_by_name[i2].project(X) + cu, cv = feat["points_px"][i2]["left"] + k2 = kfs_by_name[i2] + px_ft = k2.K[0] / float(-((X[0] - k2.C) @ k2.R_c2w)[2]) # pixels per foot at that depth + repro.append( + { + "label": feat["label"], + "from": i1, + "into": i2, + "predicted_px": [round(float(u[0]), 1), round(float(v[0]), 1)], + "clicked_px": [cu, cv], + "err_px": round(float(np.hypot(u[0] - cu, v[0] - cv)), 1), + "err_ft_at_wall": round(float(np.hypot(u[0] - cu, v[0] - cv) / px_ft), 2), + } + ) + report["reprojection"] = repro + report["coverage"] = per_band + report["features"] = feature_rows + report["meter_s_from_chain_start_ft"] = s_meter + (out / "build_report.json").write_text(json.dumps(report, indent=1, default=float) + "\n") + + if args.overlays: + draw_overlays(out, kfs, walls, ann, kfs_by_name, walls_by_id) + + if args.schema: + import jsonschema + + jsonschema.validate(scene, json.loads(args.schema.read_text())) + print(f"scene.json validates against {args.schema}") + + print( + json.dumps( + {k: report[k] for k in ("ground", "walls", "arkit_projection_check", "reprojection")}, + indent=1, + default=float, + ) + ) + print("coverage:", json.dumps(per_band)) + for r in feature_rows: + print( + f"{r['label']:<34} {r['type']:<10} {r['wall_id']} span {r['span_ft']} " + f"h {r['bottom_ft']}-{r['top_ft']} +-{r['plus_minus_ft']}" + ) + print(f"meter (INVENTED): wall {wm.id}, s = 0, pos {scene['meter']['pos']}") + print(f"wrote {out / 'scene.json'} and {out / 'scene.zip'}") + + +def draw_overlays(out, kfs, walls, ann, kfs_by_name, walls_by_id): + """Debug images outside the bundle: the chain at ground and battery height, feet ticks along + it, and each annotated outline plus its reprojection from the other image.""" + od = out / "overlays" + od.mkdir(exist_ok=True) + for k in kfs: + im = Image.open(out / "bundle" / f"{k.id}.jpg").convert("RGB") + d = ImageDraw.Draw(im) + for w in walls: + for y, col in ((0.0, (255, 0, 255)), (BATTERY_HEIGHT_FT, (0, 255, 255))): + ss = np.linspace(w.s0, w.s0 + w.length, 60) + q = w.plan_at(ss) + X = np.c_[q[:, 0], np.full(len(ss), y), q[:, 1]] + u, v, z = k.project(X) + pts = [(a, b) for a, b, c in zip(u, v, z, strict=True) if c > 0.5] + if len(pts) > 1: + d.line(pts, fill=col, width=2) + for s in np.arange(np.ceil(w.s0), w.s0 + w.length, 1.0): + q = w.plan_at(s) + u, v, z = k.project( + np.array([[q[0], 0.0, q[1]], [q[0], 1.0 if s % 5 else 2.0, q[1]]]) + ) + if (z > 0.5).all(): + d.line([(u[0], v[0]), (u[1], v[1])], fill=(255, 255, 0), width=2) + if s % 5 == 0: + d.text((u[1] + 2, v[1] - 12), f"{int(s)}", fill=(255, 255, 0)) + for feat in ann["features"]: + wall = walls_by_id[feat["wall_id"]] + for img, pts in feat["points_px"].items(): + # the measured rectangle on the wall, drawn green from this image's own clicks + # and red when measured in another image + _, _, _, pim = measure_feature( + {**feat, "points_px": {img: pts}}, kfs_by_name, walls_by_id + ) + sl, sr, yb, yt = pim[img] + q = wall.plan_at(np.array([sl, sr, sr, sl])) + X = np.c_[q[:, 0], [yt, yt, yb, yb], q[:, 1]] + u, v, z = k.project(X) + if (z > 0.5).all(): + d.polygon( + list(zip(u, v, strict=True)), + outline=(0, 255, 0) if img == k.name else (255, 60, 60), + ) + if img == k.name: + for r in POINT_ROLES: + a, b = pts[r] + d.ellipse([a - 3, b - 3, a + 3, b + 3], outline=(255, 255, 255)) + im.save(od / f"{k.name}.jpg", quality=85) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/verification/scenes/eth3d-facade/case.json b/verification/scenes/eth3d-facade/case.json new file mode 100644 index 000000000..9cffd67cd --- /dev/null +++ b/verification/scenes/eth3d-facade/case.json @@ -0,0 +1,22 @@ +{ + "id": "eth3d-facade", + "real": true, + "scene_path": "~/house-scanning-data/verify/scenes/eth3d-facade/scene.zip", + "source": "ETH3D 'facade' (dslr_undistorted + dslr_scan_eval, research-only license), south courtyard wing; built by verification/scenes/eth3d-facade/build_scene.py from annotations.json. Walls, windows and door are real; the electric meter is invented.", + "description": "Three-wall chain: w1 (52.0 ft, five ground-floor windows), w2 (11.6 ft tower side, never observed), w3 (16.2 ft tower front with the entrance door). Meter invented on w1 between windows 4 and 5. No start position can pass: every w1 window gap (5.11-5.71 ft) is at least 2.87 ft short of the 8.58 ft a battery plus two 3 ft clearances needs, w2 is unobserved, and all of w3 is past the 20 ft route. Reject is also wrong because w2 is within reach and unseen. See README.md for the arithmetic.", + "rules_assumed": {"opening": 3.0, "route_length": 20.0, "corner_allowance": 0.5, "battery_width": 2.583333333333}, + "expect": { + "decision_in": ["manual_review"], + "sweep_runs": [ + {"wall_id": "w1", "start_ft": [-42.1, -31.97], "outcome_not": "pass", "reason": "window 1 [-40.08, -34.97]: footprint within clearances.opening_ft (3 ft); starts (a - width - 3, b + 3)"}, + {"wall_id": "w1", "start_ft": [-34.84, -21.04], "outcome_not": "pass", "reason": "window 2 [-29.26, -24.04] within 3 ft"}, + {"wall_id": "w1", "start_ft": [-24.17, -10.26], "outcome_not": "pass", "reason": "window 3 [-18.59, -13.26] within 3 ft"}, + {"wall_id": "w1", "start_ft": [-13.71, 0.43], "outcome_not": "pass", "reason": "window 4 [-8.13, -2.57] within 3 ft"}, + {"wall_id": "w1", "start_ft": [-3.04, 9.9], "outcome_not": "pass", "reason": "window 5 [2.54, 7.92] within 3 ft (band runs to 10.92, clipped at the w1/w2 corner)"}, + {"wall_id": "w1", "start_ft": [-42.1, -22.58], "outcome_not": "pass", "reason": "route from meter to the battery's near (right) edge = -(start + width) exceeds route.max_ft (20 ft)"}, + {"wall_id": "w2", "start_ft": [9.9, 21.52], "outcome_not": "pass", "reason": "w2 is in no coverage.observed band (seen only beyond 75 degrees from its normal)"}, + {"wall_id": "w3", "start_ft": [21.52, 37.76], "outcome_not": "pass", "reason": "route is at least 21.52 ft along the wall + 2 corners x route.corner_allowance_ft 0.5 = 22.52 ft, over route.max_ft 20"}, + {"wall_id": "w3", "start_ft": [21.52, 34.01], "outcome_not": "pass", "reason": "entrance door [23.83, 31.01] within 3 ft; doors also fail as route crossings (route.crossing.door)"} + ] + } +} diff --git a/verification/uv.lock b/verification/uv.lock index 09fa3648b..793b85a89 100644 --- a/verification/uv.lock +++ b/verification/uv.lock @@ -27,6 +27,7 @@ source = { virtual = "." } dependencies = [ { name = "jsonschema" }, { name = "numpy" }, + { name = "pillow" }, { name = "pyyaml" }, ] @@ -40,6 +41,7 @@ dev = [ requires-dist = [ { name = "jsonschema", specifier = ">=4.23" }, { name = "numpy", specifier = ">=2.5.3" }, + { name = "pillow", specifier = ">=12.3.0" }, { name = "pyyaml", specifier = ">=6.0" }, ] @@ -167,6 +169,77 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, ] +[[package]] +name = "pillow" +version = "12.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/bf/fb3ebff8ddcb76aac5a01389251bbbb9519922a9b520d8247c1ca864a25d/pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965", size = 5345969, upload-time = "2026-07-01T11:54:06.397Z" }, + { url = "https://files.pythonhosted.org/packages/d8/66/9a386a92561f402389a4fc70c18838bf6d35eb5eb5c6850b4b2dc64f5048/pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7", size = 4780323, upload-time = "2026-07-01T11:54:09.351Z" }, + { url = "https://files.pythonhosted.org/packages/25/27/ac8f99618ffd3dde21db0f4d4b1d2ab00c0880595bfd17df103f7f39fd0c/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9", size = 6266838, upload-time = "2026-07-01T11:54:11.71Z" }, + { url = "https://files.pythonhosted.org/packages/84/21/a35af28dcc61f37ed850a2d64c65c701321dfbf25085e469d5559360cbbf/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91", size = 6940830, upload-time = "2026-07-01T11:54:13.732Z" }, + { url = "https://files.pythonhosted.org/packages/eb/51/8b08617af3ad95e33ce6d7dd2c99ed6c8298f7fb131636303956be022e25/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c", size = 6344383, upload-time = "2026-07-01T11:54:15.756Z" }, + { url = "https://files.pythonhosted.org/packages/1d/72/cf78ac9780bb93c28328f408973845a309d4d145041665f734572ced1b52/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df", size = 7052934, upload-time = "2026-07-01T11:54:17.721Z" }, + { url = "https://files.pythonhosted.org/packages/20/20/25e0f4dc178a6bc0696793720055519a0de89e7661dae886992decbd2f81/pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f", size = 6472684, upload-time = "2026-07-01T11:54:19.839Z" }, + { url = "https://files.pythonhosted.org/packages/45/89/da2f7971a317f83d807fdd4065c0af40208e59e692cc43d315a71a0e96d1/pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09", size = 7227137, upload-time = "2026-07-01T11:54:22.025Z" }, + { url = "https://files.pythonhosted.org/packages/de/47/4845a0a6c0dbf1db8456bd9fc791f13c5ced7ced20606d08a0aacfd25b49/pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510", size = 2568267, upload-time = "2026-07-01T11:54:24.051Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ac/31fb64e1e7efb5a4b50cd3d92049ba89ac6e4d8d3bb6a74e15048ca3353e/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89", size = 4161684, upload-time = "2026-07-01T11:54:25.934Z" }, + { url = "https://files.pythonhosted.org/packages/87/b4/9805e23d2b4d77842b468513841fda254ee42f0289d25088340e4ff46e2d/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace", size = 4255487, upload-time = "2026-07-01T11:54:27.935Z" }, + { url = "https://files.pythonhosted.org/packages/df/39/ecf519435a200c693fe053a6ee4d835b41cf963a4dfc2551c4e637cb2a71/pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec", size = 3696433, upload-time = "2026-07-01T11:54:29.813Z" }, + { url = "https://files.pythonhosted.org/packages/42/92/2fc3ffad878ae8dd5469ec1bc8eb83b71f48e13efdf68f02709003982a32/pillow-12.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66", size = 5345889, upload-time = "2026-07-01T11:54:31.97Z" }, + { url = "https://files.pythonhosted.org/packages/10/76/8803c13605b763d33d156c4678fc77f8443389c0c51c8aef707bb02015f4/pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35", size = 4780109, upload-time = "2026-07-01T11:54:34.026Z" }, + { url = "https://files.pythonhosted.org/packages/1f/01/e18aff37cb0b4aac47ac90f016d347a49aca667ef97f190b06ac2aabc928/pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65", size = 6263736, upload-time = "2026-07-01T11:54:36.131Z" }, + { url = "https://files.pythonhosted.org/packages/f7/62/de5bdd77d935331f4f802edc11e4d82950f642caad6cb2f949837b8560e2/pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3", size = 6937129, upload-time = "2026-07-01T11:54:38.216Z" }, + { url = "https://files.pythonhosted.org/packages/70/4d/105627a13300c5e0df1d174230b32fd1273062c96f7745fd552b945d1e1d/pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a", size = 6339562, upload-time = "2026-07-01T11:54:40.354Z" }, + { url = "https://files.pythonhosted.org/packages/6b/1d/f13de01a553988ab895ba1c722e06cf3144d4f57656fd5b81b6d881f1179/pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e", size = 7049439, upload-time = "2026-07-01T11:54:42.489Z" }, + { url = "https://files.pythonhosted.org/packages/c9/f9/066794cca041b969964f779ee5fa66a9498bbf34248ac39c5d7954e4198f/pillow-12.3.0-cp313-cp313-win32.whl", hash = "sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f", size = 6473287, upload-time = "2026-07-01T11:54:44.9Z" }, + { url = "https://files.pythonhosted.org/packages/a6/9b/7a58e61d62be561da3a356fe2384d4059a6345fc130e23ef1c36a5b81d24/pillow-12.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8", size = 7239691, upload-time = "2026-07-01T11:54:47.141Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b0/c4ed4f0ef8f8fa5ee8351537db6650bb8189f7e118842978dd6589065692/pillow-12.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b", size = 2568185, upload-time = "2026-07-01T11:54:49.137Z" }, + { url = "https://files.pythonhosted.org/packages/dc/01/001f65b68192f0228cc1dbbc8d2530ab5d58b61037ba0587f946fea607cd/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330", size = 4161736, upload-time = "2026-07-01T11:54:51.156Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d2/0219746d0fd16fc8a84498e79452375be3797d3ce4044596ce565164b84f/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217", size = 4255435, upload-time = "2026-07-01T11:54:53.414Z" }, + { url = "https://files.pythonhosted.org/packages/c8/02/8d0bc62ef0302318c46ff2a512822d2610e81c7aa46c9b3abe6cbaca5ad0/pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930", size = 3696262, upload-time = "2026-07-01T11:54:55.739Z" }, + { url = "https://files.pythonhosted.org/packages/85/e2/73c77d218410b14f5f2d565e8a998d5317b7b9c75368d29985139f7a46f0/pillow-12.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8", size = 5350344, upload-time = "2026-07-01T11:54:57.657Z" }, + { url = "https://files.pythonhosted.org/packages/c7/da/32c752228ae345f489e3a42499d817b6c3996da7e8a3bc7a04fc806b243b/pillow-12.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0", size = 4780131, upload-time = "2026-07-01T11:54:59.713Z" }, + { url = "https://files.pythonhosted.org/packages/b1/9d/8b2c807dbef61a5197c047afe99823787eb66f63daf9fb2432f91d6f0462/pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321", size = 6263757, upload-time = "2026-07-01T11:55:01.778Z" }, + { url = "https://files.pythonhosted.org/packages/5c/44/c85361f65dbe00eea8576ee467c768d25129989efb76e94f205e9ca9bb46/pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b", size = 6936962, upload-time = "2026-07-01T11:55:03.93Z" }, + { url = "https://files.pythonhosted.org/packages/18/7e/e483414b35800b86b6f08dbbc7803fb5cd52c4d6f897f47d53ea2c7e6f65/pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198", size = 6339171, upload-time = "2026-07-01T11:55:05.989Z" }, + { url = "https://files.pythonhosted.org/packages/f0/f4/68c491844841ede6bed70189546b3ee9731cf9f2cbad396faff5e1ccba45/pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130", size = 7048116, upload-time = "2026-07-01T11:55:08.131Z" }, + { url = "https://files.pythonhosted.org/packages/a3/34/77f3f793fed8efc7d243f21b33c5a3f0d1c97ee70346d3db855587e155ff/pillow-12.3.0-cp314-cp314-win32.whl", hash = "sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a", size = 6467209, upload-time = "2026-07-01T11:55:10.408Z" }, + { url = "https://files.pythonhosted.org/packages/f1/e0/492879f69d94f91f60fc8cd05ba03650e9520afebb2fb7aa12777d7c7f38/pillow-12.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d", size = 7237707, upload-time = "2026-07-01T11:55:12.745Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ac/6b11f2875f1c2ac040d84e1bbf9cf22a88038f901ca1037898b280b38365/pillow-12.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838", size = 2565995, upload-time = "2026-07-01T11:55:14.736Z" }, + { url = "https://files.pythonhosted.org/packages/52/69/c2208e56af9bfc1913afb24020297a691eb1d4ef688474c8a04913f65e04/pillow-12.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e", size = 5352503, upload-time = "2026-07-01T11:55:17.076Z" }, + { url = "https://files.pythonhosted.org/packages/07/70/e5686d753e898a45d778ff1718dba8516ead6ab6b95d85fc8c4b70650cf2/pillow-12.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17", size = 4782956, upload-time = "2026-07-01T11:55:19.448Z" }, + { url = "https://files.pythonhosted.org/packages/d5/37/25c6692f06927ee973ff18c8d9ee98ad0b4d84ee67a09610c2dd1447958e/pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385", size = 6322855, upload-time = "2026-07-01T11:55:21.613Z" }, + { url = "https://files.pythonhosted.org/packages/cc/91/420637fcb8f1bc11029e403b4538e6694744428d8246118e45719f944556/pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c", size = 6989642, upload-time = "2026-07-01T11:55:24.006Z" }, + { url = "https://files.pythonhosted.org/packages/10/08/b94d7811281ccf0d143a1cf768d1c49e1e54af63e7b708ab2ee3eb87face/pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d", size = 6391281, upload-time = "2026-07-01T11:55:26.252Z" }, + { url = "https://files.pythonhosted.org/packages/d2/87/24233f785f55474dc02ce3e739c5528a77e3a862e9333d1dd7a25cc31f70/pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931", size = 7096716, upload-time = "2026-07-01T11:55:28.318Z" }, + { url = "https://files.pythonhosted.org/packages/23/26/fcb2f6e37175b04f53570b59937867e2b80ee1685e744023153028fc14f9/pillow-12.3.0-cp314-cp314t-win32.whl", hash = "sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7", size = 6474125, upload-time = "2026-07-01T11:55:30.956Z" }, + { url = "https://files.pythonhosted.org/packages/90/de/3634abee5f1c9e13c56787b7d5517b0ba8d6de51700b95578cf338349c9f/pillow-12.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c", size = 7242939, upload-time = "2026-07-01T11:55:34.044Z" }, + { url = "https://files.pythonhosted.org/packages/ce/2a/fd13f8eb24de5714a6eb444a3d67e2842c6c576e159a43793adf23051351/pillow-12.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45", size = 2567506, upload-time = "2026-07-01T11:55:35.988Z" }, + { url = "https://files.pythonhosted.org/packages/5d/dc/8fdce34ec725a33c81c6ba122b904d6b9024e50ea9ac7bede62fab54506c/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139", size = 4162063, upload-time = "2026-07-01T11:55:37.941Z" }, + { url = "https://files.pythonhosted.org/packages/76/66/2044b9a63d3b84ff048228dfcb7cd9bf0df983e8470971bf7d4c57b693de/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402", size = 4255549, upload-time = "2026-07-01T11:55:40.022Z" }, + { url = "https://files.pythonhosted.org/packages/52/7e/1f67e6f4ece6b582ee4b539decbcc9f848dc245a93ed8cd7338bafef72f1/pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c", size = 3696331, upload-time = "2026-07-01T11:55:41.98Z" }, + { url = "https://files.pythonhosted.org/packages/12/40/d306fc2c8e4d45d7f175c77edca7063be7b86fe7fe6e68f4353bf71d808c/pillow-12.3.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f", size = 5350370, upload-time = "2026-07-01T11:55:44.028Z" }, + { url = "https://files.pythonhosted.org/packages/dd/44/668fb1437e8ce420f62d6106eb66e44a5971602a4d794615bdf79315d82d/pillow-12.3.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701", size = 4780147, upload-time = "2026-07-01T11:55:46.073Z" }, + { url = "https://files.pythonhosted.org/packages/0c/08/93fa2e70e30a2d81547e481b6ee2bb9522117221fb1e0ce4b5df70967677/pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace", size = 6273659, upload-time = "2026-07-01T11:55:48.264Z" }, + { url = "https://files.pythonhosted.org/packages/f8/6d/043e96ff814fc31a33077e4cba86082167db520c93632afdf2042febbb0c/pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4", size = 6947439, upload-time = "2026-07-01T11:55:50.503Z" }, + { url = "https://files.pythonhosted.org/packages/af/92/ba71d2ee2ac0edf3fa33bd9d5ee9ee080da70b1766f3ca3934f9938ddac9/pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39", size = 6353577, upload-time = "2026-07-01T11:55:52.697Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ce/e63064e2122923ff687c8ad792d0d736a7b3920a56a46982e81a7fdd25d6/pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71", size = 7060394, upload-time = "2026-07-01T11:55:55.149Z" }, + { url = "https://files.pythonhosted.org/packages/54/76/a09cc3ccc8d773a7283d34c38bec1708f9e3cc932093cbc4c5e71ac4060b/pillow-12.3.0-cp315-cp315-win32.whl", hash = "sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827", size = 6467375, upload-time = "2026-07-01T11:55:57.769Z" }, + { url = "https://files.pythonhosted.org/packages/3e/03/1846c49ba3b1d5550392a4bbd06d6fb4578e1cd91a803198b5c90f5f7d53/pillow-12.3.0-cp315-cp315-win_amd64.whl", hash = "sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5", size = 7237048, upload-time = "2026-07-01T11:55:59.975Z" }, + { url = "https://files.pythonhosted.org/packages/fb/bb/89f35dcc79610423f9f195504d7def7f0d1416a711541b42867e25fe3412/pillow-12.3.0-cp315-cp315-win_arm64.whl", hash = "sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658", size = 2566006, upload-time = "2026-07-01T11:56:02.143Z" }, + { url = "https://files.pythonhosted.org/packages/30/88/707027ba09942dfa2c28759b5c222d769290a41c6d20ea60ec250801941f/pillow-12.3.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf", size = 5352509, upload-time = "2026-07-01T11:56:04.2Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6d/00352fa25332c2569cd387851f568cc5a4b75a9adbfb37ac4fbce4c02eec/pillow-12.3.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64", size = 4783167, upload-time = "2026-07-01T11:56:06.631Z" }, + { url = "https://files.pythonhosted.org/packages/13/4f/9e049dfa21af7c22427275720e2490267ba8138120add5c4c574deb69782/pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e", size = 6329237, upload-time = "2026-07-01T11:56:08.868Z" }, + { url = "https://files.pythonhosted.org/packages/36/16/cf6eeaae8d0fce8dd390a33437cf68c5d5bd73834a2bc6e2f14efda0ab45/pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777", size = 6997047, upload-time = "2026-07-01T11:56:11.379Z" }, + { url = "https://files.pythonhosted.org/packages/1e/69/dbf769bdd55f48bf5733cac28edc6364ffaa072ec9ba336266e4fe66be55/pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1", size = 6400440, upload-time = "2026-07-01T11:56:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e1/ffc9cfc2eea0d178da8018e18e959301ad9d6bc9f3edb7181e748a474b97/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9", size = 7105895, upload-time = "2026-07-01T11:56:16.575Z" }, + { url = "https://files.pythonhosted.org/packages/18/f0/a5595c1e8c3ae44b9828cb2f0fa8155e5095ef04d6327b8f61cf44a3df85/pillow-12.3.0-cp315-cp315t-win32.whl", hash = "sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8", size = 6474384, upload-time = "2026-07-01T11:56:18.855Z" }, + { url = "https://files.pythonhosted.org/packages/e4/04/62bcd9f844984c5938d3b05264a61d797a29d3e0812341a8204af70bbdee/pillow-12.3.0-cp315-cp315t-win_amd64.whl", hash = "sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418", size = 7243537, upload-time = "2026-07-01T11:56:21.214Z" }, + { url = "https://files.pythonhosted.org/packages/3d/68/1f3066acedf37673694a7141381d8f811ae97f30d34413d236abe7d489f1/pillow-12.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59", size = 2567491, upload-time = "2026-07-01T11:56:23.506Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" From 9e25dbfdb0da7faf9c245fc691ee410ac83c08e4 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 03:51:37 -0500 Subject: [PATCH 19/59] verification: regenerate scoreboard Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/SCOREBOARD.md | 28 ++++----- verification/scoreboard/scoreboard.json | 77 +++++++++++++++---------- 2 files changed, 59 insertions(+), 46 deletions(-) diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index 38613fe25..542c21020 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,14 +1,14 @@ # Scoreboard -Generated 2026-09-26 03:40 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-26 03:51 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | | [#14](https://github.com/SamGu-NRX/house-scanning/pull/14) | `t3/docs-cleanup` | `9c9ceb2a` | no | 4 success | 2026-09-26 03:15 CDT | -| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `6cf8692e` | yes | 4 success | 2026-09-26 03:40 CDT | -| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `df4cce67` | yes | 5 success | 2026-09-26 03:32 CDT | +| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `820ac912` | yes | 4 success | 2026-09-26 03:50 CDT | +| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `86e3b3de` | yes | 4 success, 1 pending | 2026-09-26 03:50 CDT | | [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `44607e04` | yes | 5 success | 2026-09-26 03:37 CDT | | [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `6885b7b5` | yes | 5 success | 2026-09-26 03:07 CDT | | [#9](https://github.com/SamGu-NRX/house-scanning/pull/9) | `codex/capture-contract-probe` | `4b6c4309` | yes | 4 success | 2026-09-26 00:08 CDT | @@ -19,19 +19,19 @@ Generated 2026-09-26 03:40 CDT by `make scoreboard` (`uv run python -m hsverify. ## S1 Evals on real data -`origin/t3/evals` at [`df4cce67`](https://github.com/SamGu-NRX/house-scanning/tree/df4cce678b3b133b56e34f6c0cb910142a08f662). 1 met, 1 partial, 2 gap, 1 not run. +`origin/t3/evals` at [`86e3b3de`](https://github.com/SamGu-NRX/house-scanning/tree/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad). 2 met, 1 partial, 1 gap, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S1-1 | One command regenerates every number from a clean checkout | gap | not yet reviewed | -| S1-2 | Dataset URLs and sha256 recorded | partial | found ADVIO; missing ETH3D [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/df4cce678b3b133b56e34f6c0cb910142a08f662/experiments/evals/README.md) | +| S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/README.md) | | S1-3 | Metric code tested on hand-computed cases | not run | slow; run with --slow | -| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/df4cce678b3b133b56e34f6c0cb910142a08f662/experiments/evals/results/advio_drift.md) | -| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | gap | none found in 2 files; missing 1-3 m row, 3-10 m row, scale error, verdict | +| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/README.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/results/advio_drift.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/results/eth3d_recon.md) | +| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/results/eth3d_recon.md) | ## S2 Server and solver -`origin/t3/server` at [`44607e04`](https://github.com/SamGu-NRX/house-scanning/tree/44607e04e4f851944c47cb535c9c4b13bbad1c5b). 5 met, 1 partial, 2 not run. +`origin/t3/server` at [`44607e04`](https://github.com/SamGu-NRX/house-scanning/tree/44607e04e4f851944c47cb535c9c4b13bbad1c5b). 4 met, 1 partial, 1 gap, 2 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | @@ -40,9 +40,9 @@ Generated 2026-09-26 03:40 CDT by `make scoreboard` (`uv run python -m hsverify. | S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_properties.py) | | S2-4 | Schemas published | met | present at 44607e04 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/schemas/result.schema.json) | | S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_schemas.py), [server/tests/test_api.py](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_api.py) | -| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 733.5 (want < 1000): ~/house-scanning-data/reports/e2e/20260926-034015-44607e04/report.json | +| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 596.7 (want < 1000): ~/house-scanning-data/reports/e2e/20260926-035100-44607e04/report.json | | S2-7 | ruff clean | not run | slow; run with --slow | -| S2-8 | Black-box (verification e2e): every answer valid under C2, consistent with its decision and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric when mirrored, and never passing without coverage | met | contract_problem_count = 0 (want 0): ~/house-scanning-data/reports/e2e/20260926-034015-44607e04/report.json | +| S2-8 | Black-box (verification e2e): every answer valid under C2, consistent with its decision and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric when mirrored, and never passing without coverage | gap | contract_problem_count = 3 (want 0): ~/house-scanning-data/reports/e2e/20260926-035100-44607e04/report.json | ## S3 iOS capture app @@ -58,19 +58,19 @@ Generated 2026-09-26 03:40 CDT by `make scoreboard` (`uv run python -m hsverify. | S3-6 | Accessibility audit on every screen | gap | 0 of 1 matches needed, searched 1 files | | S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, plus symbol-name button labels) finds no issues; S4-2 shows how many screens the app reached | no evidence yet | no report matches ~/house-scanning-data/reports/a11y/*t3-ios-mvf*/report.json | | S3-7 | Screenshots of each state in the PR | gap | 0 of 9 matches in the PR body [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-8 | Exported scene.json validates against C1 | no evidence yet | no app_export_scene_valid in ~/house-scanning-data/reports/e2e/20260926-034015-44607e04/report.json | +| S3-8 | Exported scene.json validates against C1 | no evidence yet | no app_export_scene_valid in ~/house-scanning-data/reports/e2e/20260926-035100-44607e04/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/verification` at [`6cf8692e`](https://github.com/SamGu-NRX/house-scanning/tree/6cf8692ee26f3474e12a1f08fb6a8a6780e91652), `origin/t3/ios-mvf` at [`6885b7b5`](https://github.com/SamGu-NRX/house-scanning/tree/6885b7b5ad872df7f73a6b915e5837cd38a2034d). 1 met, 2 gap, 1 no evidence yet. +`origin/t3/verification` at [`820ac912`](https://github.com/SamGu-NRX/house-scanning/tree/820ac912ef0b9f15d0b07991d41dd2683c7d75cb), `origin/t3/ios-mvf` at [`6885b7b5`](https://github.com/SamGu-NRX/house-scanning/tree/6885b7b5ad872df7f73a6b915e5837cd38a2034d). 1 met, 2 gap, 1 no evidence yet. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S4-1 | End to end from a real replay to a placement result passes | no evidence yet | no app_export_passed in ~/house-scanning-data/reports/e2e/20260926-034015-44607e04/report.json | +| S4-1 | End to end from a real replay to a placement result passes | no evidence yet | no app_export_passed in ~/house-scanning-data/reports/e2e/20260926-035100-44607e04/report.json | | S4-2 | Screenshot report per app state | gap | states = length 0 (want at least 9): ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json | | S4-3 | Product description written from the code and checked against screenshots | gap | not yet reviewed | -| S4-4 | Scoreboard of every PR | met | present at 6cf8692e [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/6cf8692ee26f3474e12a1f08fb6a8a6780e91652/verification/SCOREBOARD.md) | +| S4-4 | Scoreboard of every PR | met | present at 820ac912 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/820ac912ef0b9f15d0b07991d41dd2683c7d75cb/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index 635789089..8d2d2ba70 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-09-26T03:40:54-05:00", + "generated_at": "2026-09-26T03:51:15-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": false, "prs": [ @@ -25,9 +25,9 @@ "number": 13, "title": "verification: independent end-to-end checks (S4)", "branch": "t3/verification", - "head": "6cf8692ee26f3474e12a1f08fb6a8a6780e91652", + "head": "820ac912ef0b9f15d0b07991d41dd2683c7d75cb", "draft": true, - "updated": "2026-09-26T08:40:06Z", + "updated": "2026-09-26T08:50:35Z", "ci": { "counts": { "success": 4, @@ -43,18 +43,18 @@ "number": 12, "title": "feat(evals): real-data accuracy evals, starting with an ADVIO replay session", "branch": "t3/evals", - "head": "df4cce678b3b133b56e34f6c0cb910142a08f662", + "head": "86e3b3decffc2a08cd9ec5de540a514c0d35d7ad", "draft": true, - "updated": "2026-09-26T08:32:28Z", + "updated": "2026-09-26T08:50:55Z", "ci": { "counts": { - "success": 5, + "success": 4, "failure": 0, - "pending": 0, + "pending": 1, "skipped": 0 }, "failing": [], - "text": "5 success" + "text": "4 success, 1 pending" } }, { @@ -192,7 +192,7 @@ "refs": [ { "ref": "origin/t3/evals", - "sha": "df4cce678b3b133b56e34f6c0cb910142a08f662" + "sha": "86e3b3decffc2a08cd9ec5de540a514c0d35d7ad" } ], "metrics": [ @@ -200,7 +200,7 @@ "id": "S1-1", "text": "One command regenerates every number from a clean checkout", "ref": "origin/t3/evals", - "sha": "df4cce678b3b133b56e34f6c0cb910142a08f662", + "sha": "86e3b3decffc2a08cd9ec5de540a514c0d35d7ad", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -209,13 +209,13 @@ "id": "S1-2", "text": "Dataset URLs and sha256 recorded", "ref": "origin/t3/evals", - "sha": "df4cce678b3b133b56e34f6c0cb910142a08f662", - "status": "partial", - "evidence": "found ADVIO; missing ETH3D", + "sha": "86e3b3decffc2a08cd9ec5de540a514c0d35d7ad", + "status": "met", + "evidence": "all 2 found", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/df4cce678b3b133b56e34f6c0cb910142a08f662/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/README.md" ] ] }, @@ -223,7 +223,7 @@ "id": "S1-3", "text": "Metric code tested on hand-computed cases", "ref": "origin/t3/evals", - "sha": "df4cce678b3b133b56e34f6c0cb910142a08f662", + "sha": "86e3b3decffc2a08cd9ec5de540a514c0d35d7ad", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -232,13 +232,21 @@ "id": "S1-4", "text": "ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft", "ref": "origin/t3/evals", - "sha": "df4cce678b3b133b56e34f6c0cb910142a08f662", + "sha": "86e3b3decffc2a08cd9ec5de540a514c0d35d7ad", "status": "met", "evidence": "all 5 found", "links": [ + [ + "experiments/evals/README.md", + "https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/README.md" + ], [ "experiments/evals/results/advio_drift.md", - "https://github.com/SamGu-NRX/house-scanning/blob/df4cce678b3b133b56e34f6c0cb910142a08f662/experiments/evals/results/advio_drift.md" + "https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/results/advio_drift.md" + ], + [ + "experiments/evals/results/eth3d_recon.md", + "https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/results/eth3d_recon.md" ] ] }, @@ -246,10 +254,15 @@ "id": "S1-5", "text": "ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question", "ref": "origin/t3/evals", - "sha": "df4cce678b3b133b56e34f6c0cb910142a08f662", - "status": "gap", - "evidence": "none found in 2 files; missing 1-3 m row, 3-10 m row, scale error, verdict", - "links": [] + "sha": "86e3b3decffc2a08cd9ec5de540a514c0d35d7ad", + "status": "partial", + "evidence": "found 1-3 m row, 3-10 m row, scale error; missing verdict", + "links": [ + [ + "experiments/evals/results/eth3d_recon.md", + "https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/results/eth3d_recon.md" + ] + ] } ] }, @@ -346,7 +359,7 @@ "ref": "origin/t3/server", "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", "status": "met", - "evidence": "latency_ms = 733.5 (want < 1000): ~/house-scanning-data/reports/e2e/20260926-034015-44607e04/report.json", + "evidence": "latency_ms = 596.7 (want < 1000): ~/house-scanning-data/reports/e2e/20260926-035100-44607e04/report.json", "links": [] }, { @@ -363,8 +376,8 @@ "text": "Black-box (verification e2e): every answer valid under C2, consistent with its decision and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric when mirrored, and never passing without coverage", "ref": "origin/t3/server", "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", - "status": "met", - "evidence": "contract_problem_count = 0 (want 0): ~/house-scanning-data/reports/e2e/20260926-034015-44607e04/report.json", + "status": "gap", + "evidence": "contract_problem_count = 3 (want 0): ~/house-scanning-data/reports/e2e/20260926-035100-44607e04/report.json", "links": [] } ] @@ -472,7 +485,7 @@ "ref": "origin/t3/ios-mvf", "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", "status": "no evidence yet", - "evidence": "no app_export_scene_valid in ~/house-scanning-data/reports/e2e/20260926-034015-44607e04/report.json", + "evidence": "no app_export_scene_valid in ~/house-scanning-data/reports/e2e/20260926-035100-44607e04/report.json", "links": [] }, { @@ -492,7 +505,7 @@ "refs": [ { "ref": "origin/t3/verification", - "sha": "6cf8692ee26f3474e12a1f08fb6a8a6780e91652" + "sha": "820ac912ef0b9f15d0b07991d41dd2683c7d75cb" }, { "ref": "origin/t3/ios-mvf", @@ -504,9 +517,9 @@ "id": "S4-1", "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/verification", - "sha": "6cf8692ee26f3474e12a1f08fb6a8a6780e91652", + "sha": "820ac912ef0b9f15d0b07991d41dd2683c7d75cb", "status": "no evidence yet", - "evidence": "no app_export_passed in ~/house-scanning-data/reports/e2e/20260926-034015-44607e04/report.json", + "evidence": "no app_export_passed in ~/house-scanning-data/reports/e2e/20260926-035100-44607e04/report.json", "links": [] }, { @@ -522,7 +535,7 @@ "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "6cf8692ee26f3474e12a1f08fb6a8a6780e91652", + "sha": "820ac912ef0b9f15d0b07991d41dd2683c7d75cb", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -531,13 +544,13 @@ "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "6cf8692ee26f3474e12a1f08fb6a8a6780e91652", + "sha": "820ac912ef0b9f15d0b07991d41dd2683c7d75cb", "status": "met", - "evidence": "present at 6cf8692e", + "evidence": "present at 820ac912", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/6cf8692ee26f3474e12a1f08fb6a8a6780e91652/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/820ac912ef0b9f15d0b07991d41dd2683c7d75cb/verification/SCOREBOARD.md" ] ] } From fc1858e2d1178d9fa996cda03fa0814c6636ea63 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 04:00:13 -0500 Subject: [PATCH 20/59] verification: C5 check reads the review line; margin must be within error of a line The server now publishes review_threshold_ft on checks with a review band. A pass must clear both lines, and an unsure labelled margin must lie within its error of one of them, which is what C2 means by that cause. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/hsverify/resultcheck.py | 27 +++++++++++++++++++------- verification/tests/test_resultcheck.py | 20 +++++++++++++++++++ 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 4703cd090..9ab1522a6 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -72,28 +72,41 @@ def covers(intervals: list[tuple[float, float]], a: float, b: float, slack: floa def margin_problem(check: dict) -> str | None: """The C5 rule for one check, when its numbers are all present. - PASS needs a clear pass and FAIL a clear fail. UNSURE from the margin needs neither. UNSURE - for another cause (an unobserved area, an unknown attribute, a review rule) may sit on - numbers that would pass, but not on numbers that clearly fail: a measured object inside - the clearance by more than the error fails whatever else went unseen. + PASS needs a clear pass and FAIL a clear fail. A check with a review band + (`review_threshold_ft`) passes only when it also clears that line. UNSURE for another cause + (unobserved area, unknown attribute, review rule) may sit on numbers that would pass, never + on numbers that clearly fail. UNSURE labelled `margin` must lie within its error of a line: + C2 defines that cause as "measured inside the error band". """ m, e, t, cmp = ( check.get(k) for k in ("measured_ft", "plus_minus_ft", "threshold_ft", "comparison") ) if m is None or e is None or t is None or cmp is None: return None + review = check.get("review_threshold_ft") + lines = [t] if review is None else [t, review] if cmp == "at_least": - clear_pass, clear_fail = m - e > t + EPS, m + e < t - EPS + clear_pass = all(m - e > x + EPS for x in lines) + clear_fail = m + e < t - EPS else: - clear_pass, clear_fail = m + e < t - EPS, m - e > t + EPS + clear_pass = all(m + e < x - EPS for x in lines) + clear_fail = m - e > t + EPS expected = "pass" if clear_pass else "fail" if clear_fail else "unsure" outcome, cause = check["outcome"], check.get("unsure_cause") + band = f" ({cmp} {t}" + (f", review {review})" if review is not None else ")") + if outcome == "unsure" and cause == "margin" and expected == "unsure": + if not any(abs(m - x) <= e + EPS for x in lines): + return ( + f"check {check['id']}: unsure_cause margin, but {m} ± {e} is not within its " + f"error of any line{band}; a review band is rule_requires_review" + ) + return None other_cause = outcome == "unsure" and cause not in (None, "margin") if outcome == expected or (other_cause and not clear_fail): return None because = f" ({cause})" if cause else "" return ( - f"check {check['id']}: measured {m} ± {e} {cmp} {t} should be {expected}, " + f"check {check['id']}: measured {m} ± {e}{band} should be {expected}, " f"server says {outcome}{because}" ) diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index d99d0c8bc..29b29ce6d 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -143,6 +143,26 @@ def test_unsure_for_another_cause_may_sit_on_passing_numbers_but_not_failing_one assert margin_problem(check("unsure", 5.9, 0.6, 3.0, cause="margin")) is not None +def route(outcome, measured, error=0.0, cause=None): + c = check(outcome, measured, error, 20.0, "at_most", cause) + c["review_threshold_ft"] = 15.0 + return c + + +def test_review_band_blocks_a_pass_and_is_not_a_margin(): + assert margin_problem(route("pass", 14.0)) is None + assert "should be unsure" in margin_problem(route("pass", 16.0)) + assert margin_problem(route("unsure", 16.0, cause="rule_requires_review")) is None + assert "not within its error of any line" in margin_problem( + route("unsure", 16.0, cause="margin") + ) + # Within error of either line, margin is the right cause. + assert margin_problem(route("unsure", 15.2, 0.3, cause="margin")) is None + assert margin_problem(route("unsure", 19.9, 0.3, cause="margin")) is None + assert margin_problem(route("unsure", 15.0, cause="margin")) is None # on the line + assert margin_problem(route("fail", 20.4, 0.3)) is None + + def test_margin_rule_skips_checks_without_numbers(): assert margin_problem(check("unsure", measured=None, cause="unobserved")) is None From 903820267444db104d56fea0e9bff3e47582e75e Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 04:21:43 -0500 Subject: [PATCH 21/59] verification: runner waits only for compiling builds; reports keep every app log line A prebuilt test run on the shared Mac held the runner for over ten minutes though it compiles nothing. Engine and autopilot lines now go into the report next to the screens, because they explain what a run did. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/hsverify/simrun.py | 19 ++++++++++++++++--- verification/hsverify/statelog.py | 4 +++- verification/tests/test_simrun.py | 8 +++++++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/verification/hsverify/simrun.py b/verification/hsverify/simrun.py index 7c7d3bdd9..471ea3393 100644 --- a/verification/hsverify/simrun.py +++ b/verification/hsverify/simrun.py @@ -129,13 +129,26 @@ def prepare_display(udid: str, appearance: str, content_size: str | None) -> Non # --- Build ----------------------------------------------------------------------------------- +def compiling(args: str) -> bool: + """Whether an xcodebuild command line compiles. Running prebuilt tests does not.""" + return "xcodebuild" in args and "test-without-building" not in args + + +def other_builds_running() -> bool: + listing = subprocess.run(["ps", "-Ao", "args="], capture_output=True, text=True).stdout + return any( + compiling(line) + for line in listing.splitlines() + if line.split(" ")[0].endswith("xcodebuild") + ) + + def wait_for_other_builds(max_wait_s: float) -> float: - """The Mac is shared: wait while any other xcodebuild runs, up to `max_wait_s`.""" + """The Mac is shared: wait while another xcodebuild compiles, up to `max_wait_s`.""" start = time.monotonic() announced = False while True: - busy = subprocess.run(["pgrep", "-x", "xcodebuild"], capture_output=True).returncode == 0 - if not busy: + if not other_builds_running(): return time.monotonic() - start if time.monotonic() - start > max_wait_s: raise SystemExit(f"Another xcodebuild has run for over {max_wait_s:.0f} s; try later.") diff --git a/verification/hsverify/statelog.py b/verification/hsverify/statelog.py index 1e2951ba6..8efe9974a 100644 --- a/verification/hsverify/statelog.py +++ b/verification/hsverify/statelog.py @@ -12,7 +12,9 @@ SUBSYSTEM = "dev.housescanning.housescan" CATEGORY = "state" -LOG_PREDICATE = f'subsystem == "{SUBSYSTEM}" AND category == "{CATEGORY}"' +# Every category of the app is streamed into the report (engine and autopilot lines explain what +# happened); only `state` lines are parsed as screens. +LOG_PREDICATE = f'subsystem == "{SUBSYSTEM}"' # A state name is an identifier-like token. Anything after it on the line is detail. _STATE_RE = re.compile(r"STATE=([A-Za-z0-9_.\-]+)") diff --git a/verification/tests/test_simrun.py b/verification/tests/test_simrun.py index aa68306c7..279ee6b71 100644 --- a/verification/tests/test_simrun.py +++ b/verification/tests/test_simrun.py @@ -1,6 +1,6 @@ import datetime as dt -from hsverify.simrun import seconds_since, slug +from hsverify.simrun import compiling, seconds_since, slug def test_state_time_uses_the_log_clock(): @@ -16,3 +16,9 @@ def test_unparseable_timestamp_falls_back_to_arrival_time(): def test_slug_is_filename_safe(): assert slug("origin/t3/ios-mvf") == "origin-t3-ios-mvf" assert slug("wall walk: gap!") == "wall-walk-gap" + + +def test_only_compiling_xcodebuilds_block_a_build(): + assert compiling("/Applications/Xcode.app/.../xcodebuild -project a.xcodeproj build") + assert compiling("xcodebuild test -project a.xcodeproj -scheme A") + assert not compiling("xcodebuild -project a.xcodeproj test-without-building") From 1330a82758df3a007fede909936542f5fb3fb358 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 04:21:43 -0500 Subject: [PATCH 22/59] verification: start the product description of the capture app Scope, glossary, and the two foundation documents (the flow; coverage and guidance), written from S3's integration commit 21a63e7. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/product/README.md | 79 +++++++++ .../foundations/coverage-and-guidance.md | 153 ++++++++++++++++++ verification/product/foundations/flow.md | 144 +++++++++++++++++ verification/product/glossary.md | 84 ++++++++++ 4 files changed, 460 insertions(+) create mode 100644 verification/product/README.md create mode 100644 verification/product/foundations/coverage-and-guidance.md create mode 100644 verification/product/foundations/flow.md create mode 100644 verification/product/glossary.md diff --git a/verification/product/README.md b/verification/product/README.md new file mode 100644 index 000000000..848361ad0 --- /dev/null +++ b/verification/product/README.md @@ -0,0 +1,79 @@ +# House Scan: what the homeowner experiences + +A description of the iOS capture app from the outside: what the homeowner sees on each screen, +what they can do there, and exactly what happens when they do it. It is written from the app's +code and tests, then checked against the app running in the Simulator. Anything that looks +wrong is collected in [bug-triage.md](bug-triage.md). + +## Scope decisions + +- **Surface.** The HouseScan app, one run of the homeowner flow from onboarding to the result, + default settings. The ARKit camera cannot run in the Simulator, so behavior that needs the live + camera is described from code and marked unverified. +- **Source.** `t3/ios-mvf` has not received the app yet. The source is S3's local integration + commit `21a63e7` (engine lane `e72d14b` merged with UI lane `1dcbf36` onto `6885b7b`). Every + document's footer cites it. When `t3/ios-mvf` receives the app, the documents are rechecked + against that commit and the footers updated. +- **How it is run.** `make sim REF=21a63e7 ARGS="--replay --autopilot --server-url + --extra-arg=-autopilotHold --extra-arg=3"` from `verification/`, with the server from + `t3/server` running at the URL. The replay is `advio-20-0040-0075` (C3). +- **Out of scope.** The `-uiDemo` screens (a design preview, not the product), TestFlight + signing, the server's placement logic (checked by `hsverify/e2e.py`). +- **Not yet built.** Panel close-ups, homeowner questions other than "does this window open", + and "Point at the meter like this" after an interruption. They are not described. + +## Shape of every document + +The unit of interaction is **one screen of the scan** (one `ScanPhase`). Its five phases, the +subsections of "The interaction, event by event", are: + +1. **Arriving:** what the screen shows first and what the engine starts. +2. **Leaving at once:** the ways off the screen before anything is captured or marked. +3. **First capture:** the first photo kept, tap accepted or answer given. +4. **While capturing:** what updates live and what the homeowner can still do. +5. **Advancing:** what is committed and which screen comes next. + +**Variants** (rows of every Modifiers table, columns "at arrival" and "while capturing"): +live camera or replay; autopilot on or off; a server URL or the bundled sample result; larger +text sizes; Reduce Motion. + +**Interrupts** (rows of every Cancel and interrupt table, in this order): +the homeowner's way out on the screen ("I can't get there", "Can't get a clear shot", Cancel); +Start over; tracking limited (moving fast, too little detail, too dark); tracking lost or +relocalizing, including the 20 s reset; the app backgrounded or a call arriving; camera access +off or the camera session failing; the network lost or the upload failing; the app killed. + +**Cross-cutting concerns** (paragraphs of "Interactions with other systems", in this order): +coverage and evidence; stored photos; upload and offline; accessibility (VoiceOver, Dynamic +Type, Reduce Motion); haptics and motion; verification hooks (`STATE=` logs, replay, autopilot). + +Each document then has Summary, The simple case, The interaction (with one +`stateDiagram-v2`), Modifiers, Cancel and interrupt, Interactions with other systems, Edge cases, +and Open questions and verification, ending with `Verified against house-scanning commit 21a63e7`. +[screens/meter-close-up.md](screens/meter-close-up.md) is the pilot every other document copies. + +## Writing rules + +- Describe the experience, not the code. Technical detail goes in `> Technical note:` quotes, + only where the mechanism changes what the homeowner would expect. +- Use the words in [glossary.md](glossary.md). Numbers and definitions live in the foundation + documents; other documents link to them. +- Every table cell is filled, "No effect." where that is the answer. +- Behavior that looks like a bug is said plainly under "Open questions" with the file and line. + +## Structure and coverage + +| Document | What it covers | Status | +| --- | --- | --- | +| [foundations/flow.md](foundations/flow.md) | The screens in order, what moves between them, the launch options, Start over, failures | drafted | +| [foundations/coverage-and-guidance.md](foundations/coverage-and-guidance.md) | The coverage strip, when a photo is kept, the one instruction and what overrides it | drafted | +| [screens/onboarding.md](screens/onboarding.md) | The intro pages and the failure screens (camera off, unsupported phone, unreadable replay) | not started | +| [screens/find-meter.md](screens/find-meter.md) | Finding the meter and tapping it to set the wall | not started | +| [screens/meter-close-up.md](screens/meter-close-up.md) | The self-timed meter photo (pilot) | drafted | +| [screens/wall-walk.md](screens/wall-walk.md) | Walking the wall, marking its ends, "I can't get there" | not started | +| [screens/mark-features.md](screens/mark-features.md) | Marking gas meters, doors, windows, AC units, driveways and fences; the window question | not started | +| [screens/gap-request.md](screens/gap-request.md) | One targeted extra view, from the phone or the server | not started | +| [screens/uploading.md](screens/uploading.md) | Sending the scan and waiting for the result, offline and failure | not started | +| [screens/result.md](screens/result.md) | The result, its checks and the 3D and AR views | not started | +| [verification.md](verification.md) | Checks against the running app and their results | not started | +| [bug-triage.md](bug-triage.md) | Every suspected defect, deduplicated | not started | diff --git a/verification/product/foundations/coverage-and-guidance.md b/verification/product/foundations/coverage-and-guidance.md new file mode 100644 index 000000000..aa3b449f1 --- /dev/null +++ b/verification/product/foundations/coverage-and-guidance.md @@ -0,0 +1,153 @@ +# Coverage and guidance + +## Summary + +While the homeowner walks, the app decides three things without being asked: which camera frames +to keep as photos, which stretches of wall and ground those photos have covered, and the one +thing the homeowner should do next. This document owns the rules and numbers behind all three. +The screens that use them, [the wall walk](../screens/wall-walk.md) and +[a gap request](../screens/gap-request.md), link here instead of restating them. None of it +decides whether a battery fits; covered means evidence exists, not that a spot passes. + +## The simple case + +The homeowner walks slowly along the wall with the phone pointed at it. Every half meter or so +the phone keeps a photo; the counter at the top goes up and the fog over the wall clears where +the camera has looked. The strip at the bottom fills in: gray where nothing has been seen, then +covered once a stretch has been seen from two places. The instruction at the top says where to +go ("Walk slowly to your left") and changes only when that is done or after at least 3 s. + +## The interaction, event by event + +```mermaid +stateDiagram-v2 + [*] --> unseen + unseen --> seen : a kept photo shows most of the cell + seen --> covered : a second kept photo from 0.25 m away, tracking normal + unseen --> skipped : "I can't get there" over this cell + seen --> skipped : "I can't get there" over this cell +``` + +The states are those of one 6 in cell of the [coverage strip](../glossary.md#seeing-the-wall). + +### Arriving + +Coverage starts empty when the meter is tapped: the strip is centered on the meter, both bands +gray. Nothing is kept on [finding the meter](../screens/find-meter.md) or during the +[close-up](../screens/meter-close-up.md), apart from the close-up itself. + +### Leaving at once + +A cell that no kept photo shows stays unseen for the rest of the scan. Nothing decays: a covered +cell stays covered. + +### First capture + +#### When a photo is kept + +A frame is kept only when all of these hold: + +- tracking has been normal for at least 0.5 s; +- the phone moves slower than 1.5 m/s and turns slower than 60° per second; +- the frame is at least half as sharp as the recent frames, not too dark (mean brightness at + least 40 of 255) and not glaring (under a quarter of the pixels clipped); +- at least 0.33 s has passed since the last kept photo (at most about three per second); +- the phone has moved 0.5 m or turned 15° since the last kept photo, or the frame would show at + least three cells nobody has seen yet. + +A kept photo increments the counter and gives a light haptic tap. + +### While capturing + +A cell counts as **seen** when a kept photo shows most of it (60 % of its sample points) from no +farther than 6 m and no more than 65° from straight on, away from the outer 3 % of the image. It +becomes **covered** when a second kept photo, taken from at least 0.25 m away from the first, +also sees it. Photos are kept only with normal tracking, so every kept photo counts. + +The strip shows what has been seen plus 2.5 m of fog ahead on each side, and stops at a marked +wall end. + +#### One instruction at a time + +The instruction is chosen in this order, first match wins: + +1. **Coaching** from the camera, which replaces everything else until it clears: "Move your phone + slowly" (starting up), "Slow down a little", "Aim at something with more detail", "It's too dark + to see the wall", "Hold steady" (the last frame was blurry or moving), "Point at your meter + again" (relocalizing), "Your phone lost its place". +2. **"Take a step back"** when the phone is closer than 1.2 m to the wall; satisfied at 1.4 m. +3. **Aim at a lagging band**: within 1 m either side of the phone, if one band is done over at least + 0.45 m (three cells) where the other is not, "Tilt down to show the ground" or "Tilt up to show + more wall", with a ring on the spot. +4. **Walk to the left**, until coverage runs unbroken 6.1 m (20 ft) from the meter on the left; + then **"Is this the left end of the wall?"**. Only after the left end is set: the same for the + right. +5. With both ends set, the first stretch of at least 0.45 m between them where a band is not done + (ground checked before wall), as an aim instruction. +6. **"That's the whole wall"**, with "Tap Done when you're ready". + +A new instruction replaces the current one only when the current one is satisfied or has been up +for 3 s. An aim instruction is satisfied at 80 % of the cells within 0.3 m of its spot covered; a +walk instruction when that side's end is set. + +### Advancing + +Coverage is sent with the scan: covered cells as observed stretches of each band, skipped cells +and the kind of each end (real or unexplored). The server treats anything not observed as unknown. + +## Modifiers + +| Modifier | At arrival | While capturing | +| --- | --- | --- | +| Live camera or replay | No effect on the rules; a replay's frames go through the same checks. | Same. | +| Autopilot | No effect on the rules. | The replay runs at three times speed, so the time-based limits (0.33 s between photos, 3 s per instruction) bite more often. | +| Server or sample result | No effect. | No effect. | +| Larger text sizes | No effect on coverage. | No effect. | +| Reduce Motion | The fog still fades as cells clear, without drifting. | Same. | + +## Cancel and interrupt + +| Event | Before the first kept photo | After it | +| --- | --- | --- | +| The screen's own way out | "I can't get there" marks the cells the instruction asks for as skipped; see the screens. | Same. | +| Start over | Coverage is discarded. | Coverage and photos are discarded. | +| Tracking limited | No photo is kept until tracking has been normal for 0.5 s. | No photo is kept, so coverage stops growing until tracking is normal again for 0.5 s. | +| Tracking lost or relocalizing | Guidance waits behind the coaching. | After 20 s without relocalizing, coverage and photos are discarded and the scan returns to finding the meter. | +| App backgrounded or a call | No effect on coverage. | Coverage is kept. | +| Camera off or session failed | No photos can be kept. | Coverage so far is kept but cannot grow. | +| Network lost or upload failing | No effect. | No effect. | +| App killed | Coverage is lost. | Coverage is lost. | + +## Interactions with other systems + +**Coverage and evidence.** This document. **Stored photos.** Each kept photo is written as it is +kept; see [the flow](flow.md#interactions-with-other-systems). **Upload and offline.** Coverage +travels in the uploaded scene. **Accessibility.** The strip reads to VoiceOver as "Map of the +wall"; whether it also says how much is covered is checked on [the wall walk](../screens/wall-walk.md). +**Haptics and motion.** A light tap per kept photo; the fog fades where cells clear. **Verification +hooks.** Each instruction change logs `GUIDANCE=`. + +## Edge cases + +- Walking faster than 1.5 m/s keeps no photos at all, with only "Hold steady" or no coaching. +- A stretch closer than 0.45 m to a covered stretch on both sides is never asked for. +- A walk that reaches 20 ft on the left asks for the left end whether or not the wall ends there. +- Coverage is on the one wall set by the meter tap; wall past a corner never becomes covered. +- Something standing between the phone and the wall (a bush, a bin) does not stop a cell counting + as seen: coverage records where the camera looked, not what it could see. The server re-checks + from the photos (`HouseScanKit/.../CoverageMap.swift:51`). + +## Open questions and verification + +- **Possible bug: no truthful answer at 20 ft.** At 6.1 m the walk asks "Is this the left end of the + wall?" whether or not the wall continues. The only answers are "Wall ends here" (a real end) and + "I can't get there" (an unexplored end). A homeowner whose wall simply continues has neither; + choosing "Wall ends here" records a real end, which lets the server reject the site + (`HouseScanKit/.../GuidancePlanner.swift:103`). +- The walk always sends the homeowner left first. Starting at the right end of a wall means walking + past everything to the left first. +- "About N ft to go" can never appear: the engine always passes no remaining distance + (`Runtime/ScanEngine.swift`, `step(_:)`). +- All thresholds are the code's stated hypotheses (research note), not measured values. + +Verified against house-scanning commit `21a63e7`. diff --git a/verification/product/foundations/flow.md b/verification/product/foundations/flow.md new file mode 100644 index 000000000..63a827ae8 --- /dev/null +++ b/verification/product/foundations/flow.md @@ -0,0 +1,144 @@ +# The flow + +## Summary + +A scan is one pass through a fixed sequence of screens, from an introduction to a result. The +app decides when to move on: most screens advance by themselves when their job is done, and the +homeowner moves on with a button only where a judgment is theirs (they have finished walking, +their marks are right, they want to see the spot in AR). There is no back button between +screens. "Start over", on the result and on the unsupported screen, is the only way to begin again. This document owns the order +of the screens, what moves the app from one to the next, the launch options, and what happens +when the camera, the replay or the phone cannot run the scan. + +## The simple case + +The homeowner opens the app and pages through a short introduction ("Next", or "Skip" to the last page), then taps "Allow camera". The +camera opens and asks them to find their electric meter; they aim at it and tap. The phone takes a +close-up of the meter by itself, then asks them to walk along the wall, first to the left and +then to the right, marking where the wall ends on each side. They tap "Done with this wall" and see a list of the +things they marked, where they can add a gas meter, door, window, AC unit, driveway or fence. + +When they tap "Looks complete", the phone may ask for one more view of a stretch it did not see well. Then it +sends the scan, shows progress while the server works, and shows the result: whether there is a +spot for the battery, where, how much cable it needs, and each check. From the result they can +see the spot in AR on their wall, and come back. + +## The interaction, event by event + +```mermaid +stateDiagram-v2 + [*] --> onboarding + onboarding --> findMeter : "Allow camera" + onboarding --> unsupported : phone can't track motion + findMeter --> meterCloseUp : meter tapped on a detected wall + meterCloseUp --> wallWalk : close-up taken, or "Can't get a clear shot" + wallWalk --> markFeatures : "Done with this wall" (both ends set) + markFeatures --> gapRequest : "Looks complete", and the phone's check finds a gap + markFeatures --> uploading : "Looks complete", no gap + gapRequest --> uploading : view taken, or "I can't get there" + uploading --> result : the server answered + result --> gapRequest : "Capture it now" on a missing view + result --> resultAR : "See it on your wall" + resultAR --> result : "Done" + result --> onboarding : "Start over" + unsupported --> onboarding : "Start over" + wallWalk --> findMeter : phone lost its place for over 20 s +``` + +The names are the screen names the app logs (see the [glossary](../glossary.md#verification-words)). + +### Arriving + +The app always opens on `onboarding`. At launch it checks one thing: if the phone cannot run +motion tracking at all, it goes straight to `unsupported` and stays there. Nothing else is checked +up front. Camera permission is asked the first time the camera opens, on `findMeter`. + +### Leaving at once + +There is no way to leave a scan early other than closing the app. The next launch starts a new +scan on `onboarding`; the earlier scan's photos stay in the phone's cache but are never reopened. + +### First capture + +The scan's first durable step is the meter tap on `findMeter`. It fixes the wall for the rest of +the scan: the meter's position, the wall's direction and which side is outside. Every later +distance is measured along this wall from the meter. + +### While capturing + +The camera stays open, without a blink, from `findMeter` through `wallWalk`, `markFeatures` and +`gapRequest`; only the instruction and controls above it change. Photos are kept continuously on +`wallWalk` and `gapRequest` as described in +[coverage and guidance](coverage-and-guidance.md#when-a-photo-is-kept). A counter at the top +shows how many photos have been taken, and each new one gives a light tap of haptic feedback. + +### Advancing + +A screen whose job ends by itself advances after about 1.2 s, so the homeowner sees the success: +the close-up after it is taken, a gap request after the view is covered. The result appears only +after the server has answered; nothing on `uploading` is skipped when the network fails. + +## Modifiers + +| Modifier | At arrival | While capturing | +| --- | --- | --- | +| Live camera or replay | With `-replay ` the camera never opens: frames come from a recorded session, the meter tap uses the wall recorded with it, and `unsupported` is not shown even on a phone that can't track. | A replay plays its recording on `meterCloseUp`, `wallWalk` and `gapRequest`; elsewhere it stops. | +| Autopilot | With `-autopilot` (and a replay) the app presses its own buttons through the whole flow and shows a small badge. It needs a replay; with the live camera it logs that and does nothing. | Screens stay up for `-autopilotHold` seconds (default 1.2). The replay plays at three times speed. | +| Server or sample result | With `-serverURL ` the scan is uploaded there. Without it, or with `-sampleResult`, the app answers with a built-in sample result and the result says it is a sample. | No effect. | +| Larger text sizes | Screens reflow; see each screen. | No effect. | +| Reduce Motion | Screen changes crossfade in 0.15 s instead of moving. | No effect. | + +## Cancel and interrupt + +| Event | Before the meter is tapped | After the meter is tapped | +| --- | --- | --- | +| The screen's own way out | See each screen. | See each screen. | +| Start over | Offered only on `unsupported`. | Offered only on `result`. It forgets the wall, photos, marks and result and returns to `onboarding`. | +| Tracking limited | Coaching replaces the instruction until it clears ([coverage and guidance](coverage-and-guidance.md#one-instruction-at-a-time)). | Same; no photo counts toward coverage meanwhile. | +| Tracking lost or relocalizing | "Point at your meter again". | If the phone has not found its place within 20 s, the app forgets the wall, photos and marks and returns to `findMeter`. | +| App backgrounded or a call | The screen stays; coaching says "Point at your meter again" when the camera resumes. | Same; photos and marks are kept. | +| Camera off or session failed | Suspected dead end: see Open questions. | Suspected dead end: see Open questions. | +| Network lost or upload failing | No effect. | Only `uploading` is affected; see [the upload](../screens/uploading.md). | +| App killed | The scan is lost. | The scan is lost, including one "saved on this phone" after a failed upload: its files stay in the cache, but nothing reopens or retries them. | + +## Interactions with other systems + +**Coverage and evidence.** Owned by [coverage and guidance](coverage-and-guidance.md). +**Stored photos.** Kept photos and the close-up are written to a new folder in the app's cache +(`Caches/Scans//`) as they are taken, and zipped from there for upload. The folder is never +deleted by the app and never reopened. **Upload and offline.** Only the +upload needs the network; see [the upload](../screens/uploading.md). **Accessibility.** Every +screen groups its elements in one accessibility container. Its identifier (`screen.`) is +for tests and is not spoken. **Haptics and motion.** A light tap +for each photo; a success haptic when the meter is set and when the result appears; a firmer tap +for each mark placed and a warning for a refused one. **Verification hooks.** Each screen change +logs `STATE=` publicly under subsystem `dev.housescanning.housescan`, category `state`; +guidance changes log `GUIDANCE=` under category `guidance`. + +## Edge cases + +- The flow has no back navigation. A homeowner who confirms marks too early cannot return to the + walk; they can only continue or Start over from the result. +- Start over resets the photo counter and every mark; on a replay it shows the recording's first + frame again. +- A second `STATE` line for the same screen is never logged: moving to the screen already shown is + ignored. + +## Open questions and verification + +- **Suspected bug: camera access denied is a dead end.** The failure screen text exists ("House + Scan needs your camera ... Turn on Camera for House Scan in Settings") but only the + `unsupported` screen shows failures, and the app switches to it only when motion tracking is + unsupported (`Runtime/ScanEngine.swift:93`). A denied camera, a failed camera session and an + unreadable replay set the failure (`ScanEngine.swift:89, 338, 340`) without changing screen. The + homeowner would stay on "Find your electric meter" with no camera and no Settings button. + Checkable in the Simulator for the replay case: launch with `-replay /nonexistent`. +- "Your scan is saved on this phone" is true only while the app runs: after a relaunch nothing + reopens or retries it. +- Photos of the home accumulate in `Caches/Scans/` across scans and Start over; the app never + deletes them (`Runtime/KeyframeStore.swift:25`). Worth deciding whether a finished or abandoned + scan should be removed. +- The 1.2 s pause after a finished step and the 20 s relocalization limit are read from code; + neither has been timed on a device. + +Verified against house-scanning commit `21a63e7`. diff --git a/verification/product/glossary.md b/verification/product/glossary.md new file mode 100644 index 000000000..ffb5ae6e9 --- /dev/null +++ b/verification/product/glossary.md @@ -0,0 +1,84 @@ +# Glossary + +The words the documents use, each with one meaning. Screen text is quoted as it appears in the +app; everything else is this description's own term. + +## The house and the wall + +- **Meter.** The homeowner's electric meter. Everything is measured from it. +- **Wall.** The straight outside wall the meter is on, as a line along the ground at its foot. + Set once, by the meter tap; the app does not follow the wall round a corner. +- **Left, right.** As seen by someone standing outside, facing the wall. The app's words: "to + your left", "ft left of your meter". +- **Along the wall.** Distance from the meter along the wall line, negative to the left. The app + shows it in feet and inches; the code works in meters. +- **Wall end.** Where the homeowner said the wall stops. A **real end** comes from "Wall ends + here" (the wall stops at a corner or something blocking the way); an **unexplored end** comes + from "I can't get there" during the walk (the wall may continue). + +## Seeing the wall + +- **Kept photo.** A camera frame the app saves as evidence and later uploads. The homeowner never + presses a shutter; [coverage and guidance](foundations/coverage-and-guidance.md#when-a-photo-is-kept) + says when a frame is kept. +- **Close-up.** The one deliberate photo of the meter, taken by itself when the meter is centered, + near, sharp and well exposed. See [the meter close-up](screens/meter-close-up.md). +- **Coverage strip.** The map along the bottom of the walk screen: the wall unrolled flat, meter at + zero, in two rows (bands). +- **Band.** One row of the strip. The **wall band** is the wall face from the ground up to + 2.4 m (7 ft 10 in); the **ground band** is the ground from the foot of the wall out to 1.2 m + (3 ft 11 in). +- **Cell.** A 6 in (0.1524 m) stretch of one band. Each cell is in one of four states: + - **unseen**: no kept photo shows it; drawn as fog. + - **seen**: shown in kept photos, but not yet from two positions. + - **covered**: shown from two positions at least 0.25 m apart with normal tracking. Evidence + exists; it is not a pass. + - **skipped**: the homeowner said they can't get there. Recorded for installer review, never + evidence, drawn differently from both fog and covered. +- **Fog.** The overlay on the camera image over wall and ground not yet seen. It clears where + kept photos have looked. + +## Being told what to do + +- **Instruction.** The one line (plus an optional second line) at the top of a camera screen. + There is exactly one at a time. +- **Coaching.** An instruction about the phone rather than the wall ("Slow down a little", + "Hold steady"). It replaces the guidance instruction until the problem clears. +- **Target.** A ring on the camera image where the homeowner should aim, or a chevron at the + screen edge pointing to it when it is off screen. +- **Path.** Dots on the ground showing where to stand next, at most 3 m long, 1.5 m out from the + wall. +- **Tracking.** Whether the phone knows where it is. **Normal**, **limited** (moving fast, too + little detail, starting up) or **lost**. **Relocalizing** is the phone finding its place again + after an interruption. + +## Marking + +- **Feature.** Something the homeowner marks that affects where a battery can go: gas meter, door, + window, AC unit, driveway, fence. +- **Mark.** One tap on the camera image (or on the reticle with the Mark button) placing a point of + a feature. A door or window takes two (opposite corners), a driveway or fence two (along one + edge), a gas meter or AC unit one. +- **Refusal.** A tap the app does not accept, with the reason shown ("Nothing to pin there"). + +## After the walk + +- **Gap request.** A screen asking for one more specific view ("Show the ground 3 ft right of your + meter"), from the phone's own check or from the server. +- **Upload.** Sending the scan: the scene description and the kept photos, zipped. +- **Result.** The server's answer, shown as a headline, the battery spot and its cable, and a list + of checks. +- **Check.** One rule the server evaluated at the spot, shown as **"Looks good"** (pass), **"Not + sure yet"** (unsure) or **"Doesn't work"** (fail). +- **Installer review.** What happens to anything the phone can't settle: a person at Base + decides. "An installer will take a look" is the result headline for a manual review. +- **Sample result.** A built-in result the app shows when no server is configured. The screen + says it is a sample. + +## Verification words + +- **Replay.** Running the app on a recorded session instead of the live camera (`-replay`). +- **Autopilot.** The app pressing its own buttons through the flow on a replay (`-autopilot`). +- **Screen name.** The name the app logs as `STATE=` when a screen appears: + `onboarding`, `findMeter`, `meterCloseUp`, `wallWalk`, `markFeatures`, `gapRequest`, + `uploading`, `result`, `resultAR`, `unsupported`. From e38ca04df7073985cac51e91d092bf059dd952b5 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 04:25:57 -0500 Subject: [PATCH 23/59] verification: product description pilot, first Simulator pass, triage; scoreboard evidence Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/SCOREBOARD.md | 60 ++--- verification/product/README.md | 6 +- verification/product/bug-triage.md | 75 ++++++ .../product/screens/meter-close-up.md | 134 +++++++++++ verification/product/verification.md | 32 +++ verification/scoreboard/metrics.yaml | 14 +- verification/scoreboard/scoreboard.json | 213 ++++++++++-------- 7 files changed, 408 insertions(+), 126 deletions(-) create mode 100644 verification/product/bug-triage.md create mode 100644 verification/product/screens/meter-close-up.md create mode 100644 verification/product/verification.md diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index 542c21020..c35cecde2 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,16 +1,16 @@ # Scoreboard -Generated 2026-09-26 03:51 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-26 04:25 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | | [#14](https://github.com/SamGu-NRX/house-scanning/pull/14) | `t3/docs-cleanup` | `9c9ceb2a` | no | 4 success | 2026-09-26 03:15 CDT | -| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `820ac912` | yes | 4 success | 2026-09-26 03:50 CDT | -| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `86e3b3de` | yes | 4 success, 1 pending | 2026-09-26 03:50 CDT | -| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `44607e04` | yes | 5 success | 2026-09-26 03:37 CDT | -| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `6885b7b5` | yes | 5 success | 2026-09-26 03:07 CDT | +| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `1330a827` | yes | 4 success | 2026-09-26 04:21 CDT | +| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `b5ca7af8` | yes | 5 success | 2026-09-26 03:57 CDT | +| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `6a623d8a` | yes | 5 success | 2026-09-26 04:22 CDT | +| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `90c01dde` | yes | 4 success, 1 pending | 2026-09-26 04:17 CDT | | [#9](https://github.com/SamGu-NRX/house-scanning/pull/9) | `codex/capture-contract-probe` | `4b6c4309` | yes | 4 success | 2026-09-26 00:08 CDT | | [#8](https://github.com/SamGu-NRX/house-scanning/pull/8) | `t3/testflight` | `b896d59f` | no | 4 success | 2026-09-26 03:18 CDT | | [#7](https://github.com/SamGu-NRX/house-scanning/pull/7) | `t3/measure-lab` | `45770a3b` | no | 6 success | 2026-09-26 03:19 CDT | @@ -19,58 +19,58 @@ Generated 2026-09-26 03:51 CDT by `make scoreboard` (`uv run python -m hsverify. ## S1 Evals on real data -`origin/t3/evals` at [`86e3b3de`](https://github.com/SamGu-NRX/house-scanning/tree/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad). 2 met, 1 partial, 1 gap, 1 not run. +`origin/t3/evals` at [`b5ca7af8`](https://github.com/SamGu-NRX/house-scanning/tree/b5ca7af816cf09da6deb9e480e00719031f0643c). 2 met, 2 partial, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S1-1 | One command regenerates every number from a clean checkout | gap | not yet reviewed | -| S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/README.md) | +| S1-1 | One command regenerates every number from a clean checkout | partial | make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac. | +| S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/README.md) | | S1-3 | Metric code tested on hand-computed cases | not run | slow; run with --slow | -| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/README.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/results/advio_drift.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/results/eth3d_recon.md) | -| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/results/eth3d_recon.md) | +| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/README.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/results/advio_drift.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/results/eth3d_recon.md) | +| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/README.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/frames.md](https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/results/frames.md) | ## S2 Server and solver -`origin/t3/server` at [`44607e04`](https://github.com/SamGu-NRX/house-scanning/tree/44607e04e4f851944c47cb535c9c4b13bbad1c5b). 4 met, 1 partial, 1 gap, 2 not run. +`origin/t3/server` at [`6a623d8a`](https://github.com/SamGu-NRX/house-scanning/tree/6a623d8a7e340c15072efa412dd34738818e4f7e). 4 met, 1 partial, 2 not run, 1 no evidence yet. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S2-1 | pytest green | not run | slow; run with --slow | -| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_golden_public.py) | -| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_properties.py) | -| S2-4 | Schemas published | met | present at 44607e04 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/schemas/result.schema.json) | -| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_schemas.py), [server/tests/test_api.py](https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_api.py) | -| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 596.7 (want < 1000): ~/house-scanning-data/reports/e2e/20260926-035100-44607e04/report.json | +| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_golden_public.py) | +| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_properties.py) | +| S2-4 | Schemas published | met | present at 6a623d8a [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/schemas/result.schema.json) | +| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_schemas.py), [server/tests/test_api.py](https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_api.py) | +| S2-6 | A real-derived scene returns a result in under 1 s | no evidence yet | no latency_ms in ~/house-scanning-data/reports/e2e/20260926-042213-6a623d8a/report.json | | S2-7 | ruff clean | not run | slow; run with --slow | -| S2-8 | Black-box (verification e2e): every answer valid under C2, consistent with its decision and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric when mirrored, and never passing without coverage | gap | contract_problem_count = 3 (want 0): ~/house-scanning-data/reports/e2e/20260926-035100-44607e04/report.json | +| S2-8 | Black-box (verification e2e): every answer valid under C2, consistent with its decision and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric when mirrored, and never passing without coverage | met | contract_problem_count = 0 (want 0): ~/house-scanning-data/reports/e2e/20260926-042213-6a623d8a/report.json | ## S3 iOS capture app -`origin/t3/ios-mvf` at [`6885b7b5`](https://github.com/SamGu-NRX/house-scanning/tree/6885b7b5ad872df7f73a6b915e5837cd38a2034d). 3 met, 5 gap, 2 no evidence yet. +`origin/t3/ios-mvf` at [`90c01dde`](https://github.com/SamGu-NRX/house-scanning/tree/90c01ddef93bdc7abef6a0520c41025bf8fc33be). 2 met, 3 partial, 2 gap, 1 stale, 1 not run, 1 no evidence yet. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S3-1 | CI build green | met | iOS build: success at 6885b7b5 [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/6885b7b5ad872df7f73a6b915e5837cd38a2034d/ios/project.yml) | -| S3-3 | Zero build warnings | met | build.warnings = length 0 (want 0): ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json | -| S3-4 | Unit tests for coverage, auto-capture and guidance | gap | none found in 1 files; missing coverage, auto-capture, guidance | -| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | gap | none found in 1 files; missing replay argument, autopilot argument, gap loop | -| S3-6 | Accessibility audit on every screen | gap | 0 of 1 matches needed, searched 1 files | +| S3-1 | CI build green | not run | iOS build: pending at 90c01dde [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | +| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/project.yml) | +| S3-3 | Zero build warnings | stale | report for 6885b7b5, ref at 90c01dde: ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json | +| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage; missing auto-capture, guidance [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift) | +| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanUITests/FullFlowUITests.swift) | +| S3-6 | Accessibility audit on every screen | partial | 2 matches in 1 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanUITests/FullFlowUITests.swift) | | S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, plus symbol-name button labels) finds no issues; S4-2 shows how many screens the app reached | no evidence yet | no report matches ~/house-scanning-data/reports/a11y/*t3-ios-mvf*/report.json | | S3-7 | Screenshots of each state in the PR | gap | 0 of 9 matches in the PR body [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-8 | Exported scene.json validates against C1 | no evidence yet | no app_export_scene_valid in ~/house-scanning-data/reports/e2e/20260926-035100-44607e04/report.json | +| S3-8 | Exported scene.json validates against C1 | met | app_export_scene_valid = true (want True): ~/house-scanning-data/reports/e2e/20260926-041009-b04558df/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/verification` at [`820ac912`](https://github.com/SamGu-NRX/house-scanning/tree/820ac912ef0b9f15d0b07991d41dd2683c7d75cb), `origin/t3/ios-mvf` at [`6885b7b5`](https://github.com/SamGu-NRX/house-scanning/tree/6885b7b5ad872df7f73a6b915e5837cd38a2034d). 1 met, 2 gap, 1 no evidence yet. +`origin/t3/verification` at [`1330a827`](https://github.com/SamGu-NRX/house-scanning/tree/1330a82758df3a007fede909936542f5fb3fb358), `origin/t3/ios-mvf` at [`90c01dde`](https://github.com/SamGu-NRX/house-scanning/tree/90c01ddef93bdc7abef6a0520c41025bf8fc33be). 2 met, 1 partial, 1 stale. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S4-1 | End to end from a real replay to a placement result passes | no evidence yet | no app_export_passed in ~/house-scanning-data/reports/e2e/20260926-035100-44607e04/report.json | -| S4-2 | Screenshot report per app state | gap | states = length 0 (want at least 9): ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json | -| S4-3 | Product description written from the code and checked against screenshots | gap | not yet reviewed | -| S4-4 | Scoreboard of every PR | met | present at 820ac912 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/820ac912ef0b9f15d0b07991d41dd2683c7d75cb/verification/SCOREBOARD.md) | +| S4-1 | End to end from a real replay to a placement result passes | met | app_export_passed = true (want True): ~/house-scanning-data/reports/e2e/20260926-041009-b04558df/report.json | +| S4-2 | Screenshot report per app state | stale | report for 6885b7b5, ref at 90c01dde: ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json | +| S4-3 | Product description written from the code and checked against screenshots | partial | verification/product: scope, glossary, both foundations and the pilot drafted from S3's integration commit 21a63e7; one Simulator pass recorded (verification.md); 7 suspected defects in bug-triage.md. Seven screen documents to go. | +| S4-4 | Scoreboard of every PR | met | present at 1330a827 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/1330a82758df3a007fede909936542f5fb3fb358/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/product/README.md b/verification/product/README.md index 848361ad0..6bb6e7c18 100644 --- a/verification/product/README.md +++ b/verification/product/README.md @@ -69,11 +69,11 @@ and Open questions and verification, ending with `Verified against house-scannin | [foundations/coverage-and-guidance.md](foundations/coverage-and-guidance.md) | The coverage strip, when a photo is kept, the one instruction and what overrides it | drafted | | [screens/onboarding.md](screens/onboarding.md) | The intro pages and the failure screens (camera off, unsupported phone, unreadable replay) | not started | | [screens/find-meter.md](screens/find-meter.md) | Finding the meter and tapping it to set the wall | not started | -| [screens/meter-close-up.md](screens/meter-close-up.md) | The self-timed meter photo (pilot) | drafted | +| [screens/meter-close-up.md](screens/meter-close-up.md) | The self-timed meter photo (pilot) | drafted; checked on the replay | | [screens/wall-walk.md](screens/wall-walk.md) | Walking the wall, marking its ends, "I can't get there" | not started | | [screens/mark-features.md](screens/mark-features.md) | Marking gas meters, doors, windows, AC units, driveways and fences; the window question | not started | | [screens/gap-request.md](screens/gap-request.md) | One targeted extra view, from the phone or the server | not started | | [screens/uploading.md](screens/uploading.md) | Sending the scan and waiting for the result, offline and failure | not started | | [screens/result.md](screens/result.md) | The result, its checks and the 3D and AR views | not started | -| [verification.md](verification.md) | Checks against the running app and their results | not started | -| [bug-triage.md](bug-triage.md) | Every suspected defect, deduplicated | not started | +| [verification.md](verification.md) | Checks against the running app and their results | pass 1 recorded | +| [bug-triage.md](bug-triage.md) | Every suspected defect, deduplicated | 7 entries | diff --git a/verification/product/bug-triage.md b/verification/product/bug-triage.md new file mode 100644 index 000000000..c3594a038 --- /dev/null +++ b/verification/product/bug-triage.md @@ -0,0 +1,75 @@ +# Suspected defects + +From the documents' open questions and [verification.md](verification.md), deduplicated. Source +`21a63e7` (S3 integration commit) unless stated. + +| ID | Severity | Where the homeowner meets it | Decision | +| --- | --- | --- | --- | +| B-01 | Blocker | Uploading: every scan fails with 404 | fix (S3) | +| B-02 | Major | Uploading: a failure shows raw server JSON | fix (S3) | +| B-03 | Blocker | Camera denied, camera failure or unreadable replay: no message, no way on | fix (S3) | +| B-04 | Blocker | Meter close-up: "Can't get a clear shot" may never appear | fix (S3) | +| B-05 | Major, cause unconfirmed | Camera screens: cards render without text in Simulator screenshots | investigate (S3, S4) | +| B-06 | Major | Walk at 20 ft: "Is this the left end?" has no truthful answer when the wall continues | product call | +| B-07 | Minor | Photos of the home stay in the phone's cache after every scan | product call | + +## B-01. The app uploads to a route the server does not have + +- **What happens.** The app posts the scan to `POST {serverURL}/v1/scenes`; the server serves + `POST /v1/placements`. Every upload fails with 404. +- **Evidence.** Server log: `"POST /v1/scenes HTTP/1.1" 404 Not Found`. The same bundle posted to + `/v1/placements` as `application/zip` returns 200 with a result, and passes every check in + `hsverify/e2e.py` (report `20260926-041009-b04558df`). +- **Cause.** `ios/HouseScan/Runtime/ResultClient.swift:27`; the comment above it calls the route a + placeholder. Contracts C1 and C2 fix the body and the answer but not the path. + +## B-02. Upload failures show raw error text + +- **What happens.** "That didn't go through" is followed by `The server answered 404: + {"error":{"code":"not_found",...}}`. +- **Cause.** `ios/HouseScan/UI/Copy/ScanCopy.swift`, `upload(_:)` puts the error's description in + the detail line; `Runtime/ScanEngine.swift` passes `String(describing: error)`. +- **Expected.** Plain words ("We couldn't reach the server. Your scan is saved; try again.") and + the "Try again" button, which the screenshot does not show (see B-05). + +## B-03. Failures other than an unsupported phone never reach the screen + +- **What happens.** Only the `unsupported` screen shows a failure, and the app switches to it only + when motion tracking is unsupported. A denied camera, a failed camera session or an unreadable + replay records the failure and stays on the current screen. +- **Cause.** `Runtime/ScanEngine.swift:89, 93, 338, 340`. +- **Reproduce.** Launch with `-replay /nonexistent`: expected "This recording can't be opened"; + predicted from code: onboarding, then a camera screen with nothing on it. Not yet run. + +## B-04. The close-up's way out can fail to appear + +- **What happens.** A failed try counts only when one problem lasts 4 s with no good frame between. + Frames that alternate between good and blurry neither fill the 0.6 s ring nor count a failure, + so "Can't get a clear shot" never appears. +- **Cause.** `ios/HouseScanKit/Sources/HouseScanKit/Capture/CloseUpGate.swift`, `evaluate`. + S3's full-flow test cannot catch it: the autopilot calls the skip action directly. +- **Suggested fix.** Count a failed try per 4 s on the screen without a photo. + +## B-05. Camera-screen chrome renders without its text in Simulator screenshots + +- **What happens.** On `meterCloseUp` and `wallWalk` the instruction card, the pills and the strip + are empty panels; on `markFeatures` the text is black on the camera image with no panel; on + `uploading` after a failure the emblem and button are not visible. +- **Unknown.** Whether this is the app or the capture: screenshots were taken with `simctl io + screenshot` with the Simulator in light appearance, while camera screens force dark. Next: the + same screens captured from inside the app's process (XCUITest), and in dark appearance. + +## B-06. The walk asks for a wall end at 20 ft whether or not the wall ends + +- **What happens.** At 6.1 m of coverage the instruction is "Is this the left end of the wall?". + If the wall continues, the homeowner can only answer "Wall ends here" (recorded as a real end, + which lets the server reject) or "I can't get there" (an unexplored end). +- **Cause.** `ios/HouseScanKit/Sources/HouseScanKit/Guidance/GuidancePlanner.swift`, + `preferredTask`. A third answer ("The wall keeps going") recorded as unexplored-beyond-reach + would be truthful. + +## B-07. Scan photos are never deleted + +- **What happens.** Each scan writes its photos to `Caches/Scans//`; nothing deletes them + after upload, Start over or relaunch. iOS may purge caches, but only under storage pressure. +- **Cause.** `ios/HouseScan/Runtime/KeyframeStore.swift:25`. diff --git a/verification/product/screens/meter-close-up.md b/verification/product/screens/meter-close-up.md new file mode 100644 index 000000000..27ff0b0a2 --- /dev/null +++ b/verification/product/screens/meter-close-up.md @@ -0,0 +1,134 @@ +# The meter close-up + +## Summary + +The meter close-up takes one clear photo of the electric meter, so an installer can read it +later. The homeowner does not press a shutter: they hold the meter inside a circle and the phone +takes the photo itself once the meter is centered, near, sharp and well lit. It is the screen +right after the meter tap on [finding the meter](find-meter.md) (screen name `meterCloseUp`), +and it ends on its own. A homeowner who cannot get a usable photo can skip it after the second +failed try. + +## The simple case + +The circle appears in the middle of the camera view and the instruction reads "Hold your meter in +the circle", with "Your phone takes the photo by itself." The homeowner points the phone at the +meter from within about 1.5 m. While everything is right the circle's ring fills; after 0.6 s of +holding still it is full, the screen flashes white, a check mark appears in the circle and the +photo shrinks into the counter at the top right. About 1.2 s later the walk begins +([the wall walk](wall-walk.md)). + +## The interaction, event by event + +```mermaid +stateDiagram-v2 + [*] --> aiming + aiming --> holding : meter centered, near, sharp, well lit + holding --> aiming : any problem (ring empties) + holding --> taken : held 0.6 s (photo saved) + aiming --> aiming : a problem lasting 4 s (one failed try) + aiming --> skipped : "Can't get a clear shot" (from the second failed try) + taken --> [*] : after 1.2 s, the walk + skipped --> [*] : the walk at once +``` + +### Arriving + +The circle scales in and fades up around the middle of the screen. The ring is empty, the +failed-try count is zero and nothing is saved. The meter's position comes from the tap that +opened this screen; the close-up checks where that point appears in each camera frame. + +### Leaving at once + +There is no way off this screen before the second failed try other than taking the photo. No +back button, no skip. Nothing is recorded until the photo is taken. + +### First capture + +The ring starts filling on the first frame where all of these hold: tracking is normal; the meter +appears within the middle half of the image in both directions; the phone is within 1.5 m of it; +the image is not too dark (mean brightness at least 40 of 255) or glaring (mean above 225, or +more than a fifth of the pixels clipped); and the frame is at least half as sharp as the last 15. +While a check fails, one short fix shows under the instruction, first failing check first: + +| Problem | Shown | +| --- | --- | +| Tracking not normal | "Move slowly" | +| Meter outside the middle of the view, or behind the phone | "Center the meter in the circle" | +| Farther than 1.5 m | "Move closer to the meter" | +| Too dark | "Too dark to read. Try your phone's flashlight." | +| Too bright or glaring | "Too much glare. Tilt the phone a little." | +| Blurry | "Hold still" | + +### While capturing + +The ring fills over 0.6 s while every check passes; any failing frame empties it and the hold +starts again. A problem that lasts 4 s without a break counts as one failed try. From the second +failed try on, a button "Can't get a clear shot" appears below the fix. VoiceOver reads the circle +as "Photo of your meter" with its progress ("40 percent ready"). + +### Advancing + +When the ring is full the photo is saved as the scan's meter close-up. If saving fails, that +counts as a failed try and the fix reads "Hold still". Otherwise the screen flashes, the check +mark appears, the counter goes up with a light haptic tap, and 1.2 s later the walk begins. +"Can't get a clear shot" goes straight to the walk without a photo; the scan then carries no +meter close-up and an installer reads the meter instead. + +## Modifiers + +| Modifier | At arrival | While capturing | +| --- | --- | --- | +| Live camera or replay | A replay plays its recording from the start through this screen; it only succeeds if the recording shows the meter. | No effect. | +| Autopilot | The autopilot waits 4 s for the photo, then presses "Can't get a clear shot" itself, whether or not the button is showing. | No effect. | +| Server or sample result | No effect. | No effect. | +| Larger text sizes | The fix and the button grow; the circle does not. | No effect. | +| Reduce Motion | The circle fades in over 0.15 s instead of scaling. | The flash still shows; the photo does not fly into the counter. | + +## Cancel and interrupt + +| Event | Before the second failed try | From the second failed try | +| --- | --- | --- | +| The screen's own way out | None. | "Can't get a clear shot": the walk starts without a close-up. | +| Start over | Not offered. | Not offered. | +| Tracking limited | "Move slowly"; the ring empties and a 4 s problem counts as a failed try. | Same. | +| Tracking lost or relocalizing | Coaching replaces the instruction; the ring stays empty. After 20 s the scan returns to finding the meter. | Same. | +| App backgrounded or a call | The screen stays; the ring restarts when the camera resumes. | Same. | +| Camera off or session failed | Suspected dead end ([the flow](../foundations/flow.md#open-questions-and-verification)). | Same. | +| Network lost or upload failing | No effect. | No effect. | +| App killed | The scan is lost. | The scan is lost. | + +## Interactions with other systems + +**Coverage and evidence.** The close-up is not a walk photo and covers no cells. **Stored +photos.** Saved as the scan's meter close-up and uploaded with it. **Upload and offline.** No +effect here. **Accessibility.** The circle is one element with a spoken progress value; the fix +appears as a label; the button carries the hint "Skips the close-up. An installer will read the +meter instead." **Haptics and motion.** A success haptic as the screen opens (the meter is +set) and a light tap when the photo is taken. **Verification hooks.** The log records "close-up skipped after N +failed attempts" when the homeowner skips. + +## Edge cases + +- The circle is fixed in the middle of the screen, but the check uses the middle half of the + camera image. On a tall screen these differ: the meter can sit inside the circle and still be + "not centered", or outside it and pass. +- Two meters side by side: only the tapped one is checked. +- Saving can fail after the ring fills; that is the only failed try that is not 4 s long. + +## Open questions and verification + +- **Suspected bug: the way out can fail to appear.** A failed try counts only when one problem + lasts 4 s without a single good frame (`HouseScanKit/.../CloseUpGate.swift`, `evaluate`), while + the hold needs 0.6 s of good frames in a row. A homeowner whose frames alternate between good + and blurry (a shaky hand, flickering light) never fills the ring and never reaches a failed try, + so "Can't get a clear shot" never appears. That is the dead end the first-try note rules out. + Suggested rule: count attempts by elapsed time on the screen without a photo. +- The autopilot skips after 4 s by calling the skip action directly + (`Runtime/Autopilot.swift`), so S3's full-flow UI test cannot catch the dead end above. +- Observed in the Simulator on the ADVIO replay (no meter in it): the screen showed for 4 s and the + app logged "close-up skipped after 2 failed attempts". Its instruction card rendered without + its text in that screenshot; see [verification](../verification.md). +- The circle-versus-image mismatch above is read from code; it needs a device to confirm. + +Verified against house-scanning commit `21a63e7`. diff --git a/verification/product/verification.md b/verification/product/verification.md new file mode 100644 index 000000000..bdcc8a128 --- /dev/null +++ b/verification/product/verification.md @@ -0,0 +1,32 @@ +# Checks against the running app + +Each row is one claim from a document, checked in the Simulator on `21a63e7`. A failed row is not +automatically an app bug: sometimes the document is wrong, and the note says which. Screenshots +stay in `~/house-scanning-data/reports/sim/` because replay frames come from a non-commercial +dataset. + +## How a pass is run + +`make sim REF=21a63e7 ARGS="--replay ~/house-scanning-data/replays/advio-20-0040-0075 --autopilot +--server-url http://127.0.0.1:8765 --extra-arg=-autopilotHold --extra-arg=3"`, with the server +from `t3/server` on port 8765. The runner screenshots each screen and records the app's own log. +What this covers: the order of screens, what each shows, and what the app logged. What it does +not cover: anything needing the live camera (tracking, the meter tap on a real wall, the close-up +succeeding), haptics, or timing felt by a person. + +## Pass 1: 2026-09-26 04:06, report `20260926-040635-21a63e7-21a63e72-int-replay` + +| ID | Claim | Result | +| --- | --- | --- | +| FLOW-01 | Screens appear in the order of [the flow](foundations/flow.md#the-interaction-event-by-event) | Pass up to `uploading`: onboarding, findMeter, meterCloseUp, wallWalk, markFeatures, gapRequest, uploading. | +| FLOW-02 | The result appears only after the server answered | Pass, in the bad sense: the upload got 404 (see B-01) and the app stayed on `uploading`. | +| UP-01 | A failed upload offers "Try again" | Fail: the screenshot shows "That didn't go through" and the raw server JSON, with no button and no emblem visible (see B-02, B-05). | +| CU-01 | Without a usable photo, the close-up is skipped after the second failed try | Pass on the replay: logged "close-up skipped after 2 failed attempts" after 4 s. | +| CU-02 | The instruction card shows "Hold your meter in the circle" | Fail: the card is an empty panel in the screenshot (see B-05). | +| WW-01 | The walk shows one instruction, the strip and its labels | Fail: every card and the strip render without text (see B-05). | +| MF-01 | The feature list is readable over the dimmed camera | Fail: black text directly on the camera image, no panel (see B-05). | +| GAP-01 | A gap request appears when the phone's check finds a gap | Pass: shown, then skipped by the autopilot because this replay has no held-back frames. | + +Not checked yet: `result` and `resultAR` (unreachable while B-01 stands; a `-sampleResult` pass +is next), larger text sizes, dark appearance, and failure screens (launch with `-replay +/nonexistent` for B-03). diff --git a/verification/scoreboard/metrics.yaml b/verification/scoreboard/metrics.yaml index b2225d606..fcb8b9bf1 100644 --- a/verification/scoreboard/metrics.yaml +++ b/verification/scoreboard/metrics.yaml @@ -20,7 +20,12 @@ metrics: workstream: S1 ref: origin/t3/evals text: One command regenerates every number from a clean checkout - manual: {status: gap, evidence: not yet reviewed} + manual: + status: partial + evidence: >- + make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json + byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory + has not been bounded on this shared Mac. - id: S1-2 workstream: S1 @@ -273,7 +278,12 @@ metrics: workstream: S4 ref: origin/t3/verification text: Product description written from the code and checked against screenshots - manual: {status: gap, evidence: not yet reviewed} + manual: + status: partial + evidence: >- + verification/product: scope, glossary, both foundations and the pilot drafted from S3's + integration commit 21a63e7; one Simulator pass recorded (verification.md); 7 suspected + defects in bug-triage.md. Seven screen documents to go. - id: S4-4 workstream: S4 diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index 8d2d2ba70..b85390a39 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-09-26T03:51:15-05:00", + "generated_at": "2026-09-26T04:25:25-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": false, "prs": [ @@ -25,9 +25,9 @@ "number": 13, "title": "verification: independent end-to-end checks (S4)", "branch": "t3/verification", - "head": "820ac912ef0b9f15d0b07991d41dd2683c7d75cb", + "head": "1330a82758df3a007fede909936542f5fb3fb358", "draft": true, - "updated": "2026-09-26T08:50:35Z", + "updated": "2026-09-26T09:21:51Z", "ci": { "counts": { "success": 4, @@ -41,29 +41,29 @@ }, { "number": 12, - "title": "feat(evals): real-data accuracy evals, starting with an ADVIO replay session", + "title": "feat(evals): real-data accuracy evals for ARKit drift and photo-based wall reconstruction", "branch": "t3/evals", - "head": "86e3b3decffc2a08cd9ec5de540a514c0d35d7ad", + "head": "b5ca7af816cf09da6deb9e480e00719031f0643c", "draft": true, - "updated": "2026-09-26T08:50:55Z", + "updated": "2026-09-26T08:57:45Z", "ci": { "counts": { - "success": 4, + "success": 5, "failure": 0, - "pending": 1, + "pending": 0, "skipped": 0 }, "failing": [], - "text": "4 success, 1 pending" + "text": "5 success" } }, { "number": 11, "title": "server: placement solver, rules and API (S2)", "branch": "t3/server", - "head": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", + "head": "6a623d8a7e340c15072efa412dd34738818e4f7e", "draft": true, - "updated": "2026-09-26T08:37:02Z", + "updated": "2026-09-26T09:22:28Z", "ci": { "counts": { "success": 5, @@ -79,18 +79,18 @@ "number": 10, "title": "ios: guided wall scan, from meter to battery spot", "branch": "t3/ios-mvf", - "head": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", + "head": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", "draft": true, - "updated": "2026-09-26T08:07:57Z", + "updated": "2026-09-26T09:17:54Z", "ci": { "counts": { - "success": 5, + "success": 4, "failure": 0, - "pending": 0, + "pending": 1, "skipped": 0 }, "failing": [], - "text": "5 success" + "text": "4 success, 1 pending" } }, { @@ -192,7 +192,7 @@ "refs": [ { "ref": "origin/t3/evals", - "sha": "86e3b3decffc2a08cd9ec5de540a514c0d35d7ad" + "sha": "b5ca7af816cf09da6deb9e480e00719031f0643c" } ], "metrics": [ @@ -200,22 +200,22 @@ "id": "S1-1", "text": "One command regenerates every number from a clean checkout", "ref": "origin/t3/evals", - "sha": "86e3b3decffc2a08cd9ec5de540a514c0d35d7ad", - "status": "gap", - "evidence": "not yet reviewed", + "sha": "b5ca7af816cf09da6deb9e480e00719031f0643c", + "status": "partial", + "evidence": "make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac.", "links": [] }, { "id": "S1-2", "text": "Dataset URLs and sha256 recorded", "ref": "origin/t3/evals", - "sha": "86e3b3decffc2a08cd9ec5de540a514c0d35d7ad", + "sha": "b5ca7af816cf09da6deb9e480e00719031f0643c", "status": "met", "evidence": "all 2 found", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/README.md" ] ] }, @@ -223,7 +223,7 @@ "id": "S1-3", "text": "Metric code tested on hand-computed cases", "ref": "origin/t3/evals", - "sha": "86e3b3decffc2a08cd9ec5de540a514c0d35d7ad", + "sha": "b5ca7af816cf09da6deb9e480e00719031f0643c", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -232,21 +232,21 @@ "id": "S1-4", "text": "ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft", "ref": "origin/t3/evals", - "sha": "86e3b3decffc2a08cd9ec5de540a514c0d35d7ad", + "sha": "b5ca7af816cf09da6deb9e480e00719031f0643c", "status": "met", "evidence": "all 5 found", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/README.md" ], [ "experiments/evals/results/advio_drift.md", - "https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/results/advio_drift.md" + "https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/results/advio_drift.md" ], [ "experiments/evals/results/eth3d_recon.md", - "https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/results/eth3d_recon.md" + "https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/results/eth3d_recon.md" ] ] }, @@ -254,13 +254,21 @@ "id": "S1-5", "text": "ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question", "ref": "origin/t3/evals", - "sha": "86e3b3decffc2a08cd9ec5de540a514c0d35d7ad", + "sha": "b5ca7af816cf09da6deb9e480e00719031f0643c", "status": "partial", "evidence": "found 1-3 m row, 3-10 m row, scale error; missing verdict", "links": [ + [ + "experiments/evals/README.md", + "https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/README.md" + ], [ "experiments/evals/results/eth3d_recon.md", - "https://github.com/SamGu-NRX/house-scanning/blob/86e3b3decffc2a08cd9ec5de540a514c0d35d7ad/experiments/evals/results/eth3d_recon.md" + "https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/results/eth3d_recon.md" + ], + [ + "experiments/evals/results/frames.md", + "https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/results/frames.md" ] ] } @@ -272,7 +280,7 @@ "refs": [ { "ref": "origin/t3/server", - "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b" + "sha": "6a623d8a7e340c15072efa412dd34738818e4f7e" } ], "metrics": [ @@ -280,7 +288,7 @@ "id": "S2-1", "text": "pytest green", "ref": "origin/t3/server", - "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", + "sha": "6a623d8a7e340c15072efa412dd34738818e4f7e", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -289,13 +297,13 @@ "id": "S2-2", "text": "Every public golden test 01-14 of docs/research/t3-lane-c-review.md present", "ref": "origin/t3/server", - "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", + "sha": "6a623d8a7e340c15072efa412dd34738818e4f7e", "status": "met", "evidence": "all 14 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_golden_public.py" ] ] }, @@ -303,13 +311,13 @@ "id": "S2-3", "text": "Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail", "ref": "origin/t3/server", - "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", + "sha": "6a623d8a7e340c15072efa412dd34738818e4f7e", "status": "partial", "evidence": "found missing coverage, mirror; missing equality unsure, larger clearance", "links": [ [ "server/tests/test_properties.py", - "https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_properties.py" + "https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_properties.py" ] ] }, @@ -317,17 +325,17 @@ "id": "S2-4", "text": "Schemas published", "ref": "origin/t3/server", - "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", + "sha": "6a623d8a7e340c15072efa412dd34738818e4f7e", "status": "met", - "evidence": "present at 44607e04", + "evidence": "present at 6a623d8a", "links": [ [ "server/schemas/scene.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/schemas/scene.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/schemas/scene.schema.json" ], [ "server/schemas/result.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/schemas/result.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/schemas/result.schema.json" ] ] }, @@ -335,21 +343,21 @@ "id": "S2-5", "text": "Schemas validated in tests", "ref": "origin/t3/server", - "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", + "sha": "6a623d8a7e340c15072efa412dd34738818e4f7e", "status": "met", "evidence": "all 2 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_golden_public.py" ], [ "server/tests/test_schemas.py", - "https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_schemas.py" + "https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_schemas.py" ], [ "server/tests/test_api.py", - "https://github.com/SamGu-NRX/house-scanning/blob/44607e04e4f851944c47cb535c9c4b13bbad1c5b/server/tests/test_api.py" + "https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_api.py" ] ] }, @@ -357,16 +365,16 @@ "id": "S2-6", "text": "A real-derived scene returns a result in under 1 s", "ref": "origin/t3/server", - "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", - "status": "met", - "evidence": "latency_ms = 596.7 (want < 1000): ~/house-scanning-data/reports/e2e/20260926-035100-44607e04/report.json", + "sha": "6a623d8a7e340c15072efa412dd34738818e4f7e", + "status": "no evidence yet", + "evidence": "no latency_ms in ~/house-scanning-data/reports/e2e/20260926-042213-6a623d8a/report.json", "links": [] }, { "id": "S2-7", "text": "ruff clean", "ref": "origin/t3/server", - "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", + "sha": "6a623d8a7e340c15072efa412dd34738818e4f7e", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -375,9 +383,9 @@ "id": "S2-8", "text": "Black-box (verification e2e): every answer valid under C2, consistent with its decision and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric when mirrored, and never passing without coverage", "ref": "origin/t3/server", - "sha": "44607e04e4f851944c47cb535c9c4b13bbad1c5b", - "status": "gap", - "evidence": "contract_problem_count = 3 (want 0): ~/house-scanning-data/reports/e2e/20260926-035100-44607e04/report.json", + "sha": "6a623d8a7e340c15072efa412dd34738818e4f7e", + "status": "met", + "evidence": "contract_problem_count = 0 (want 0): ~/house-scanning-data/reports/e2e/20260926-042213-6a623d8a/report.json", "links": [] } ] @@ -388,7 +396,7 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d" + "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be" } ], "metrics": [ @@ -396,9 +404,9 @@ "id": "S3-1", "text": "CI build green", "ref": "origin/t3/ios-mvf", - "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", - "status": "met", - "evidence": "iOS build: success at 6885b7b5", + "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "status": "not run", + "evidence": "iOS build: pending at 90c01dde", "links": [ [ "#10", @@ -410,13 +418,13 @@ "id": "S3-2", "text": "Built as Swift 6 with strict concurrency", "ref": "origin/t3/ios-mvf", - "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", + "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", "status": "met", "evidence": "all 2 found", "links": [ [ "ios/project.yml", - "https://github.com/SamGu-NRX/house-scanning/blob/6885b7b5ad872df7f73a6b915e5837cd38a2034d/ios/project.yml" + "https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/project.yml" ] ] }, @@ -424,43 +432,66 @@ "id": "S3-3", "text": "Zero build warnings", "ref": "origin/t3/ios-mvf", - "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", - "status": "met", - "evidence": "build.warnings = length 0 (want 0): ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json", + "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "status": "stale", + "evidence": "report for 6885b7b5, ref at 90c01dde: ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json", "links": [] }, { "id": "S3-4", "text": "Unit tests for coverage, auto-capture and guidance", "ref": "origin/t3/ios-mvf", - "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", - "status": "gap", - "evidence": "none found in 1 files; missing coverage, auto-capture, guidance", - "links": [] + "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "status": "partial", + "evidence": "found coverage; missing auto-capture, guidance", + "links": [ + [ + "ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift", + "https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" + ], + [ + "ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift", + "https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift" + ], + [ + "ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift", + "https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift" + ] + ] }, { "id": "S3-5", "text": "UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop", "ref": "origin/t3/ios-mvf", - "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", - "status": "gap", - "evidence": "none found in 1 files; missing replay argument, autopilot argument, gap loop", - "links": [] + "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "status": "partial", + "evidence": "all 3 found (the tests exist; passing is S3-1)", + "links": [ + [ + "ios/HouseScanUITests/FullFlowUITests.swift", + "https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanUITests/FullFlowUITests.swift" + ] + ] }, { "id": "S3-6", "text": "Accessibility audit on every screen", "ref": "origin/t3/ios-mvf", - "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", - "status": "gap", - "evidence": "0 of 1 matches needed, searched 1 files", - "links": [] + "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "status": "partial", + "evidence": "2 matches in 1 files (an audit exists; which screens it covers needs a read)", + "links": [ + [ + "ios/HouseScanUITests/FullFlowUITests.swift", + "https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanUITests/FullFlowUITests.swift" + ] + ] }, { "id": "S3-6b", "text": "Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, plus symbol-name button labels) finds no issues; S4-2 shows how many screens the app reached", "ref": "origin/t3/ios-mvf", - "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", + "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", "status": "no evidence yet", "evidence": "no report matches ~/house-scanning-data/reports/a11y/*t3-ios-mvf*/report.json", "links": [] @@ -469,7 +500,7 @@ "id": "S3-7", "text": "Screenshots of each state in the PR", "ref": "origin/t3/ios-mvf", - "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", + "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", "status": "gap", "evidence": "0 of 9 matches in the PR body", "links": [ @@ -483,16 +514,16 @@ "id": "S3-8", "text": "Exported scene.json validates against C1", "ref": "origin/t3/ios-mvf", - "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", - "status": "no evidence yet", - "evidence": "no app_export_scene_valid in ~/house-scanning-data/reports/e2e/20260926-035100-44607e04/report.json", + "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "status": "met", + "evidence": "app_export_scene_valid = true (want True): ~/house-scanning-data/reports/e2e/20260926-041009-b04558df/report.json", "links": [] }, { "id": "S3-9", "text": "Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design", "ref": "origin/t3/ios-mvf", - "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", + "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -505,11 +536,11 @@ "refs": [ { "ref": "origin/t3/verification", - "sha": "820ac912ef0b9f15d0b07991d41dd2683c7d75cb" + "sha": "1330a82758df3a007fede909936542f5fb3fb358" }, { "ref": "origin/t3/ios-mvf", - "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d" + "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be" } ], "metrics": [ @@ -517,40 +548,40 @@ "id": "S4-1", "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/verification", - "sha": "820ac912ef0b9f15d0b07991d41dd2683c7d75cb", - "status": "no evidence yet", - "evidence": "no app_export_passed in ~/house-scanning-data/reports/e2e/20260926-035100-44607e04/report.json", + "sha": "1330a82758df3a007fede909936542f5fb3fb358", + "status": "met", + "evidence": "app_export_passed = true (want True): ~/house-scanning-data/reports/e2e/20260926-041009-b04558df/report.json", "links": [] }, { "id": "S4-2", "text": "Screenshot report per app state", "ref": "origin/t3/ios-mvf", - "sha": "6885b7b5ad872df7f73a6b915e5837cd38a2034d", - "status": "gap", - "evidence": "states = length 0 (want at least 9): ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json", + "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "status": "stale", + "evidence": "report for 6885b7b5, ref at 90c01dde: ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json", "links": [] }, { "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "820ac912ef0b9f15d0b07991d41dd2683c7d75cb", - "status": "gap", - "evidence": "not yet reviewed", + "sha": "1330a82758df3a007fede909936542f5fb3fb358", + "status": "partial", + "evidence": "verification/product: scope, glossary, both foundations and the pilot drafted from S3's integration commit 21a63e7; one Simulator pass recorded (verification.md); 7 suspected defects in bug-triage.md. Seven screen documents to go.", "links": [] }, { "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "820ac912ef0b9f15d0b07991d41dd2683c7d75cb", + "sha": "1330a82758df3a007fede909936542f5fb3fb358", "status": "met", - "evidence": "present at 820ac912", + "evidence": "present at 1330a827", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/820ac912ef0b9f15d0b07991d41dd2683c7d75cb/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/1330a82758df3a007fede909936542f5fb3fb358/verification/SCOREBOARD.md" ] ] } From 7c2c9ab5a5f4fc937d10cd8de0fd7904e61664e8 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 04:28:41 -0500 Subject: [PATCH 24/59] verification: make sim-app and make e2e; reports record peak memory sim-app runs the app at a ref on the real ADVIO replay with the server started from its own ref, one screenshot per STATE. e2e sends every case and the ETH3D scene through the server's API. Both reports record the harness's peak memory and that of the largest child it waited for. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/Makefile | 16 +++++++++++++++- verification/hsverify/e2e.py | 2 ++ verification/hsverify/memory.py | 15 +++++++++++++++ verification/hsverify/simrun.py | 20 +++++++++++++++++++- verification/tests/test_memory.py | 8 ++++++++ 5 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 verification/hsverify/memory.py create mode 100644 verification/tests/test_memory.py diff --git a/verification/Makefile b/verification/Makefile index 79e8d3cf6..ea4c6e888 100644 --- a/verification/Makefile +++ b/verification/Makefile @@ -2,9 +2,11 @@ # ~/house-scanning-data/reports/ (outside git) because runs can show dataset frames. REF ?= origin/t3/ios-mvf +SERVER_REF ?= origin/t3/server +REPLAY ?= $(HOME)/house-scanning-data/replays/advio-20-0040-0075 ARGS ?= -.PHONY: test sim sim-probe e2e scoreboard +.PHONY: test sim sim-app sim-probe e2e scoreboard test: uv run ruff check . @@ -16,6 +18,18 @@ test: sim: uv run python -m hsverify.simrun --ref $(REF) $(ARGS) +# The app at REF on the real ADVIO replay (C3), driven by its autopilot, uploading to the server +# started from SERVER_REF: a screenshot per STATE in ~/house-scanning-data/reports/sim/. +sim-app: + uv run python -m hsverify.simrun --ref $(REF) --replay $(REPLAY) --autopilot \ + --server-ref $(SERVER_REF) --extra-arg=-autopilotHold --extra-arg=3 \ + --idle 60 --timeout 900 --label replay $(ARGS) + +# Every case in e2e/cases/ plus the ETH3D facade scene through the server's API at SERVER_REF; +# each result is validated against server/schemas/result.schema.json at the same ref. +e2e: + uv run python -m hsverify.e2e --server-ref $(SERVER_REF) --scene scenes/eth3d-facade/case.json $(ARGS) + # Self-test of the runner against a fixture app that logs a known STATE sequence. sim-probe: uv run python -m hsverify.simrun --ref HEAD --project verification/fixtures/state-probe/StateProbe.xcodeproj \ diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index 2747efd00..aad933d7b 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -42,6 +42,7 @@ from pathlib import Path from hsverify import gitref +from hsverify.memory import peak_rss_mb from hsverify.resultcheck import ( assumption_mismatches, comparable, @@ -529,6 +530,7 @@ def main(argv: list[str] | None = None) -> int: "command": " ".join([sys.executable, "-m", "hsverify.e2e", *(argv or sys.argv[1:])]), # Latency on this shared Mac depends on what else runs; keep the load with the numbers. "load_average_1_5_15": [round(x, 1) for x in os.getloadavg()], + "peak_memory": peak_rss_mb(), "cases_sha256": hashlib.sha256(b"".join(i.raw for i in items)).hexdigest(), } report = write_report(out, meta, records) diff --git a/verification/hsverify/memory.py b/verification/hsverify/memory.py new file mode 100644 index 000000000..9b8e0e314 --- /dev/null +++ b/verification/hsverify/memory.py @@ -0,0 +1,15 @@ +"""Peak memory of this harness process and of the child processes it waited for. + +The Mac is shared by several agents and one oversized process froze it once, so every report +records these two numbers. macOS reports `ru_maxrss` in bytes. +""" + +from __future__ import annotations + +import resource + + +def peak_rss_mb() -> dict[str, float]: + own = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + children = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + return {"harness_mb": round(own / 2**20, 1), "largest_child_mb": round(children / 2**20, 1)} diff --git a/verification/hsverify/simrun.py b/verification/hsverify/simrun.py index 471ea3393..8199a3e05 100644 --- a/verification/hsverify/simrun.py +++ b/verification/hsverify/simrun.py @@ -15,6 +15,7 @@ from __future__ import annotations import argparse +import contextlib import datetime as dt import fcntl import json @@ -31,6 +32,8 @@ from pathlib import Path from hsverify import gitref, report +from hsverify.e2e import server_from_ref +from hsverify.memory import peak_rss_mb from hsverify.statelog import LOG_PREDICATE, RedactedStateError, parse_ndjson_line DEVICE_NAME = "HouseScan Verify" @@ -65,6 +68,8 @@ class RunReport: problems: list[str] = field(default_factory=list) crash_reports: list[str] = field(default_factory=list) final_screenshot: str | None = None + server: dict | None = None + peak_memory: dict | None = None def run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess: @@ -420,7 +425,11 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--scheme", default="HouseScan", help="also the .app name") parser.add_argument("--replay", type=Path, help="replay session folder (contract C3)") parser.add_argument("--autopilot", action="store_true") - parser.add_argument("--server-url") + server = parser.add_mutually_exclusive_group() + server.add_argument("--server-url", help="a running placement server") + server.add_argument( + "--server-ref", help="start the placement server from this ref for the run (S2 API)" + ) parser.add_argument("--extra-arg", action="append", default=[], help="more launch args") parser.add_argument("--until", action="append", default=[], help="state that ends the run") parser.add_argument("--settle", type=float, default=1.2, help="seconds before a screenshot") @@ -481,11 +490,18 @@ def main(argv: list[str] | None = None) -> int: static_c4=static, ) started = time.time() + services = contextlib.ExitStack() try: if app is None: rep.end_reason = "build failed" rep.problems.append("The app did not build; see build.log.") else: + if args.server_ref: + server_sha = gitref.resolve(args.server_ref) + url = services.enter_context(server_from_ref(server_sha, out / "server.log")) + launch_args += ["-serverURL", url] + rep.server = {"ref": args.server_ref, "sha": server_sha, "url": url} + print(f"Server {args.server_ref} at {server_sha[:12]} on {url}", flush=True) bundle_id = bundle_id_of(app) rep.device["bundle_id"] = bundle_id udid = device["udid"] @@ -504,6 +520,8 @@ def main(argv: list[str] | None = None) -> int: if rep.crash_reports: rep.problems.append(f"Crash reports: {', '.join(rep.crash_reports)}") finally: + services.close() + rep.peak_memory = peak_rss_mb() if not args.keep_booted: subprocess.run(["xcrun", "simctl", "shutdown", device["udid"]], capture_output=True) data = asdict(rep) diff --git a/verification/tests/test_memory.py b/verification/tests/test_memory.py new file mode 100644 index 000000000..c162b65ff --- /dev/null +++ b/verification/tests/test_memory.py @@ -0,0 +1,8 @@ +from hsverify.memory import peak_rss_mb + + +def test_peak_memory_is_reported_in_megabytes(): + peak = peak_rss_mb() + # A Python test process uses tens of MB, not bytes or gigabytes. + assert 5 < peak["harness_mb"] < 2000 + assert peak["largest_child_mb"] >= 0 From e18a2eb7a213a5bd9d752a4fadccf09c50bcebf2 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 05:10:39 -0500 Subject: [PATCH 25/59] verification: product description and UX review at t3/ios-mvf 0876e03; cleanup The product description covers every screen, written from S3's code and checked in the Simulator at t3/ios-mvf 0876e03 with the server's real answer; bug-triage.md lists 16 suspected defects, 2 resolved. ux/review.md judges every state against the checklist, with evidence from default, largest-text and dark runs and Apple's audit. The solver shim is gone now that the server has its own API; a deslop pass tidied the harness, the case generator and the ETH3D builder without changing their output (cases and scene hash identical). Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/README.md | 140 ++++---- verification/SCOREBOARD.md | 58 +-- verification/e2e/cases/generate.py | 7 +- verification/hsverify/e2e.py | 27 +- verification/hsverify/resultcheck.py | 4 +- verification/hsverify/scoreboard.py | 21 +- verification/hsverify/simrun.py | 10 +- verification/hsverify/solver_shim.py | 77 ---- verification/product/README.md | 41 +-- verification/product/bug-triage.md | 187 ++++++---- .../foundations/coverage-and-guidance.md | 46 ++- verification/product/foundations/flow.md | 21 +- verification/product/glossary.md | 32 +- verification/product/screens/find-meter.md | 176 +++++++++ verification/product/screens/gap-request.md | 191 ++++++++++ verification/product/screens/mark-features.md | 214 +++++++++++ .../product/screens/meter-close-up.md | 7 +- verification/product/screens/onboarding.md | 176 +++++++++ verification/product/screens/result.md | 221 ++++++++++++ verification/product/screens/uploading.md | 178 ++++++++++ verification/product/screens/wall-walk.md | 335 ++++++++++++++++++ verification/product/verification.md | 55 +-- .../scenes/eth3d-facade/build_scene.py | 34 +- verification/scoreboard/scoreboard.json | 192 ++++++---- verification/ux/review.md | 86 +++++ 25 files changed, 2067 insertions(+), 469 deletions(-) delete mode 100644 verification/hsverify/solver_shim.py create mode 100644 verification/product/screens/find-meter.md create mode 100644 verification/product/screens/gap-request.md create mode 100644 verification/product/screens/mark-features.md create mode 100644 verification/product/screens/onboarding.md create mode 100644 verification/product/screens/result.md create mode 100644 verification/product/screens/uploading.md create mode 100644 verification/product/screens/wall-walk.md create mode 100644 verification/ux/review.md diff --git a/verification/README.md b/verification/README.md index 1fd15abf7..67c7856c1 100644 --- a/verification/README.md +++ b/verification/README.md @@ -1,83 +1,65 @@ # Verification (S4) -Independent checks of what works end to end, run against the exact commits the other -workstreams push. Nothing here fixes their code; findings go back to the owning thread. +Independent checks of what the other workstreams' commits do, each tied to a commit SHA. Nothing +here fixes their code; findings go to the owning thread. Reports are written to +`~/house-scanning-data/reports/`, outside git, because replay frames come from non-commercial +datasets. Requirements: uv, and Xcode 26 or newer with an iOS Simulator runtime. -| Check | Command | Output | +| Check | Command, from `verification/` | Output | | --- | --- | --- | -| Unit tests for these tools | `make test` | pass or fail | -| App in the Simulator, a screenshot per `STATE` | `make sim REF=origin/t3/ios-mvf ARGS="--replay --autopilot --server-url "` | `~/house-scanning-data/reports/sim//index.html` | -| Runner self-test | `make sim-probe` | same, for `fixtures/state-probe` | -| A replay session (C3) is well formed, and how far its poses are from ground truth | `uv run python -m hsverify.replaycheck ` | printed | -| Apple's accessibility audit on every screen the app reaches | `uv run python -m hsverify.a11yaudit --ref origin/t3/ios-mvf --replay --autopilot` | `~/house-scanning-data/reports/a11y//index.html` | -| Scenes through the server's HTTP API, judged against C1, C2 and C5 | `uv run python -m hsverify.e2e --server-ref origin/t3/server [--real scene.zip]` | `~/house-scanning-data/reports/e2e//report.md` | - -Requirements: uv, Xcode 26 or newer with an iOS Simulator runtime. - -## Simulator runner - -`hsverify/simrun.py` checks the ref out into a detached worktree under `/tmp`, builds it for -the Simulator, installs it on its own device ("HouseScan Verify", so it never touches a -Simulator another worker is using), and launches it with the contract C4 arguments. It -attaches `log stream` before launch and follows `STATE=` lines from subsystem -`dev.housescanning.housescan`, category `state`. - -A state is screenshotted after it has been on screen for `--settle` seconds (default 1.2). If -the next state arrives first, the screenshot is taken at once and marked *transient*, since it -may already show the next screen. The run ends at an `--until` state, after `--idle` seconds -with no new state, or at `--timeout`. It reports a build failure, a crash, markers logged as -``, and a run with no markers at all as problems. - -Builds wait while any other `xcodebuild` runs, because four workers share this Mac. - -The report folder holds `index.html` (screenshots in state order), `report.json`, -`report.md`, the build log, the raw log stream and the app's stdout and stderr. - -`fixtures/state-probe` is a ten-line app that logs a fixed sequence, including a 0.3 s state, -so the runner can be checked without the real app. - -## Replay check - -`hsverify/replaycheck.py` reads a Measure Lab v2 session and checks what the app and server -rely on: every listed JPEG exists at the stated size, poses are rigid, timestamps increase, -and motion keyframes respect the spacing gate the session declares. With `ground_truth.json` -beside it, it aligns ground truth to ARKit by heading and origin (both worlds are gravity -aligned) and reports the error after the best fit, after also removing scale, and when -anchored at the first keyframe, as the app anchors on the meter. The ARKit-to-truth path -length ratio is printed on its own because a scale difference dominates the other numbers. -It does not say which track is right: ADVIO's ground truth has its own scale error (see -`experiments/evals` on `t3/evals`). - -## End-to-end server check - -`hsverify/e2e.py` starts the server from a ref in a `/tmp` worktree, or uses `--server-url`, -and reads the scene endpoint from its OpenAPI document. It stops rather than guesses when -more than one POST could take a scene. Every scene is validated against the scene schema at -the same ref first, so bad input is not blamed on the server. Every answer must validate -against the result schema and pass `hsverify/resultcheck.py`: - -- `pass` needs a passing spot, every check passing, no photo request and an approved policy; - `reject` has no spot and no photo request; `manual_review` gives reasons. -- Each check's outcome follows from its numbers (C5): at least T needs measured − error > T - to pass and measured + error < T to fail; anything else is unsure. At most T is mirrored. -- No battery start passes unless the wall and cable route back to the meter, and the ground - under the battery out to its depth, were observed. No photo request covers an observed area. -- Counts add up, `input_sha256` is the hash of the bytes sent, and the spot's offset from the - meter equals its centre minus the meter. - -Each scene is also sent again (identical result apart from timing), mirrored left to right -(same decision and the same length of passing, unsure and failing starts), and without its -coverage (no pass anywhere). Case files in `e2e/cases/` add the outcomes their geometry -forces. Each one names the thresholds it assumes, and a case whose assumption differs from the -server's rules is reported as an assumption mismatch, not as a pass or fail. Scenes passed -with `--real` must answer within 1 s. - -## Accessibility audit - -`hsverify/a11yaudit.py` builds and installs the app as the Simulator runner does, then runs the -UI test bundle in `fixtures/a11y-audit`. The bundle launches the installed app by bundle id with -the C4 arguments and calls `performAccessibilityAudit(for: .all)` on each distinct screen (a -screen is new when the labels of its texts and buttons change). Apple's audit does not object -to a button whose label SwiftUI took from an SF Symbol name, which VoiceOver reads aloud, so the -wrapper flags those too. The probe app's middle screen has an unlabelled 12 pt button as a -negative control: an audit of the probe must report it, and does (hit area, symbol-name label). +| The app at `t3/ios-mvf` on the real ADVIO replay, uploading to a server started from `t3/server`; a screenshot per `STATE` | `make sim-app` (`REF=`, `SERVER_REF=`, `ARGS=` to vary) | `reports/sim//index.html` | +| Every case, the ETH3D facade scene and optional app exports through the server's API | `make e2e` (`ARGS="--app-export "`) | `reports/e2e//report.md` | +| Apple's accessibility audit on every screen the app reaches | `uv run python -m hsverify.a11yaudit --ref --replay --autopilot` | `reports/a11y//index.html` | +| A replay session is well formed; its poses against ground truth | `uv run python -m hsverify.replaycheck ` | printed | +| Every open PR and plan metric with its evidence | `make scoreboard` (`ARGS=--slow` also runs test suites) | `SCOREBOARD.md` | +| These tools' own tests | `make test` | pass or fail | +| The Simulator runner against a probe app with a known sequence | `make sim-probe` | `reports/sim//` | + +Written results: [the UX review](ux/review.md) against [the checklist](ux/checklist.md), and +[the product description](product/README.md) with its [bug triage](product/bug-triage.md). + +## What the checks guarantee + +**Simulator runner** (`hsverify/simrun.py`). Builds the ref in a `/tmp` worktree, installs it on +its own Simulator ("HouseScan Verify") and launches it with the C4 arguments. It follows +`STATE=` from subsystem `dev.housescanning.housescan` and screenshots each state once it has +settled for 1.2 s; a state replaced sooner is captured at once and marked *transient*. The report +lists a failed build, a crash, markers logged as ``, and a run with no markers as +problems, and keeps every app log line. It waits while another `xcodebuild` compiles, because +several workers share the Mac. + +**End-to-end check** (`hsverify/e2e.py`, `hsverify/resultcheck.py`). Starts the server from a ref +and finds its placement endpoint in the OpenAPI document, refusing to guess between candidates. +Each scene is validated against `server/schemas/scene.schema.json` at that ref first, so bad input +is not blamed on the server. Each answer must validate against `server/schemas/result.schema.json` +and satisfy, for any policy: + +- `pass` has a passing spot, all checks passing, no photo request and an approved policy; + `reject` has no spot; `manual_review` gives reasons. +- Each check's outcome follows from its numbers (C5): a minimum T passes only when measured − + error > T and fails only when measured + error < T, a maximum mirrored, a review line also + cleared for a pass; an unsure labelled `margin` lies within its error of a line. +- No battery start passes over wall, cable route or ground that was not observed, and no photo + request covers an observed area. +- Counts add up, `input_sha256` is the hash of the bytes sent, the spot's offset equals its centre + minus the meter. +- The same scene sent again answers the same; mirrored left to right it gives the same decision + and the same length of each outcome; without coverage nothing passes. + +The 40 cases in [`e2e/cases/`](e2e/cases/README.md) were written from the public goldens and C5 +without reading the server's tests. Each names the thresholds it assumes; a mismatch is reported +as such rather than as a pass or fail. [`scenes/eth3d-facade`](scenes/eth3d-facade/README.md) +builds a scene from real laser-scanned geometry; real scenes must answer within 1 s. + +**Accessibility audit** (`hsverify/a11yaudit.py`). Runs the UI test bundle in +`fixtures/a11y-audit`, which launches the installed app by bundle id and calls +`performAccessibilityAudit(for: .all)` on each distinct screen. It also flags buttons labelled +with an SF Symbol name, which Apple's audit accepts. The probe app's middle screen carries an +unlabelled 12 pt button; an audit of the probe must find it. + +**Replay check** (`hsverify/replaycheck.py`). Checks the fields, JPEG sizes, rigid poses and +spacing gate of a Measure Lab v2 session, then compares ARKit with `ground_truth.json` after +aligning heading and origin. It reports the path-length ratio separately and does not say which +track is right: ADVIO's ground truth has its own scale error. + +Every report records the harness's peak memory and that of the largest child it waited for. diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index c35cecde2..088ea4635 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,16 +1,18 @@ # Scoreboard -Generated 2026-09-26 04:25 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-26 05:09 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | +| [#16](https://github.com/SamGu-NRX/house-scanning/pull/16) | `t3/meter-closeup` | `60a912bc` | yes | 4 success | 2026-09-26 04:49 CDT | +| [#15](https://github.com/SamGu-NRX/house-scanning/pull/15) | `t3/web-review` | `c69f8880` | yes | 5 success | 2026-09-26 05:06 CDT | | [#14](https://github.com/SamGu-NRX/house-scanning/pull/14) | `t3/docs-cleanup` | `9c9ceb2a` | no | 4 success | 2026-09-26 03:15 CDT | -| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `1330a827` | yes | 4 success | 2026-09-26 04:21 CDT | -| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `b5ca7af8` | yes | 5 success | 2026-09-26 03:57 CDT | -| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `6a623d8a` | yes | 5 success | 2026-09-26 04:22 CDT | -| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `90c01dde` | yes | 4 success, 1 pending | 2026-09-26 04:17 CDT | +| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `7c2c9ab5` | yes | 4 success | 2026-09-26 04:28 CDT | +| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `f6d081d2` | yes | 5 success | 2026-09-26 04:57 CDT | +| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `26d28707` | yes | 5 success | 2026-09-26 05:06 CDT | +| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `24af4342` | yes | 4 success, 1 pending | 2026-09-26 05:02 CDT | | [#9](https://github.com/SamGu-NRX/house-scanning/pull/9) | `codex/capture-contract-probe` | `4b6c4309` | yes | 4 success | 2026-09-26 00:08 CDT | | [#8](https://github.com/SamGu-NRX/house-scanning/pull/8) | `t3/testflight` | `b896d59f` | no | 4 success | 2026-09-26 03:18 CDT | | [#7](https://github.com/SamGu-NRX/house-scanning/pull/7) | `t3/measure-lab` | `45770a3b` | no | 6 success | 2026-09-26 03:19 CDT | @@ -19,58 +21,58 @@ Generated 2026-09-26 04:25 CDT by `make scoreboard` (`uv run python -m hsverify. ## S1 Evals on real data -`origin/t3/evals` at [`b5ca7af8`](https://github.com/SamGu-NRX/house-scanning/tree/b5ca7af816cf09da6deb9e480e00719031f0643c). 2 met, 2 partial, 1 not run. +`origin/t3/evals` at [`f6d081d2`](https://github.com/SamGu-NRX/house-scanning/tree/f6d081d20df1063491a169c9365f1e295452b069). 2 met, 2 partial, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S1-1 | One command regenerates every number from a clean checkout | partial | make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac. | -| S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/README.md) | +| S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/README.md) | | S1-3 | Metric code tested on hand-computed cases | not run | slow; run with --slow | -| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/README.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/results/advio_drift.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/results/eth3d_recon.md) | -| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/README.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/frames.md](https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/results/frames.md) | +| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/README.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/results/advio_drift.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/results/eth3d_recon.md) | +| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/README.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/results/eth3d_visibility_sensitivity.md) | ## S2 Server and solver -`origin/t3/server` at [`6a623d8a`](https://github.com/SamGu-NRX/house-scanning/tree/6a623d8a7e340c15072efa412dd34738818e4f7e). 4 met, 1 partial, 2 not run, 1 no evidence yet. +`origin/t3/server` at [`26d28707`](https://github.com/SamGu-NRX/house-scanning/tree/26d28707473f25f7c7d1f3660fbbf8321704f609). 5 met, 1 partial, 2 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S2-1 | pytest green | not run | slow; run with --slow | -| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_golden_public.py) | -| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_properties.py) | -| S2-4 | Schemas published | met | present at 6a623d8a [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/schemas/result.schema.json) | -| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_schemas.py), [server/tests/test_api.py](https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_api.py) | -| S2-6 | A real-derived scene returns a result in under 1 s | no evidence yet | no latency_ms in ~/house-scanning-data/reports/e2e/20260926-042213-6a623d8a/report.json | +| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_golden_public.py) | +| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_properties.py) | +| S2-4 | Schemas published | met | present at 26d28707 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/schemas/result.schema.json) | +| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_schemas.py), [server/tests/test_api.py](https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_api.py) | +| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 970.4 (want < 1000): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | | S2-7 | ruff clean | not run | slow; run with --slow | -| S2-8 | Black-box (verification e2e): every answer valid under C2, consistent with its decision and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric when mirrored, and never passing without coverage | met | contract_problem_count = 0 (want 0): ~/house-scanning-data/reports/e2e/20260926-042213-6a623d8a/report.json | +| S2-8 | Black-box (verification e2e): every answer valid under C2, consistent with its decision and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric when mirrored, and never passing without coverage | met | contract_problem_count = 0 (want 0): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | ## S3 iOS capture app -`origin/t3/ios-mvf` at [`90c01dde`](https://github.com/SamGu-NRX/house-scanning/tree/90c01ddef93bdc7abef6a0520c41025bf8fc33be). 2 met, 3 partial, 2 gap, 1 stale, 1 not run, 1 no evidence yet. +`origin/t3/ios-mvf` at [`24af4342`](https://github.com/SamGu-NRX/house-scanning/tree/24af4342bf760c4af7ed3f942c330533bed1f804). 2 met, 3 partial, 2 gap, 1 stale, 1 not run, 1 no evidence yet. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S3-1 | CI build green | not run | iOS build: pending at 90c01dde [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/project.yml) | -| S3-3 | Zero build warnings | stale | report for 6885b7b5, ref at 90c01dde: ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json | -| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage; missing auto-capture, guidance [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift) | -| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanUITests/FullFlowUITests.swift) | -| S3-6 | Accessibility audit on every screen | partial | 2 matches in 1 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanUITests/FullFlowUITests.swift) | +| S3-1 | CI build green | not run | iOS build: pending at 24af4342 [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | +| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/project.yml) | +| S3-3 | Zero build warnings | stale | report for 0876e03c, ref at 24af4342: ~/house-scanning-data/reports/sim/20260926-050058-t3-ios-mvf-0876e03c-replay/report.json | +| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage; missing auto-capture, guidance [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift) | +| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanUITests/ScreenStatesUITests.swift) | | S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, plus symbol-name button labels) finds no issues; S4-2 shows how many screens the app reached | no evidence yet | no report matches ~/house-scanning-data/reports/a11y/*t3-ios-mvf*/report.json | | S3-7 | Screenshots of each state in the PR | gap | 0 of 9 matches in the PR body [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-8 | Exported scene.json validates against C1 | met | app_export_scene_valid = true (want True): ~/house-scanning-data/reports/e2e/20260926-041009-b04558df/report.json | +| S3-8 | Exported scene.json validates against C1 | met | app_export_scene_valid = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/verification` at [`1330a827`](https://github.com/SamGu-NRX/house-scanning/tree/1330a82758df3a007fede909936542f5fb3fb358), `origin/t3/ios-mvf` at [`90c01dde`](https://github.com/SamGu-NRX/house-scanning/tree/90c01ddef93bdc7abef6a0520c41025bf8fc33be). 2 met, 1 partial, 1 stale. +`origin/t3/verification` at [`7c2c9ab5`](https://github.com/SamGu-NRX/house-scanning/tree/7c2c9ab5a5f4fc937d10cd8de0fd7904e61664e8), `origin/t3/ios-mvf` at [`24af4342`](https://github.com/SamGu-NRX/house-scanning/tree/24af4342bf760c4af7ed3f942c330533bed1f804). 2 met, 1 partial, 1 stale. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S4-1 | End to end from a real replay to a placement result passes | met | app_export_passed = true (want True): ~/house-scanning-data/reports/e2e/20260926-041009-b04558df/report.json | -| S4-2 | Screenshot report per app state | stale | report for 6885b7b5, ref at 90c01dde: ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json | +| S4-1 | End to end from a real replay to a placement result passes | met | app_export_passed = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | +| S4-2 | Screenshot report per app state | stale | report for 0876e03c, ref at 24af4342: ~/house-scanning-data/reports/sim/20260926-050058-t3-ios-mvf-0876e03c-replay/report.json | | S4-3 | Product description written from the code and checked against screenshots | partial | verification/product: scope, glossary, both foundations and the pilot drafted from S3's integration commit 21a63e7; one Simulator pass recorded (verification.md); 7 suspected defects in bug-triage.md. Seven screen documents to go. | -| S4-4 | Scoreboard of every PR | met | present at 1330a827 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/1330a82758df3a007fede909936542f5fb3fb358/verification/SCOREBOARD.md) | +| S4-4 | Scoreboard of every PR | met | present at 7c2c9ab5 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/7c2c9ab5a5f4fc937d10cd8de0fd7904e61664e8/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/e2e/cases/generate.py b/verification/e2e/cases/generate.py index ed3b4ae48..ed3717ddb 100644 --- a/verification/e2e/cases/generate.py +++ b/verification/e2e/cases/generate.py @@ -120,7 +120,6 @@ def polyline_wall(wall_id, s_start, segments): x, z = pts[-1] pts.append((x + length * math.cos(a), z + length * math.sin(a))) s_marks.append(s_marks[-1] + length) - # Find the point at s = 0 and shift it to the origin. for i in range(len(segments)): if s_marks[i] <= 0 <= s_marks[i + 1]: t = (0 - s_marks[i]) / (s_marks[i + 1] - s_marks[i]) @@ -418,7 +417,7 @@ def g10_scene(d, e, wall_pm=0.0): ) # g11b: both walls walked to real ends, every start fails gas. -z1 = D + 1.0 +strip_out = D + 1.0 g11b = { "schema_version": "1.0", "meter": meter(0.0, 0.0), @@ -427,8 +426,8 @@ def g10_scene(d, e, wall_pm=0.0): {"id": "w2", "baseline": [[8, 0], [8, -6]], "plus_minus_ft": 0.0}, ], "objects": [ - gas("w1", (-3, 8), rect(-10, 12, z1, z1 + 0.5)), - gas("w2", (8, 14), [[8 + z1, -10], [8 + z1 + 0.5, -10], [8 + z1 + 0.5, 3], [8 + z1, 3]]), + gas("w1", (-3, 8), rect(-10, 12, strip_out, strip_out + 0.5)), + gas("w2", (8, 14), rect(8 + strip_out, 8 + strip_out + 0.5, -10, 3)), ], "ground": [concrete([[-3, 0], [8, 0], [8, -6], [18, -6], [18, 10], [-3, 10]])], "facing": [facing("w1", (-3, 8)), facing("w2", (8, 14))], diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index aad933d7b..13e821726 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -146,21 +146,15 @@ def find_app(server_dir: Path) -> str: return found[0] -SHIM = Path(__file__).with_name("solver_shim.py") - - @contextlib.contextmanager -def server_from_ref(sha: str, log_path: Path, via_shim: bool = False) -> Iterator[str]: - """The server at `sha` on a free port: its own FastAPI app, or `solver_shim` around `solve`.""" +def server_from_ref(sha: str, log_path: Path) -> Iterator[str]: + """The server at `sha` running its own FastAPI app on a free port.""" with gitref.detached_worktree(sha) as tree: server_dir = tree / "server" subprocess.run(["uv", "sync", "--locked", "--quiet"], cwd=server_dir, check=True) port = free_port() url = f"http://127.0.0.1:{port}" - if via_shim: - target = ["python", str(SHIM), "--server-dir", str(server_dir), "--port", str(port)] - else: - target = ["uvicorn", find_app(server_dir), "--host", "127.0.0.1", "--port", str(port)] + target = ["uvicorn", find_app(server_dir), "--host", "127.0.0.1", "--port", str(port)] with log_path.open("w") as log: proc = subprocess.Popen( ["uv", "run", "--quiet", *target], @@ -414,7 +408,7 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: lines = [ f"# End-to-end run against `{meta['server_ref']}` at `{meta['server_sha'][:12]}`", "", - f"{meta['started_at']}. Endpoint `{meta['endpoint']}` via {meta['transport']}. " + f"{meta['started_at']}. Endpoint `{meta['endpoint']}`. " f"Load average {meta['load_average_1_5_15']} on {os.cpu_count()} cores. " + ", ".join(f"{v} {k}" for k, v in sorted(counts.items())), "", @@ -439,11 +433,6 @@ def main(argv: list[str] | None = None) -> int: where.add_argument("--server-url", help="use a running server instead of starting one") parser.add_argument("--schema-ref", help="ref for the schemas (default: --server-ref)") parser.add_argument("--endpoint", help="POST path, when discovery is ambiguous") - parser.add_argument( - "--via-shim", - action="store_true", - help="serve the ref's solver.solve through solver_shim (before the ref has an API)", - ) parser.add_argument("--cases", type=Path, default=DEFAULT_CASES) parser.add_argument( "--scene", @@ -470,7 +459,8 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) gitref.fetch() - schema_ref = args.schema_ref or (args.server_ref if not args.server_url else "origin/t3/server") + # --server-ref keeps its default when --server-url is given, so it also names the schemas then. + schema_ref = args.schema_ref or args.server_ref schema_sha = gitref.resolve(schema_ref) schemas = {} for key, path in (("scene", SCENE_SCHEMA), ("result", RESULT_SCHEMA)): @@ -498,7 +488,7 @@ def main(argv: list[str] | None = None) -> int: with contextlib.ExitStack() as stack: url = args.server_url or stack.enter_context( - server_from_ref(server_sha, out / "server.log", via_shim=args.via_shim) + server_from_ref(server_sha, out / "server.log") ) openapi = json.loads(urllib.request.urlopen(f"{url}/openapi.json", timeout=10).read()) endpoint = discover_endpoint(openapi, args.endpoint) @@ -524,9 +514,8 @@ def main(argv: list[str] | None = None) -> int: "server_sha": server_sha, "schema_ref": schema_ref, "schema_sha": schema_sha, - "sha": server_sha, + "sha": server_sha, # the scoreboard matches a report to a branch head by this key "endpoint": f"POST {endpoint.path} ({endpoint.content_type})", - "transport": "solver shim (HTTP layer not tested)" if args.via_shim else "server API", "command": " ".join([sys.executable, "-m", "hsverify.e2e", *(argv or sys.argv[1:])]), # Latency on this shared Mac depends on what else runs; keep the load with the numbers. "load_average_1_5_15": [round(x, 1) for x in os.getloadavg()], diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 9ab1522a6..ea8493566 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -6,8 +6,8 @@ - decision consistency: what `pass`, `reject` and `manual_review` each require of the rest of the record; - the margin rule: a check's outcome must follow from its measurement, error and threshold - (PASS only when the margin beats the error, FAIL only when past the threshold by more - than the error, otherwise UNSURE); + (PASS only when the margin beats the error, and clears the review line when the check has + one; FAIL only when past the threshold by more than the error; otherwise UNSURE); - coverage: no battery position whose wall stretch, cable route or ground was never observed may pass, and nothing may ask for photos of an area the scene says was observed; - bookkeeping: counts add up, the input hash matches what was sent, the spot's offset from diff --git a/verification/hsverify/scoreboard.py b/verification/hsverify/scoreboard.py index d5297d0eb..afa1fd06a 100644 --- a/verification/hsverify/scoreboard.py +++ b/verification/hsverify/scoreboard.py @@ -179,12 +179,12 @@ def _check_probe(mid: str, kind: str, probe: dict) -> None: raise MetricsError(f"{mid}: report needs exactly one of {', '.join(COMPARATORS)}") if kind == "manual" and probe["status"] not in MANUAL_STATUSES: raise MetricsError(f"{mid}: manual status must be one of {', '.join(MANUAL_STATUSES)}") + if kind == "grep_at_ref" and probe.get("each") and "{each}" not in probe["pattern"]: + raise MetricsError(f"{mid}: pattern must contain {{each}} when each is set") if kind in ("grep_at_ref", "pr_body"): - if kind == "grep_at_ref" and probe.get("each") and "{each}" not in probe["pattern"]: - raise MetricsError(f"{mid}: pattern must contain {{each}} when each is set") - for fragment in _each_items(probe) or [("", "")]: + for _, fragment in _each_items(probe) or [("", "")]: try: - re.compile(probe["pattern"].replace("{each}", fragment[1])) + re.compile(probe["pattern"].replace("{each}", fragment)) except re.error as e: raise MetricsError(f"{mid}: bad pattern: {e}") from e if probe.get("max_status") not in (None, "partial"): @@ -456,7 +456,7 @@ def _read_report(path: Path) -> dict | None: return data if isinstance(data, dict) else None -def _value_or_none(data: dict, key: str): +def _value_or_none(data: dict, key: str) -> Any: try: return lookup(data, key) except KeyError: @@ -624,9 +624,9 @@ def build_board( rows = [] for m in (m for m in metrics if m.workstream == ws): sha = shas[m.ref] - o = evaluate(m, sha, ctx) - links = [list(link) for link in o.links] - rows.append(Row(m.id, m.text, m.ref, sha, o.status, o.evidence, links)) + outcome = evaluate(m, sha, ctx) + links = [list(link) for link in outcome.links] + rows.append(Row(m.id, m.text, m.ref, sha, outcome.status, outcome.evidence, links)) refs = list(dict.fromkeys(r.ref for r in rows)) streams.append( { @@ -664,8 +664,7 @@ def render_markdown(board: Board) -> str: "# Scoreboard", "", f"Generated {_local(board.generated_at)} by `make scoreboard` (`{board.command}`) in " - f"`verification/`; " - f"{slow}. Probes are defined in `verification/scoreboard/metrics.yaml`.", + f"`verification/`; {slow}. Probes are defined in `verification/scoreboard/metrics.yaml`.", "", "## Open pull requests", "", @@ -692,7 +691,7 @@ def render_markdown(board: Board) -> str: else f"`{r['ref']}` (missing)" for r in ws["refs"] ) - counts = {s: 0 for s in STATUSES} + counts = dict.fromkeys(STATUSES, 0) for m in ws["metrics"]: counts[m["status"]] += 1 tally = ", ".join(f"{n} {s}" for s, n in counts.items() if n) diff --git a/verification/hsverify/simrun.py b/verification/hsverify/simrun.py index 8199a3e05..5d989a33a 100644 --- a/verification/hsverify/simrun.py +++ b/verification/hsverify/simrun.py @@ -1,7 +1,7 @@ """Build the iOS app from a git ref, run it in a dedicated Simulator, screenshot every STATE. - uv run python -m hsverify.simrun --ref origin/t3/ios-mvf \ - --replay ~/house-scanning-data/replays/ --autopilot --server-url http://127.0.0.1:8000 + make sim-app # t3/ios-mvf on the ADVIO replay, uploading to a server started from t3/server + uv run python -m hsverify.simrun --ref --replay --autopilot --server-ref The app is launched with the contract C4 arguments (`-replay `, `-autopilot`, `-serverURL `) and the runner follows its `STATE=` log markers. Each state gets @@ -102,11 +102,7 @@ def ensure_device() -> dict: break else: udid = simctl("create", DEVICE_NAME, DEVICE_TYPE, runtime) - state = json.loads(simctl("list", "devices", "-j"))["devices"][runtime] - booted = any(d["udid"] == udid and d["state"] == "Booted" for d in state) - if not booted: - simctl("boot", udid) - simctl("bootstatus", udid, "-b") + simctl("bootstatus", udid, "-b") # boots the device if it isn't already booted return {"name": DEVICE_NAME, "udid": udid, "runtime": runtime, "type": DEVICE_TYPE} diff --git a/verification/hsverify/solver_shim.py b/verification/hsverify/solver_shim.py deleted file mode 100644 index bf69d67e1..000000000 --- a/verification/hsverify/solver_shim.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Serve the server branch's `solve` over HTTP until that branch has its own API. - -Runs inside the server's own environment (`uv run --project /server python this.py`), -imports its `rules`, `scene` and `solver` modules unchanged, and exposes one POST endpoint -plus a minimal OpenAPI document so `hsverify.e2e` can discover it. Only the public rules load: -the worktree has no `private/` folder and the override variable is cleared. - -Results through this shim test the solver and the ingest code, not the server's HTTP layer. -Stdlib only, because it runs in someone else's environment. -""" - -from __future__ import annotations - -import argparse -import json -import os -import sys -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - -OPENAPI = { - "openapi": "3.1.0", - "info": {"title": "hsverify solver shim", "version": "1"}, - "paths": { - "/shim/scene": {"post": {"requestBody": {"content": {"application/json": {"schema": {}}}}}} - }, -} - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--server-dir", required=True) - parser.add_argument("--port", type=int, required=True) - args = parser.parse_args() - os.environ.pop("HOUSESCAN_PRIVATE_RULES", None) - sys.path.insert(0, args.server_dir) - from rules import load_rules - from scene import SceneError, parse_scene - from solver import solve - - loaded = load_rules(None) - - class Handler(BaseHTTPRequestHandler): - def _send(self, status: int, payload: object) -> None: - body = json.dumps(payload).encode() - self.send_response(status) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def do_GET(self) -> None: - if self.path == "/openapi.json": - self._send(200, OPENAPI) - else: - self._send(404, {"detail": "not found"}) - - def do_POST(self) -> None: - if self.path != "/shim/scene": - self._send(404, {"detail": "not found"}) - return - body = self.rfile.read(int(self.headers["Content-Length"])) - try: - scene = parse_scene(json.loads(body), loaded.rules, input_bytes=body) - self._send(200, solve(scene, loaded)) - except (SceneError, ValueError) as exc: - self._send(422, {"detail": str(exc)}) - except Exception as exc: # a solver crash is a finding, not a shim failure - self._send(500, {"detail": f"{type(exc).__name__}: {exc}"}) - - def log_message(self, *args: object) -> None: - pass - - ThreadingHTTPServer(("127.0.0.1", args.port), Handler).serve_forever() - - -if __name__ == "__main__": - main() diff --git a/verification/product/README.md b/verification/product/README.md index 6bb6e7c18..7c76c2f4c 100644 --- a/verification/product/README.md +++ b/verification/product/README.md @@ -10,17 +10,14 @@ wrong is collected in [bug-triage.md](bug-triage.md). - **Surface.** The HouseScan app, one run of the homeowner flow from onboarding to the result, default settings. The ARKit camera cannot run in the Simulator, so behavior that needs the live camera is described from code and marked unverified. -- **Source.** `t3/ios-mvf` has not received the app yet. The source is S3's local integration - commit `21a63e7` (engine lane `e72d14b` merged with UI lane `1dcbf36` onto `6885b7b`). Every - document's footer cites it. When `t3/ios-mvf` receives the app, the documents are rechecked - against that commit and the footers updated. -- **How it is run.** `make sim REF=21a63e7 ARGS="--replay --autopilot --server-url - --extra-arg=-autopilotHold --extra-arg=3"` from `verification/`, with the server from - `t3/server` running at the URL. The replay is `advio-20-0040-0075` (C3). +- **Source.** `t3/ios-mvf` at `0876e03`, which has the engine and the UI lane's screens. Every + footer cites it. +- **How it is run.** `make sim-app REF=` from `verification/`: the real ADVIO replay + `advio-20-0040-0075` (C3), the autopilot, and the server started from `t3/server`. - **Out of scope.** The `-uiDemo` screens (a design preview, not the product), TestFlight signing, the server's placement logic (checked by `hsverify/e2e.py`). -- **Not yet built.** Panel close-ups, homeowner questions other than "does this window open", - and "Point at the meter like this" after an interruption. They are not described. +- **Not yet built.** Panel close-ups and homeowner questions other than "does this window open". + They are not described. ## Shape of every document @@ -49,7 +46,7 @@ Type, Reduce Motion); haptics and motion; verification hooks (`STATE=` logs, rep Each document then has Summary, The simple case, The interaction (with one `stateDiagram-v2`), Modifiers, Cancel and interrupt, Interactions with other systems, Edge cases, -and Open questions and verification, ending with `Verified against house-scanning commit 21a63e7`. +and Open questions and verification, ending with the commit it was verified against. [screens/meter-close-up.md](screens/meter-close-up.md) is the pilot every other document copies. ## Writing rules @@ -65,15 +62,15 @@ and Open questions and verification, ending with `Verified against house-scannin | Document | What it covers | Status | | --- | --- | --- | -| [foundations/flow.md](foundations/flow.md) | The screens in order, what moves between them, the launch options, Start over, failures | drafted | -| [foundations/coverage-and-guidance.md](foundations/coverage-and-guidance.md) | The coverage strip, when a photo is kept, the one instruction and what overrides it | drafted | -| [screens/onboarding.md](screens/onboarding.md) | The intro pages and the failure screens (camera off, unsupported phone, unreadable replay) | not started | -| [screens/find-meter.md](screens/find-meter.md) | Finding the meter and tapping it to set the wall | not started | -| [screens/meter-close-up.md](screens/meter-close-up.md) | The self-timed meter photo (pilot) | drafted; checked on the replay | -| [screens/wall-walk.md](screens/wall-walk.md) | Walking the wall, marking its ends, "I can't get there" | not started | -| [screens/mark-features.md](screens/mark-features.md) | Marking gas meters, doors, windows, AC units, driveways and fences; the window question | not started | -| [screens/gap-request.md](screens/gap-request.md) | One targeted extra view, from the phone or the server | not started | -| [screens/uploading.md](screens/uploading.md) | Sending the scan and waiting for the result, offline and failure | not started | -| [screens/result.md](screens/result.md) | The result, its checks and the 3D and AR views | not started | -| [verification.md](verification.md) | Checks against the running app and their results | pass 1 recorded | -| [bug-triage.md](bug-triage.md) | Every suspected defect, deduplicated | 7 entries | +| [foundations/flow.md](foundations/flow.md) | The screens in order, what moves between them, the launch options, Start over, failures | drafted; checked in the Simulator | +| [foundations/coverage-and-guidance.md](foundations/coverage-and-guidance.md) | The coverage strip, when a photo is kept, the one instruction and what overrides it | drafted; checked in the Simulator | +| [screens/onboarding.md](screens/onboarding.md) | The intro pages and the failure screens (camera off, unsupported phone, unreadable replay) | drafted; checked in the Simulator | +| [screens/find-meter.md](screens/find-meter.md) | Finding the meter and tapping it to set the wall | drafted; checked in the Simulator | +| [screens/meter-close-up.md](screens/meter-close-up.md) | The self-timed meter photo (pilot) | drafted; checked in the Simulator | +| [screens/wall-walk.md](screens/wall-walk.md) | Walking the wall, marking its ends, "I can't get there" | drafted; checked in the Simulator | +| [screens/mark-features.md](screens/mark-features.md) | Marking gas meters, doors, windows, AC units, driveways and fences; the window question | drafted; checked in the Simulator | +| [screens/gap-request.md](screens/gap-request.md) | One targeted extra view, from the phone or the server | drafted; checked in the Simulator | +| [screens/uploading.md](screens/uploading.md) | Sending the scan and waiting for the result, offline and failure | drafted; checked in the Simulator | +| [screens/result.md](screens/result.md) | The result, its checks and the 3D and AR views | drafted; checked in the Simulator | +| [verification.md](verification.md) | Checks against the running app and their results | current pass at `0876e03` | +| [bug-triage.md](bug-triage.md) | Every suspected defect, deduplicated | 16 entries, 2 resolved | diff --git a/verification/product/bug-triage.md b/verification/product/bug-triage.md index c3594a038..c663016e8 100644 --- a/verification/product/bug-triage.md +++ b/verification/product/bug-triage.md @@ -1,75 +1,116 @@ # Suspected defects -From the documents' open questions and [verification.md](verification.md), deduplicated. Source -`21a63e7` (S3 integration commit) unless stated. - -| ID | Severity | Where the homeowner meets it | Decision | -| --- | --- | --- | --- | -| B-01 | Blocker | Uploading: every scan fails with 404 | fix (S3) | -| B-02 | Major | Uploading: a failure shows raw server JSON | fix (S3) | -| B-03 | Blocker | Camera denied, camera failure or unreadable replay: no message, no way on | fix (S3) | -| B-04 | Blocker | Meter close-up: "Can't get a clear shot" may never appear | fix (S3) | -| B-05 | Major, cause unconfirmed | Camera screens: cards render without text in Simulator screenshots | investigate (S3, S4) | -| B-06 | Major | Walk at 20 ft: "Is this the left end?" has no truthful answer when the wall continues | product call | -| B-07 | Minor | Photos of the home stay in the phone's cache after every scan | product call | - -## B-01. The app uploads to a route the server does not have - -- **What happens.** The app posts the scan to `POST {serverURL}/v1/scenes`; the server serves - `POST /v1/placements`. Every upload fails with 404. -- **Evidence.** Server log: `"POST /v1/scenes HTTP/1.1" 404 Not Found`. The same bundle posted to - `/v1/placements` as `application/zip` returns 200 with a result, and passes every check in - `hsverify/e2e.py` (report `20260926-041009-b04558df`). -- **Cause.** `ios/HouseScan/Runtime/ResultClient.swift:27`; the comment above it calls the route a - placeholder. Contracts C1 and C2 fix the body and the answer but not the path. - -## B-02. Upload failures show raw error text - -- **What happens.** "That didn't go through" is followed by `The server answered 404: - {"error":{"code":"not_found",...}}`. -- **Cause.** `ios/HouseScan/UI/Copy/ScanCopy.swift`, `upload(_:)` puts the error's description in - the detail line; `Runtime/ScanEngine.swift` passes `String(describing: error)`. -- **Expected.** Plain words ("We couldn't reach the server. Your scan is saved; try again.") and - the "Try again" button, which the screenshot does not show (see B-05). - -## B-03. Failures other than an unsupported phone never reach the screen - -- **What happens.** Only the `unsupported` screen shows a failure, and the app switches to it only - when motion tracking is unsupported. A denied camera, a failed camera session or an unreadable - replay records the failure and stays on the current screen. -- **Cause.** `Runtime/ScanEngine.swift:89, 93, 338, 340`. -- **Reproduce.** Launch with `-replay /nonexistent`: expected "This recording can't be opened"; - predicted from code: onboarding, then a camera screen with nothing on it. Not yet run. - -## B-04. The close-up's way out can fail to appear - -- **What happens.** A failed try counts only when one problem lasts 4 s with no good frame between. - Frames that alternate between good and blurry neither fill the 0.6 s ring nor count a failure, - so "Can't get a clear shot" never appears. -- **Cause.** `ios/HouseScanKit/Sources/HouseScanKit/Capture/CloseUpGate.swift`, `evaluate`. - S3's full-flow test cannot catch it: the autopilot calls the skip action directly. -- **Suggested fix.** Count a failed try per 4 s on the screen without a photo. - -## B-05. Camera-screen chrome renders without its text in Simulator screenshots - -- **What happens.** On `meterCloseUp` and `wallWalk` the instruction card, the pills and the strip - are empty panels; on `markFeatures` the text is black on the camera image with no panel; on - `uploading` after a failure the emblem and button are not visible. -- **Unknown.** Whether this is the app or the capture: screenshots were taken with `simctl io - screenshot` with the Simulator in light appearance, while camera screens force dark. Next: the - same screens captured from inside the app's process (XCUITest), and in dark appearance. - -## B-06. The walk asks for a wall end at 20 ft whether or not the wall ends - -- **What happens.** At 6.1 m of coverage the instruction is "Is this the left end of the wall?". - If the wall continues, the homeowner can only answer "Wall ends here" (recorded as a real end, - which lets the server reject) or "I can't get there" (an unexplored end). -- **Cause.** `ios/HouseScanKit/Sources/HouseScanKit/Guidance/GuidancePlanner.swift`, - `preferredTask`. A third answer ("The wall keeps going") recorded as unexplored-beyond-reach - would be truthful. - -## B-07. Scan photos are never deleted - -- **What happens.** Each scan writes its photos to `Caches/Scans//`; nothing deletes them - after upload, Start over or relaunch. iOS may purge caches, but only under storage pressure. -- **Cause.** `ios/HouseScan/Runtime/KeyframeStore.swift:25`. +Collected from every document's open questions and from [verification.md](verification.md), +deduplicated. Source `t3/ios-mvf` at `0876e03` unless an entry names another commit; paths are +under `ios/`. **Seen** means observed in the Simulator; **code** means read from the +source and not run. Every blocker and major entry was re-read in the source by a second reader. + +| ID | Severity | Where the homeowner meets it | Evidence | Decision | +| --- | --- | --- | --- | --- | +| B-03 | Blocker | Camera denied, camera failure, unreadable replay: no message, no way on | seen (replay) | fix | +| B-04 | Blocker | Meter close-up: "Can't get a clear shot" may never appear | code | fix | +| B-09 | Blocker | Feature list: "Add something" does nothing | code | fix | +| B-06 | Major | Walk: wall ends can't be recorded truthfully | code | product call | +| B-08 | Major | "Can't get there" is never sent to the server | code | fix (with S2) | +| B-10 | Major | Result, upload or list: the scan resets after 20 s of relocalizing | code | fix | +| B-11 | Major | Meter tap can land on another wall's extended plane | code | fix | +| B-12 | Major | "Capture it now" past a wall end turns it unexplored; a request beyond a marked end can't finish | code | fix | +| B-02 | Major | Upload and camera failures show raw error text | seen at `21a63e7` | fix | +| B-13 | Minor | 3D and AR tint zones stop one battery width short | code | fix | +| B-14 | Minor | Result rule lines hide whether a limit is a minimum or maximum | seen | fix | +| B-15 | Minor | AR view does not say a sample result is a sample | seen | fix | +| B-16 | Minor | Wording and VoiceOver: "Tap the ac unit"; "ft" and "in" read as letters | seen in part | fix | +| B-01 | Resolved | Every upload failed with 404 | seen fixed at `0876e03` | none | +| B-05 | Resolved | Camera-screen text missing at `21a63e7` | seen fixed | none | +| B-07 | Minor | Scan photos are never deleted from the phone | code | product call | + +## Blockers + +**B-03. Failures other than an unsupported phone never reach the screen.** Only the failure screen +shows a failure, and the app switches to it only for an unsupported phone +(`HouseScan/Runtime/ScanEngine.swift:91-93`). A denied camera, a failed session or an unreadable +replay set the failure without changing screen (lines 105, 379, 381). The homeowner stays on +"Find your electric meter" with no camera and taps do nothing; "Open Settings" is unreachable. +Seen with `-replay /nonexistent`: the app logged "replay unreadable: cannot read +/nonexistent/session.json" and stayed on onboarding (report +`sim/20260926-044618-5520229-5520229d-preview-bad-replay`). + +**B-04. The close-up's way out can fail to appear.** A failed try counts only when one problem +lasts 4 s with no good frame in between (`HouseScanKit/.../Capture/CloseUpGate.swift`, `evaluate`), +and the photo needs 0.6 s of good frames in a row. Frames that alternate between good and blurry +satisfy neither, so "Can't get a clear shot" never appears. S3's full-flow test cannot catch it: +its autopilot calls skip directly. Suggested: count a failed try per 4 s without a photo. + +**B-09. "Add something" on the feature list does nothing.** The six chips start a mark +(`HouseScan/UI/Screens/MarkFeaturesScreen.swift:64`), but the list never shows marking: the circle, +prompt and Mark button exist only on the walk (`WallWalkScreen.swift:28-45`). The homeowner taps a +chip and nothing visible happens; "Looks complete" then uploads without it. Adding a missed +feature is what this screen is for. + +## Major + +**B-06. Wall ends can't be recorded truthfully.** "Wall ends here" is offered only while the walk +asks "Is this the left end of the wall?", which happens only at 6.1 m of coverage +(`HouseScanKit/.../Guidance/GuidancePlanner.swift`, `preferredTask`), and "Can't get there" is not +offered then (`WallWalkScreen.swift:105`, `asksForArea`). A wall that ends at 10 ft can only be recorded as +unexplored; one that continues past 20 ft only as a real end, which lets the server reject the +site. Decision needed: offer "Wall ends here" whenever an end is in view, and a third answer at +20 ft ("The wall keeps going") recorded as unexplored. + +**B-08. "Can't get there" is never sent.** The upload carries only covered stretches and each end's +kind (`HouseScan/Runtime/ScanEngine+Export.swift:48-58`, `HouseScanKit/.../Scene/SceneExport.swift`), +so a stretch the homeowner can't reach looks to the server like one nobody looked at. The gap +screen's hint promises "An installer will look at this part instead"; the server may ask for the +same view again. C1 has no field for it, so this needs S2 and S3 together. + +**B-10. The scan resets under the result.** The 20 s relocalization reset runs on every screen while +the camera session runs (`ScanEngine.swift:189, 357-364`), including the feature list, the upload +and the result, where no coaching is shown. After a call, the homeowner can come back to "Find your +electric meter" with every photo, mark and the result gone. The reset also keeps a half-finished +mark and the first close-up (`ScanEngine.swift:386-410`, `KeyframeStore.swift:79`). + +**B-11. The meter tap can guess a depth.** The raycast falls back to `existingPlaneInfinite` +(`HouseScan/Runtime/LiveCapture.swift:76`): a tap at a meter whose wall is not detected yet can pin +it on another wall's plane extended past its edge. The code's own comment says a tap never guesses a +depth. The ground height also stays a guess (1.4 m below the phone) if no ground plane was seen at +the tap (`ScanEngine+Actions.swift:48-57`). + +**B-12. Requests past a wall end.** "Capture it now" on a past-end request first clears that end +(`ScanEngine+Actions.swift:206-209`) and nothing sets it again, so the next upload reports the side +as unexplored even after the homeowner looked past it, and the server can ask again. A server band +request whose stretch lies beyond an end that is still marked can never fill: the coverage map +records no cells past a marked end (`HouseScanKit/.../Coverage/CoverageMap.swift:113-118`), so its +bar stays at 0% and "I can't get there" is the only way out. + +**B-02. Failures show raw error text.** "That didn't go through" is followed by the error's +description, for example `The server answered 404: {"error":{"code":"not_found",...}}` (seen at +`21a63e7`; same code at `5520229`: `UI/Copy/ScanCopy.swift:177`, `ScanEngine.swift:533, 560`). The +camera-failure text does the same (`ScanCopy.swift:240`) once B-03 is fixed. + +## Minor + +- **B-13.** Tint zones in the 3D model and the AR view use the sweep's range of battery *start* + positions as if it were where the battery stands, so each ends one battery width short + (`ScanEngine+Export.swift:124`). +- **B-14.** Seen on the sample result: "Measured 3 ft. The rule is 20 ft, and the measurement can be + off by about 4 in." A maximum reads like a minimum; the server's `comparison` and + `review_threshold_ft` are not used (`UI/Copy/ScanCopy.swift:210-218`). +- **B-15.** The AR view says "Your battery could go here" over the sample result's spot without the + sample badge the result screen shows (`ResultARScreen.swift:43`). Seen in pass 2. +- **B-16.** The AC unit prompt reads "Tap the ac unit", and VoiceOver hears "Mark ac unit" and + "Remove ac unit" (`ScanCopy.swift:122`, `WallWalkScreen.swift:258`, `MarkFeaturesScreen.swift:121`; + "Add ac unit" seen in the accessibility audit). + Distances are spoken as "ft" and "in" in the 3D description and measurement lines, though + `Distance.spoken` exists (`ResultScene3D.swift:296-321`, `ResultScreen.swift:237`). +- **B-07.** Each scan's photos stay in `Caches/Scans//` after upload, Start over or relaunch; + `discardKeyframes` forgets the list but not the files (`KeyframeStore.swift:79`). +- **B-01 (resolved).** Until `0876e03` the app posted to `/v1/scenes`, which the server does not + serve, and every upload failed with 404. At `0876e03` it posts to `/v1/placements`: seen `POST + /v1/placements 200` and a result (report `sim/20260926-050058-t3-ios-mvf-0876e03c-replay`). +- **B-05 (resolved).** At `21a63e7` the camera screens' cards rendered without text; at `5520229` the + same capture shows every word. The UI lane added the missing colour assets. + +Smaller notes stay in each document's open questions: refusal wording on the meter tap, "Wall ends +here" aiming at a circle that is not drawn, the kind picker sliding under Reduce Motion, two taps on +"Can't get there" finishing an empty walk, and the result appearing more than 1.2 s after the app +logs it on first load. diff --git a/verification/product/foundations/coverage-and-guidance.md b/verification/product/foundations/coverage-and-guidance.md index aa3b449f1..7a50ee839 100644 --- a/verification/product/foundations/coverage-and-guidance.md +++ b/verification/product/foundations/coverage-and-guidance.md @@ -55,7 +55,8 @@ A frame is kept only when all of these hold: - the phone has moved 0.5 m or turned 15° since the last kept photo, or the frame would show at least three cells nobody has seen yet. -A kept photo increments the counter and gives a light haptic tap. +A kept photo counts, and coverage moves, only once its file is written to the phone, so the strip +never claims a view the upload lacks. Walk photos give no haptic; the counter flashes instead. ### While capturing @@ -72,16 +73,22 @@ wall end. The instruction is chosen in this order, first match wins: 1. **Coaching** from the camera, which replaces everything else until it clears: "Move your phone - slowly" (starting up), "Slow down a little", "Aim at something with more detail", "It's too dark - to see the wall", "Hold steady" (the last frame was blurry or moving), "Point at your meter - again" (relocalizing), "Your phone lost its place". + slowly" (starting up), "Slow down" ("Walk a little slower so the photos stay sharp."), "Aim at + a corner or somewhere with more texture" ("A plain wall or the sky gives your phone nothing to + follow."), "It's too dark to see the wall", "Hold steady" (the last frame was blurry or + moving), "Point at the meter like this." ("Your phone lost its place for a moment.", + relocalizing), "Your phone lost its place". + While tracking is not normal, the walk and the gap request hide everything drawn on the camera + image (the fog, the path, the target ring, the meter dot, the ends and the marks): drawn from + a position the phone does not trust, they would sit in the wrong place. They fade out and back + over 0.2 s. The strip is not drawn on the camera image and stays. 2. **"Take a step back"** when the phone is closer than 1.2 m to the wall; satisfied at 1.4 m. 3. **Aim at a lagging band**: within 1 m either side of the phone, if one band is done over at least 0.45 m (three cells) where the other is not, "Tilt down to show the ground" or "Tilt up to show more wall", with a ring on the spot. 4. **Walk to the left**, until coverage runs unbroken 6.1 m (20 ft) from the meter on the left; - then **"Is this the left end of the wall?"**. Only after the left end is set: the same for the - right. + then **"Is this the left end of the wall?"**, the only moment "Wall ends here" is offered. + Only after the left end is set: the same for the right. 5. With both ends set, the first stretch of at least 0.45 m between them where a band is not done (ground checked before wall), as an aim instruction. 6. **"That's the whole wall"**, with "Tap Done when you're ready". @@ -92,8 +99,9 @@ walk instruction when that side's end is set. ### Advancing -Coverage is sent with the scan: covered cells as observed stretches of each band, skipped cells -and the kind of each end (real or unexplored). The server treats anything not observed as unknown. +Coverage is sent with the scan as the covered stretches of each band and the kind of each end +(real or unexplored). Skipped cells are not sent: to the server a stretch the homeowner said they +can't reach looks the same as one nobody looked at, and it treats both as unknown. ## Modifiers @@ -109,10 +117,10 @@ and the kind of each end (real or unexplored). The server treats anything not ob | Event | Before the first kept photo | After it | | --- | --- | --- | -| The screen's own way out | "I can't get there" marks the cells the instruction asks for as skipped; see the screens. | Same. | +| The screen's own way out | "Can't get there" (walk) or "I can't get there" (gap request) marks the cells the instruction asks for as skipped; see the screens. | Same. | | Start over | Coverage is discarded. | Coverage and photos are discarded. | -| Tracking limited | No photo is kept until tracking has been normal for 0.5 s. | No photo is kept, so coverage stops growing until tracking is normal again for 0.5 s. | -| Tracking lost or relocalizing | Guidance waits behind the coaching. | After 20 s without relocalizing, coverage and photos are discarded and the scan returns to finding the meter. | +| Tracking limited | No photo is kept until tracking has been normal for 0.5 s. The fog is hidden. | No photo is kept, so coverage stops growing until tracking is normal again for 0.5 s. The fog is hidden until tracking is normal. | +| Tracking lost or relocalizing | Guidance waits behind the coaching; the fog is hidden. | After 20 s without relocalizing, coverage and photos are discarded and the scan returns to finding the meter. | | App backgrounded or a call | No effect on coverage. | Coverage is kept. | | Camera off or session failed | No photos can be kept. | Coverage so far is kept but cannot grow. | | Network lost or upload failing | No effect. | No effect. | @@ -124,13 +132,16 @@ and the kind of each end (real or unexplored). The server treats anything not ob kept; see [the flow](flow.md#interactions-with-other-systems). **Upload and offline.** Coverage travels in the uploaded scene. **Accessibility.** The strip reads to VoiceOver as "Map of the wall"; whether it also says how much is covered is checked on [the wall walk](../screens/wall-walk.md). -**Haptics and motion.** A light tap per kept photo; the fog fades where cells clear. **Verification +**Haptics and motion.** No haptic per walk photo; the counter flashes and the fog fades where +cells clear. **Verification hooks.** Each instruction change logs `GUIDANCE=`. ## Edge cases - Walking faster than 1.5 m/s keeps no photos at all, with only "Hold steady" or no coaching. - A stretch closer than 0.45 m to a covered stretch on both sides is never asked for. +- "Can't get there" is not sent to the server (see Advancing), so the server may ask for the same + stretch again on the result. - A walk that reaches 20 ft on the left asks for the left end whether or not the wall ends there. - Coverage is on the one wall set by the meter tap; wall past a corner never becomes covered. - Something standing between the phone and the wall (a bush, a bin) does not stop a cell counting @@ -139,15 +150,14 @@ hooks.** Each instruction change logs `GUIDANCE=`. ## Open questions and verification -- **Possible bug: no truthful answer at 20 ft.** At 6.1 m the walk asks "Is this the left end of the - wall?" whether or not the wall continues. The only answers are "Wall ends here" (a real end) and - "I can't get there" (an unexplored end). A homeowner whose wall simply continues has neither; - choosing "Wall ends here" records a real end, which lets the server reject the site - (`HouseScanKit/.../GuidancePlanner.swift:103`). +- **Suspected bug: wall ends can't be recorded truthfully** (triage B-06). "Wall ends here" is + offered only when the walk asks for the end, which happens at 6.1 m of coverage; "Can't get + there" is not offered then. A wall that really ends sooner can only be recorded as unexplored, + and one that continues past 20 ft only as a real end, which lets the server reject the site. - The walk always sends the homeowner left first. Starting at the right end of a wall means walking past everything to the left first. - "About N ft to go" can never appear: the engine always passes no remaining distance (`Runtime/ScanEngine.swift`, `step(_:)`). - All thresholds are the code's stated hypotheses (research note), not measured values. -Verified against house-scanning commit `21a63e7`. +Verified against house-scanning commit `0876e03` (t3/ios-mvf). diff --git a/verification/product/foundations/flow.md b/verification/product/foundations/flow.md index 63a827ae8..f8583398d 100644 --- a/verification/product/foundations/flow.md +++ b/verification/product/foundations/flow.md @@ -6,7 +6,7 @@ A scan is one pass through a fixed sequence of screens, from an introduction to app decides when to move on: most screens advance by themselves when their job is done, and the homeowner moves on with a button only where a judgment is theirs (they have finished walking, their marks are right, they want to see the spot in AR). There is no back button between -screens. "Start over", on the result and on the unsupported screen, is the only way to begin again. This document owns the order +screens. "Start over", on the result, is the only way to begin again. This document owns the order of the screens, what moves the app from one to the next, the launch options, and what happens when the camera, the replay or the phone cannot run the scan. @@ -41,7 +41,6 @@ stateDiagram-v2 result --> resultAR : "See it on your wall" resultAR --> result : "Done" result --> onboarding : "Start over" - unsupported --> onboarding : "Start over" wallWalk --> findMeter : phone lost its place for over 20 s ``` @@ -70,7 +69,7 @@ The camera stays open, without a blink, from `findMeter` through `wallWalk`, `ma `gapRequest`; only the instruction and controls above it change. Photos are kept continuously on `wallWalk` and `gapRequest` as described in [coverage and guidance](coverage-and-guidance.md#when-a-photo-is-kept). A counter at the top -shows how many photos have been taken, and each new one gives a light tap of haptic feedback. +shows how many photos have been taken; it goes up once each photo is written to the phone. ### Advancing @@ -93,10 +92,10 @@ after the server has answered; nothing on `uploading` is skipped when the networ | Event | Before the meter is tapped | After the meter is tapped | | --- | --- | --- | | The screen's own way out | See each screen. | See each screen. | -| Start over | Offered only on `unsupported`. | Offered only on `result`. It forgets the wall, photos, marks and result and returns to `onboarding`. | +| Start over | Not offered. The failure screen has it for failures other than an unsupported phone, but those never reach it (Open questions). | Offered only on `result`. It forgets the wall, photos, marks and result and returns to `onboarding`. | | Tracking limited | Coaching replaces the instruction until it clears ([coverage and guidance](coverage-and-guidance.md#one-instruction-at-a-time)). | Same; no photo counts toward coverage meanwhile. | -| Tracking lost or relocalizing | "Point at your meter again". | If the phone has not found its place within 20 s, the app forgets the wall, photos and marks and returns to `findMeter`. | -| App backgrounded or a call | The screen stays; coaching says "Point at your meter again" when the camera resumes. | Same; photos and marks are kept. | +| Tracking lost or relocalizing | "Point at the meter like this." (relocalizing) or "Your phone lost its place". | If the phone has not found its place within 20 s, the app forgets the wall, photos and marks and returns to `findMeter`, on any screen while the camera runs, including the result. | +| App backgrounded or a call | The screen stays; coaching says "Point at the meter like this." when the camera resumes. | Same; photos and marks are kept. | | Camera off or session failed | Suspected dead end: see Open questions. | Suspected dead end: see Open questions. | | Network lost or upload failing | No effect. | Only `uploading` is affected; see [the upload](../screens/uploading.md). | | App killed | The scan is lost. | The scan is lost, including one "saved on this phone" after a failed upload: its files stay in the cache, but nothing reopens or retries them. | @@ -110,10 +109,12 @@ deleted by the app and never reopened. **Upload and offline.** Only the upload needs the network; see [the upload](../screens/uploading.md). **Accessibility.** Every screen groups its elements in one accessibility container. Its identifier (`screen.`) is for tests and is not spoken. **Haptics and motion.** A light tap -for each photo; a success haptic when the meter is set and when the result appears; a firmer tap -for each mark placed and a warning for a refused one. **Verification hooks.** Each screen change +for a deliberate photo (the close-up, a requested view) but not for walk photos; a success haptic +when the meter is set and when the result appears; a firmer tap for each mark placed and a warning +for a refused one. **Verification hooks.** Each screen change logs `STATE=` publicly under subsystem `dev.housescanning.housescan`, category `state`; -guidance changes log `GUIDANCE=` under category `guidance`. +guidance changes log `GUIDANCE=` under category `guidance`. `-autopilotGate ` makes +the autopilot wait on each screen until a file named after it appears, for UI tests. ## Edge cases @@ -141,4 +142,4 @@ guidance changes log `GUIDANCE=` under category `guidance`. - The 1.2 s pause after a finished step and the 20 s relocalization limit are read from code; neither has been timed on a device. -Verified against house-scanning commit `21a63e7`. +Verified against house-scanning commit `0876e03` (t3/ios-mvf). diff --git a/verification/product/glossary.md b/verification/product/glossary.md index ffb5ae6e9..846422b6a 100644 --- a/verification/product/glossary.md +++ b/verification/product/glossary.md @@ -14,7 +14,7 @@ app; everything else is this description's own term. shows it in feet and inches; the code works in meters. - **Wall end.** Where the homeowner said the wall stops. A **real end** comes from "Wall ends here" (the wall stops at a corner or something blocking the way); an **unexplored end** comes - from "I can't get there" during the walk (the wall may continue). + from "Can't get there" during the walk (the wall may continue). ## Seeing the wall @@ -22,7 +22,8 @@ app; everything else is this description's own term. presses a shutter; [coverage and guidance](foundations/coverage-and-guidance.md#when-a-photo-is-kept) says when a frame is kept. - **Close-up.** The one deliberate photo of the meter, taken by itself when the meter is centered, - near, sharp and well exposed. See [the meter close-up](screens/meter-close-up.md). + near, sharp and well exposed. See [the meter close-up](screens/meter-close-up.md). The walk + shows it again while the phone is finding its place, as the picture to aim at. - **Coverage strip.** The map along the bottom of the walk screen: the wall unrolled flat, meter at zero, in two rows (bands). - **Band.** One row of the strip. The **wall band** is the wall face from the ground up to @@ -33,8 +34,8 @@ app; everything else is this description's own term. - **seen**: shown in kept photos, but not yet from two positions. - **covered**: shown from two positions at least 0.25 m apart with normal tracking. Evidence exists; it is not a pass. - - **skipped**: the homeowner said they can't get there. Recorded for installer review, never - evidence, drawn differently from both fog and covered. + - **skipped**: the homeowner said they can't get there. Drawn differently from both fog and + covered, never evidence, and not sent to the server. - **Fog.** The overlay on the camera image over wall and ground not yet seen. It clears where kept photos have looked. @@ -42,8 +43,13 @@ app; everything else is this description's own term. - **Instruction.** The one line (plus an optional second line) at the top of a camera screen. There is exactly one at a time. -- **Coaching.** An instruction about the phone rather than the wall ("Slow down a little", +- **Coaching.** An instruction about the phone rather than the wall ("Slow down", "Hold steady"). It replaces the guidance instruction until the problem clears. +- **Reticle.** The small circle in the middle of the camera view that a tap on Mark (or "This is + my meter") aims with, when the homeowner does not tap the image itself. +- **Mode badge.** The yellow "REPLAY · AUTOPILOT" label shown when the app runs on a recording or + drives itself. +- **Tap ring.** The ring that blooms where a finger touched the camera image. - **Target.** A ring on the camera image where the homeowner should aim, or a chevron at the screen edge pointing to it when it is off screen. - **Path.** Dots on the ground showing where to stand next, at most 3 m long, 1.5 m out from the @@ -60,14 +66,30 @@ app; everything else is this description's own term. a feature. A door or window takes two (opposite corners), a driveway or fence two (along one edge), a gas meter or AC unit one. - **Refusal.** A tap the app does not accept, with the reason shown ("Nothing to pin there"). +- **Kind picker.** The "What do you see?" panel behind "Mark something" on the walk, with one tile + per feature. +- **Feature list.** The `markFeatures` screen, "Anything else near your meter?": the marked + features, the window question and "Add something". ## After the walk +- **Can't get there.** The walk's way out ("Can't get there"); the gap request's is "I can't get + there". Both mark the asked-for stretch as skipped. - **Gap request.** A screen asking for one more specific view ("Show the ground 3 ft right of your meter"), from the phone's own check or from the server. +- **Phone request, server request.** A gap request from the phone's own check after the feature + list, or from "Capture it now" on the result. A **past-end request** asks for the ground beyond a + wall end. The **requested stretch** is highlighted in amber. - **Upload.** Sending the scan: the scene description and the kept photos, zipped. - **Result.** The server's answer, shown as a headline, the battery spot and its cable, and a list of checks. +- **Placement line.** The line under the result headline: where the spot is and how much cable. +- **3D model, AR view.** The small 3D model of the wall on the result, and the `resultAR` screen + that draws the spot on the camera image. +- **Sweep.** The server's outcome for every stretch of wall where it tried the battery, drawn as + tinted zones in the 3D model and the AR view. +- **"Still needed".** The result's list of missing views, each with "Capture it now" or "An + installer will check this". - **Check.** One rule the server evaluated at the spot, shown as **"Looks good"** (pass), **"Not sure yet"** (unsure) or **"Doesn't work"** (fail). - **Installer review.** What happens to anything the phone can't settle: a person at Base diff --git a/verification/product/screens/find-meter.md b/verification/product/screens/find-meter.md new file mode 100644 index 000000000..aea1dab87 --- /dev/null +++ b/verification/product/screens/find-meter.md @@ -0,0 +1,176 @@ +# Finding the meter + +## Summary + +Finding the meter is where the scan fixes its one wall. The homeowner points the phone at their +electric meter and either taps "This is my meter", which pins the meter at the circle in the +middle of the screen, or taps the meter itself on the camera image. The phone accepts the tap +only where it has detected a real vertical surface; that surface becomes the wall for the rest of +the scan. It is the first camera screen (screen name `findMeter`), reached from "Allow camera" on +the introduction, and again after the phone loses its place for over 20 s +([the flow](../foundations/flow.md#the-interaction-event-by-event)). Nothing is photographed here. +An accepted tap goes straight to [the meter close-up](meter-close-up.md). + +## The simple case + +The camera opens. A white ring with a dot at its center sits in the middle of the view, and the +instruction reads "Find your electric meter", with "A gray box with a round glass dial or a small +screen, usually on an outside wall." A blue "This is my meter" button fills the bottom of the +screen. The homeowner stands a few steps from the wall, puts the ring on the meter and taps the +button. A ring blooms from the middle of the screen and fades, the phone gives a success haptic, +and the screen crossfades to the close-up. + +## The interaction, event by event + +```mermaid +stateDiagram-v2 + [*] --> looking + looking --> coached : tracking not normal + coached --> looking : tracking normal again + looking --> refused : tap with no detected wall there + coached --> refused : any tap (refusal hidden behind the coaching) + refused --> refused : another refused tap (only the ring shows) + looking --> [*] : tap on a detected wall, the close-up + refused --> [*] : tap on a detected wall, the close-up +``` + +### Arriving + +The camera fades in behind the screen and stays open through the next screens. The instruction is +"Find your electric meter" and nothing is set: no wall, no photos, and no photo counter on this +screen. On the first arrival the phone starts motion tracking, and iOS asks for camera access if +it has not been granted. While tracking starts up, [coaching](../foundations/coverage-and-guidance.md#one-instruction-at-a-time) +replaces the instruction, usually "Move your phone slowly" ("It's getting its bearings."), with an +amber icon beside it. + +Arriving after a 20 s loss of place, the phone has already thrown away the wall, photos and marks +and restarted tracking with a fresh map, so it has to detect the wall again before a tap can +succeed. + +### Leaving at once + +The only way off this screen is an accepted tap. There is no back button, no skip and no Start +over. Nothing is recorded until the tap is accepted. + +### First capture + +Two gestures do the same thing: + +- **"This is my meter"** pins the point under the ring in the middle of the screen. A ring blooms + at the screen's center. +- **A tap on the camera image** pins the point under the finger. A ring blooms where the finger + touched. + +The ring blooms at once for every tap, accepted or not. The tap is accepted when both hold: + +1. Tracking is normal. +2. The point lies on a vertical surface the phone has detected, or on that surface continued past + the part it has detected so far. A surface the phone would only be guessing at does not count. + +If the first check fails, the tap is refused and nothing changes while the coaching is up. If the +second fails, the instruction changes to "Step a little closer to the wall", with "Then aim at +your meter again." The refusal has no warning icon and no haptic, and the new instruction stays +until a tap is accepted. + +> Technical note: when the tap is accepted, the wall's outside is the side the phone is on, and +> the ground at the wall is the highest detected level surface at least 0.3 m (1 ft) below the +> meter whose detected extent comes within 2 m (6 ft 7 in) of it. With no such surface yet, the +> ground is taken as 1.4 m (4 ft 7 in) below the phone. This happens once, at the tap +> (`Runtime/ScanEngine+Actions.swift:39`, `:51`); see Open questions. + +### While capturing + +Nothing is captured on this screen. Only the coaching updates while it is up, as the phone's +tracking changes: "Slow down", "Aim at a corner or somewhere with more texture", "Point at the +meter like this.", "Your phone lost its place". This screen never shows "Hold steady" or "It's too dark to +see the wall", which belong to the walk. The homeowner can tap as often as they like; each refused +tap shows the ring and nothing else. + +### Advancing + +An accepted tap sets the wall: the meter's position, which way the wall runs and which side is +outside. Coverage starts empty, centered on the meter. The phone pins the meter to the wall so its +position is refined as tracking improves. The success haptic fires and the close-up appears at +once, with no pause on this screen. + +## Modifiers + +| Modifier | At arrival | While capturing | +| --- | --- | --- | +| Live camera or replay | A replay shows the recording's first frame as a still. No coaching appears, because that frame is shown for tapping and is not checked. | Any tap or "This is my meter" is accepted at once, wherever it lands. The wall comes from the recording (its recorded wall, or one assumed from the path the camera took), not from the tap. | +| Autopilot | A badge reads "Replay · Autopilot". The screen stays up for the autopilot's hold ([the flow](../foundations/flow.md#modifiers)), then the autopilot presses "This is my meter" itself. With the live camera it does nothing, and the badge reads "Autopilot". | No effect. | +| Server or sample result | No effect. | No effect. | +| Larger text sizes | The instruction and the button label grow. When they no longer fit, the screen scrolls instead of shrinking the text. The ring in the middle does not grow. | No effect. | +| Reduce Motion | The screen crossfades in over 0.15 s. | The ring from a tap fades in place without growing. A changed instruction crossfades without blurring. | + +## Cancel and interrupt + +| Event | Before any tap | After a refused tap | +| --- | --- | --- | +| The screen's own way out | None. | None. | +| Start over | Not offered. | Not offered. | +| Tracking limited | Coaching replaces the instruction, and taps are refused without any change on screen. | Same. When the coaching clears, "Step a little closer to the wall" shows, even if the refusal was about tracking. | +| Tracking lost or relocalizing | "Point at the meter like this." or "Your phone lost its place". Taps are refused. After 20 s of relocalizing, tracking restarts with a fresh map and the screen stays. | Same. The "Step a little closer to the wall" instruction stays after the restart. | +| App backgrounded or a call | The screen stays. When the camera resumes, "Point at the meter like this." shows until tracking is normal. | Same. | +| Camera off or session failed | Suspected dead end: the screen stays with no camera image, and taps only show the ring ([the flow](../foundations/flow.md#open-questions-and-verification)). | Same. | +| Network lost or upload failing | No effect. | No effect. | +| App killed | The scan is lost. The next launch starts at the introduction. | Same. | + +## Interactions with other systems + +**Coverage and evidence.** Coverage begins at the accepted tap, empty and centered on the meter; +the tap's wall is the only wall coverage is ever measured on. **Stored photos.** None; the camera +keeps no frames on this screen. **Upload and offline.** No effect. The uploaded scene measures +everything from the meter, wall direction and ground set here. **Accessibility.** The ring and +the camera image are hidden from VoiceOver, so the only way to pin the meter is "This is my meter", +whose hint is "Pins your electric meter at the circle in the middle of the screen". The +instruction and its second line are read as one element, marked as updating often. **Haptics and +motion.** A success haptic when a tap is accepted and nothing for a refusal. A 0.55 s ring blooms +at every tap, and the button shrinks slightly while pressed. **Verification hooks.** `STATE=findMeter` +on arrival. A refused tap with no wall logs "meter tap refused: no vertical plane" (category +`engine`); a refusal for tracking logs nothing. No `GUIDANCE=` line is logged on this screen. A +replay logs where its wall came from when it loads. + +## Edge cases + +- The meter is placed where the tap landed, not at the meter's middle. A tap on the meter's edge + or just above it moves the meter's height and position by that much. +- A meter on a wall with little texture (smooth stucco, fresh paint) may never be detected as a + surface. Taps are then refused over and over with the same instruction. +- Tapping past the edge of the part of the wall the phone has detected is accepted, because the + surface counts as continuing (see Open questions). +- A replay started with `-replay` whose homeowner taps "Allow camera" before the recording has + loaded starts the live camera as well (`Runtime/ScanEngine.swift:168`). Read from code only. +- A replay that could not be opened leaves this screen dark, with no way on: every tap is + ignored. + +## Open questions and verification + +- **Suspected bug: the ground can be guessed for the whole scan.** If no level surface has been + detected near the meter when the tap is accepted, the ground is set 1.4 m below the phone and + never corrected, though the code's own description says "until one appears" + (`Runtime/ScanEngine+Actions.swift:48-57`; nothing else writes it). A homeowner who taps quickly, + or holds the phone low or high, gets every height and the ground strip offset for the rest of + the scan. +- **Suspected bug: the refusal message is misleading and silent on repeat.** Both refusals show + "Step a little closer to the wall" (`Runtime/ScanEngine+Actions.swift:26`, `:33`), though + neither is about distance: one is tracking, and the other is a surface not yet detected, which + can happen at any range. A second refused tap produces the same text with no haptic + (`UI/ScanRootView.swift:95` warns only for feature marks), so it looks as if the tap did + nothing. +- **Question: the wall can be extended into empty space.** The tap may land on a detected wall's + continuation past what was detected (`Runtime/LiveCapture.swift:76`, `existingPlaneInfinite`). A + tap at a meter on a surface that has not been detected yet can pin the meter on another detected + wall's continuation, at the wrong depth. The code's comment says a tap "never guesses a depth". +- **Suspected bug: a 20 s loss of place anywhere returns here.** The check runs on every screen + while the camera session exists, including the upload and the result (`Runtime/ScanEngine.swift:357-364`). + A homeowner who leaves the app during the upload and comes back to a phone lying face down + could lose the whole scan and land here. The reset also leaves an unfinished feature mark in + place (`Runtime/ScanEngine.swift:386-410` does not clear it). +- **Question: "Point at the meter like this." with nothing to match.** The relocalizing coaching + (`UI/Copy/ScanCopy.swift:65-66`) is written for the walk, which shows the saved close-up under + it. Here no close-up exists yet and no picture is shown, so "like this" points at nothing. +- The live tap, the refusals, the coaching and the camera prompt need the live camera and have not + been seen running. The Simulator shows only the replay path, where every tap is accepted. + +Verified against house-scanning commit `0876e03` (t3/ios-mvf). diff --git a/verification/product/screens/gap-request.md b/verification/product/screens/gap-request.md new file mode 100644 index 000000000..7ff1b25b4 --- /dev/null +++ b/verification/product/screens/gap-request.md @@ -0,0 +1,191 @@ +# A gap request + +## Summary + +A gap request asks the homeowner for one more specific view of the wall or the ground, such as +"Show the ground about 4 ft right of your meter". It comes from two places. The phone asks once, +after "Looks complete" on [marking features](mark-features.md), when its own check finds a stretch +near the meter that no two kept photos show. The server asks when the homeowner taps "Capture it +now" under "Still needed" on [the result](result.md). Either way the screen (screen name +`gapRequest`) is the walk's camera with one target: the requested stretch glows amber, a bar fills +as it is covered, and at 80 % a green check lands and the scan is sent again. "I can't get there" +leaves at any time before that. There is no other way off the screen. + +## The simple case + +The homeowner taps "Looks complete". The camera stays open and the instruction changes to "Show the +ground about 3 ft left of your meter", with "This might be a spot for the battery, so the ground +there needs a clear look from two places." A stretch of ground on the camera image turns amber, a +ring marks its middle, and dots on the ground lead there. Below the instruction a bar reads "0%", +and the strip at the bottom outlines the same stretch in amber. The homeowner walks over and points +the phone down at it. Photos are kept as on [the wall walk](wall-walk.md), each with a light tap, +and the amber clears cell by cell as the bar climbs. At 80 % a large green check appears, the +instruction reads "Got it, thanks" with "That's the view we needed.", and the phone gives a success +haptic. About 1.2 s later [the upload](uploading.md) begins. + +## The interaction, event by event + +```mermaid +stateDiagram-v2 + [*] --> asking : "Looks complete" and the phone finds a gap, or "Capture it now" + asking --> asking : a photo kept (bar moves) + asking --> complete : 80 % of the stretch covered, with a new photo + asking --> [*] : "I can't get there" (upload at once) + asking --> [*] : phone relocalizing for over 20 s (finding the meter) + complete --> [*] : after 1.2 s, the upload +``` + +### Arriving + +The phone's own check runs when the homeowner taps "Looks complete". It looks between the two +marked [wall ends](../glossary.md#the-house-and-the-wall), no farther than 6.1 m (20 ft) from the +meter, for stretches of at least 0.45 m (three cells) that are unseen or seen but not yet +[covered](../foundations/coverage-and-guidance.md#while-capturing). Ground comes first: the ground +stretch nearest the meter is asked for, and only if the ground is complete, the nearest wall +stretch. If nothing is missing, the app goes straight to the upload and this screen never shows. +The phone asks at most once per walk. + +A server request starts from a "Still needed" card that has "Capture it now". A wall or ground +stretch asks for the span the server named. A request to look past a wall end asks for the ground +from that end to 2 m (6 ft 7 in) beyond it, and the app first forgets that end so the stretch +beyond can count. + +The screen crossfades in over the open camera. What it shows: + +| Part | Phone request | Server request | +| --- | --- | --- | +| Instruction | "Show the ground ..." with "This might be a spot for the battery, so the ground there needs a clear look from two places.", or "Show the wall ..." with "Tilt up so the wall above this spot is in view." | "Show the ground ..." or "Show the wall ...", with the server's message as the second line. | +| Place | "around your meter" when the stretch includes the meter or its middle is within 0.3 m; otherwise "about N ft left of your meter" or "right", whole feet from the middle of the stretch. | Same. | + +Under the instruction, inside its card, is "I can't get there". Below it is the progress bar with a +target icon and the percent already covered, and the strip. On the camera image, the requested +cells that are not yet covered glow amber instead of fog. A ring sits on the middle of the +stretch, 1.2 m up the wall or 0.6 m out on the ground, and dots lead from where the homeowner +stands along the wall to it, 1.5 m out. A phone request starts at "0%"; a server request can start +higher. + +### Leaving at once + +"I can't get there" is the only way off before the view is complete. It marks every cell of the +stretch that is not yet covered as skipped (hatched on the camera and slashed on the strip) and +starts the upload at once, with no pause and no haptic. The upload does not carry the skip: see +Open questions. Nothing else leaves the screen short of the 20 s tracking reset or closing the +app. There is no back button and no Start over. + +### First capture + +The first photo is kept by the same rules as on the walk +([when a photo is kept](../foundations/coverage-and-guidance.md#when-a-photo-is-kept)). It gives a +light tap, the counter at the top rises and its camera icon flashes. Any requested cell it covers +loses its amber, and the bar moves. + +### While capturing + +The bar shows the share of the stretch's cells that are covered, rounded to a whole percent. It +counts only covered cells, so a cell seen from one place adds nothing until a second photo from +0.25 m away sees it. The instruction does not change while the request is open; only +[coaching](../foundations/coverage-and-guidance.md#one-instruction-at-a-time) replaces it, until +the problem clears. The ring and the dots follow the homeowner as they move. "I can't get there" +stays available, coaching or not. + +### Advancing + +The view is complete when 80 % of the stretch is covered and at least one photo has been kept since +the screen opened. At that moment the bar turns green and shows at least "80%", its icon becomes a +check, a large green check mark appears in the middle of the screen, the instruction reads "Got it, +thanks" with "That's the view we needed.", "I can't get there" disappears and the phone gives a +success haptic. After the pause described in [the flow](../foundations/flow.md#advancing) the upload +starts with every photo so far, including any kept during the pause. The server checks the whole +scan again and a new result replaces the old one. The phone never asks a second time; the server can, +through "Still needed" on the new result. + +## Modifiers + +| Modifier | At arrival | While capturing | +| --- | --- | --- | +| Live camera or replay | A replay plays again from where the autopilot held frames back for this request (two frames before them); with no held-back frames, the whole recording from the start. Frames already kept on the walk are not kept twice. | Whether a replay can complete the view depends on the recording. | +| Autopilot | The autopilot waits only on the phone's request. With no held-back frames it presses "I can't get there" after `-autopilotHold` seconds; otherwise it waits up to 150 s for the upload and then presses it. It never taps "Capture it now". | The replay runs at three times speed; the pause after completion is `-autopilotHold` seconds, at least 1.2. | +| Server or sample result | The sample result always lists two capturable views: the ground from 3 to 5.5 ft right of the meter ("Sample: film the ground in front of the spot.") and past the left end. | The upload returns the same sample, so both cards come back after every request. | +| Larger text sizes | The instruction, its button and the bar's percent grow; the chrome scrolls rather than squeeze when it outgrows the screen. | No effect. | +| Reduce Motion | The screen crossfades in 0.15 s; instruction changes fade instead of blurring. | The green check fades in over 0.15 s instead of scaling up; the fog fades without drifting. | + +## Cancel and interrupt + +| Event | Before the view is complete | During the pause after it | +| --- | --- | --- | +| The screen's own way out | "I can't get there": the rest of the stretch is marked skipped and the upload starts at once. | None; the button is gone and the upload follows. | +| Start over | Not offered. | Not offered. | +| Tracking limited | Coaching replaces the instruction; no photo is kept, so the bar stops. The amber, fog, ring and dots fade off the camera image until tracking is normal. "I can't get there" still works. | Coaching replaces "Got it, thanks"; the upload still starts. | +| Tracking lost or relocalizing | "Your phone lost its place" or "Point at the meter like this.", with no meter photo (see Open questions). The amber, fog, ring and dots fade off the camera image. After 20 s the request is dropped with the wall, photos and marks, and the scan returns to [finding the meter](find-meter.md). | The pause is shorter than 20 s, so the upload starts. | +| App backgrounded or a call | The request and its progress stay; "Point at the meter like this." shows until tracking returns. | The upload starts once the pause has run. | +| Camera off or session failed | Suspected dead end ([the flow](../foundations/flow.md#open-questions-and-verification)), but "I can't get there" still leaves to the upload. | The upload starts. | +| Network lost or upload failing | No effect here; the upload that follows needs it. | Same. | +| App killed | The scan is lost. | The scan is lost. | + +## Interactions with other systems + +**Coverage and evidence.** The stretch is chosen from [coverage](../foundations/coverage-and-guidance.md), +and photos here add coverage anywhere they look, not only on the stretch. Covered stretches go to +the server; skipped cells and the request itself do not. **Stored photos.** Photos kept here are +stored like walk photos and uploaded with them. **Upload and offline.** The screen needs no +network. A server request is followed by a new upload; if it fails, the earlier result cannot be +reopened, only retried with "Try again" ([the upload](uploading.md)). **Accessibility.** The bar +is one element, "Captured so far" (then "View captured") with a value such as "40 percent". The +button's hint is "Skips this view. An installer will look at this part instead." The green check is +hidden from VoiceOver; the instruction is marked as frequently updating. The strip reads "Map of the wall" +with the percent seen on each side, and says nothing about the requested stretch. **Haptics and +motion.** A light tap for every photo kept on this screen (walk photos get none) and a success +haptic when the view is complete. The check springs in over 0.45 s with a slight bounce, the bar +eases over 0.25 s and its icon swaps. **Verification hooks.** `STATE=gapRequest` on arrival, +`GUIDANCE=gap` once; the engine logs "gap N satisfied" or "gap N skipped". The autopilot logs +"skipping the gap: no held-back frames to show it" or "gap not satisfied by the held-back frames; +skipping it". With `-autopilotGate`, the upload after a completed view waits for the test's `gapRequest` file. + +## Edge cases + +- A server stretch that is already 80 % covered shows its percent at once but completes only when + one new photo is kept, from anywhere. +- A server stretch lying beyond a marked wall end can never fill: cells past an end are never + recorded, so the bar stays at "0%" and only "I can't get there" leaves. +- The dots run straight from the homeowner to the stretch, however far; they are not limited to the + walk's 3 m. +- A phone request covers only what is between the marked ends and within 20 ft of the meter, even + on a longer wall. +- After "I can't get there" on a phone request the scan uploads without asking for any other gap, + even if one exists. + +## Open questions and verification + +- **Suspected bug: "I can't get there" is not sent.** The button's hint promises "An installer will + look at this part instead.", and the cells turn skipped, but the upload carries only covered + stretches and the end kinds (`Runtime/ScanEngine+Export.swift:52-58`, + `HouseScanKit/.../SceneExport.swift:273-293`). The server cannot tell a skipped stretch from one + never looked at, and a server request the homeowner skipped can come back on the next result. + This also contradicts the Advancing section of + [coverage and guidance](../foundations/coverage-and-guidance.md#advancing), which says skipped + cells are sent. +- **Suspected bug: a past-end request turns a real end into an unexplored one.** "Capture it now" + forgets the end before the request starts (`Runtime/ScanEngine+Actions.swift:206-209`, + `Runtime/ScanEngine.swift:463-467`), and nothing marks it again. The next upload reports that + side as unexplored (`ScanEngine+Export.swift:53-54`) even when the homeowner has now shown the + ground beyond it, so the server may ask for the same view again. +- **Suspected bug: a second past-end request asks for the wrong place.** With the end forgotten, + the plan falls back to the meter (`HouseScanKit/.../GapPlanner.swift:127, 130`): the ground from + the meter to 2 m toward that side, "about 3 ft left of your meter", which is usually covered + already. Checkable in the Simulator with the sample result: tap "Capture it now" on the left-end + card twice, completing or skipping in between. +- **Suspected bug: "Capture it now" can do nothing.** The card offers the button for any wall or + ground item and any past-end item (`ScanEngine+Export.swift:134`), but the request needs a span + or a side (`GapPlanner.swift:113, 124`). Without one, the tap is ignored with no message + (`ScanEngine+Actions.swift:205`). +- The phone's wording "This might be a spot for the battery" is stronger than its choice: it asks + for the nearest missing ground, not a place it has judged a candidate. +- The 20 s reset keeps the list of skipped requests (`ScanEngine.swift:386-410`); after + re-walking, a phone request identical to a skipped one would be passed over (`ScanEngine.swift:475`). +- **Question: "Point at the meter like this." without the picture.** The walk shows the saved + close-up under this coaching (`UI/Screens/WallWalkScreen.swift:23-27`); this screen shows the + words alone, even when a close-up was taken. +- Nothing here has run in the Simulator. The completion moment, the amber highlight and the + dots need the replay with `-autopilot`, which reaches this screen only when held-back frames exist. + +Verified against house-scanning commit `0876e03` (t3/ios-mvf). diff --git a/verification/product/screens/mark-features.md b/verification/product/screens/mark-features.md new file mode 100644 index 000000000..6a28302d1 --- /dev/null +++ b/verification/product/screens/mark-features.md @@ -0,0 +1,214 @@ +# Marking features + +## Summary + +Some things near the meter change where a battery can go: a gas meter, a door, a window, an AC +unit, a driveway, a fence. The camera does not find them; the homeowner marks each one by tapping +it on the camera image during [the wall walk](wall-walk.md). After "Done with this wall" the +list screen (screen name `markFeatures`) shows every feature marked, lets the homeowner remove one +and answer "Does this window open?" for each window, and ends with "Looks complete". This +document covers that screen and the marking interaction. Where a battery may go is decided by the +server from these marks; the phone only records them. + +## The simple case + +On the walk the homeowner sees a window and taps "Mark something". A chooser headed "What do you +see?" offers six kinds; they tap "Window". The chooser closes, a circle appears in the middle of +the camera image and the instruction reads "Tap the window's bottom-left corner", with "Put the +circle on it and tap Mark, or tap it on screen." They tap the corner on screen; a ring blooms +where the finger landed and the instruction becomes "Now tap its top-right corner". The second tap +records the window: a dashed outline appears on the wall, a pin at each tap, a window icon on the +coverage strip, and the phone gives a firm tap. When the walk is done they tap "Done with this +wall". The camera dims and a panel appears along the bottom: "Anything else near your +meter?", a row "Window", "About 4 ft right of your meter", then "Does this window open?" with +"It opens" and "It stays shut". They tap "It opens", then "Looks complete". + +## The interaction, event by event + +```mermaid +stateDiagram-v2 + [*] --> walking + walking --> chooser : "Mark something" + chooser --> walking : Close + chooser --> firstTap : a kind picked + firstTap --> firstTap : tap refused + firstTap --> secondTap : tap accepted (door, window, driveway, fence) + firstTap --> walking : tap accepted (gas meter, AC unit), feature recorded + secondTap --> secondTap : tap refused + secondTap --> walking : tap accepted, feature recorded + firstTap --> walking : "Cancel" + secondTap --> walking : "Cancel" + walking --> list : "Done with this wall" + list --> list : remove, answer a window, "Add something" (nothing visible) + list --> [*] : "Looks complete" +``` + +`walking` to `list` is the walk ending; the rest of the walk is in [the wall walk](wall-walk.md). + +### Arriving + +Marking starts from "Mark something" on the walk, shown next to "Wall ends here" or "Done with +this wall" when those are offered, and alone otherwise. It opens the chooser "What do you see?": +"Gas meter", "Door", "Window", "AC unit", "Driveway", "Fence", three to a row, with a close +button. Picking one closes the chooser and starts marking that kind. While marking, a circle sits +in the middle of the camera image, the dotted path and the target ring disappear, the instruction +shows the marking prompt, and the controls become "Cancel" and "Mark". + +The list screen arrives when "Done with this wall" is tapped with both wall ends set (it is not +offered while a mark is in progress). The camera stays on screen, darkened, with a light panel +along the bottom: the title "Anything else near your meter?", the line "Gas meters, doors, windows, AC units, +driveways and fences all change where a battery can go.", then one row per feature in the order +marked, or "Nothing marked yet." if there are none. Each row shows the kind's icon, its name and +where it is ("About 4 ft right of your meter", or "Around your meter" when it spans the meter or +its middle is within 0.3 m of it), and a trash button. A window row adds "Does this window open?" +with two full-width answers stacked one above the other, "It opens" and "It stays shut", neither +selected. Below the list, "Add something" offers the six kinds as +buttons, and "Looks complete" stays pinned at the bottom of the panel. There is no instruction +card, no photo counter and no coverage strip on this screen, and no pins on the camera image. + +### Leaving at once + +While marking, "Cancel" forgets the taps so far and returns to the walk; nothing is recorded. The +chooser's close button returns to the walk without starting. + +On the list, "Looks complete" is the only way on. It needs nothing: an empty list and unanswered +windows are both accepted. There is no way back to the walk ([the flow](../foundations/flow.md#edge-cases)). + +### First capture + +A tap counts when it lands on the open camera image (not on a card or button), or when "Mark" is +tapped, which uses the center of the circle. Each tap is checked in this order; the first failure +is a refusal: + +| Check | Refusal shown | +| --- | --- | +| Tracking not normal | "One moment, your phone is still finding its place." | +| The phone is on the far side of the wall's line | "That spot is behind the wall. Tap something on this side." | +| The tap meets no surface: the wall's plane (gas meter, door, window, AC unit) or the ground (driveway, fence) | "Nothing to pin there. Aim at the wall or the ground and try again." | +| A ground tap more than 8 m out from the wall, or behind its line | "That's too far from the wall to matter. Tap something closer." | + +A refusal replaces the instruction with its words in a warning style, keeps the prompt as the +second line, and gives a warning haptic. The step does not advance. The next accepted tap clears +it. What each kind asks for and records: + +| Kind | Taps and prompts | What the mark records | +| --- | --- | --- | +| Gas meter, AC unit | One: "Tap the gas meter" / "Tap the ac unit", with "Put the circle on it and tap Mark, or tap it on screen." | The tapped point on the wall. The list and strip treat it as 0.3 m wide, centered on the tap; that width is nominal, not measured. | +| Door, window | Two: "Tap the door's bottom-left corner" (or "window's"), with the same second line; then "Now tap its top-right corner". | A rectangle on the wall: from the lower tap's height (never below the ground) to the higher, and between the two taps along the wall. Any two opposite corners give the same rectangle. | +| Driveway | Two: "Tap one end of the driveway's edge", with "Use the edge closest to the wall."; then "Now tap the other end of that edge". | The two ground points: one edge. | +| Fence | Two: "Tap the bottom of the fence at one end", with "Where it meets the ground."; then "Now tap the bottom at the other end". | The two ground points where the fence meets the ground. | + +### While capturing + +Between the taps of a two-tap mark only the ripple confirms the first tap; no pin appears until +the mark is complete. The walk carries on underneath: photos are still kept and coverage still +grows, but coaching, the "Can't get there" reply, the path and the target ring stay hidden until +the mark ends. A finished mark draws on the camera image (a dashed outline for a door or window, a +dashed line for a driveway or fence, a pin at every tap) and as its icon on the coverage strip. + +On the list, the trash button removes its row at once, with no confirmation and no undo; the row +fades out. "It opens" or "It stays shut" fills in when chosen and can be changed; once one is chosen there is no +way back to unanswered. The "Add something" buttons do nothing the homeowner can see (see Open +questions). + +### Advancing + +A mark is recorded on its last accepted tap: the marking ends, the walk's controls return, and +the new feature is added to the end of the list. "Looks complete" runs the phone's own gap check +and goes to a [gap request](gap-request.md) or straight to [the upload](uploading.md) (see +[the flow](../foundations/flow.md#the-interaction-event-by-event)). The features are sent in the +scene description with the scan: + +- a door or window as an opening with its span along the wall, bottom and top; for a window, + whether it opens (yes for "It opens", no for "It stays shut", or left empty when unanswered); a door always left empty; +- a gas meter or AC unit as the tapped point; +- a fence as its two foot points, a driveway as its two edge points. + +Heights are sent as height above the ground at the wall. + +## Modifiers + +| Modifier | At arrival | While capturing | +| --- | --- | --- | +| Live camera or replay | On the list, the live camera keeps running dimmed but keeps no photos; a replay stops on the frame it was showing. | On a replay a tap is measured against the recorded frame on screen. | +| Autopilot | On the walk the autopilot marks a gas meter and a window on covered wall, answers that the window opens, and on the list waits `-autopilotHold` seconds before pressing "Looks complete". It calls the actions directly, never the chooser or "Mark". A mode badge reads "Replay · Autopilot", in capitals. | If the window is still incomplete after two tries at its corners, it cancels it and logs the refusal. | +| Server or sample result | No effect. | No effect. | +| Larger text sizes | At accessibility sizes the chooser and the "Add something" buttons go to one column (from three and two), "Mark something" spans the width, and the panel scrolls with "Looks complete" still pinned. The window answers are stacked at every size. | No effect. | +| Reduce Motion | The screen crossfades in 0.15 s. Rows still fade and scale slightly as they appear and go. | The tap ripple fades without growing. | + +## Cancel and interrupt + +| Event | While a mark is in progress (on the walk) | On the list | +| --- | --- | --- | +| The screen's own way out | "Cancel": the taps so far are forgotten, nothing is recorded. | None; "Looks complete" is the only way on. | +| Start over | Not offered. | Not offered. | +| Tracking limited | Every tap is refused with "One moment, your phone is still finding its place."; the prompt hides the coaching. | Nothing is shown; the list is unaffected. | +| Tracking lost or relocalizing | Taps are refused as above. After 20 s the scan returns to finding the meter and every feature is forgotten, but the half-done mark is not (see Open questions). | No coaching is shown. After 20 s the scan returns to finding the meter and the list is gone, without warning (see Open questions). | +| App backgrounded or a call | The mark in progress and its taps are kept; taps are refused until tracking is normal again. | The list and answers are kept. | +| Camera off or session failed | Suspected dead end ([the flow](../foundations/flow.md#open-questions-and-verification)). A tap is measured against the last frame the camera delivered. | The list still works and "Looks complete" still leads on. | +| Network lost or upload failing | No effect. | No effect here; see [the upload](uploading.md). | +| App killed | The scan is lost. | The scan is lost. | + +## Interactions with other systems + +**Coverage and evidence.** A mark is a claim by the homeowner, not evidence: it keeps no photo and +covers no cell, and nothing checks that the marked stretch was ever seen. Photos keep being kept +during marking on the walk ([coverage and guidance](../foundations/coverage-and-guidance.md#when-a-photo-is-kept)). +**Stored photos.** None are taken for a mark. **Upload and offline.** Marking and the list need no +network; the features travel in the uploaded scene description. **Accessibility.** The title and +"Add something" are headings. A row reads as one element, its name then its place; the trash +button reads "Remove window"; the kind buttons read "Add window" on the list and "Mark window" in +the chooser; the chosen answer carries the selected trait. "Mark" has the +hint "Marks the point under the circle in the middle of the screen"; "Mark something" has "Pin a +gas meter, door, window, AC unit, driveway or fence". Tapping on the camera image is not +available to VoiceOver; "Mark" is the only way. **Haptics and motion.** A firm tap when a feature +is recorded, a warning when a tap is refused, nothing for a removal or an answer. A ring blooms +and fades at each tap in 0.55 s. **Verification hooks.** `STATE=markFeatures` when the list +appears; marking itself logs nothing. Identifiers: `action.markSomething`, `action.closeTray`, +`feature.` (in the chooser and on the list), `action.markPoint`, `action.cancelMarking`, +`action.deleteFeature`, `window.opens.yes`, `window.opens.no`, `action.confirmFeatures`. + +## Edge cases + +- Tapping the same refused spot again repeats the same refusal: the text does not change and the + warning haptic does not fire again; only the ripple shows the tap was received. +- Two taps at the same height make a door or window with no height; nothing checks the size. +- Wall taps have no distance limit. A tap near the horizon can land far along the wall or high up + it, and is recorded there. Only ground taps are limited, to 8 m out. +- A door or window on the wall around a corner is recorded where the tap meets this wall's plane, + extended: at the wrong place. +- A feature marked beyond a wall end is still recorded and sent. +- Removing every row brings back "Nothing marked yet." +- Features cannot be edited after "Looks complete": not on a gap request, the upload or the result. + +## Open questions and verification + +- **Suspected bug: "Add something" does nothing visible.** The buttons start a mark + (`UI/Screens/MarkFeaturesScreen.swift:64`, accepted on this screen by + `Runtime/ScanEngine+Actions.swift:73`), but the list screen never shows the circle, the prompt, + "Mark" or camera taps; those exist only on the walk (`UI/Screens/WallWalkScreen.swift:28-45, + 119-133`). The homeowner who spots a missed gas meter here presses a button and nothing + happens. "Looks complete" does not end the stray mark either (`ScanEngine+Actions.swift:155-158`). +- **Suspected bug: "Tap the ac unit".** The prompt lowercases the kind's name + (`UI/Copy/ScanCopy.swift:122`), so "AC unit" becomes "ac unit" on screen, and VoiceOver hears + "Remove ac unit", "Add ac unit", "Mark ac unit" (`MarkFeaturesScreen.swift:121, 189`, + `WallWalkScreen.swift:258`). +- **Suspected bug: the list can vanish without a word.** Relocalization is timed on every screen + (`Runtime/ScanEngine.swift:189, 357-368`), but the list shows no coaching. After a call, a + homeowner reading the list with the phone pointed away for 20 s is sent back to finding the + meter with every mark and photo gone, never having seen "Point at the meter like this.". +- **Suspected bug: a half-done mark survives the 20 s reset.** `resetSpatialState` clears the + features but not the mark in progress or its taps (`ScanEngine.swift:386-410`, taps held at + `ScanEngine.swift:62`). After the meter is tapped again the walk opens mid-mark, and a first tap + from the old, discarded map can be joined to a new one. The same holds for a stray mark from + "Add something". +- **Wrong words for the wrong-side refusal.** The check is where the phone is, not where the tap + lands (`ScanEngine+Actions.swift:88`). "Tap something on this side" cannot help; stepping back + out in front of the wall's line does. +- The place in a row reads "ft" to VoiceOver (`MarkFeaturesScreen.swift:107`), though the app has + a spelled-out form for this (`UI/Copy/Distance.swift:25`). +- Unverified: all marking needs the live camera or the autopilot, which bypasses the chooser and + "Mark". MF-01 in [verification](../verification.md) was observed on an earlier commit and has + not been rechecked here. + +Verified against house-scanning commit `0876e03` (t3/ios-mvf). diff --git a/verification/product/screens/meter-close-up.md b/verification/product/screens/meter-close-up.md index 27ff0b0a2..ded89b7e3 100644 --- a/verification/product/screens/meter-close-up.md +++ b/verification/product/screens/meter-close-up.md @@ -14,7 +14,7 @@ failed try. The circle appears in the middle of the camera view and the instruction reads "Hold your meter in the circle", with "Your phone takes the photo by itself." The homeowner points the phone at the meter from within about 1.5 m. While everything is right the circle's ring fills; after 0.6 s of -holding still it is full, the screen flashes white, a check mark appears in the circle and the +holding still it is full, the camera image flashes white under the words, a check mark appears in the circle and the photo shrinks into the counter at the top right. About 1.2 s later the walk begins ([the wall walk](wall-walk.md)). @@ -101,7 +101,8 @@ meter close-up and an installer reads the meter instead. ## Interactions with other systems **Coverage and evidence.** The close-up is not a walk photo and covers no cells. **Stored -photos.** Saved as the scan's meter close-up and uploaded with it. **Upload and offline.** No +photos.** Saved as the scan's meter close-up and uploaded with it. The walk shows it again +while the phone is finding its place ([the wall walk](wall-walk.md#while-capturing)). **Upload and offline.** No effect here. **Accessibility.** The circle is one element with a spoken progress value; the fix appears as a label; the button carries the hint "Skips the close-up. An installer will read the meter instead." **Haptics and motion.** A success haptic as the screen opens (the meter is @@ -131,4 +132,4 @@ failed attempts" when the homeowner skips. its text in that screenshot; see [verification](../verification.md). - The circle-versus-image mismatch above is read from code; it needs a device to confirm. -Verified against house-scanning commit `21a63e7`. +Verified against house-scanning commit `0876e03` (t3/ios-mvf). diff --git a/verification/product/screens/onboarding.md b/verification/product/screens/onboarding.md new file mode 100644 index 000000000..83ca40881 --- /dev/null +++ b/verification/product/screens/onboarding.md @@ -0,0 +1,176 @@ +# The onboarding + +## Summary + +The onboarding is the first thing the homeowner sees on every launch and after every Start over +(screen name `onboarding`). Three intro pages say what the scan is, that the phone takes the +photos, and how to stay safe; the last page's "Allow camera" opens [finding the +meter](find-meter.md), where iOS asks for the camera. The same document covers the failure +screen (screen name `unsupported`), which has one message per failure: an unsupported phone, +camera access denied, a failed camera session, an unreadable replay. At this commit only the +unsupported phone ever reaches it; the other three are set but never shown (see Open questions). + +## The simple case + +Page 1 shows a drawn wall with a gray meter, a row of blue dots on the ground and a phone sliding +back and forth under an "About 2 min" tag. Its title reads "Let's find a spot for your battery", +with "Walk along the wall by your electric meter for about 2 minutes. Your phone measures as you +go." The homeowner taps "Next". Page 2, "Your phone takes the photos", shows haze clearing off +the drawn wall, with "Just walk slowly. The haze on the wall clears as your phone sees it." They +tap "Next" again. Page 3, "Stay safe out there", lists three safety rules, and "Next" becomes +"Allow camera" with the note "Your phone will ask to use the camera." They tap it, the camera +screen appears with "Find your electric meter", and iOS shows its camera prompt: "House Scan uses +the camera to measure the outside wall around your electric meter." + +## The interaction, event by event + +```mermaid +stateDiagram-v2 + [*] --> page1 + [*] --> unsupported : phone can't track motion (live camera only) + page1 --> page2 : "Next" or swipe + page2 --> page3 : "Next" or swipe + page1 --> page3 : "Skip" + page2 --> page3 : "Skip" + page3 --> page2 : swipe back + page2 --> page1 : swipe back + page3 --> [*] : "Allow camera" (finding the meter, camera prompt) + unsupported --> unsupported : no way out +``` + +### Arriving + +The app opens on page 1 with the first of three page dots filled. At the same moment it checks +one thing: with the live camera, if the phone cannot run motion tracking, it switches straight to +the failure screen, crossfading over the intro in 0.25 s. With a replay it instead starts loading +the recording in the background. Nothing else is checked here; in particular the camera +permission is not asked. A top bar holds "Skip" on the right and, on a replay or with the +autopilot, a small badge on the left ("Replay", "Autopilot" or "Replay · Autopilot"). + +### Leaving at once + +"Skip" (top right, on pages 1 and 2) jumps to page 3; it does not leave the onboarding. There is +no way past the intro except "Allow camera", and no back button to anything before it. Closing +the app loses nothing, since nothing has been captured. + +On the failure screen for an unsupported phone there is no button at all: the homeowner can only +close the app. + +### First capture + +Nothing is captured here. The one committed action is "Allow camera": it moves to finding the +meter, and only then does the app start the camera. The first time, starting the camera makes iOS +show its permission prompt over "Find your electric meter". The app does not ask for permission +itself and does not wait for the answer before switching screens. + +### While capturing + +The pages can also be swiped in either direction; the button below them follows the current page +("Next" on pages 1 and 2, "Allow camera" on page 3) and the dots follow too, the current one +stretching into a pill. Each illustration animates only while its page is showing: the phone on +page 1 walks one lap every 6 s; the haze on page 2 clears column by column over 4 s, holds clear, +then rolls back in, every 6.5 s. Page 3 has no illustration, only three cards: "Stay on the +ground" ("No ladders, roofs or climbing."), "Keep clear of gas pipes" ("Don't lean on or step +over the gas meter.") and "Don't open the meter" ("Leave covers closed and wires alone."). + +### Advancing + +"Allow camera" always goes to finding the meter. What follows depends on the permission answer: + +| Answer | What the homeowner sees | +| --- | --- | +| Allow, or already allowed | The camera image appears behind "Find your electric meter". | +| Declined, now or earlier | Suspected dead end: the screen stays on "Find your electric meter" with no camera image, no message and no way to Settings. Tapping does nothing. | +| The camera session fails | Suspected dead end: the same screen, frozen. | + +The failure screen shows one of these, with an icon above the title: + +| Failure | Title | Detail | Button | Reached at this commit | +| --- | --- | --- | --- | --- | +| Unsupported phone | "This phone can't measure walls" | "House Scan needs an iPhone that supports motion tracking with the camera. Try another iPhone from the last few years." | None | At launch, live camera only | +| Camera denied | "House Scan needs your camera" | "It uses the camera to measure the wall around your meter. Turn on Camera for House Scan in Settings." | "Open Settings" | No | +| Session failed | "The camera stopped" | The system's error text | "Start over" | No | +| Unreadable replay | "This recording can't be opened" | The loader's error text | "Start over" | No | + +"Open Settings" opens the app's page in iOS Settings. "Start over" would do what it does on the +result ([the flow](../foundations/flow.md#cancel-and-interrupt)) and return to page 1. + +## Modifiers + +| Modifier | At arrival | While capturing | +| --- | --- | --- | +| Live camera or replay | With a replay the motion-tracking check is skipped, so the failure screen never appears at launch, and "Allow camera" never opens the camera or triggers the prompt; the replay's first frame shows behind finding the meter. | No effect. | +| Autopilot | Needs a replay. It waits up to 60 s for the recording to load, holds page 1 for the hold time ([the flow](../foundations/flow.md#modifiers)), then moves on as if "Allow camera" were tapped, without paging. If the recording fails to load it stops and the app stays on page 1. | No effect. | +| Server or sample result | No effect. | No effect. | +| Larger text sizes | Each page's title and body grow and the page scrolls; the illustrations stay 260 pt tall. The failure screen scrolls when its text no longer fits. | The buttons and dots stay fixed below the page. | +| Reduce Motion | The illustrations hold still: the phone stops mid-wall and the haze stops partly cleared. | "Next" slides the page in 0.15 s instead of 0.25 s; "Skip" and swiping are unchanged. | + +## Cancel and interrupt + +| Event | On the intro pages | On the failure screen | +| --- | --- | --- | +| The screen's own way out | "Skip" to page 3; "Allow camera" on it. | Unsupported phone: none. The other three would offer "Open Settings" or "Start over", but are never shown. | +| Start over | Not offered. Arriving here after Start over shows page 1 again. | Not offered for an unsupported phone. | +| Tracking limited | No effect: the camera is not running. | No effect. | +| Tracking lost or relocalizing | No effect. | No effect. | +| App backgrounded or a call | The current page is kept. | No effect. | +| Camera off or session failed | No effect here; the failure lands after "Allow camera", on finding the meter, as a suspected dead end. | Would show "House Scan needs your camera" or "The camera stopped", but never does. | +| Network lost or upload failing | No effect. | No effect. | +| App killed | Nothing is lost; the next launch opens page 1. | The next launch shows the same screen. | + +## Interactions with other systems + +**Coverage and evidence.** None; nothing is seen or covered before the meter tap. **Stored +photos.** None are taken. **Upload and offline.** No effect; nothing here needs the network. +**Accessibility.** Each page title is a heading; the illustrations and the page dots are hidden +from VoiceOver; each safety card is read as one element; "Allow camera" carries the hint "Your +phone will ask to use the camera". On the failure screen the icon is hidden and the title is a +heading. The mode badge is read aloud. **Haptics and motion.** No haptics on either screen. Pages +change with a slide, the button swaps with a fade and the dots settle with a 0.4 s spring. +**Verification hooks.** The log records `STATE=onboarding` at launch and `STATE=unsupported` +when the phone check fails; an unreadable replay logs "replay unreadable: ..." without a screen +change. Tests find the controls as `action.onboardingNext`, `action.onboardingSkip`, +`action.finishOnboarding`, `action.openSettings` and `action.startOver`. + +## Edge cases + +- The app declares motion tracking (`arkit`) as a required device capability + (`ios/Config/Info.plist`), so the App Store and TestFlight will not install it on a phone that + lacks it. The unsupported-phone screen is in practice seen only in the Simulator without a + replay. +- On a live launch the intro is drawn for a moment before the phone check runs, so an unsupported + phone briefly shows page 1 before crossfading away. +- The note "Your phone will ask to use the camera." shows on every pass, but iOS asks only once. + After Start over, or on any later launch, no prompt follows. +- Swiping back from page 3 brings "Next" and "Skip" back; nothing on page 3 is committed until + "Allow camera". + +## Open questions and verification + +- **Suspected bug: camera denied, a failed session and an unreadable replay never reach the + screen** (triaged as [B-03](../bug-triage.md)). The engine switches to the failure screen only + for an unsupported phone (`Runtime/ScanEngine.swift:91-93`). A denied camera and a failed + session set the failure (`ScanEngine.swift:379, 381`) and an unreadable replay does too + (`ScanEngine.swift:105`), with no screen change. The homeowner who declines the prompt sits on + "Find your electric meter" with no camera image; the "Open Settings" and "Start over" buttons on + `UI/Screens/UnsupportedScreen.swift:35-48` are unreachable outside the `-uiDemo` preview. +- **Suspected bug: an unsupported phone has no way out.** `UnsupportedScreen.swift:44` hides + "Start over" for that failure, so the screen has no button. That fits (nothing can fix it), but + it contradicts [the flow](../foundations/flow.md#summary), which says "Start over" is offered on + the unsupported screen. One of the two needs updating. +- **Suspected bug: "Allow camera" during replay loading starts the live camera.** On a replay, + `startSourceIfNeeded` (`ScanEngine.swift:169`) starts the camera whenever the replay has not + loaded yet and no failure is set. A homeowner who taps "Skip" and "Allow camera" before a slow + recording loads gets a live session as well as the replay. Affects replays only; not observed. +- **"The camera stopped" would show raw system error text** as its detail + (`UI/Copy/ScanCopy.swift:241`), the same pattern as B-02, if the screen were ever reached. +- **Reduce Motion is applied to "Next" but not to "Skip"** (`OnboardingScreen.swift:37` versus + `:75`). Even "Next" still slides the page sideways under Reduce Motion, only faster. +- The page view hides its dots from VoiceOver on the grounds that it announces its own position + (`OnboardingScreen.swift:158`), but the index display is set to never show (`:54`). Whether + VoiceOver announces the page number needs checking on a device. +- Read from code, not observed: the camera prompt's timing, ARKit reporting a denied camera as a + session error, and the Reduce Motion stills. The unsupported screen and the + `-replay /nonexistent` case can be checked in the Simulator. + +Verified against house-scanning commit `0876e03` (t3/ios-mvf). diff --git a/verification/product/screens/result.md b/verification/product/screens/result.md new file mode 100644 index 000000000..5cb632776 --- /dev/null +++ b/verification/product/screens/result.md @@ -0,0 +1,221 @@ +# The result + +## Summary + +The result is the server's answer, shown once the [upload](uploading.md) has finished. It covers +two screens. The result screen (screen name `result`) has a small 3D model of the homeowner's own +wall with the battery on it, a headline, where the battery goes and how much cable it needs, and +each check with its outcome. From there the homeowner can open the AR view (screen name +`resultAR`), which draws the battery onto the live camera at the chosen spot. They can also fill +in a missing view with "Capture it now", or begin again with "Start over". Nothing is captured on +either screen, and there is no way back to the walk. + +## The simple case + +After "Checking your wall" the result fades in with a success haptic. At the top, a 3D model of +the wall turns once and comes to rest on the battery, with the cable run from the meter and the +ground in front of each stretch tinted by outcome. Below it the headline reads "There's a spot for +your battery" and, in blue, where it is, for example "4 ft 3 in right of your meter, 3 ft of cable". Next come +the server's one-sentence summary, then "See it on your wall". Under that are a note that the +electrical panel still needs an electrician's review and "What we checked", one row per check, and the "Start over" button. The homeowner taps "See it on your wall", points the +phone at the wall and sees the battery standing there, then taps "Done" to go back. + +## The interaction, event by event + +```mermaid +stateDiagram-v2 + [*] --> result : the server answered (success haptic) + result --> resultAR : "See it on your wall" (only with a spot) + resultAR --> result : "Done" + result --> gapRequest : "Capture it now" + result --> onboarding : "Start over" + result --> findMeter : phone relocalizing for over 20 s + resultAR --> findMeter : phone relocalizing for over 20 s +``` + +### Arriving + +The result screen opens only after the server has answered ([the flow](../foundations/flow.md)). +A success haptic plays, except when coming back from the AR view. The 3D model fills the top +340 points of the screen; the rest scrolls beneath it, in this order: + +1. The headline, with an icon in the decision's color: + + | Decision | Headline | + | --- | --- | + | pass | "There's a spot for your battery" | + | manual review | "An installer will take a look" | + | reject | "This wall doesn't have a spot" | + +2. The placement line, only when the server chose a spot: the spot's middle measured + along the wall ("4 ft 3 in right of your meter", or "At your meter" within 3 in of it), then + the cable length in whole feet ("3 ft of cable"). The server never sends a spot with a reject. +3. The server's summary sentence. If the summary is empty on a reject, the reason of the first + failing check stands in. +4. "See it on your wall", only when there is a spot. +5. "The placement rules aren't final yet, so an installer reviews every result for now.", when + the server's rules are not approved for automatic decisions. +6. "A closer spot may exist on the left of your meter. The scan didn't reach that side." (or + right). It shows when the server asks for a view past a wall end, or when either wall end is + an unexplored end. +7. "Your electrical panel still needs an electrician's review. This scan only covers where the + battery can go.", on every result: the server's answer covers where the battery goes, not the + panel. +8. "What we checked", one row per check, in the server's order (see + [While capturing](#while-capturing)). +9. "Still needed", one card per missing view the server lists (see [Advancing](#advancing)). +10. "Start over". + +The headline block fades in and rises 12 points after 0.35 s. The 3D model sweeps over 1.6 s from +a high three-quarter view down to its resting angle, centered on the battery (or on the meter +when there is no spot). + +### Leaving at once + +"Start over" is always at the bottom. It forgets the wall, photos, marks and result and returns +to the introduction ([the flow](../foundations/flow.md#cancel-and-interrupt)). It asks for no +confirmation. The screen has no back button to the walk or the marks. + +### First capture + +Nothing is captured here. The first thing most homeowners do is turn the model or open the AR +view. + +**The 3D model.** Dragging turns it, up to 70° either side and between 8° and 45° above the +ground; the first drag stops the opening sweep where it is. Pinching zooms between 0.6 and 1.6 +times. The model shows the wall (to the marked wall ends, or 1.5 m past everything the result +mentions), the meter, every marked feature, the cable as a blue tube, and the battery as a white +box with a blue light bar. The server also sends its sweep: the stretches of wall where it tried +the battery, each with one outcome. Each stretch is a translucent tint on the ground in front of +the wall, as deep as the battery: green where a battery passes, amber where it is unsure, red +where it fails. + +**The AR view.** "See it on your wall" crossfades to the camera. The instruction reads "Your +battery could go here" with the placement line under it. On the camera image the app draws each +sweep stretch as a tinted ground strip with a dashed outline, the cable as a blue line edged in +white, and the battery as a box with a soft shadow and a blue light bar across its front. The box +rises out of the ground as the view opens. Everything is placed relative to the meter's anchor, +so it stays on the wall as the homeowner moves. "Done" is the only control. The drawing is shown +only while tracking is normal (see [While capturing](#while-capturing)). + +### While capturing + +Each row of "What we checked" shows an icon (check, question mark or cross), the check's name +from the server, its reason, and a measurement line when the server measured something: +"Measured 3 ft 1 in. The rule is 3 ft, and the measurement can be off by about 4 in." The second +sentence drops its error clause when the server gives no error, and disappears when there is no +rule value. An unsure row adds one note: "An installer will check this" when the doubt is a +narrow margin, an unknown attribute or a rule that always needs review, or "One more photo would +settle this" when the area was not seen. The rows have no buttons of their own. + +On the AR view the drawing is redrawn from every camera frame. Whenever tracking is not normal +(limited, lost or relocalizing), the drawing disappears at once and the instruction reads "Point +at your meter", with "The battery comes back once your phone finds its place." When tracking is +normal again the drawing returns, without rising again, and the instruction goes back to "Your +battery could go here". +No other coaching appears here: the homeowner is not told why tracking is limited. + +### Advancing + +"Done" returns to the result screen. The screen is built again, so the headline fades in again +and the model replays its sweep and loses any turn or zoom. + +Each "Still needed" card shows the server's message. Under it is "Capture it now" when the view +is of the wall, the ground or past a wall end, and "An installer will check this" otherwise. +"Capture it now" opens a [gap request](gap-request.md) for that stretch, using the server's +message as the second line. For a view past a wall end, that end is forgotten first, because the +wall may go on. When the view is taken, or the homeowner taps "I can't get there", the scan is +uploaded again and a new result replaces this one. The old result cannot be reopened meanwhile. + +## Modifiers + +| Modifier | At arrival | While capturing | +| --- | --- | --- | +| Live camera or replay | A replay shows the "Replay" badge over the model. The AR view on a replay shows one still: the recorded frame that best shows the spot, with the battery drawn on it. | On a replay the AR picture does not move. Live, the drawing follows the phone. | +| Autopilot | The autopilot waits `-autopilotHold` seconds, opens the AR view itself (even when the result has no spot), waits again, taps "Done" and logs "AUTOPILOT done". | No effect. | +| Server or sample result | The sample result shows "Sample result, not from the server" over the model. It is always the same: "An installer will take a look", "4 ft 3 in right of your meter, 3 ft of cable", the rules note, the left-side note, the panel note, four checks and two "Capture it now" cards. | "Capture it now" works, but the upload returns the same sample, so nothing changes. | +| Larger text sizes | The headline, notes and rows wrap and the page scrolls; the model stays 340 points tall. On the AR view the instruction grows, and at the largest sizes the chrome scrolls. | No effect. | +| Reduce Motion | The headline fades in over 0.2 s without rising. The model starts at its resting angle with no sweep. The AR battery rises over 0.2 s instead of springing up. | Dragging and pinching the model work as usual. | + +## Cancel and interrupt + +| Event | On the result | On the AR view | +| --- | --- | --- | +| The screen's own way out | "Start over"; "Capture it now" leaves for a gap request. | "Done", back to the result. | +| Start over | Offered at the bottom; see [the flow](../foundations/flow.md#cancel-and-interrupt). | Not offered. | +| Tracking limited | Nothing shown; the camera is hidden but keeps tracking. | The drawing disappears and the instruction reads "Point at your meter" with "The battery comes back once your phone finds its place." until tracking is normal. | +| Tracking lost or relocalizing | After 20 s of relocalizing the app leaves the result for finding the meter, with the wall, photos and marks gone. Suspected bug; see Open questions. | The drawing disappears and "Point at your meter" shows, as above. After 20 s of relocalizing, the same reset as on the result. | +| App backgrounded or a call | The screen stays. When the app returns the phone relocalizes, and the 20 s limit above applies. | Same, with the camera resuming behind the drawing. | +| Camera off or session failed | No visible effect. | Suspected dead end, as in [the flow](../foundations/flow.md#open-questions-and-verification): no message, no picture. | +| Network lost or upload failing | No effect; the result is already on the phone. "Capture it now" leads to a new upload, which needs the network ([the upload](uploading.md)). | No effect. | +| App killed | The result is lost; nothing reopens it. | The result is lost. | + +## Interactions with other systems + +**Coverage and evidence.** Nothing here adds coverage; "Capture it now" hands over to a gap +request, which follows [coverage and guidance](../foundations/coverage-and-guidance.md). +**Stored photos.** No photos are kept on either screen. **Upload and offline.** The result needs +no network once shown; only "Capture it now" leads to another upload. **Accessibility.** The +headline block is read as one heading. The model is one element, "3D view of your wall", with a +value such as "Battery 3 ft right of your meter, cable 3 ft" and the hint "Drag to turn the view". +Each check row reads as "Distance from the window: Not sure yet", followed by its reason, +measurement line and note. The outcome words are "Looks good", "Not sure yet" and "Doesn't work". +The AR drawing is hidden from VoiceOver. **Haptics and motion.** A success haptic when the result +appears, not when returning from the AR view. **Verification hooks.** `STATE=result` and +`STATE=resultAR`. Identifiers: `result.headline`, `result.placement`, `result.rulesNotFinal`, +`result.unseenSide`, `result.panelReview`, `result.sampleBadge`, `check.`, `action.showAR`, +`action.captureMissing`, `action.startOver`, `action.closeAR`. + +## Edge cases + +- A manual review can come without a spot. The headline then reads "An installer will take a + look", with no placement line and no "See it on your wall". +- On a reject the checks are those of the nearest spot the server considered, and the model + centers on the meter. +- The left-or-right note says "A closer spot may exist" even on a reject, where no spot was found. +- Opened without a spot (only the autopilot can do this), the AR view draws the sweep and says + "Point at your meter", with no second line while tracking is normal. +- The AR battery is drawn over everything in the picture, including a bush or a person standing + in front of the spot. +- With the sample result on a real phone, the AR view places a battery 4 ft 3 in right of the + real meter, whatever the wall is like. +- A "Still needed" item about the wall or the ground that comes without a stretch still shows + "Capture it now", but the button does nothing. + +## Open questions and verification + +- **Suspected bug: the result can disappear after a call or a trip to another app.** The 20 s + relocalization limit runs on every screen (`Runtime/ScanEngine.swift:189`, `357-364`). The live + camera keeps running under the result, so a phone that fails to relocalize within 20 s sends a + homeowner who is reading their result back to "Find your electric meter". On the result + screen the camera is hidden, so nothing even tells them to aim at the meter; only the AR view + says "Point at your meter". The reset + (`ScanEngine.swift:386-409`) also leaves the old result in memory. The limit should apply only + to the capture screens. +- **Suspected bug: the tinted stretches are in the wrong place.** Each sweep run's `start_ft` is + the range of the battery's left edge (`HouseScanKit/.../PlacementResult.swift:450`; the result + schema says the same). The app tints exactly that range (`Runtime/ScanEngine+Export.swift:124`), + so every stretch stops one battery width short on the right. +- **Suspected bug: the AR view does not say a sample is a sample.** The sample badge is only on the + result screen (`UI/Screens/ResultScreen.swift:96-105`). The AR view says "Your battery could go + here" over a made-up spot (`UI/Screens/ResultARScreen.swift:43`). +- The measurement line says "The rule is 20 ft" for a limit that is a maximum and "The rule is + 3 ft" for a minimum. The app decodes the server's `comparison` but never shows it, and ignores + `review_threshold_ft` (`UI/Copy/ScanCopy.swift:214-218`). +- "Capture it now" does nothing when the server's item has no stretch + (`Runtime/ScanEngine+Actions.swift:205`, `HouseScanKit/.../GapPlanner.swift:113`), although + the card offers it (`ScanEngine+Export.swift:134`). +- After "Capture it now" the old result is gone from view. If the new upload fails, the homeowner + has only "Try again" on the upload screen and cannot reread the earlier answer. +- VoiceOver reads the model's value and the measurement lines with "ft" and "in", which it reads + as letters (`UI/Result/ResultScene3D.swift:296-321`, `UI/Screens/ResultScreen.swift:240`); + `Distance.spoken` exists for this. +- The autopilot opens the AR view without checking for a spot (`Runtime/Autopilot.swift:93`), a + path no homeowner can take. +- Dragging the model sits inside a scrolling page; whether a vertical drag turns the model or + scrolls the page needs a device. +- Both screens were seen at `0876e03` with the real server's answer and with the sample result + ([verification](../verification.md), RES-01 to AR-01). Whether the AR drawing lines up with a + real wall, and the relocalization reset, need the live camera. + +Verified against house-scanning commit `0876e03` (t3/ios-mvf). diff --git a/verification/product/screens/uploading.md b/verification/product/screens/uploading.md new file mode 100644 index 000000000..d3dd0bf46 --- /dev/null +++ b/verification/product/screens/uploading.md @@ -0,0 +1,178 @@ +# The upload + +## Summary + +The upload screen (screen name `uploading`) sends the scan and waits for the result. The +homeowner does nothing here unless it fails. It shows a large symbol, a title and three steps +that tick off in order: "Get photos ready", "Send photos", "Check clearances". It appears after +"Looks complete" on [marking features](mark-features.md) when the phone finds no gap, or after a +[gap request](gap-request.md) is taken or skipped, and it ends on its own when the answer arrives +([the result](result.md)). If the upload fails, the steps give way to one button, "Try again". +There is no other way off the screen. + +## The simple case + +The camera closes and the screen reads "Getting your photos ready", with a spinner beside "Get +photos ready". A moment later the first step becomes "Photos ready" with a check mark and the +title changes to "Sending your photos", with "Keep the app open. It takes about a minute." The +second step counts up: "Sending photos, 40%". When the server's answer arrives the title briefly +becomes "Checking your wall" ("Measuring clearances around your meter."), and the result appears +with a success haptic. + +## The interaction, event by event + +```mermaid +stateDiagram-v2 + [*] --> packaging + packaging --> sending : scan zipped + packaging --> failed : zipping failed + sending --> analyzing : the server answered (or the sample reached 100%) + sending --> failed : connection error or a non-2xx answer + analyzing --> failed : the answer is not a readable result + analyzing --> [*] : the result + failed --> packaging : "Try again" +``` + +### Arriving + +The screen fades in over 0.25 s on a plain background; the camera view is gone. The symbol is a +stack of photos in a pale circle and pulses while work is going on. The title is "Getting your +photos ready" with no second line. Of the three steps, the first has a spinner and the other two +an empty circle in a muted color. The app first waits for photos still being written to finish, +for up to 10 s, then zips the scene description (`scene.json`) with the photos. + +### Leaving at once + +None. There is no Cancel, no back and no Start over on this screen. Until an answer or a +failure, the homeowner can only wait or close the app. + +### First capture + +Nothing is captured here. The first durable step is the zip: the scene description, every +[kept photo](../glossary.md#seeing-the-wall) of the scan and the [close-up](meter-close-up.md) +if one was taken. Once it is written the symbol becomes an upward arrow, the first step reads +"Photos ready" with a check mark, the title becomes "Sending your photos" and the second step +reads "Sending photos, 0%". + +### While capturing + +The percentage follows the bytes sent to the server, rounded to a whole number and capped at 99% +until the server answers. The digits roll as they change. When the answer arrives, the symbol +becomes a ruler, the second step reads "Photos sent", the third gets the spinner and the title +becomes "Checking your wall" with "Measuring clearances around your meter." With a real server +this lasts only as long as the app takes to read the answer (see Open questions). + +Any failure replaces the three steps with a "Try again" button, stops the pulse and turns the +symbol to the caution color: + +| What failed | Symbol | Title | Second line | +| --- | --- | --- | --- | +| Any connection error (no network, server unreachable, timeout) | Wi-Fi crossed out | "You're offline" | "Your scan is saved on this phone. Try again when you have signal." | +| The server answered with an error, the answer was unreadable, or zipping failed | Warning triangle | "That didn't go through" | The error's own text, for example "The server answered", the status code and the start of the server's reply | + +"Try again" returns to "Getting your photos ready", zips the scan again and sends all of it. + +### Advancing + +When the answer is a readable result, the app stores it, the title becomes "Done" with a check +mark symbol, and the result screen appears with a success haptic +([the flow](../foundations/flow.md#interactions-with-other-systems)). In normal use the switch +is immediate, so "Done" is not seen. Nothing advances without an answer: a failure stays on +screen until "Try again" succeeds. + +## Modifiers + +| Modifier | At arrival | While capturing | +| --- | --- | --- | +| Live camera or replay | A replay stops playing; the scan is zipped the same way. | On the live camera the phone still watches tracking behind this screen (see Cancel and interrupt); a replay does not. | +| Autopilot | A badge at the top left reads "Replay · Autopilot" (in capitals). With the sample result, the `-autopilotHold` time replaces the 1.2 s in the timings below. | If no result arrives within 240 s and the upload has failed, the autopilot presses "Try again" once and waits 90 s more. With `-autopilotGate`, "Done" stays up until the test's gate file appears, at most 120 s. | +| Server or sample result | With `-serverURL` the zip is posted to that server. Without it, or with `-sampleResult`, the scan is still zipped but not sent. | The sample shows 25%, 50% and 75% about 0.24 s apart, then "Checking your wall" for 1.2 s, then the [sample result](../glossary.md#after-the-walk). It fails only if the sample file is missing from the app. | +| Larger text sizes | Title, second line and steps grow and wrap; the screen scrolls when they no longer fit. The symbol stays 132 pt. | No effect. | +| Reduce Motion | The screen fades in over 0.15 s; the symbol does not pulse. | Check marks still scale in and the symbol still swaps with an animation. | + +## Cancel and interrupt + +| Event | While zipping or sending | After a failure | +| --- | --- | --- | +| The screen's own way out | None. | "Try again": zips and sends the whole scan again. | +| Start over | Not offered. | Not offered. | +| Tracking limited | No effect; this screen shows no coaching. | No effect. | +| Tracking lost or relocalizing | Suspected bug: after 20 s of relocalizing on the live camera, the scan is forgotten and the app returns to finding the meter; the answer, when it comes, is dropped. | Same: the failure gives way to finding the meter. | +| App backgrounded or a call | The app asks for no background time, so iOS may suspend the transfer; a broken connection would then show "You're offline". Unverified. | The failure stays. | +| Camera off or session failed | No effect; the upload does not need the camera. | No effect. | +| Network lost or upload failing | The steps give way to the failure and "Try again" (table above). | "Try again" fails the same way until the network or server is back. | +| App killed | The scan is lost. The zip and photos stay in the cache, never reopened. | Same, although the screen said "Your scan is saved on this phone". | + +## Interactions with other systems + +**Coverage and evidence.** Nothing is captured here. The scene description carries the wall, +the covered stretches of each band, whether each wall end is a real end, the marked features, +and each kept photo's camera position and lens data +([coverage and guidance](../foundations/coverage-and-guidance.md)). The server evaluates the whole +scan again on every upload. **Stored photos.** The zip, `scan.zip`, is written into the scan's +cache folder ([the flow](../foundations/flow.md#interactions-with-other-systems)) and replaced on +each attempt. It holds `scene.json`, every kept photo as `k00001.jpg`, `k00002.jpg` and so on +(landscape JPEGs, unrotated, as the camera produced them) and the close-up as `meter_close.jpg`. +**Upload and offline.** The app posts the raw zip to `{serverURL}/v1/placements` with +`Content-Type: application/zip` and `Accept: application/json`. The request times out after +120 s without data. Any answer outside 200–299 is a failure; so is one that does not read as a +result: a schema version other than 1.0, a missing required field, a wrong type, a pair of the +wrong length, or a value outside a fixed set such as the decision or an outcome +(`HouseScanKit/.../PlacementResult.swift:1-11`). Fields the app does not know are ignored, so the server can add +optional ones without breaking the upload. Only connection errors count as offline. **Accessibility.** The title is a header. The +symbol is hidden from VoiceOver. The three steps are one element, read together. No change of +step or percentage is announced, and neither is a failure. **Haptics and motion.** None on +arrival, failure or "Try again"; a success haptic when the result appears. The symbol pulses while +working and swaps with a replace animation; check marks scale in; the title crossfades in 0.2 s. +**Verification hooks.** `STATE=uploading` on arrival. The log records "bundle with N +keyframes" after zipping, and "packaging failed: …" or "upload failed: …" with the error. The +autopilot logs "upload failed; retrying once". Test identifiers: `screen.uploading`, +`upload.steps`, `action.retryUpload`. + +## Edge cases + +- The screen can appear more than once in a scan: after each gap request, including one started + with "Capture it now" on the result. Each visit starts again at "Getting your photos ready" and + sends every photo, not only the new ones. +- A photo whose write has not finished after the 10 s wait is left out of the zip without notice. +- After a relocalization reset earlier in the scan, only photos kept since the reset are sent; + the older files stay in the folder. +- A wrong server URL, a stopped server or a slow server that goes quiet for 120 s all show "You're + offline" and "Try again when you have signal", even with full signal. +- "Try again" does nothing once pressed until the attempt fails again; a second tap is ignored. + +## Open questions and verification + +- The wrong upload route ([bug triage](../bug-triage.md) B-01) is fixed in `0876e03`: the app + posts to `/v1/placements` (`Runtime/ResultClient.swift:24`), and the server answered 200 in the + Simulator run at this commit ([verification](../verification.md), FLOW-02). +- **Suspected bug: raw error text on screen.** The second line of "That didn't go through" is the + error's developer description (`UI/Copy/ScanCopy.swift:177`, fed by + `Runtime/ScanEngine.swift:533, 560`): HTTP status codes, server JSON, decoder errors. B-02. +- **Suspected bug: "You're offline" for any connection error.** `Runtime/ScanEngine.swift:560` + treats every `URLError` as offline, including an unreachable host, refused connection and + timeout. The copy then tells a homeowner with full signal to wait for signal. +- **Suspected bug: "Checking your wall" is skipped with a real server.** Progress is capped at + 99% while sending (`Runtime/ResultClient.swift:68`) and reaches 100% only after the server has + answered (`:40`); the step changes only then (`Runtime/ScanEngine.swift:542, 546`). The + homeowner watches "Sending photos, 99%" for the whole analysis, then the result replaces + "Checking your wall" almost at once. Only the sample result shows the third step. +- **Suspected bug: a relocalization reset can wipe the scan during the upload.** The + relocalization check runs on every live frame whatever the screen (`Runtime/ScanEngine.swift:189`, + `364`). After 20 s it forgets the wall, marks and photo list and goes to `findMeter` (`:386–409`), + and the upload's answer is dropped (the `generation` guards at `:545, 552`). A homeowner who + locks the phone mid-upload and returns somewhere the phone can't recognise would lose the scan. + Unverified: needs a device, and whether the AR session keeps delivering frames while the camera + view is off screen is not settled by the code. +- The app asks for no background execution time (no `beginBackgroundTask`), so "Keep the app + open" is the only guard for the transfer. What iOS does to the transfer when the phone locks + has not been tested. +- "It takes about a minute" has no measurement behind it; no upload has been timed with a real + scan. +- The comment on `writeBundle` (`Runtime/KeyframeStore.swift:83–84`) says the zip stays so a + failed upload can be retried, but "Try again" rebuilds it (`Runtime/ScanEngine+Actions.swift:194–196`). + Harmless, but the kept zip is never reused. +- Pass 1 (`21a63e7`) saw "That didn't go through" with no button or symbol visible + ([verification](../verification.md) UP-01, B-05). Not rechecked at this commit. + +Verified against house-scanning commit `0876e03` (t3/ios-mvf). diff --git a/verification/product/screens/wall-walk.md b/verification/product/screens/wall-walk.md new file mode 100644 index 000000000..4838d2042 --- /dev/null +++ b/verification/product/screens/wall-walk.md @@ -0,0 +1,335 @@ +# The wall walk + +## Summary + +The wall walk is where the scan collects its evidence. The homeowner walks along the wall with +the phone pointed at it, first to the left of the meter and then to the right, and the phone keeps +photos by itself. Fog over the camera image clears where the phone has looked, the coverage strip +at the bottom fills in, and one instruction at the top says what to do next. The homeowner says +where the wall stops on each side ("Wall ends here", or "Can't get there" when they cannot go +further), can mark features on the way ("Mark something"), and leaves with "Done with this wall" +once both ends are set. It follows [the meter close-up](meter-close-up.md) (screen name +`wallWalk`) and leads to [marking features](mark-features.md). If the phone loses its place for +too long, the scan starts again from [finding the meter](find-meter.md). + +## The simple case + +The screen opens over the same camera view. The instruction reads "Walk slowly to your left", with +"Keep the wall and the ground in view." Blue dots on the ground show where to walk, and a ring on +the wall shows where to aim. The homeowner walks. The counter at the top right goes up, the fog +lifts off the wall and ground behind them, and the strip turns from gray to amber to green. + +When enough of the left side is covered, the instruction asks "Is this the left end of the wall?" +and a "Wall ends here" button appears. The homeowner aims the middle of the screen at the corner +and taps it. A white line appears on the wall there, and the instruction switches to "Walk slowly +to your right". They walk back past the meter and do the same on the right. With both ends set, +"Done with this wall" appears; any stretch between the ends that is still thin gets one more aim +instruction, and then "That's the whole wall", with "Tap Done when you're ready." Tapping "Done with +this wall" opens the list of marks. + +## The interaction, event by event + +```mermaid +stateDiagram-v2 + [*] --> walking + state walking { + [*] --> walkLeft + walkLeft --> askLeftEnd : coverage runs far enough left + walkLeft --> walkRight : "Can't get there" (unexplored left end) + askLeftEnd --> walkRight : "Wall ends here" (real left end) + walkRight --> askRightEnd : coverage runs far enough right + walkRight --> filling : "Can't get there" (unexplored right end) + askRightEnd --> filling : "Wall ends here" (real right end) + filling --> complete : nothing lags between the ends + } + walking --> kindPicker : "Mark something" + kindPicker --> walking : close + kindPicker --> marking : a kind tapped + marking --> walking : last mark placed, or "Cancel" + walking --> [*] : "Done with this wall" (both ends set) + walking --> findMeter : relocalizing for over 20 s +``` + +`filling` and `complete` both show "Done with this wall". "Take a step back" and the tilt +instructions can interrupt any walking state; they are left out of the diagram. + +### Arriving + +The camera stays open from the close-up; only the chrome changes. At the top: a mode badge on a +replay, the photo counter (it already counts the close-up, if one was taken), and the instruction +card. At the bottom: a "Mark something" button and the coverage strip. Over the camera image: +fog on the wall band and ground band around the meter, a dot on the meter, and the wayfinding +cues. The first instruction is chosen from the first camera frame, in the order set out in +[coverage and guidance](../foundations/coverage-and-guidance.md#one-instruction-at-a-time). A +homeowner who has just held the meter close is usually nearer the wall than the walk wants, so +the first line they read is likely "Take a step back", with "Your phone needs to see more of the +wall at once." + +The instructions this screen can show, with their second lines: + +| Instruction | Second line | Cue on the camera | +| --- | --- | --- | +| "Take a step back" | "Your phone needs to see more of the wall at once." | None. | +| "Tilt down to show the ground" | "The strip along the wall, 3 ft 4 in left of your meter." (the distance varies) | Ring on the ground, dots toward it. | +| "Tilt up to show more wall" | "Around 3 ft 4 in left of your meter." (the distance varies) | Ring on the wall, dots toward it. | +| "Walk slowly to your left" (or right) | "Keep the wall and the ground in view." | Ring on the wall past the covered stretch, dots toward it. | +| "Is this the left end of the wall?" (or right) | "Aim at the corner, or where something blocks your way, and tap Wall ends here." | Ring at the end of the covered stretch, no dots. | +| "That's the whole wall" | "Tap Done when you're ready." | None. | + +A distance within 3 in of the meter reads "at your meter". Coaching replaces all of these while +the phone has a problem (see [Cancel and interrupt](#cancel-and-interrupt)). + +The wayfinding cues work together. The **path** is a line of blue dots on the ground, 1.5 m out +from the wall, from where the homeowner stands toward the next place to stand; the dots shrink +with distance like paint on the ground. The **target** is a white-and-blue ring, sized to the +distance, that pulses slowly while its point is comfortably on screen. Near the edges or off +screen it becomes a blue chevron at the side of the screen, pointing toward the spot, kept clear +of the instruction card and the buttons. + +### Leaving at once + +There is no back button and no way to leave without setting both ends. The fastest way off is +"Can't get there" under "Walk slowly to your left" and again under "Walk slowly to your right": +each sets that side's end where coverage stops, which on arrival is the meter itself. The +instruction then reads "That's the whole wall" and "Done with this wall" moves on with no walk +photos at all (see [Open questions](#open-questions-and-verification)). The only other way off is +the phone losing its place for over 20 s, which throws the walk away. + +### First capture + +The first photo is kept by the rules in +[coverage and guidance](../foundations/coverage-and-guidance.md#when-a-photo-is-kept). When it is +saved, the counter's camera icon flashes green and the number rolls up by one. There is no haptic +tap for walk photos in this build. Wall and ground cells the photo shows most of go from fog to a +thinner haze: their fog fades over 0.7 s while drifting upward, like mist lifting. On the strip +those cells turn from gray to amber. A cell seen a second time from a different position turns +green on the strip and its haze clears completely. + +### While capturing + +**The strip.** A dark rounded panel above the bottom edge. The top row is the wall band, the lower, +thinner row the ground band. Cells are gray while unseen, amber once seen, green once covered, and +slate with a white slash when skipped. A blue line and a lightning-bolt badge mark the meter, a +white pointer under the strip marks where the homeowner stands, and small ticks along the bottom +mark every foot, longer every fifth. A marked end is a tall white cap; a marked feature shows its +icon above the rows. The strip always spans at least 3.5 m (about 11 ft), and its scale changes as +coverage, the ends or the homeowner move outward. + +**The fog.** A frosted haze painted on the wall face (up to the top of the wall band) and on the +ground in front of it, over what has been seen plus the fog ahead of it on each side. Unseen cells are thick, seen cells +thinner, covered cells clear. Skipped cells are not fogged: they get a slate tint with white +diagonal hatching, so they read neither as clear nor as unseen. Set ends are white vertical lines +on the wall with a white dot at the foot. + +**The instruction.** It changes only when the current one is satisfied or has been up long enough +([coverage and guidance](../foundations/coverage-and-guidance.md#one-instruction-at-a-time)). The +old line blurs out and the new one blurs in. + +**"Can't get there".** A small pill inside the instruction card, under the text. It shows only +under "Walk slowly to your …", "Tilt down to show the ground" and "Tilt up to show more wall", +and never while coaching is showing, the kind picker is open or a mark is in progress. + +- Under a tilt instruction it marks the cells not yet covered in a 1 m stretch centered on the + spot as skipped, in that band only. The fog there turns to hatching, the strip shows slate, and + the next instruction is chosen at once. +- Under a walk instruction it sets that side's end as an unexplored end, at the point where + coverage stops running unbroken from the meter. The walk turns to the other side, or to the + thin stretches between the ends. + +**"Wall ends here".** Shown, with "Mark something" beside it, only while the instruction asks "Is +this the left end of the wall?" or the right one. Tapping it takes the point in the middle of the +screen, finds where it meets the wall's line, and sets a real end there. Which end it sets depends +on which side of the meter that point is, not on which end was asked for. The white end line +appears on the camera, the cap appears on the strip, and the instruction moves on. Nothing on the +screen marks the middle of the screen at this moment, and if the middle of the screen does not +meet the wall line (the phone aimed away from the wall) the tap does nothing. + +**"Mark something".** Always available while walking. It replaces the buttons with the kind +picker: a panel headed "What do you see?", a close button, and six tiles, each an icon and a +name: "Gas meter", "Door", "Window", "AC unit", "Driveway", "Fence". While it is open, +"Can't get there" hides. The close button returns the buttons. Tapping a tile starts marking that +kind and closes the picker. + +**Marking.** The ring and the path disappear, a white circle with a center dot appears in the +middle of the screen, and the bottom row becomes "Cancel" and "Mark". The instruction becomes the +marking prompt: + +| Kind | First prompt | Second prompt | +| --- | --- | --- | +| Door, window | "Tap the door's bottom-left corner" (or window's), with "Put the circle on it and tap Mark, or tap it on screen." | "Now tap its top-right corner" | +| Driveway | "Tap one end of the driveway's edge", with "Use the edge closest to the wall." | "Now tap the other end of that edge" | +| Fence | "Tap the bottom of the fence at one end", with "Where it meets the ground." | "Now tap the bottom at the other end" | +| Gas meter, AC unit | "Tap the gas meter" (the AC unit prompt reads "Tap the ac unit"), with "Put the circle on it and tap Mark, or tap it on screen." | None: one mark. | + +The homeowner can tap the spot on the camera image or put the circle on it and tap "Mark". Either +way a white ring blooms out from the tapped point (the middle of the screen for "Mark"). Doors, +windows, gas meters and AC units are placed on the wall face; driveways and fences on the ground. +A mark the app accepts moves the prompt on; the last one adds the feature, gives a firm haptic +tap, draws it on the camera (white dots on the marks, a dashed outline for a door or window, a +dashed line for a driveway or fence) and adds its icon to the strip. Marking then ends and the +walk buttons return. + +A refused mark keeps the step. The instruction shows the reason with a red warning icon, the +prompt drops to the second line, and the phone gives a warning haptic: + +| Refusal | When | +| --- | --- | +| "One moment, your phone is still finding its place." | Tracking is not normal. | +| "That spot is behind the wall. Tap something on this side." | The phone itself is on the far side of the wall line. | +| "Nothing to pin there. Aim at the wall or the ground and try again." | The tap's line of sight never meets the wall face (or the ground, for a driveway or fence). | +| "That's too far from the wall to matter. Tap something closer." | A driveway or fence mark lands more than 8 m out from the wall, or behind it. | + +"Cancel" drops the marks placed so far for this feature and returns to walking. While marking, +the marking prompt takes the instruction slot even over coaching, and new photos keep being +taken if the homeowner moves. + +**Finding its place again.** While tracking is not normal, the fog, path, ring, meter dot, ends +and marks fade off the camera image over 0.2 s and fade back once it is normal +([coverage and guidance](../foundations/coverage-and-guidance.md#one-instruction-at-a-time)). +While the phone is relocalizing, the instruction reads "Point at the meter like this.", with "Your +phone lost its place for a moment.", and the meter close-up taken at the start appears in the +middle of the screen: a square photo with rounded corners, a white border and a shadow, fading in +and out over 0.2 s. It cannot be tapped. If the close-up was skipped ("Can't get a clear shot") or its +thumbnail could not be read back, there is no photo and only the instruction shows. + +### Advancing + +"Done with this wall" appears once both ends are set, whatever the coverage between them. Tapping +it ends any marking and moves to [marking features](mark-features.md). What moves with it: the +coverage (covered, skipped and unseen cells), each end with its kind (real or unexplored), the +features marked so far and the walk photos. Coverage between the ends that was never covered +stays unseen; the server treats it as unknown +([coverage and guidance](../foundations/coverage-and-guidance.md#advancing)). There is no way back +to the walk from the next screen. + +## Modifiers + +| Modifier | At arrival | While capturing | +| --- | --- | --- | +| Live camera or replay | A replay starts playing its recording from the first frame (leaving out any frames the autopilot holds back for a gap request); the meter's wall comes from the recording. | When the recording ends the image freezes on its last frame and the instruction stops changing. The 20 s reset never happens on a replay. | +| Autopilot | The walk plays at three times speed and the badge reads "Replay · Autopilot". | After the recording ends, the autopilot marks a gas meter and a window by calling the marking actions at projected points, sets both ends (falling back to setting them directly when no frame shows the spot), and finishes the walk. It never opens the kind picker or presses "Can't get there". | +| Server or sample result | No effect. | No effect. | +| Larger text sizes | The instruction card and buttons grow; the chrome scrolls when it no longer fits. At accessibility sizes "Mark something" fills the row. | The kind picker shows one tile per row at accessibility sizes instead of three. The strip does not grow. | +| Reduce Motion | The instruction crossfades instead of blurring; the button rows change with a 0.15 s fade. | The fog fades without drifting, the target ring does not pulse, the counter icon flashes without growing, the tap ring fades without expanding. The kind picker still slides up from the bottom. | + +## Cancel and interrupt + +| Event | Before both ends are set | With both ends set | +| --- | --- | --- | +| The screen's own way out | "Can't get there" under a walk instruction sets an unexplored end; under a tilt instruction it skips that patch. "Cancel" leaves marking. | "Done with this wall". "Can't get there" still skips a patch under a tilt instruction; "Cancel" leaves marking. | +| Start over | Not offered. | Not offered. | +| Tracking limited | Coaching ("Slow down", "Aim at a corner or somewhere with more texture", "Move your phone slowly") replaces the instruction and hides "Can't get there". The fog, path, ring and marks fade off the camera image. No photos are kept. "Wall ends here" still works. Marks are refused with "One moment, your phone is still finding its place." | Same; "Done with this wall" still works. | +| Tracking lost or relocalizing | "Point at the meter like this.", with "Your phone lost its place for a moment." and the saved meter close-up in the middle of the screen (or "Your phone lost its place" when tracking is lost, with no photo). The fog, path, ring and marks fade off the camera image. After 20 s of relocalizing, the app forgets the wall, coverage, ends, marks and walk photos, sets the counter to 0 and goes back to "Find your electric meter", with no message saying why. | Same; the ends are lost too. | +| App backgrounded or a call | The camera stops; on return the coaching reads "Point at the meter like this." over the saved close-up until the phone finds its place. Photos, coverage, ends and marks are kept. Relocalizing time counts toward the 20 s. | Same. | +| Camera off or session failed | The screen stays with no new frames; coverage cannot grow. Suspected dead end ([the flow](../foundations/flow.md#open-questions-and-verification)). | Same; "Done with this wall" still moves on. | +| Network lost or upload failing | No effect. | No effect. | +| App killed | The scan is lost. | The scan is lost. | + +## Interactions with other systems + +**Coverage and evidence.** This screen is where coverage grows, by the rules in +[coverage and guidance](../foundations/coverage-and-guidance.md). Skipped cells and unexplored ends +are recorded for installer review, never as evidence. + +**Stored photos.** Each walk photo is written to the scan's folder as it is kept; the counter and +coverage move only after the write succeeds. See [the flow](../foundations/flow.md#interactions-with-other-systems). + +**Upload and offline.** Nothing is sent from this screen. + +**Accessibility.** VoiceOver reads the strip as "Map of the wall" with a value such as "Left of +your meter: 40 percent seen. Right of your meter: 0 percent seen. You are 6 feet 2 inches left of +your meter", ending "Both ends marked" once both are set. The counter reads "12 photos taken". The +instruction card is one element that reads its text and marks itself as updating often; the fog, +path, ring and marks are hidden from VoiceOver. Hints: "Pin a gas meter, door, window, AC unit, +driveway or fence" on "Mark something"; "Marks the end of the wall at the circle in the middle of +the screen" on "Wall ends here"; "Marks the point under the circle in the middle of the screen" on +"Mark"; "Skips this part of the wall. An installer will look at it instead." on "Can't get there". +The picker's heading is a header, its close button reads "Close", and each tile reads "Mark" and +the kind in lower case. The saved close-up, while relocalizing, reads "Your photo of the meter +from the start of the scan". + +**Haptics and motion.** A firm tap for each feature added and a warning for each refused mark. +No haptic for walk photos, for setting an end or for "Can't get there". Fog lifts over 0.7 s; +button rows change with a spring of 0.4 s; the target ring pulses. + +**Verification hooks.** `STATE=wallWalk` on arrival. Each instruction change logs +`GUIDANCE=`: `walk.left`, `walk.right`, `markEnd.left`, `markEnd.right`, `aimAtGround`, +`aimAtWall`, `stepBack`, `walkComplete`. The engine log (category `engine`) records each end +("end left at s=… (limit)" or "(unexplored)"), "cannot access area during ", and "spatial +reset: relocalization timed out". Accessibility identifiers: `action.markSomething`, +`action.markEnd`, `action.finishWalk`, `action.cannotAccess`, `action.markPoint`, +`action.cancelMarking`, `action.closeTray`, `feature.`, `wallTape`, `photoCount`, +`instruction`, `relocalize.meterPhoto`. + +## Edge cases + +- Asked for the left end while aiming right of the meter, "Wall ends here" sets the right end. The + question about the left end stays up. +- A wall end or a door, window, gas meter or AC unit mark lands wherever the aim meets the wall's + line extended, with no limit: past a corner, above the roof line or below ground. A door's bottom + is raised to ground level. +- "Nothing to pin there" appears for a wall mark only when the phone points away from the wall's + line; aiming at the sky over the wall is accepted. +- The same refusal twice in a row gives the warning haptic only the first time. +- A tilt instruction can come back after "Can't get there": the skip covers 1 m, while the lagging + stretch that raised the instruction can be longer. +- The kind picker stays open if the instruction changes underneath it, including to "Is this the + left end of the wall?"; the end buttons return only when it is closed. +- "Done with this wall" shows while a tilt instruction is still up; the homeowner can leave thin + stretches for the server to call unknown. +- VoiceOver's "percent seen" counts only the wall band, and counts amber (seen once) cells as seen. + A side can read 100 percent while none of it is covered and its ground is unseen. +- Standing at the meter, VoiceOver says "You are 0 inches right of your meter". +- "About N ft to go" can never appear on the walk instruction: the engine always passes no + remaining distance (`Runtime/ScanEngine.swift:644`). + +## Open questions and verification + +- **Suspected bug: a real wall end closer than the reach cannot be recorded.** "Wall ends here" is + shown only while the instruction is "Is this the … end of the wall?" + (`UI/Screens/WallWalkScreen.swift:99`, `:141`), and the planner asks that only once coverage + runs unbroken to its reach (`HouseScanKit/Guidance/GuidancePlanner.swift:113`). A homeowner + whose wall turns a corner 10 ft from the meter is told "Walk slowly to your left" with only + "Can't get there" to answer, which records an unexplored end instead of a real one. +- **Suspected bug: no "Can't get there" at "Is this the … end of the wall?".** The pill is limited + to walk and tilt instructions (`WallWalkScreen.swift:105-110`), although the engine handles it + for this question (`Runtime/ScanEngine+Actions.swift:177`). A wall that keeps going can only be + answered with "Wall ends here", a real end. This sharpens B-06 in [bug-triage.md](../bug-triage.md), + which assumes both answers are offered. +- **Suspected bug: "Wall ends here" aims at a circle that is not drawn.** The end goes where the + middle of the screen meets the wall (`ScanEngine+Actions.swift:68`), but the center circle is + drawn only while marking (`WallWalkScreen.swift:28-33`). The ring on screen is the planner's + guess at the end (`GuidancePlanner.swift:187-188`), so a homeowner who lines the ring up with the + corner sets the end elsewhere. The hint promises "the circle in the middle of the screen". +- **Suspected bug: "Wall ends here" is unchecked and silent.** It accepts a tap with limited + tracking, unlike marks (`ScanEngine+Actions.swift:66-70` against `:80`); it does nothing and says + nothing when the aim misses the wall (`:68`); and it gives no haptic when it succeeds. +- **Suspected bug: the 20 s reset keeps a mark in progress.** `resetSpatialState` + (`Runtime/ScanEngine.swift:386-410`) clears the wall, marks and photos but not the marking prompt + or its placed taps. After the meter is found again, the walk opens straight into the old prompt + ("Now tap its top-right corner"), and the finished feature joins a point from the lost world + frame to one from the new frame. +- **Suspected bug: two taps on "Can't get there" finish an empty walk.** On arrival each tap sets an + end at the meter (`ScanEngine+Actions.swift:179-181`); with both ends at the same point the + planner finds nothing to fill (`GuidancePlanner.swift:138`) and says "That's the whole wall". +- **Suspected bug: the AC unit prompt reads "Tap the ac unit".** The prompt lower-cases the kind's + name (`UI/Copy/ScanCopy.swift:122`, `:137`); the picker tile's VoiceOver label has the same + problem ("Mark ac unit", `WallWalkScreen.swift:258`). +- **Possible bug: "That spot is behind the wall" judges the phone, not the tap.** The check is on + where the phone stands (`ScanEngine+Actions.swift:88`), so the message blames the spot when it is + the homeowner who is behind the wall line. +- The kind picker slides in under Reduce Motion (`WallWalkScreen.swift:140`). +- The reset returns to "Find your electric meter" without telling the homeowner that the walk was + lost. The old meter close-up stays in the scan (`Runtime/KeyframeStore.swift:79-81` clears only + walk photos); if the second close-up is skipped, the first one is uploaded. +- The foundations say each kept photo gives a light haptic tap. In this build walk photos give none + (`UI/ScanRootView.swift:91`, `:101-104`); [the flow](../foundations/flow.md) and + [coverage and guidance](../foundations/coverage-and-guidance.md) need updating. +- The walk's reply reads "Can't get there"; the gap request's reads "I can't get there" + (`UI/Screens/GapRequestScreen.swift:26`). The glossary uses the second for both. +- Everything on this screen that needs the live camera is read from code: the fog's look on a real + wall, the ring and chevron placement, the overlays hiding and the saved close-up while the phone +finds its place, the 20 s reset, and interruption recovery. The first + Simulator pass showed this screen's chrome without text (B-05 in [bug-triage.md](../bug-triage.md)). + +Verified against house-scanning commit `0876e03` (t3/ios-mvf). diff --git a/verification/product/verification.md b/verification/product/verification.md index bdcc8a128..bc45f4fce 100644 --- a/verification/product/verification.md +++ b/verification/product/verification.md @@ -1,32 +1,43 @@ # Checks against the running app -Each row is one claim from a document, checked in the Simulator on `21a63e7`. A failed row is not -automatically an app bug: sometimes the document is wrong, and the note says which. Screenshots -stay in `~/house-scanning-data/reports/sim/` because replay frames come from a non-commercial -dataset. +Each row is one claim from a document, checked in the Simulator. A failed row is not automatically +an app bug: sometimes the document is wrong, and the note says which. Screenshots stay in +`~/house-scanning-data/reports/sim/` because replay frames come from a non-commercial dataset. ## How a pass is run -`make sim REF=21a63e7 ARGS="--replay ~/house-scanning-data/replays/advio-20-0040-0075 --autopilot ---server-url http://127.0.0.1:8765 --extra-arg=-autopilotHold --extra-arg=3"`, with the server -from `t3/server` on port 8765. The runner screenshots each screen and records the app's own log. -What this covers: the order of screens, what each shows, and what the app logged. What it does -not cover: anything needing the live camera (tracking, the meter tap on a real wall, the close-up -succeeding), haptics, or timing felt by a person. +`make sim-app` from `verification/` builds `t3/ios-mvf`, runs it on the ADVIO replay +`advio-20-0040-0075` with the autopilot, and uploads to the server started from `t3/server`. Add +`ARGS="--extra-arg=-sampleResult"` for the built-in result, or `--content-size +accessibility-extra-extra-extra-large --appearance dark` for large text. `python -m +hsverify.a11yaudit` runs Apple's audit on every screen. What these cover: the order of screens, +what each shows and speaks, what the app logged, and what the server answered. What they do not +cover: anything needing the live camera (tracking, the meter tap on a real wall, a close-up that +succeeds), haptics, or timing a person feels. -## Pass 1: 2026-09-26 04:06, report `20260926-040635-21a63e7-21a63e72-int-replay` +## Current pass: `t3/ios-mvf` `0876e03`, server `6a623d8` + +Report `sim/20260926-050058-t3-ios-mvf-0876e03c-replay`, plus the variant runs on the preview +`5520229` (the same UI before S3's last fixes) named in [the UX review](../ux/review.md#evidence). | ID | Claim | Result | | --- | --- | --- | -| FLOW-01 | Screens appear in the order of [the flow](foundations/flow.md#the-interaction-event-by-event) | Pass up to `uploading`: onboarding, findMeter, meterCloseUp, wallWalk, markFeatures, gapRequest, uploading. | -| FLOW-02 | The result appears only after the server answered | Pass, in the bad sense: the upload got 404 (see B-01) and the app stayed on `uploading`. | -| UP-01 | A failed upload offers "Try again" | Fail: the screenshot shows "That didn't go through" and the raw server JSON, with no button and no emblem visible (see B-02, B-05). | -| CU-01 | Without a usable photo, the close-up is skipped after the second failed try | Pass on the replay: logged "close-up skipped after 2 failed attempts" after 4 s. | -| CU-02 | The instruction card shows "Hold your meter in the circle" | Fail: the card is an empty panel in the screenshot (see B-05). | -| WW-01 | The walk shows one instruction, the strip and its labels | Fail: every card and the strip render without text (see B-05). | -| MF-01 | The feature list is readable over the dimmed camera | Fail: black text directly on the camera image, no panel (see B-05). | -| GAP-01 | A gap request appears when the phone's check finds a gap | Pass: shown, then skipped by the autopilot because this replay has no held-back frames. | +| FLOW-01 | Screens appear in the order of [the flow](foundations/flow.md#the-interaction-event-by-event) | Pass: all ten, onboarding to result, AR and back. | +| FLOW-02 | The result appears only after the server answered | Pass: `POST /v1/placements 200`, then `result`. | +| FLOW-03 | A failure other than an unsupported phone reaches the failure screen | Fail, as the document predicts: `-replay /nonexistent` logs "replay unreadable" and stays on onboarding ([B-03](bug-triage.md#blockers)). | +| FLOW-04 | An unsupported phone sees "This phone can't measure walls" | Pass (no replay, Simulator). | +| CU-01 | Without a usable photo, the close-up is skipped after the second failed try | Pass on the replay: logged "close-up skipped after 2 failed attempts". | +| CU-02 | One instruction and one named fix under it | Pass: "Hold your meter in the circle", "Center the meter in the circle". | +| WW-01 | One instruction, the reply inside the card, the strip below | Pass at default size; fails at the largest text size (UX-01). | +| WW-02 | Guidance moves from walking to tilting to stepping back | Pass: the audit saw "Walk slowly to your left", "Tilt down to show the ground", "Take a step back". | +| MF-01 | The list shows each mark with its distance, the window question and "Add something" | Pass at default size; hidden behind "Looks complete" at the largest text size (UX-01). | +| GAP-01 | A gap request states what to show and why, with a progress bar and "I can't get there" | Pass ("Show the ground around your meter", "0%"). | +| UP-01 | Upload progress is shown in steps apart from coverage | Pass: "Photos ready", "Sending photos, 99%", "Check clearances". | +| RES-01 | The result shows the headline, placement line, the server's summary, the rules-not-final note and the panel notice | Pass: "An installer will take a look", "13 ft 3 in left of your meter, 12 ft of cable". | +| RES-02 | A borderline check shows the measurement, the rule and the error | Pass on the sample result: "Measured 3 ft 1 in. The rule is 3 ft, and the measurement can be off by about 4 in." | +| RES-03 | A maximum reads as a maximum | Fail: "Measured 3 ft. The rule is 20 ft" (B-14). | +| RES-04 | The result is visible when the app logs `STATE=result` | Fail, minor: the upload screen was still showing 1.2 s later on first load, in both passes (UX-07). | +| AR-01 | The AR view draws the spot and cable with one headline and "Done" | Pass: "Your battery could go here". On the sample result it has no sample badge (B-15). | -Not checked yet: `result` and `resultAR` (unreachable while B-01 stands; a `-sampleResult` pass -is next), larger text sizes, dark appearance, and failure screens (launch with `-replay -/nonexistent` for B-03). +Superseded: at `21a63e7` the upload went to `/v1/scenes` and failed with 404, and the camera +screens rendered without text. Both are fixed on `t3/ios-mvf`. diff --git a/verification/scenes/eth3d-facade/build_scene.py b/verification/scenes/eth3d-facade/build_scene.py index 5d3d0b94b..c05bfe24b 100644 --- a/verification/scenes/eth3d-facade/build_scene.py +++ b/verification/scenes/eth3d-facade/build_scene.py @@ -24,6 +24,7 @@ from dataclasses import dataclass from pathlib import Path +import jsonschema import numpy as np from PIL import Image, ImageDraw @@ -410,10 +411,6 @@ def extent_end(plan, h, c, d, rough_end, side): return c + te * d -def chain_segments(walls): - return [(w.p0, w.p1, w.id) for w in walls] - - def seg_cross(a, b, c, d): def orient(p, q, r): return np.sign((q[0] - p[0]) * (r[1] - p[1]) - (q[1] - p[1]) * (r[0] - p[0])) @@ -588,8 +585,9 @@ def measure_feature(feat, kfs_by_name, walls_by_id): ) X = backproject(kfs_by_name[img], wall, [pts[r] for r in POINT_ROLES]) s = wall.s_of(X[:, [0, 2]]) - per_image[img] = np.array([s[0], s[1], X[2, 1], X[3, 1]]) - if not (per_image[img][0] < per_image[img][1] and per_image[img][2] < per_image[img][3]): + s_left, s_right, bottom, top = s[0], s[1], X[2, 1], X[3, 1] + per_image[img] = np.array([s_left, s_right, bottom, top]) + if not (s_left < s_right and bottom < top): raise SystemExit(f"{feat['label']} @ {img}: left/right or bottom/top came out swapped") vals = np.array(list(per_image.values())) mean = vals.mean(0) @@ -600,6 +598,10 @@ def measure_feature(feat, kfs_by_name, walls_by_id): # -------------------------------------------------------------------------------------------- +def round3(a): + return [round(float(x), 3) for x in a] + + def main(): ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter @@ -766,14 +768,17 @@ def main(): observed.append(entry) # 7. Assemble ---------------------------------------------------------------------------- - r3 = lambda a: [round(float(x), 3) for x in a] # noqa: E731 scene = { "schema_version": "1.0", - "meter": {"pos": r3(meter_pos), "wall_id": wm.id, "plus_minus_ft": mt["plus_minus_ft"]}, + "meter": { + "pos": round3(meter_pos), + "wall_id": wm.id, + "plus_minus_ft": mt["plus_minus_ft"], + }, "walls": [ { "id": w.id, - "baseline": [r3(w.p0), r3(w.p1)], + "baseline": [round3(w.p0), round3(w.p1)], "height_ft": w.height_ft, "plus_minus_ft": round(max(0.1, 3 * w.fit_rms_ft), 2), } @@ -840,10 +845,11 @@ def main(): i1, i2 = feat["left_is_bottom_left_corner_in"] wall = walls_by_id[feat["wall_id"]] X = backproject(kfs_by_name[i1], wall, [feat["points_px"][i1]["left"]]) - u, v, _ = kfs_by_name[i2].project(X) - cu, cv = feat["points_px"][i2]["left"] k2 = kfs_by_name[i2] + u, v, _ = k2.project(X) + cu, cv = feat["points_px"][i2]["left"] px_ft = k2.K[0] / float(-((X[0] - k2.C) @ k2.R_c2w)[2]) # pixels per foot at that depth + err_px = float(np.hypot(u[0] - cu, v[0] - cv)) repro.append( { "label": feat["label"], @@ -851,8 +857,8 @@ def main(): "into": i2, "predicted_px": [round(float(u[0]), 1), round(float(v[0]), 1)], "clicked_px": [cu, cv], - "err_px": round(float(np.hypot(u[0] - cu, v[0] - cv)), 1), - "err_ft_at_wall": round(float(np.hypot(u[0] - cu, v[0] - cv) / px_ft), 2), + "err_px": round(err_px, 1), + "err_ft_at_wall": round(err_px / px_ft, 2), } ) report["reprojection"] = repro @@ -865,8 +871,6 @@ def main(): draw_overlays(out, kfs, walls, ann, kfs_by_name, walls_by_id) if args.schema: - import jsonschema - jsonschema.validate(scene, json.loads(args.schema.read_text())) print(f"scene.json validates against {args.schema}") diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index b85390a39..78190db2c 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,8 +1,44 @@ { - "generated_at": "2026-09-26T04:25:25-05:00", + "generated_at": "2026-09-26T05:09:23-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": false, "prs": [ + { + "number": 16, + "title": "experiments: meter close-up readability eval (S5)", + "branch": "t3/meter-closeup", + "head": "60a912bc884dd6d6defa4f4a94297e6ee86352ea", + "draft": true, + "updated": "2026-09-26T09:49:58Z", + "ci": { + "counts": { + "success": 4, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "4 success" + } + }, + { + "number": 15, + "title": "web: placement review page", + "branch": "t3/web-review", + "head": "c69f88809c06175373079d7ec3b6b414194a73cf", + "draft": true, + "updated": "2026-09-26T10:06:25Z", + "ci": { + "counts": { + "success": 5, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "5 success" + } + }, { "number": 14, "title": "docs: trim AGENTS.md and fix stale decisions in the team docs", @@ -25,9 +61,9 @@ "number": 13, "title": "verification: independent end-to-end checks (S4)", "branch": "t3/verification", - "head": "1330a82758df3a007fede909936542f5fb3fb358", + "head": "7c2c9ab5a5f4fc937d10cd8de0fd7904e61664e8", "draft": true, - "updated": "2026-09-26T09:21:51Z", + "updated": "2026-09-26T09:28:49Z", "ci": { "counts": { "success": 4, @@ -43,9 +79,9 @@ "number": 12, "title": "feat(evals): real-data accuracy evals for ARKit drift and photo-based wall reconstruction", "branch": "t3/evals", - "head": "b5ca7af816cf09da6deb9e480e00719031f0643c", + "head": "f6d081d20df1063491a169c9365f1e295452b069", "draft": true, - "updated": "2026-09-26T08:57:45Z", + "updated": "2026-09-26T09:57:22Z", "ci": { "counts": { "success": 5, @@ -61,9 +97,9 @@ "number": 11, "title": "server: placement solver, rules and API (S2)", "branch": "t3/server", - "head": "6a623d8a7e340c15072efa412dd34738818e4f7e", + "head": "26d28707473f25f7c7d1f3660fbbf8321704f609", "draft": true, - "updated": "2026-09-26T09:22:28Z", + "updated": "2026-09-26T10:06:21Z", "ci": { "counts": { "success": 5, @@ -79,9 +115,9 @@ "number": 10, "title": "ios: guided wall scan, from meter to battery spot", "branch": "t3/ios-mvf", - "head": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "head": "24af4342bf760c4af7ed3f942c330533bed1f804", "draft": true, - "updated": "2026-09-26T09:17:54Z", + "updated": "2026-09-26T10:02:50Z", "ci": { "counts": { "success": 4, @@ -192,7 +228,7 @@ "refs": [ { "ref": "origin/t3/evals", - "sha": "b5ca7af816cf09da6deb9e480e00719031f0643c" + "sha": "f6d081d20df1063491a169c9365f1e295452b069" } ], "metrics": [ @@ -200,7 +236,7 @@ "id": "S1-1", "text": "One command regenerates every number from a clean checkout", "ref": "origin/t3/evals", - "sha": "b5ca7af816cf09da6deb9e480e00719031f0643c", + "sha": "f6d081d20df1063491a169c9365f1e295452b069", "status": "partial", "evidence": "make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac.", "links": [] @@ -209,13 +245,13 @@ "id": "S1-2", "text": "Dataset URLs and sha256 recorded", "ref": "origin/t3/evals", - "sha": "b5ca7af816cf09da6deb9e480e00719031f0643c", + "sha": "f6d081d20df1063491a169c9365f1e295452b069", "status": "met", "evidence": "all 2 found", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/README.md" ] ] }, @@ -223,7 +259,7 @@ "id": "S1-3", "text": "Metric code tested on hand-computed cases", "ref": "origin/t3/evals", - "sha": "b5ca7af816cf09da6deb9e480e00719031f0643c", + "sha": "f6d081d20df1063491a169c9365f1e295452b069", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -232,21 +268,21 @@ "id": "S1-4", "text": "ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft", "ref": "origin/t3/evals", - "sha": "b5ca7af816cf09da6deb9e480e00719031f0643c", + "sha": "f6d081d20df1063491a169c9365f1e295452b069", "status": "met", "evidence": "all 5 found", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/README.md" ], [ "experiments/evals/results/advio_drift.md", - "https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/results/advio_drift.md" + "https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/results/advio_drift.md" ], [ "experiments/evals/results/eth3d_recon.md", - "https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/results/eth3d_recon.md" + "https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/results/eth3d_recon.md" ] ] }, @@ -254,21 +290,21 @@ "id": "S1-5", "text": "ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question", "ref": "origin/t3/evals", - "sha": "b5ca7af816cf09da6deb9e480e00719031f0643c", + "sha": "f6d081d20df1063491a169c9365f1e295452b069", "status": "partial", "evidence": "found 1-3 m row, 3-10 m row, scale error; missing verdict", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/README.md" ], [ "experiments/evals/results/eth3d_recon.md", - "https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/results/eth3d_recon.md" + "https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/results/eth3d_recon.md" ], [ - "experiments/evals/results/frames.md", - "https://github.com/SamGu-NRX/house-scanning/blob/b5ca7af816cf09da6deb9e480e00719031f0643c/experiments/evals/results/frames.md" + "experiments/evals/results/eth3d_visibility_sensitivity.md", + "https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/results/eth3d_visibility_sensitivity.md" ] ] } @@ -280,7 +316,7 @@ "refs": [ { "ref": "origin/t3/server", - "sha": "6a623d8a7e340c15072efa412dd34738818e4f7e" + "sha": "26d28707473f25f7c7d1f3660fbbf8321704f609" } ], "metrics": [ @@ -288,7 +324,7 @@ "id": "S2-1", "text": "pytest green", "ref": "origin/t3/server", - "sha": "6a623d8a7e340c15072efa412dd34738818e4f7e", + "sha": "26d28707473f25f7c7d1f3660fbbf8321704f609", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -297,13 +333,13 @@ "id": "S2-2", "text": "Every public golden test 01-14 of docs/research/t3-lane-c-review.md present", "ref": "origin/t3/server", - "sha": "6a623d8a7e340c15072efa412dd34738818e4f7e", + "sha": "26d28707473f25f7c7d1f3660fbbf8321704f609", "status": "met", "evidence": "all 14 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_golden_public.py" ] ] }, @@ -311,13 +347,13 @@ "id": "S2-3", "text": "Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail", "ref": "origin/t3/server", - "sha": "6a623d8a7e340c15072efa412dd34738818e4f7e", + "sha": "26d28707473f25f7c7d1f3660fbbf8321704f609", "status": "partial", "evidence": "found missing coverage, mirror; missing equality unsure, larger clearance", "links": [ [ "server/tests/test_properties.py", - "https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_properties.py" + "https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_properties.py" ] ] }, @@ -325,17 +361,17 @@ "id": "S2-4", "text": "Schemas published", "ref": "origin/t3/server", - "sha": "6a623d8a7e340c15072efa412dd34738818e4f7e", + "sha": "26d28707473f25f7c7d1f3660fbbf8321704f609", "status": "met", - "evidence": "present at 6a623d8a", + "evidence": "present at 26d28707", "links": [ [ "server/schemas/scene.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/schemas/scene.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/schemas/scene.schema.json" ], [ "server/schemas/result.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/schemas/result.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/schemas/result.schema.json" ] ] }, @@ -343,21 +379,21 @@ "id": "S2-5", "text": "Schemas validated in tests", "ref": "origin/t3/server", - "sha": "6a623d8a7e340c15072efa412dd34738818e4f7e", + "sha": "26d28707473f25f7c7d1f3660fbbf8321704f609", "status": "met", "evidence": "all 2 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_golden_public.py" ], [ "server/tests/test_schemas.py", - "https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_schemas.py" + "https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_schemas.py" ], [ "server/tests/test_api.py", - "https://github.com/SamGu-NRX/house-scanning/blob/6a623d8a7e340c15072efa412dd34738818e4f7e/server/tests/test_api.py" + "https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_api.py" ] ] }, @@ -365,16 +401,16 @@ "id": "S2-6", "text": "A real-derived scene returns a result in under 1 s", "ref": "origin/t3/server", - "sha": "6a623d8a7e340c15072efa412dd34738818e4f7e", - "status": "no evidence yet", - "evidence": "no latency_ms in ~/house-scanning-data/reports/e2e/20260926-042213-6a623d8a/report.json", + "sha": "26d28707473f25f7c7d1f3660fbbf8321704f609", + "status": "met", + "evidence": "latency_ms = 970.4 (want < 1000): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json", "links": [] }, { "id": "S2-7", "text": "ruff clean", "ref": "origin/t3/server", - "sha": "6a623d8a7e340c15072efa412dd34738818e4f7e", + "sha": "26d28707473f25f7c7d1f3660fbbf8321704f609", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -383,9 +419,9 @@ "id": "S2-8", "text": "Black-box (verification e2e): every answer valid under C2, consistent with its decision and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric when mirrored, and never passing without coverage", "ref": "origin/t3/server", - "sha": "6a623d8a7e340c15072efa412dd34738818e4f7e", + "sha": "26d28707473f25f7c7d1f3660fbbf8321704f609", "status": "met", - "evidence": "contract_problem_count = 0 (want 0): ~/house-scanning-data/reports/e2e/20260926-042213-6a623d8a/report.json", + "evidence": "contract_problem_count = 0 (want 0): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json", "links": [] } ] @@ -396,7 +432,7 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be" + "sha": "24af4342bf760c4af7ed3f942c330533bed1f804" } ], "metrics": [ @@ -404,9 +440,9 @@ "id": "S3-1", "text": "CI build green", "ref": "origin/t3/ios-mvf", - "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", "status": "not run", - "evidence": "iOS build: pending at 90c01dde", + "evidence": "iOS build: pending at 24af4342", "links": [ [ "#10", @@ -418,13 +454,13 @@ "id": "S3-2", "text": "Built as Swift 6 with strict concurrency", "ref": "origin/t3/ios-mvf", - "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", "status": "met", "evidence": "all 2 found", "links": [ [ "ios/project.yml", - "https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/project.yml" + "https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/project.yml" ] ] }, @@ -432,30 +468,30 @@ "id": "S3-3", "text": "Zero build warnings", "ref": "origin/t3/ios-mvf", - "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", "status": "stale", - "evidence": "report for 6885b7b5, ref at 90c01dde: ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json", + "evidence": "report for 0876e03c, ref at 24af4342: ~/house-scanning-data/reports/sim/20260926-050058-t3-ios-mvf-0876e03c-replay/report.json", "links": [] }, { "id": "S3-4", "text": "Unit tests for coverage, auto-capture and guidance", "ref": "origin/t3/ios-mvf", - "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", "status": "partial", "evidence": "found coverage; missing auto-capture, guidance", "links": [ [ "ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift" ] ] }, @@ -463,13 +499,17 @@ "id": "S3-5", "text": "UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop", "ref": "origin/t3/ios-mvf", - "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", "status": "partial", "evidence": "all 3 found (the tests exist; passing is S3-1)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanUITests/FullFlowUITests.swift" + ], + [ + "ios/HouseScanUITests/ScreenStatesUITests.swift", + "https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -477,13 +517,17 @@ "id": "S3-6", "text": "Accessibility audit on every screen", "ref": "origin/t3/ios-mvf", - "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", "status": "partial", - "evidence": "2 matches in 1 files (an audit exists; which screens it covers needs a read)", + "evidence": "3 matches in 2 files (an audit exists; which screens it covers needs a read)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/90c01ddef93bdc7abef6a0520c41025bf8fc33be/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanUITests/FullFlowUITests.swift" + ], + [ + "ios/HouseScanUITests/ScreenStatesUITests.swift", + "https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -491,7 +535,7 @@ "id": "S3-6b", "text": "Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, plus symbol-name button labels) finds no issues; S4-2 shows how many screens the app reached", "ref": "origin/t3/ios-mvf", - "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", "status": "no evidence yet", "evidence": "no report matches ~/house-scanning-data/reports/a11y/*t3-ios-mvf*/report.json", "links": [] @@ -500,7 +544,7 @@ "id": "S3-7", "text": "Screenshots of each state in the PR", "ref": "origin/t3/ios-mvf", - "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", "status": "gap", "evidence": "0 of 9 matches in the PR body", "links": [ @@ -514,16 +558,16 @@ "id": "S3-8", "text": "Exported scene.json validates against C1", "ref": "origin/t3/ios-mvf", - "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", "status": "met", - "evidence": "app_export_scene_valid = true (want True): ~/house-scanning-data/reports/e2e/20260926-041009-b04558df/report.json", + "evidence": "app_export_scene_valid = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json", "links": [] }, { "id": "S3-9", "text": "Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design", "ref": "origin/t3/ios-mvf", - "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -536,11 +580,11 @@ "refs": [ { "ref": "origin/t3/verification", - "sha": "1330a82758df3a007fede909936542f5fb3fb358" + "sha": "7c2c9ab5a5f4fc937d10cd8de0fd7904e61664e8" }, { "ref": "origin/t3/ios-mvf", - "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be" + "sha": "24af4342bf760c4af7ed3f942c330533bed1f804" } ], "metrics": [ @@ -548,25 +592,25 @@ "id": "S4-1", "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/verification", - "sha": "1330a82758df3a007fede909936542f5fb3fb358", + "sha": "7c2c9ab5a5f4fc937d10cd8de0fd7904e61664e8", "status": "met", - "evidence": "app_export_passed = true (want True): ~/house-scanning-data/reports/e2e/20260926-041009-b04558df/report.json", + "evidence": "app_export_passed = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json", "links": [] }, { "id": "S4-2", "text": "Screenshot report per app state", "ref": "origin/t3/ios-mvf", - "sha": "90c01ddef93bdc7abef6a0520c41025bf8fc33be", + "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", "status": "stale", - "evidence": "report for 6885b7b5, ref at 90c01dde: ~/house-scanning-data/reports/sim/20260926-031352-t3-ios-mvf-6885b7b5-replay/report.json", + "evidence": "report for 0876e03c, ref at 24af4342: ~/house-scanning-data/reports/sim/20260926-050058-t3-ios-mvf-0876e03c-replay/report.json", "links": [] }, { "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "1330a82758df3a007fede909936542f5fb3fb358", + "sha": "7c2c9ab5a5f4fc937d10cd8de0fd7904e61664e8", "status": "partial", "evidence": "verification/product: scope, glossary, both foundations and the pilot drafted from S3's integration commit 21a63e7; one Simulator pass recorded (verification.md); 7 suspected defects in bug-triage.md. Seven screen documents to go.", "links": [] @@ -575,13 +619,13 @@ "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "1330a82758df3a007fede909936542f5fb3fb358", + "sha": "7c2c9ab5a5f4fc937d10cd8de0fd7904e61664e8", "status": "met", - "evidence": "present at 1330a827", + "evidence": "present at 7c2c9ab5", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/1330a82758df3a007fede909936542f5fb3fb358/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/7c2c9ab5a5f4fc937d10cd8de0fd7904e61664e8/verification/SCOREBOARD.md" ] ] } diff --git a/verification/ux/review.md b/verification/ux/review.md new file mode 100644 index 000000000..9fd34fd6f --- /dev/null +++ b/verification/ux/review.md @@ -0,0 +1,86 @@ +# UX review of every app state + +Each state of the HouseScan app judged against [checklist.md](checklist.md). Screenshots and +audit reports stay in `~/house-scanning-data/reports/` (replay frames are non-commercial data). + +## Evidence + +| Run | What it shows | +| --- | --- | +| `sim/20260926-050058-t3-ios-mvf-0876e03c-replay` | Every state at `t3/ios-mvf` `0876e03` with the real server's result | +| `sim/20260926-043152-5520229-5520229d-preview-sample` | Every state of the flow, default text size, light appearance, sample result | +| `sim/20260926-044328-5520229-5520229d-preview-ax5-dark` | The same at the largest accessibility text size (AX5), dark appearance | +| `sim/20260926-044527-5520229-5520229d-preview-unsupported` | The unsupported-phone screen | +| `a11y/20260926-044005-5520229-5520229d-preview-sample` | Apple's accessibility audit on 17 distinct screens, with every label | +| Source at `0876e03` | Items that need the live camera | + +`5520229` is `t3/ios-mvf` `90c01dd` merged with the UI lane `e5dde35`; `0876e03` is the same UI +merged by S3, plus the fixes listed under "Fixed since the preview". The Simulator cannot run +ARKit, so tracking loss, relocalization and the live meter tap are judged from source. + +## Verdict: Block + +Blockers that fail: R9 (camera access denied leads nowhere, [B-03](../product/bug-triage.md#blockers)), +T5 (the close-up's way out can fail to appear, B-04) and R1 (a stretch the homeowner can't reach +is not recorded, B-08). The feature list's "Add something" doing nothing (B-09) blocks adding a +missed feature. + +## By state + +| State | Result | Findings | +| --- | --- | --- | +| `onboarding` | Pass at default size: one message, one primary button, "About 2 min". At AX5 the pill reads "About 2 mi" and the body runs under the page dots. | UX-01 | +| `unsupported` | Pass: "This phone can't measure walls" with what to do instead. Camera denied never reaches this screen. | B-03 | +| `findMeter` | Pass at default size: one instruction, the reticle, "This is my meter". At AX5 the button is pushed partly off screen. Refusal wording ("Step a little closer to the wall") also covers tracking refusals. | UX-01, B-11 | +| `meterCloseUp` | Pass: one instruction, no shutter, a named fix ("Center the meter in the circle"). The problem pill fails contrast; the way out can fail to appear. | UX-03, B-04 | +| `wallWalk` | One instruction with its reply inside the card, as the checklist asks. The strip's states differ by colour alone, the counter fails contrast, and at AX5 the card covers most of the camera and pushes "Mark something" and the strip off screen. Wall ends can't be recorded truthfully. | UX-01, UX-02, UX-03, UX-05, B-06 | +| `markFeatures` | Pass at default size: clear list, remove buttons labelled, window question with two full-width answers. At AX5 the list is hidden behind "Looks complete". "Add something" does nothing. | UX-01, B-09 | +| `gapRequest` | Pass: one line with the reason and target, a progress bar, "I can't get there". The requested stretch reuses the amber that means "seen". | UX-05, UX-06, B-08 | +| `uploading` | Pass: progress in its own steps, apart from coverage (R7). Four text elements do not scale. A failure shows raw error text. | UX-04, B-02 | +| `result` | Pass on honesty: "An installer will take a look", the rules-not-final note, the unseen side disclosed, the borderline window check with measurement, rule and error, and "An installer will check this". A maximum reads like a minimum; the first load leaves the upload screen up for over 1.2 s. | B-14, UX-07 | +| `resultAR` | The spot and cable over the camera with one headline and "Done". It says "Your battery could go here" without the sample badge. | B-15 | + +Checks that passed across the flow: one primary instruction per camera screen (I1); plain words, +no jargon in any on-screen string (I3); no shutter where capture is automatic (I5); every control +has a spoken name (A1, from the audit's labels); no hit area below 44 pt (A6, audit). + +## Findings + +**UX-01. Major. Large text breaks the camera screens and the feature list (A4).** At AX5: +"This is my meter" is pushed partly off screen; the feature list shows its heading and "Looks +complete" but not the marked features or the window question; on the walk and a gap request the +instruction card covers most of the camera, and "Mark something", "I can't get there" and the strip +fall off screen; onboarding's "About 2 min" truncates to "About 2 mi". Seen on the preview; the +only layout change at `0876e03` stacks the window answers, which makes the list taller. The audit flags clipped text +on the close-up and walk cards. Suggested: cap the instruction card's type at a large size with +the rest scrollable, keep the primary button pinned above the home indicator, and let the list +scroll behind a pinned "Looks complete". + +**UX-02. Major. Coverage is shown by colour alone (A9).** Covered cells are green and seen cells +amber; in grayscale both are the same light gray, and the strip has no legend or row labels. A +homeowner with red-green colour blindness cannot tell done from not done. Suggested: a pattern or +height difference per state, and "Wall" and "Ground" row labels. + +**UX-03. Minor. Contrast over the camera (A5).** The audit fails contrast on the photo counter +during the walk and on the close-up's problem pill. + +**UX-04. Minor. Upload text that does not scale (A4).** The audit reports four elements on the +upload screen whose font size cannot change. + +**UX-05. Minor. Two names for one action (V8).** The walk's reply is "Can't get there"; the gap +request's is "I can't get there". + +**UX-06. Minor. One colour, two meanings.** The requested stretch is outlined in the amber that +marks "seen but not enough" cells on the same strip. + +**UX-07. Minor. The result appears late the first time.** On the first visit the upload screen +("Checking your wall") was still showing 1.2 s after the app logged the result; after returning +from AR the result appeared at once. The 3D model's first load is the likely cause. + +## Fixed since the preview (`0876e03`) + +Seen: the window question answers "It opens" and "It stays shut", stacked (I4), and the result +says the electrical panel still needs an electrician's review (H7). From source: tracking coaching says +"Slow down" and "Aim at a corner or somewhere with more texture" (T1). While relocalizing, the +saved meter photo shows with "Point at the meter like this." (T7). Overlays hide while tracking is +not normal (T3). From 60fe0d327dfd0ac56b5570ffdc9daf24d8760a39 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 05:40:43 -0500 Subject: [PATCH 26/59] verification: recheck at t3/ios-mvf 525ea40; fix a race in the audit harness The audit bundle read screen labels in two queries, so a button vanishing between them failed the whole test; it now reads one snapshot of the tree. The product description, UX review and triage follow S3's latest changes (the wall-end question, sample-result wording, walk coaching) and were checked in the Simulator at 525ea40 against the server's real answer. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/SCOREBOARD.md | 64 +++--- .../AccessibilityAuditTests.swift | 24 ++- verification/product/README.md | 6 +- verification/product/bug-triage.md | 21 +- .../foundations/coverage-and-guidance.md | 50 +++-- verification/product/foundations/flow.md | 6 +- verification/product/glossary.md | 11 +- verification/product/screens/find-meter.md | 8 +- verification/product/screens/gap-request.md | 12 +- verification/product/screens/mark-features.md | 18 +- .../product/screens/meter-close-up.md | 2 +- verification/product/screens/onboarding.md | 12 +- verification/product/screens/result.md | 34 +-- verification/product/screens/uploading.md | 25 +-- verification/product/screens/wall-walk.md | 121 ++++++----- verification/product/verification.md | 11 +- verification/scoreboard/metrics.yaml | 8 +- verification/scoreboard/scoreboard.json | 202 +++++++++--------- verification/ux/review.md | 24 ++- 19 files changed, 367 insertions(+), 292 deletions(-) diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index 088ea4635..d4c8cacc6 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,82 +1,82 @@ # Scoreboard -Generated 2026-09-26 05:09 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-26 05:40 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | -| [#16](https://github.com/SamGu-NRX/house-scanning/pull/16) | `t3/meter-closeup` | `60a912bc` | yes | 4 success | 2026-09-26 04:49 CDT | +| [#16](https://github.com/SamGu-NRX/house-scanning/pull/16) | `t3/meter-closeup` | `03e04b54` | yes | 5 success | 2026-09-26 05:36 CDT | | [#15](https://github.com/SamGu-NRX/house-scanning/pull/15) | `t3/web-review` | `c69f8880` | yes | 5 success | 2026-09-26 05:06 CDT | -| [#14](https://github.com/SamGu-NRX/house-scanning/pull/14) | `t3/docs-cleanup` | `9c9ceb2a` | no | 4 success | 2026-09-26 03:15 CDT | -| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `7c2c9ab5` | yes | 4 success | 2026-09-26 04:28 CDT | -| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `f6d081d2` | yes | 5 success | 2026-09-26 04:57 CDT | -| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `26d28707` | yes | 5 success | 2026-09-26 05:06 CDT | -| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `24af4342` | yes | 4 success, 1 pending | 2026-09-26 05:02 CDT | +| [#14](https://github.com/SamGu-NRX/house-scanning/pull/14) | `t3/docs-cleanup` | `ab03d7fd` | no | 4 success | 2026-09-26 05:17 CDT | +| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `e18a2eb7` | yes | 4 success | 2026-09-26 05:11 CDT | +| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `7c5318d6` | yes | 3 success, 1 failure; failing: Vercel | 2026-09-26 05:38 CDT | +| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `f2705dd5` | no | 5 success | 2026-09-26 05:35 CDT | +| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `525ea401` | yes | 5 success | 2026-09-26 05:21 CDT | | [#9](https://github.com/SamGu-NRX/house-scanning/pull/9) | `codex/capture-contract-probe` | `4b6c4309` | yes | 4 success | 2026-09-26 00:08 CDT | | [#8](https://github.com/SamGu-NRX/house-scanning/pull/8) | `t3/testflight` | `b896d59f` | no | 4 success | 2026-09-26 03:18 CDT | | [#7](https://github.com/SamGu-NRX/house-scanning/pull/7) | `t3/measure-lab` | `45770a3b` | no | 6 success | 2026-09-26 03:19 CDT | -| [#6](https://github.com/SamGu-NRX/house-scanning/pull/6) | `t3/research` | `a7d91f16` | no | 4 success | 2026-09-26 02:11 CDT | +| [#6](https://github.com/SamGu-NRX/house-scanning/pull/6) | `t3/research` | `9737e3f0` | no | 4 success | 2026-09-26 05:13 CDT | | [#4](https://github.com/SamGu-NRX/house-scanning/pull/4) | `t3/scoring-harness` | `77ce830d` | no | 5 success | 2026-09-26 03:17 CDT | ## S1 Evals on real data -`origin/t3/evals` at [`f6d081d2`](https://github.com/SamGu-NRX/house-scanning/tree/f6d081d20df1063491a169c9365f1e295452b069). 2 met, 2 partial, 1 not run. +`origin/t3/evals` at [`7c5318d6`](https://github.com/SamGu-NRX/house-scanning/tree/7c5318d6610a06e657a98b0c8c7d1cc2acc53738). 2 met, 2 partial, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S1-1 | One command regenerates every number from a clean checkout | partial | make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac. | -| S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/README.md) | +| S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/README.md) | | S1-3 | Metric code tested on hand-computed cases | not run | slow; run with --slow | -| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/README.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/results/advio_drift.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/results/eth3d_recon.md) | -| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/README.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/results/eth3d_visibility_sensitivity.md) | +| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/README.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/advio_drift.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/eth3d_recon.md) | +| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/README.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/eth3d_visibility_sensitivity.md) | ## S2 Server and solver -`origin/t3/server` at [`26d28707`](https://github.com/SamGu-NRX/house-scanning/tree/26d28707473f25f7c7d1f3660fbbf8321704f609). 5 met, 1 partial, 2 not run. +`origin/t3/server` at [`f2705dd5`](https://github.com/SamGu-NRX/house-scanning/tree/f2705dd5580ccd8c18ebd34a743b448bcb59ae03). 3 met, 1 partial, 2 stale, 2 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S2-1 | pytest green | not run | slow; run with --slow | -| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_golden_public.py) | -| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_properties.py) | -| S2-4 | Schemas published | met | present at 26d28707 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/schemas/result.schema.json) | -| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_schemas.py), [server/tests/test_api.py](https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_api.py) | -| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 970.4 (want < 1000): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | +| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_golden_public.py) | +| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_properties.py) | +| S2-4 | Schemas published | met | present at f2705dd5 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/schemas/result.schema.json) | +| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_schemas.py), [server/tests/test_api.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_api.py) | +| S2-6 | A real-derived scene returns a result in under 1 s | stale | report for 26d28707, ref at f2705dd5: ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | | S2-7 | ruff clean | not run | slow; run with --slow | -| S2-8 | Black-box (verification e2e): every answer valid under C2, consistent with its decision and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric when mirrored, and never passing without coverage | met | contract_problem_count = 0 (want 0): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | +| S2-8 | Black-box (verification e2e): every answer valid under C2, consistent with its decision and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric when mirrored, and never passing without coverage | stale | report for 26d28707, ref at f2705dd5: ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | ## S3 iOS capture app -`origin/t3/ios-mvf` at [`24af4342`](https://github.com/SamGu-NRX/house-scanning/tree/24af4342bf760c4af7ed3f942c330533bed1f804). 2 met, 3 partial, 2 gap, 1 stale, 1 not run, 1 no evidence yet. +`origin/t3/ios-mvf` at [`525ea401`](https://github.com/SamGu-NRX/house-scanning/tree/525ea401641553fad581925023ed1bb82c0f7352). 4 met, 3 partial, 3 gap. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S3-1 | CI build green | not run | iOS build: pending at 24af4342 [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/project.yml) | -| S3-3 | Zero build warnings | stale | report for 0876e03c, ref at 24af4342: ~/house-scanning-data/reports/sim/20260926-050058-t3-ios-mvf-0876e03c-replay/report.json | -| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage; missing auto-capture, guidance [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift) | -| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, plus symbol-name button labels) finds no issues; S4-2 shows how many screens the app reached | no evidence yet | no report matches ~/house-scanning-data/reports/a11y/*t3-ios-mvf*/report.json | +| S3-1 | CI build green | met | iOS build: success at 525ea401 [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | +| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/project.yml) | +| S3-3 | Zero build warnings | met | build.warnings = length 0 (want 0): ~/house-scanning-data/reports/sim/20260926-052535-t3-ios-mvf-525ea401-replay/report.json | +| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage; missing auto-capture, guidance [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift) | +| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, plus symbol-name button labels) finds no issues; S4-2 shows how many screens the app reached | gap | issue_count = 32 (want 0): ~/house-scanning-data/reports/a11y/20260926-052756-t3-ios-mvf-525ea401-head-sample/report.json | | S3-7 | Screenshots of each state in the PR | gap | 0 of 9 matches in the PR body [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | | S3-8 | Exported scene.json validates against C1 | met | app_export_scene_valid = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/verification` at [`7c2c9ab5`](https://github.com/SamGu-NRX/house-scanning/tree/7c2c9ab5a5f4fc937d10cd8de0fd7904e61664e8), `origin/t3/ios-mvf` at [`24af4342`](https://github.com/SamGu-NRX/house-scanning/tree/24af4342bf760c4af7ed3f942c330533bed1f804). 2 met, 1 partial, 1 stale. +`origin/t3/verification` at [`e18a2eb7`](https://github.com/SamGu-NRX/house-scanning/tree/e18a2eb7a213a5bd9d752a4fadccf09c50bcebf2), `origin/t3/ios-mvf` at [`525ea401`](https://github.com/SamGu-NRX/house-scanning/tree/525ea401641553fad581925023ed1bb82c0f7352). 4 met. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S4-1 | End to end from a real replay to a placement result passes | met | app_export_passed = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | -| S4-2 | Screenshot report per app state | stale | report for 0876e03c, ref at 24af4342: ~/house-scanning-data/reports/sim/20260926-050058-t3-ios-mvf-0876e03c-replay/report.json | -| S4-3 | Product description written from the code and checked against screenshots | partial | verification/product: scope, glossary, both foundations and the pilot drafted from S3's integration commit 21a63e7; one Simulator pass recorded (verification.md); 7 suspected defects in bug-triage.md. Seven screen documents to go. | -| S4-4 | Scoreboard of every PR | met | present at 7c2c9ab5 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/7c2c9ab5a5f4fc937d10cd8de0fd7904e61664e8/verification/SCOREBOARD.md) | +| S4-2 | Screenshot report per app state | met | states = length 10 (want at least 9): ~/house-scanning-data/reports/sim/20260926-052535-t3-ios-mvf-525ea401-replay/report.json | +| S4-3 | Product description written from the code and checked against screenshots | met | verification/product covers every screen of t3/ios-mvf at 525ea40, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state. | +| S4-4 | Scoreboard of every PR | met | present at e18a2eb7 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/e18a2eb7a213a5bd9d752a4fadccf09c50bcebf2/verification/SCOREBOARD.md) | ## M Maintenance -`origin/t3/docs-cleanup` at [`9c9ceb2a`](https://github.com/SamGu-NRX/house-scanning/tree/9c9ceb2a608e9762bd002f20f1b979694bb0c92f). 1 met, 1 gap. +`origin/t3/docs-cleanup` at [`ab03d7fd`](https://github.com/SamGu-NRX/house-scanning/tree/ab03d7fd735884346f9182e3cb4cbf4aee60994a). 1 met, 1 gap. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | diff --git a/verification/fixtures/a11y-audit/AccessibilityAudit/AccessibilityAuditTests.swift b/verification/fixtures/a11y-audit/AccessibilityAudit/AccessibilityAuditTests.swift index c6e833f6e..b2639d7e7 100644 --- a/verification/fixtures/a11y-audit/AccessibilityAudit/AccessibilityAuditTests.swift +++ b/verification/fixtures/a11y-audit/AccessibilityAudit/AccessibilityAuditTests.swift @@ -34,7 +34,10 @@ final class AccessibilityAuditTests: XCTestCase { emit("A11Y_SCREEN", ["index": seen.count + 1, "error": "app not running"]) break } - let labels = screenLabels(app) + guard let labels = screenLabels(app) else { + RunLoop.current.run(until: Date().addingTimeInterval(0.5)) + continue + } let key = labels.joined(separator: "\n") if !seen.contains(key) { seen.append(key) @@ -46,10 +49,23 @@ final class AccessibilityAuditTests: XCTestCase { } } + /// Labels from one snapshot of the accessibility tree. Querying elements and then reading + /// each label is two round trips, and a button that disappears in between (the screen + /// changing) fails the whole test. @MainActor - private func screenLabels(_ app: XCUIApplication) -> [String] { - let texts = app.staticTexts.allElementsBoundByIndex.map { "text:" + $0.label } - let buttons = app.buttons.allElementsBoundByIndex.map { "button:" + $0.label } + private func screenLabels(_ app: XCUIApplication) -> [String]? { + guard let root = try? app.snapshot() else { return nil } + var texts: [String] = [] + var buttons: [String] = [] + func walk(_ node: any XCUIElementSnapshot) { + switch node.elementType { + case .staticText: texts.append("text:" + node.label) + case .button: buttons.append("button:" + node.label) + default: break + } + node.children.forEach(walk) + } + walk(root) return texts + buttons } diff --git a/verification/product/README.md b/verification/product/README.md index 7c76c2f4c..5f0ea16dc 100644 --- a/verification/product/README.md +++ b/verification/product/README.md @@ -10,8 +10,8 @@ wrong is collected in [bug-triage.md](bug-triage.md). - **Surface.** The HouseScan app, one run of the homeowner flow from onboarding to the result, default settings. The ARKit camera cannot run in the Simulator, so behavior that needs the live camera is described from code and marked unverified. -- **Source.** `t3/ios-mvf` at `0876e03`, which has the engine and the UI lane's screens. Every - footer cites it. +- **Source.** `t3/ios-mvf`. Line references and footers are at `525ea40`, and the app was checked + in the Simulator at `525ea40`. - **How it is run.** `make sim-app REF=` from `verification/`: the real ADVIO replay `advio-20-0040-0075` (C3), the autopilot, and the server started from `t3/server`. - **Out of scope.** The `-uiDemo` screens (a design preview, not the product), TestFlight @@ -72,5 +72,5 @@ and Open questions and verification, ending with the commit it was verified agai | [screens/gap-request.md](screens/gap-request.md) | One targeted extra view, from the phone or the server | drafted; checked in the Simulator | | [screens/uploading.md](screens/uploading.md) | Sending the scan and waiting for the result, offline and failure | drafted; checked in the Simulator | | [screens/result.md](screens/result.md) | The result, its checks and the 3D and AR views | drafted; checked in the Simulator | -| [verification.md](verification.md) | Checks against the running app and their results | current pass at `0876e03` | +| [verification.md](verification.md) | Checks against the running app and their results | current pass at `525ea40` | | [bug-triage.md](bug-triage.md) | Every suspected defect, deduplicated | 16 entries, 2 resolved | diff --git a/verification/product/bug-triage.md b/verification/product/bug-triage.md index c663016e8..b20efc10a 100644 --- a/verification/product/bug-triage.md +++ b/verification/product/bug-triage.md @@ -10,7 +10,7 @@ source and not run. Every blocker and major entry was re-read in the source by a | B-03 | Blocker | Camera denied, camera failure, unreadable replay: no message, no way on | seen (replay) | fix | | B-04 | Blocker | Meter close-up: "Can't get a clear shot" may never appear | code | fix | | B-09 | Blocker | Feature list: "Add something" does nothing | code | fix | -| B-06 | Major | Walk: wall ends can't be recorded truthfully | code | product call | +| B-06 | Major | Walk: an end closer than 20 ft can't be marked (answer half fixed at `525ea40`) | code | fix | | B-08 | Major | "Can't get there" is never sent to the server | code | fix (with S2) | | B-10 | Major | Result, upload or list: the scan resets after 20 s of relocalizing | code | fix | | B-11 | Major | Meter tap can land on another wall's extended plane | code | fix | @@ -18,10 +18,10 @@ source and not run. Every blocker and major entry was re-read in the source by a | B-02 | Major | Upload and camera failures show raw error text | seen at `21a63e7` | fix | | B-13 | Minor | 3D and AR tint zones stop one battery width short | code | fix | | B-14 | Minor | Result rule lines hide whether a limit is a minimum or maximum | seen | fix | -| B-15 | Minor | AR view does not say a sample result is a sample | seen | fix | | B-16 | Minor | Wording and VoiceOver: "Tap the ac unit"; "ft" and "in" read as letters | seen in part | fix | | B-01 | Resolved | Every upload failed with 404 | seen fixed at `0876e03` | none | | B-05 | Resolved | Camera-screen text missing at `21a63e7` | seen fixed | none | +| B-15 | Resolved | AR view did not say a sample result is a sample | seen fixed at `525ea40` | none | | B-07 | Minor | Scan photos are never deleted from the phone | code | product call | ## Blockers @@ -49,13 +49,12 @@ feature is what this screen is for. ## Major -**B-06. Wall ends can't be recorded truthfully.** "Wall ends here" is offered only while the walk -asks "Is this the left end of the wall?", which happens only at 6.1 m of coverage -(`HouseScanKit/.../Guidance/GuidancePlanner.swift`, `preferredTask`), and "Can't get there" is not -offered then (`WallWalkScreen.swift:105`, `asksForArea`). A wall that ends at 10 ft can only be recorded as -unexplored; one that continues past 20 ft only as a real end, which lets the server reject the -site. Decision needed: offer "Wall ends here" whenever an end is in view, and a third answer at -20 ft ("The wall keeps going") recorded as unexplored. +**B-06. Wall ends.** Half fixed at `525ea40`: after "Wall ends here" the walk asks "What's at the +left end?", and only "something blocks it" records a real end; a corner or no answer stays +unexplored. Still open: "Wall ends here" is offered only while the walk asks for the end, which +happens at 6.1 m of coverage (`HouseScanKit/.../Guidance/GuidancePlanner.swift`, `preferredTask`), +so a wall that ends at 10 ft can only be recorded through "Can't get there". Suggested: offer +"Wall ends here" whenever the wall's end is in view. **B-08. "Can't get there" is never sent.** The upload carries only covered stretches and each end's kind (`HouseScan/Runtime/ScanEngine+Export.swift:48-58`, `HouseScanKit/.../Scene/SceneExport.swift`), @@ -95,8 +94,6 @@ camera-failure text does the same (`ScanCopy.swift:240`) once B-03 is fixed. - **B-14.** Seen on the sample result: "Measured 3 ft. The rule is 20 ft, and the measurement can be off by about 4 in." A maximum reads like a minimum; the server's `comparison` and `review_threshold_ft` are not used (`UI/Copy/ScanCopy.swift:210-218`). -- **B-15.** The AR view says "Your battery could go here" over the sample result's spot without the - sample badge the result screen shows (`ResultARScreen.swift:43`). Seen in pass 2. - **B-16.** The AC unit prompt reads "Tap the ac unit", and VoiceOver hears "Mark ac unit" and "Remove ac unit" (`ScanCopy.swift:122`, `WallWalkScreen.swift:258`, `MarkFeaturesScreen.swift:121`; "Add ac unit" seen in the accessibility audit). @@ -107,6 +104,8 @@ camera-failure text does the same (`ScanCopy.swift:240`) once B-03 is fixed. - **B-01 (resolved).** Until `0876e03` the app posted to `/v1/scenes`, which the server does not serve, and every upload failed with 404. At `0876e03` it posts to `/v1/placements`: seen `POST /v1/placements 200` and a result (report `sim/20260926-050058-t3-ios-mvf-0876e03c-replay`). +- **B-15 (resolved).** At `525ea40` the AR view on a sample result says "Example spot, not your + result" and "No server checked this scan.", and the upload screen says "Making a sample result". - **B-05 (resolved).** At `21a63e7` the camera screens' cards rendered without text; at `5520229` the same capture shows every word. The UI lane added the missing colour assets. diff --git a/verification/product/foundations/coverage-and-guidance.md b/verification/product/foundations/coverage-and-guidance.md index 7a50ee839..4bbfeb11d 100644 --- a/verification/product/foundations/coverage-and-guidance.md +++ b/verification/product/foundations/coverage-and-guidance.md @@ -72,35 +72,42 @@ wall end. The instruction is chosen in this order, first match wins: -1. **Coaching** from the camera, which replaces everything else until it clears: "Move your phone +1. **The end question** on the walk, right after "Wall ends here": "What's at the left end?" (or + right), with "This tells the installer whether the wall keeps going." Nothing else is chosen + until it is answered ([the wall walk](../screens/wall-walk.md#while-capturing)). +2. **Coaching** from the camera, which replaces everything below it until it clears: "Move your phone slowly" (starting up), "Slow down" ("Walk a little slower so the photos stay sharp."), "Aim at a corner or somewhere with more texture" ("A plain wall or the sky gives your phone nothing to - follow."), "It's too dark to see the wall", "Hold steady" (the last frame was blurry or - moving), "Point at the meter like this." ("Your phone lost its place for a moment.", - relocalizing), "Your phone lost its place". + follow."), "It's too dark to see the wall", "Point at the meter like this." ("Your phone lost its place for a moment.", + relocalizing), "Your phone lost its place". During the walk, frames kept back for being blurry, + moving or dark show "Slow down" or "It's too dark to see the wall" only once the problem has + lasted 0.7 s, and the coaching clears after 0.5 s without it. While tracking is not normal, the walk and the gap request hide everything drawn on the camera image (the fog, the path, the target ring, the meter dot, the ends and the marks): drawn from a position the phone does not trust, they would sit in the wrong place. They fade out and back over 0.2 s. The strip is not drawn on the camera image and stays. -2. **"Take a step back"** when the phone is closer than 1.2 m to the wall; satisfied at 1.4 m. -3. **Aim at a lagging band**: within 1 m either side of the phone, if one band is done over at least +3. **"Take a step back"** when the phone is closer than 1.2 m to the wall; satisfied at 1.4 m. +4. **Aim at a lagging band**: within 1 m either side of the phone, if one band is done over at least 0.45 m (three cells) where the other is not, "Tilt down to show the ground" or "Tilt up to show more wall", with a ring on the spot. -4. **Walk to the left**, until coverage runs unbroken 6.1 m (20 ft) from the meter on the left; - then **"Is this the left end of the wall?"**, the only moment "Wall ends here" is offered. - Only after the left end is set: the same for the right. -5. With both ends set, the first stretch of at least 0.45 m between them where a band is not done +5. **Walk to the left**, until coverage runs unbroken 6.1 m (20 ft) from the meter on the left; + then **"Is this the left end of the wall?"**, with "Aim where the wall stops or turns a + corner, and tap Wall ends here.", the only moment "Wall ends here" is offered. Only after the + left end is set: the same for the right. +6. With both ends set, the first stretch of at least 0.45 m between them where a band is not done (ground checked before wall), as an aim instruction. -6. **"That's the whole wall"**, with "Tap Done when you're ready". +7. **"That's the whole wall"**, with "Tap Done when you're ready". A new instruction replaces the current one only when the current one is satisfied or has been up for 3 s. An aim instruction is satisfied at 80 % of the cells within 0.3 m of its spot covered; a -walk instruction when that side's end is set. +walk instruction when that side's end is set; an end question when it is answered. ### Advancing -Coverage is sent with the scan as the covered stretches of each band and the kind of each end -(real or unexplored). Skipped cells are not sent: to the server a stretch the homeowner said they +Coverage is sent with the scan as the covered stretches of each band and the kind of each end. An +end is **real** only when it was marked with "Wall ends here" and answered "Something blocks it". +It is **unexplored** when answered "It turns a corner", when left unanswered, or when set with +"Can't get there": the wall may go on past it. Skipped cells are not sent: to the server a stretch the homeowner said they can't reach looks the same as one nobody looked at, and it treats both as unknown. ## Modifiers @@ -138,7 +145,7 @@ hooks.** Each instruction change logs `GUIDANCE=`. ## Edge cases -- Walking faster than 1.5 m/s keeps no photos at all, with only "Hold steady" or no coaching. +- Walking faster than 1.5 m/s keeps no photos at all; "Slow down" appears after 0.7 s. - A stretch closer than 0.45 m to a covered stretch on both sides is never asked for. - "Can't get there" is not sent to the server (see Advancing), so the server may ask for the same stretch again on the result. @@ -150,14 +157,17 @@ hooks.** Each instruction change logs `GUIDANCE=`. ## Open questions and verification -- **Suspected bug: wall ends can't be recorded truthfully** (triage B-06). "Wall ends here" is - offered only when the walk asks for the end, which happens at 6.1 m of coverage; "Can't get - there" is not offered then. A wall that really ends sooner can only be recorded as unexplored, - and one that continues past 20 ft only as a real end, which lets the server reject the site. +- **Suspected bug: a blocked end nearer than 20 ft can't be recorded** (triage B-06, half + resolved). The end question makes a marked end's kind truthful, but "Wall ends here" is still + offered only when the walk asks for the end, at 6.1 m of coverage + (`HouseScanKit/.../GuidancePlanner.swift:27`, `:113`). A wall blocked by a fence sooner can only + be recorded as unexplored, through "Can't get there". A wall that goes on straight past 20 ft is + recorded as unexplored only by answering "It turns a corner", which is not what the homeowner + sees. - The walk always sends the homeowner left first. Starting at the right end of a wall means walking past everything to the left first. - "About N ft to go" can never appear: the engine always passes no remaining distance (`Runtime/ScanEngine.swift`, `step(_:)`). - All thresholds are the code's stated hypotheses (research note), not measured values. -Verified against house-scanning commit `0876e03` (t3/ios-mvf). +Verified against house-scanning commit `525ea40` (t3/ios-mvf). diff --git a/verification/product/foundations/flow.md b/verification/product/foundations/flow.md index f8583398d..ac9ab05e6 100644 --- a/verification/product/foundations/flow.md +++ b/verification/product/foundations/flow.md @@ -130,8 +130,8 @@ the autopilot wait on each screen until a file named after it appears, for UI te - **Suspected bug: camera access denied is a dead end.** The failure screen text exists ("House Scan needs your camera ... Turn on Camera for House Scan in Settings") but only the `unsupported` screen shows failures, and the app switches to it only when motion tracking is - unsupported (`Runtime/ScanEngine.swift:93`). A denied camera, a failed camera session and an - unreadable replay set the failure (`ScanEngine.swift:89, 338, 340`) without changing screen. The + unsupported (`Runtime/ScanEngine.swift:99`). A denied camera, a failed camera session and an + unreadable replay set the failure (`ScanEngine.swift:111, 424, 426`) without changing screen. The homeowner would stay on "Find your electric meter" with no camera and no Settings button. Checkable in the Simulator for the replay case: launch with `-replay /nonexistent`. - "Your scan is saved on this phone" is true only while the app runs: after a relaunch nothing @@ -142,4 +142,4 @@ the autopilot wait on each screen until a file named after it appears, for UI te - The 1.2 s pause after a finished step and the 20 s relocalization limit are read from code; neither has been timed on a device. -Verified against house-scanning commit `0876e03` (t3/ios-mvf). +Verified against house-scanning commit `525ea40` (t3/ios-mvf). diff --git a/verification/product/glossary.md b/verification/product/glossary.md index 846422b6a..082a04a81 100644 --- a/verification/product/glossary.md +++ b/verification/product/glossary.md @@ -12,9 +12,12 @@ app; everything else is this description's own term. your left", "ft left of your meter". - **Along the wall.** Distance from the meter along the wall line, negative to the left. The app shows it in feet and inches; the code works in meters. -- **Wall end.** Where the homeowner said the wall stops. A **real end** comes from "Wall ends - here" (the wall stops at a corner or something blocking the way); an **unexplored end** comes - from "Can't get there" during the walk (the wall may continue). +- **Wall end.** Where the homeowner said the wall stops. A **real end** is one marked with "Wall + ends here" and answered "Something blocks it" (a fence, gate or neighbor's yard). An + **unexplored end** is one answered "It turns a corner", one left unanswered, or one set with + "Can't get there" during the walk: the wall may continue. +- **End question.** "What's at the left end?" (or right), asked right after "Wall ends here", with + the answers "It turns a corner" and "Something blocks it". ## Seeing the wall @@ -44,7 +47,7 @@ app; everything else is this description's own term. - **Instruction.** The one line (plus an optional second line) at the top of a camera screen. There is exactly one at a time. - **Coaching.** An instruction about the phone rather than the wall ("Slow down", - "Hold steady"). It replaces the guidance instruction until the problem clears. + "Move your phone slowly"). It replaces the guidance instruction until the problem clears. - **Reticle.** The small circle in the middle of the camera view that a tap on Mark (or "This is my meter") aims with, when the homeowner does not tap the image itself. - **Mode badge.** The yellow "REPLAY · AUTOPILOT" label shown when the app runs on a recording or diff --git a/verification/product/screens/find-meter.md b/verification/product/screens/find-meter.md index aea1dab87..490a9bb36 100644 --- a/verification/product/screens/find-meter.md +++ b/verification/product/screens/find-meter.md @@ -140,7 +140,7 @@ replay logs where its wall came from when it loads. - Tapping past the edge of the part of the wall the phone has detected is accepted, because the surface counts as continuing (see Open questions). - A replay started with `-replay` whose homeowner taps "Allow camera" before the recording has - loaded starts the live camera as well (`Runtime/ScanEngine.swift:168`). Read from code only. + loaded starts the live camera as well (`Runtime/ScanEngine.swift:174`). Read from code only. - A replay that could not be opened leaves this screen dark, with no way on: every tap is ignored. @@ -163,14 +163,14 @@ replay logs where its wall came from when it loads. tap at a meter on a surface that has not been detected yet can pin the meter on another detected wall's continuation, at the wrong depth. The code's comment says a tap "never guesses a depth". - **Suspected bug: a 20 s loss of place anywhere returns here.** The check runs on every screen - while the camera session exists, including the upload and the result (`Runtime/ScanEngine.swift:357-364`). + while the camera session exists, including the upload and the result (`Runtime/ScanEngine.swift:402-409`). A homeowner who leaves the app during the upload and comes back to a phone lying face down could lose the whole scan and land here. The reset also leaves an unfinished feature mark in - place (`Runtime/ScanEngine.swift:386-410` does not clear it). + place (`Runtime/ScanEngine.swift:431-456` does not clear it). - **Question: "Point at the meter like this." with nothing to match.** The relocalizing coaching (`UI/Copy/ScanCopy.swift:65-66`) is written for the walk, which shows the saved close-up under it. Here no close-up exists yet and no picture is shown, so "like this" points at nothing. - The live tap, the refusals, the coaching and the camera prompt need the live camera and have not been seen running. The Simulator shows only the replay path, where every tap is accepted. -Verified against house-scanning commit `0876e03` (t3/ios-mvf). +Verified against house-scanning commit `525ea40` (t3/ios-mvf). diff --git a/verification/product/screens/gap-request.md b/verification/product/screens/gap-request.md index 7ff1b25b4..049b4e26e 100644 --- a/verification/product/screens/gap-request.md +++ b/verification/product/screens/gap-request.md @@ -165,8 +165,8 @@ skipping it". With `-autopilotGate`, the upload after a completed view waits for [coverage and guidance](../foundations/coverage-and-guidance.md#advancing), which says skipped cells are sent. - **Suspected bug: a past-end request turns a real end into an unexplored one.** "Capture it now" - forgets the end before the request starts (`Runtime/ScanEngine+Actions.swift:206-209`, - `Runtime/ScanEngine.swift:463-467`), and nothing marks it again. The next upload reports that + forgets the end before the request starts (`Runtime/ScanEngine+Actions.swift:219-222`, + `Runtime/ScanEngine.swift:516-521`), and nothing marks it again. The next upload reports that side as unexplored (`ScanEngine+Export.swift:53-54`) even when the homeowner has now shown the ground beyond it, so the server may ask for the same view again. - **Suspected bug: a second past-end request asks for the wrong place.** With the end forgotten, @@ -177,15 +177,15 @@ skipping it". With `-autopilotGate`, the upload after a completed view waits for - **Suspected bug: "Capture it now" can do nothing.** The card offers the button for any wall or ground item and any past-end item (`ScanEngine+Export.swift:134`), but the request needs a span or a side (`GapPlanner.swift:113, 124`). Without one, the tap is ignored with no message - (`ScanEngine+Actions.swift:205`). + (`ScanEngine+Actions.swift:218`). - The phone's wording "This might be a spot for the battery" is stronger than its choice: it asks for the nearest missing ground, not a place it has judged a candidate. -- The 20 s reset keeps the list of skipped requests (`ScanEngine.swift:386-410`); after - re-walking, a phone request identical to a skipped one would be passed over (`ScanEngine.swift:475`). +- The 20 s reset keeps the list of skipped requests (`ScanEngine.swift:431-456`); after + re-walking, a phone request identical to a skipped one would be passed over (`ScanEngine.swift:529`). - **Question: "Point at the meter like this." without the picture.** The walk shows the saved close-up under this coaching (`UI/Screens/WallWalkScreen.swift:23-27`); this screen shows the words alone, even when a close-up was taken. - Nothing here has run in the Simulator. The completion moment, the amber highlight and the dots need the replay with `-autopilot`, which reaches this screen only when held-back frames exist. -Verified against house-scanning commit `0876e03` (t3/ios-mvf). +Verified against house-scanning commit `525ea40` (t3/ios-mvf). diff --git a/verification/product/screens/mark-features.md b/verification/product/screens/mark-features.md index 6a28302d1..84f158442 100644 --- a/verification/product/screens/mark-features.md +++ b/verification/product/screens/mark-features.md @@ -185,25 +185,25 @@ appears; marking itself logs nothing. Identifiers: `action.markSomething`, `acti - **Suspected bug: "Add something" does nothing visible.** The buttons start a mark (`UI/Screens/MarkFeaturesScreen.swift:64`, accepted on this screen by - `Runtime/ScanEngine+Actions.swift:73`), but the list screen never shows the circle, the prompt, + `Runtime/ScanEngine+Actions.swift:86`), but the list screen never shows the circle, the prompt, "Mark" or camera taps; those exist only on the walk (`UI/Screens/WallWalkScreen.swift:28-45, - 119-133`). The homeowner who spots a missed gas meter here presses a button and nothing - happens. "Looks complete" does not end the stray mark either (`ScanEngine+Actions.swift:155-158`). + 121-135`). The homeowner who spots a missed gas meter here presses a button and nothing + happens. "Looks complete" does not end the stray mark either (`ScanEngine+Actions.swift:168-171`). - **Suspected bug: "Tap the ac unit".** The prompt lowercases the kind's name (`UI/Copy/ScanCopy.swift:122`), so "AC unit" becomes "ac unit" on screen, and VoiceOver hears "Remove ac unit", "Add ac unit", "Mark ac unit" (`MarkFeaturesScreen.swift:121, 189`, - `WallWalkScreen.swift:258`). + `WallWalkScreen.swift:282`). - **Suspected bug: the list can vanish without a word.** Relocalization is timed on every screen - (`Runtime/ScanEngine.swift:189, 357-368`), but the list shows no coaching. After a call, a + (`Runtime/ScanEngine.swift:195, 402-413`), but the list shows no coaching. After a call, a homeowner reading the list with the phone pointed away for 20 s is sent back to finding the meter with every mark and photo gone, never having seen "Point at the meter like this.". - **Suspected bug: a half-done mark survives the 20 s reset.** `resetSpatialState` clears the - features but not the mark in progress or its taps (`ScanEngine.swift:386-410`, taps held at - `ScanEngine.swift:62`). After the meter is tapped again the walk opens mid-mark, and a first tap + features but not the mark in progress or its taps (`ScanEngine.swift:431-456`, taps held at + `ScanEngine.swift:65`). After the meter is tapped again the walk opens mid-mark, and a first tap from the old, discarded map can be joined to a new one. The same holds for a stray mark from "Add something". - **Wrong words for the wrong-side refusal.** The check is where the phone is, not where the tap - lands (`ScanEngine+Actions.swift:88`). "Tap something on this side" cannot help; stepping back + lands (`ScanEngine+Actions.swift:101`). "Tap something on this side" cannot help; stepping back out in front of the wall's line does. - The place in a row reads "ft" to VoiceOver (`MarkFeaturesScreen.swift:107`), though the app has a spelled-out form for this (`UI/Copy/Distance.swift:25`). @@ -211,4 +211,4 @@ appears; marking itself logs nothing. Identifiers: `action.markSomething`, `acti "Mark". MF-01 in [verification](../verification.md) was observed on an earlier commit and has not been rechecked here. -Verified against house-scanning commit `0876e03` (t3/ios-mvf). +Verified against house-scanning commit `525ea40` (t3/ios-mvf). diff --git a/verification/product/screens/meter-close-up.md b/verification/product/screens/meter-close-up.md index ded89b7e3..ba21405a0 100644 --- a/verification/product/screens/meter-close-up.md +++ b/verification/product/screens/meter-close-up.md @@ -132,4 +132,4 @@ failed attempts" when the homeowner skips. its text in that screenshot; see [verification](../verification.md). - The circle-versus-image mismatch above is read from code; it needs a device to confirm. -Verified against house-scanning commit `0876e03` (t3/ios-mvf). +Verified against house-scanning commit `525ea40` (t3/ios-mvf). diff --git a/verification/product/screens/onboarding.md b/verification/product/screens/onboarding.md index 83ca40881..7f16e2f58 100644 --- a/verification/product/screens/onboarding.md +++ b/verification/product/screens/onboarding.md @@ -149,9 +149,9 @@ change. Tests find the controls as `action.onboardingNext`, `action.onboardingSk - **Suspected bug: camera denied, a failed session and an unreadable replay never reach the screen** (triaged as [B-03](../bug-triage.md)). The engine switches to the failure screen only - for an unsupported phone (`Runtime/ScanEngine.swift:91-93`). A denied camera and a failed - session set the failure (`ScanEngine.swift:379, 381`) and an unreadable replay does too - (`ScanEngine.swift:105`), with no screen change. The homeowner who declines the prompt sits on + for an unsupported phone (`Runtime/ScanEngine.swift:97-99`). A denied camera and a failed + session set the failure (`ScanEngine.swift:424, 426`) and an unreadable replay does too + (`ScanEngine.swift:111`), with no screen change. The homeowner who declines the prompt sits on "Find your electric meter" with no camera image; the "Open Settings" and "Start over" buttons on `UI/Screens/UnsupportedScreen.swift:35-48` are unreachable outside the `-uiDemo` preview. - **Suspected bug: an unsupported phone has no way out.** `UnsupportedScreen.swift:44` hides @@ -159,11 +159,11 @@ change. Tests find the controls as `action.onboardingNext`, `action.onboardingSk it contradicts [the flow](../foundations/flow.md#summary), which says "Start over" is offered on the unsupported screen. One of the two needs updating. - **Suspected bug: "Allow camera" during replay loading starts the live camera.** On a replay, - `startSourceIfNeeded` (`ScanEngine.swift:169`) starts the camera whenever the replay has not + `startSourceIfNeeded` (`ScanEngine.swift:175`) starts the camera whenever the replay has not loaded yet and no failure is set. A homeowner who taps "Skip" and "Allow camera" before a slow recording loads gets a live session as well as the replay. Affects replays only; not observed. - **"The camera stopped" would show raw system error text** as its detail - (`UI/Copy/ScanCopy.swift:241`), the same pattern as B-02, if the screen were ever reached. + (`UI/Copy/ScanCopy.swift:260`), the same pattern as B-02, if the screen were ever reached. - **Reduce Motion is applied to "Next" but not to "Skip"** (`OnboardingScreen.swift:37` versus `:75`). Even "Next" still slides the page sideways under Reduce Motion, only faster. - The page view hides its dots from VoiceOver on the grounds that it announces its own position @@ -173,4 +173,4 @@ change. Tests find the controls as `action.onboardingNext`, `action.onboardingSk session error, and the Reduce Motion stills. The unsupported screen and the `-replay /nonexistent` case can be checked in the Simulator. -Verified against house-scanning commit `0876e03` (t3/ios-mvf). +Verified against house-scanning commit `525ea40` (t3/ios-mvf). diff --git a/verification/product/screens/result.md b/verification/product/screens/result.md index 5cb632776..f57c1f62c 100644 --- a/verification/product/screens/result.md +++ b/verification/product/screens/result.md @@ -91,7 +91,9 @@ the wall, as deep as the battery: green where a battery passes, amber where it i where it fails. **The AR view.** "See it on your wall" crossfades to the camera. The instruction reads "Your -battery could go here" with the placement line under it. On the camera image the app draws each +battery could go here" for an approved pass and "The spot an installer will check" otherwise, +with the placement line under it. With the sample result it reads "Example spot, not your +result", with "No server checked this scan." and the placement line. On the camera image the app draws each sweep stretch as a tinted ground strip with a dashed outline, the cable as a blue line edged in white, and the battery as a box with a soft shadow and a blue light bar across its front. The box rises out of the ground as the view opens. Everything is placed relative to the meter's anchor, @@ -111,8 +113,8 @@ settle this" when the area was not seen. The rows have no buttons of their own. On the AR view the drawing is redrawn from every camera frame. Whenever tracking is not normal (limited, lost or relocalizing), the drawing disappears at once and the instruction reads "Point at your meter", with "The battery comes back once your phone finds its place." When tracking is -normal again the drawing returns, without rising again, and the instruction goes back to "Your -battery could go here". +normal again the drawing returns, without rising again, and the instruction goes back to the +headline above. No other coaching appears here: the homeowner is not told why tracking is limited. ### Advancing @@ -133,7 +135,7 @@ uploaded again and a new result replaces this one. The old result cannot be reop | --- | --- | --- | | Live camera or replay | A replay shows the "Replay" badge over the model. The AR view on a replay shows one still: the recorded frame that best shows the spot, with the battery drawn on it. | On a replay the AR picture does not move. Live, the drawing follows the phone. | | Autopilot | The autopilot waits `-autopilotHold` seconds, opens the AR view itself (even when the result has no spot), waits again, taps "Done" and logs "AUTOPILOT done". | No effect. | -| Server or sample result | The sample result shows "Sample result, not from the server" over the model. It is always the same: "An installer will take a look", "4 ft 3 in right of your meter, 3 ft of cable", the rules note, the left-side note, the panel note, four checks and two "Capture it now" cards. | "Capture it now" works, but the upload returns the same sample, so nothing changes. | +| Server or sample result | The sample result shows "Sample result, not from the server" over the model. It is always the same: "An installer will take a look", "4 ft 3 in right of your meter, 3 ft of cable", the rules note, the left-side note, the panel note, four checks and two "Capture it now" cards. The AR view says "Example spot, not your result", with "No server checked this scan." | "Capture it now" works, but the upload returns the same sample, so nothing changes. | | Larger text sizes | The headline, notes and rows wrap and the page scrolls; the model stays 340 points tall. On the AR view the instruction grows, and at the largest sizes the chrome scrolls. | No effect. | | Reduce Motion | The headline fades in over 0.2 s without rising. The model starts at its resting angle with no sweep. The AR battery rises over 0.2 s instead of springing up. | Dragging and pinching the model work as usual. | @@ -178,32 +180,36 @@ appears, not when returning from the AR view. **Verification hooks.** `STATE=res - The AR battery is drawn over everything in the picture, including a bush or a person standing in front of the spot. - With the sample result on a real phone, the AR view places a battery 4 ft 3 in right of the - real meter, whatever the wall is like. + real meter, whatever the wall is like, under "Example spot, not your result". - A "Still needed" item about the wall or the ground that comes without a stretch still shows "Capture it now", but the button does nothing. ## Open questions and verification - **Suspected bug: the result can disappear after a call or a trip to another app.** The 20 s - relocalization limit runs on every screen (`Runtime/ScanEngine.swift:189`, `357-364`). The live + relocalization limit runs on every screen (`Runtime/ScanEngine.swift:195`, `402-409`). The live camera keeps running under the result, so a phone that fails to relocalize within 20 s sends a homeowner who is reading their result back to "Find your electric meter". On the result screen the camera is hidden, so nothing even tells them to aim at the meter; only the AR view says "Point at your meter". The reset - (`ScanEngine.swift:386-409`) also leaves the old result in memory. The limit should apply only + (`ScanEngine.swift:431-455`) also leaves the old result in memory. The limit should apply only to the capture screens. - **Suspected bug: the tinted stretches are in the wrong place.** Each sweep run's `start_ft` is the range of the battery's left edge (`HouseScanKit/.../PlacementResult.swift:450`; the result schema says the same). The app tints exactly that range (`Runtime/ScanEngine+Export.swift:124`), so every stretch stops one battery width short on the right. -- **Suspected bug: the AR view does not say a sample is a sample.** The sample badge is only on the - result screen (`UI/Screens/ResultScreen.swift:96-105`). The AR view says "Your battery could go - here" over a made-up spot (`UI/Screens/ResultARScreen.swift:43`). +- The AR view now names a sample spot as an example (`UI/Screens/ResultARScreen.swift:49-51`), + fixed in `525ea40`. It still has no badge like the result screen's + (`UI/Screens/ResultScreen.swift:96-105`). +- **Question: an end answered "It turns a corner" may read as unseen.** The left-or-right note + shows when the server reports an end as unexplored (`Runtime/ScanEngine+Export.swift:141-144`), + and a corner answer is sent as unexplored. A homeowner who walked to the corner would then read + that "the scan didn't reach that side". Depends on what the server echoes back; not checked. - The measurement line says "The rule is 20 ft" for a limit that is a maximum and "The rule is 3 ft" for a minimum. The app decodes the server's `comparison` but never shows it, and ignores - `review_threshold_ft` (`UI/Copy/ScanCopy.swift:214-218`). + `review_threshold_ft` (`UI/Copy/ScanCopy.swift:233-237`). - "Capture it now" does nothing when the server's item has no stretch - (`Runtime/ScanEngine+Actions.swift:205`, `HouseScanKit/.../GapPlanner.swift:113`), although + (`Runtime/ScanEngine+Actions.swift:218`, `HouseScanKit/.../GapPlanner.swift:113`), although the card offers it (`ScanEngine+Export.swift:134`). - After "Capture it now" the old result is gone from view. If the new upload fails, the homeowner has only "Try again" on the upload screen and cannot reread the earlier answer. @@ -214,8 +220,8 @@ appears, not when returning from the AR view. **Verification hooks.** `STATE=res path no homeowner can take. - Dragging the model sits inside a scrolling page; whether a vertical drag turns the model or scrolls the page needs a device. -- Both screens were seen at `0876e03` with the real server's answer and with the sample result +- Both screens were seen at `0876e03`, `24af434` and `525ea40` with the real server's answer, and with the sample result ([verification](../verification.md), RES-01 to AR-01). Whether the AR drawing lines up with a real wall, and the relocalization reset, need the live camera. -Verified against house-scanning commit `0876e03` (t3/ios-mvf). +Verified against house-scanning commit `525ea40` (t3/ios-mvf). diff --git a/verification/product/screens/uploading.md b/verification/product/screens/uploading.md index d3dd0bf46..58aaaa388 100644 --- a/verification/product/screens/uploading.md +++ b/verification/product/screens/uploading.md @@ -86,7 +86,7 @@ screen until "Try again" succeeds. | --- | --- | --- | | Live camera or replay | A replay stops playing; the scan is zipped the same way. | On the live camera the phone still watches tracking behind this screen (see Cancel and interrupt); a replay does not. | | Autopilot | A badge at the top left reads "Replay · Autopilot" (in capitals). With the sample result, the `-autopilotHold` time replaces the 1.2 s in the timings below. | If no result arrives within 240 s and the upload has failed, the autopilot presses "Try again" once and waits 90 s more. With `-autopilotGate`, "Done" stays up until the test's gate file appears, at most 120 s. | -| Server or sample result | With `-serverURL` the zip is posted to that server. Without it, or with `-sampleResult`, the scan is still zipped but not sent. | The sample shows 25%, 50% and 75% about 0.24 s apart, then "Checking your wall" for 1.2 s, then the [sample result](../glossary.md#after-the-walk). It fails only if the sample file is missing from the app. | +| Server or sample result | With `-serverURL` the zip is posted to that server. Without it, or with `-sampleResult`, the scan is still zipped but not sent, and the screen says so: the title is "Making a sample result", with "No server is connected, so your photos stay on this phone. The result you'll see is an example, not a check of your wall.", and the steps read "Get photos ready", "Load the sample result" and "Show the example". | The title stays the same until the result. "Load the sample result" stays up for about 1 s (four pauses of 0.24 s, no percentage shown), then reads "Sample loaded", and "Show the example" holds for 1.2 s before the [sample result](../glossary.md#after-the-walk). A failure still shows the failure titles. It fails only if the sample file is missing from the app. | | Larger text sizes | Title, second line and steps grow and wrap; the screen scrolls when they no longer fit. The symbol stays 132 pt. | No effect. | | Reduce Motion | The screen fades in over 0.15 s; the symbol does not pulse. | Check marks still scale in and the symbol still swaps with an animation. | @@ -118,7 +118,7 @@ each attempt. It holds `scene.json`, every kept photo as `k00001.jpg`, `k00002.j 120 s without data. Any answer outside 200–299 is a failure; so is one that does not read as a result: a schema version other than 1.0, a missing required field, a wrong type, a pair of the wrong length, or a value outside a fixed set such as the decision or an outcome -(`HouseScanKit/.../PlacementResult.swift:1-11`). Fields the app does not know are ignored, so the server can add +(`HouseScanKit/.../PlacementResult.swift:3-12`). Fields the app does not know are ignored, so the server can add optional ones without breaking the upload. Only connection errors count as offline. **Accessibility.** The title is a header. The symbol is hidden from VoiceOver. The three steps are one element, read together. No change of step or percentage is announced, and neither is a failure. **Haptics and motion.** None on @@ -147,20 +147,21 @@ autopilot logs "upload failed; retrying once". Test identifiers: `screen.uploadi posts to `/v1/placements` (`Runtime/ResultClient.swift:24`), and the server answered 200 in the Simulator run at this commit ([verification](../verification.md), FLOW-02). - **Suspected bug: raw error text on screen.** The second line of "That didn't go through" is the - error's developer description (`UI/Copy/ScanCopy.swift:177`, fed by - `Runtime/ScanEngine.swift:533, 560`): HTTP status codes, server JSON, decoder errors. B-02. -- **Suspected bug: "You're offline" for any connection error.** `Runtime/ScanEngine.swift:560` + error's developer description (`UI/Copy/ScanCopy.swift:196`, fed by + `Runtime/ScanEngine.swift:587, 614`): HTTP status codes, server JSON, decoder errors. B-02. +- **Suspected bug: "You're offline" for any connection error.** `Runtime/ScanEngine.swift:614` treats every `URLError` as offline, including an unreachable host, refused connection and timeout. The copy then tells a homeowner with full signal to wait for signal. - **Suspected bug: "Checking your wall" is skipped with a real server.** Progress is capped at 99% while sending (`Runtime/ResultClient.swift:68`) and reaches 100% only after the server has - answered (`:40`); the step changes only then (`Runtime/ScanEngine.swift:542, 546`). The + answered (`:40`); the step changes only then (`Runtime/ScanEngine.swift:596, 600`). The homeowner watches "Sending photos, 99%" for the whole analysis, then the result replaces - "Checking your wall" almost at once. Only the sample result shows the third step. + "Checking your wall" almost at once. Only the sample result lingers on the third step, where it + reads "Show the example". - **Suspected bug: a relocalization reset can wipe the scan during the upload.** The - relocalization check runs on every live frame whatever the screen (`Runtime/ScanEngine.swift:189`, - `364`). After 20 s it forgets the wall, marks and photo list and goes to `findMeter` (`:386–409`), - and the upload's answer is dropped (the `generation` guards at `:545, 552`). A homeowner who + relocalization check runs on every live frame whatever the screen (`Runtime/ScanEngine.swift:195`, + `409`). After 20 s it forgets the wall, marks and photo list and goes to `findMeter` (`:431–455`), + and the upload's answer is dropped (the `generation` guards at `:599, 606`). A homeowner who locks the phone mid-upload and returns somewhere the phone can't recognise would lose the scan. Unverified: needs a device, and whether the AR session keeps delivering frames while the camera view is off screen is not settled by the code. @@ -170,9 +171,9 @@ autopilot logs "upload failed; retrying once". Test identifiers: `screen.uploadi - "It takes about a minute" has no measurement behind it; no upload has been timed with a real scan. - The comment on `writeBundle` (`Runtime/KeyframeStore.swift:83–84`) says the zip stays so a - failed upload can be retried, but "Try again" rebuilds it (`Runtime/ScanEngine+Actions.swift:194–196`). + failed upload can be retried, but "Try again" rebuilds it (`Runtime/ScanEngine+Actions.swift:207–209`). Harmless, but the kept zip is never reused. - Pass 1 (`21a63e7`) saw "That didn't go through" with no button or symbol visible ([verification](../verification.md) UP-01, B-05). Not rechecked at this commit. -Verified against house-scanning commit `0876e03` (t3/ios-mvf). +Verified against house-scanning commit `525ea40` (t3/ios-mvf). diff --git a/verification/product/screens/wall-walk.md b/verification/product/screens/wall-walk.md index 4838d2042..76ee457c4 100644 --- a/verification/product/screens/wall-walk.md +++ b/verification/product/screens/wall-walk.md @@ -6,8 +6,8 @@ The wall walk is where the scan collects its evidence. The homeowner walks along the phone pointed at it, first to the left of the meter and then to the right, and the phone keeps photos by itself. Fog over the camera image clears where the phone has looked, the coverage strip at the bottom fills in, and one instruction at the top says what to do next. The homeowner says -where the wall stops on each side ("Wall ends here", or "Can't get there" when they cannot go -further), can mark features on the way ("Mark something"), and leaves with "Done with this wall" +where the wall stops on each side ("Wall ends here", then whether it turns a corner or something +blocks it; or "Can't get there" when they cannot go further), can mark features on the way ("Mark something"), and leaves with "Done with this wall" once both ends are set. It follows [the meter close-up](meter-close-up.md) (screen name `wallWalk`) and leads to [marking features](mark-features.md). If the phone loses its place for too long, the scan starts again from [finding the meter](find-meter.md). @@ -21,8 +21,10 @@ lifts off the wall and ground behind them, and the strip turns from gray to ambe When enough of the left side is covered, the instruction asks "Is this the left end of the wall?" and a "Wall ends here" button appears. The homeowner aims the middle of the screen at the corner -and taps it. A white line appears on the wall there, and the instruction switches to "Walk slowly -to your right". They walk back past the meter and do the same on the right. With both ends set, +and taps it. A white line appears on the wall there, and the instruction asks "What's at the left +end?", with "This tells the installer whether the wall keeps going." and two buttons, "It turns a +corner" and "Something blocks it". They tap "It turns a corner", and the instruction switches to +"Walk slowly to your right". They walk back past the meter and do the same on the right. With both ends set, "Done with this wall" appears; any stretch between the ends that is still thin gets one more aim instruction, and then "That's the whole wall", with "Tap Done when you're ready." Tapping "Done with this wall" opens the list of marks. @@ -36,10 +38,12 @@ stateDiagram-v2 [*] --> walkLeft walkLeft --> askLeftEnd : coverage runs far enough left walkLeft --> walkRight : "Can't get there" (unexplored left end) - askLeftEnd --> walkRight : "Wall ends here" (real left end) + askLeftEnd --> whatAtLeft : "Wall ends here" + whatAtLeft --> walkRight : "It turns a corner" (unexplored) or "Something blocks it" (real end) walkRight --> askRightEnd : coverage runs far enough right walkRight --> filling : "Can't get there" (unexplored right end) - askRightEnd --> filling : "Wall ends here" (real right end) + askRightEnd --> whatAtRight : "Wall ends here" + whatAtRight --> filling : "It turns a corner" (unexplored) or "Something blocks it" (real end) filling --> complete : nothing lags between the ends } walking --> kindPicker : "Mark something" @@ -73,11 +77,12 @@ The instructions this screen can show, with their second lines: | "Tilt down to show the ground" | "The strip along the wall, 3 ft 4 in left of your meter." (the distance varies) | Ring on the ground, dots toward it. | | "Tilt up to show more wall" | "Around 3 ft 4 in left of your meter." (the distance varies) | Ring on the wall, dots toward it. | | "Walk slowly to your left" (or right) | "Keep the wall and the ground in view." | Ring on the wall past the covered stretch, dots toward it. | -| "Is this the left end of the wall?" (or right) | "Aim at the corner, or where something blocks your way, and tap Wall ends here." | Ring at the end of the covered stretch, no dots. | +| "Is this the left end of the wall?" (or right) | "Aim where the wall stops or turns a corner, and tap Wall ends here." | Ring at the end of the covered stretch, no dots. | +| "What's at the left end?" (or right) | "This tells the installer whether the wall keeps going." | The ring and dots of the previous instruction stay until the answer. | | "That's the whole wall" | "Tap Done when you're ready." | None. | -A distance within 3 in of the meter reads "at your meter". Coaching replaces all of these while -the phone has a problem (see [Cancel and interrupt](#cancel-and-interrupt)). +A distance within 3 in of the meter reads "at your meter". Coaching replaces all of these except +the end question while the phone has a problem (see [Cancel and interrupt](#cancel-and-interrupt)). The wayfinding cues work together. The **path** is a line of blue dots on the ground, 1.5 m out from the wall, from where the homeowner stands toward the next place to stand; the dots shrink @@ -116,8 +121,11 @@ icon above the rows. The strip always spans at least 3.5 m (about 11 ft), and it coverage, the ends or the homeowner move outward. **The fog.** A frosted haze painted on the wall face (up to the top of the wall band) and on the -ground in front of it, over what has been seen plus the fog ahead of it on each side. Unseen cells are thick, seen cells -thinner, covered cells clear. Skipped cells are not fogged: they get a slate tint with white +ground in front of it, over what has been seen plus the fog ahead of it on each side. It is a +thin frosted blur under a light white wash (35 % opacity), both scaled per cell: unseen cells at +75 % strength, so the wall's shapes still show through, seen cells at 35 %, covered cells clear. +These strengths were picked by eye on the demo wall and a synthetic replay; the code says to tune +them on a phone in sun. Skipped cells are not fogged: they get a slate tint with white diagonal hatching, so they read neither as clear nor as unseen. Set ends are white vertical lines on the wall with a white dot at the foot. @@ -138,11 +146,20 @@ and never while coaching is showing, the kind picker is open or a mark is in pro **"Wall ends here".** Shown, with "Mark something" beside it, only while the instruction asks "Is this the left end of the wall?" or the right one. Tapping it takes the point in the middle of the -screen, finds where it meets the wall's line, and sets a real end there. Which end it sets depends -on which side of the meter that point is, not on which end was asked for. The white end line -appears on the camera, the cap appears on the strip, and the instruction moves on. Nothing on the -screen marks the middle of the screen at this moment, and if the middle of the screen does not -meet the wall line (the phone aimed away from the wall) the tap does nothing. +screen, finds where it meets the wall's line, and sets an end there, unexplored until the +homeowner says otherwise. Which end it sets depends on which side of the meter that point is, not +on which end was asked for. The white end line appears on the camera, the cap appears on the +strip, and the end question follows. Nothing on the screen marks the middle of the screen at this +moment, and if the middle of the screen does not meet the wall line (the phone aimed away from the +wall) the tap does nothing. + +**The end question.** The instruction reads "What's at the left end?" (or right, for the end just +set), with "This tells the installer whether the wall keeps going." The buttons give way to two +full-width answers, one above the other: "It turns a corner" and "Something blocks it". "It turns +a corner" keeps the end unexplored: the wall may go on. "Something blocks it" makes it a real end. +Until one is tapped the walk waits: the instruction does not change, coaching does not replace +it, and "Mark something", "Can't get there" and "Done with this wall" are hidden. The answer gives +no haptic; the next instruction is chosen at once. **"Mark something".** Always available while walking. It replaces the buttons with the kind picker: a panel headed "What do you see?", a close button, and six tiles, each an icon and a @@ -196,7 +213,8 @@ thumbnail could not be read back, there is no photo and only the instruction sho "Done with this wall" appears once both ends are set, whatever the coverage between them. Tapping it ends any marking and moves to [marking features](mark-features.md). What moves with it: the -coverage (covered, skipped and unseen cells), each end with its kind (real or unexplored), the +coverage (covered, skipped and unseen cells), each end with its kind (real only when the +homeowner answered "Something blocks it", otherwise unexplored), the features marked so far and the walk photos. Coverage between the ends that was never covered stays unseen; the server treats it as unknown ([coverage and guidance](../foundations/coverage-and-guidance.md#advancing)). There is no way back @@ -207,19 +225,19 @@ to the walk from the next screen. | Modifier | At arrival | While capturing | | --- | --- | --- | | Live camera or replay | A replay starts playing its recording from the first frame (leaving out any frames the autopilot holds back for a gap request); the meter's wall comes from the recording. | When the recording ends the image freezes on its last frame and the instruction stops changing. The 20 s reset never happens on a replay. | -| Autopilot | The walk plays at three times speed and the badge reads "Replay · Autopilot". | After the recording ends, the autopilot marks a gas meter and a window by calling the marking actions at projected points, sets both ends (falling back to setting them directly when no frame shows the spot), and finishes the walk. It never opens the kind picker or presses "Can't get there". | +| Autopilot | The walk plays at three times speed and the badge reads "Replay · Autopilot". | After the recording ends, the autopilot marks a gas meter and a window by calling the marking actions at projected points, sets both ends, answering "It turns a corner" to each end question after `-autopilotHold` seconds (when no frame shows the spot it sets a real end directly, with no question), and finishes the walk. It never opens the kind picker or presses "Can't get there". | | Server or sample result | No effect. | No effect. | -| Larger text sizes | The instruction card and buttons grow; the chrome scrolls when it no longer fits. At accessibility sizes "Mark something" fills the row. | The kind picker shows one tile per row at accessibility sizes instead of three. The strip does not grow. | +| Larger text sizes | The instruction card and buttons grow; the chrome scrolls when it no longer fits. At accessibility sizes "Mark something" fills the row. The two end answers are stacked and full width at every size. | The kind picker shows one tile per row at accessibility sizes instead of three. The strip does not grow. | | Reduce Motion | The instruction crossfades instead of blurring; the button rows change with a 0.15 s fade. | The fog fades without drifting, the target ring does not pulse, the counter icon flashes without growing, the tap ring fades without expanding. The kind picker still slides up from the bottom. | ## Cancel and interrupt | Event | Before both ends are set | With both ends set | | --- | --- | --- | -| The screen's own way out | "Can't get there" under a walk instruction sets an unexplored end; under a tilt instruction it skips that patch. "Cancel" leaves marking. | "Done with this wall". "Can't get there" still skips a patch under a tilt instruction; "Cancel" leaves marking. | +| The screen's own way out | "Can't get there" under a walk instruction sets an unexplored end; under a tilt instruction it skips that patch. "Cancel" leaves marking. The end question has no way out but an answer. | "Done with this wall". "Can't get there" still skips a patch under a tilt instruction; "Cancel" leaves marking. | | Start over | Not offered. | Not offered. | | Tracking limited | Coaching ("Slow down", "Aim at a corner or somewhere with more texture", "Move your phone slowly") replaces the instruction and hides "Can't get there". The fog, path, ring and marks fade off the camera image. No photos are kept. "Wall ends here" still works. Marks are refused with "One moment, your phone is still finding its place." | Same; "Done with this wall" still works. | -| Tracking lost or relocalizing | "Point at the meter like this.", with "Your phone lost its place for a moment." and the saved meter close-up in the middle of the screen (or "Your phone lost its place" when tracking is lost, with no photo). The fog, path, ring and marks fade off the camera image. After 20 s of relocalizing, the app forgets the wall, coverage, ends, marks and walk photos, sets the counter to 0 and goes back to "Find your electric meter", with no message saying why. | Same; the ends are lost too. | +| Tracking lost or relocalizing | "Point at the meter like this.", with "Your phone lost its place for a moment." and the saved meter close-up in the middle of the screen (or "Your phone lost its place" when tracking is lost, with no photo). The fog, path, ring and marks fade off the camera image. After 20 s of relocalizing, the app forgets the wall, coverage, ends, an unanswered end question, marks and walk photos, sets the counter to 0 and goes back to "Find your electric meter", with no message saying why. | Same; the ends are lost too. | | App backgrounded or a call | The camera stops; on return the coaching reads "Point at the meter like this." over the saved close-up until the phone finds its place. Photos, coverage, ends and marks are kept. Relocalizing time counts toward the 20 s. | Same. | | Camera off or session failed | The screen stays with no new frames; coverage cannot grow. Suspected dead end ([the flow](../foundations/flow.md#open-questions-and-verification)). | Same; "Done with this wall" still moves on. | | Network lost or upload failing | No effect. | No effect. | @@ -243,28 +261,31 @@ instruction card is one element that reads its text and marks itself as updating path, ring and marks are hidden from VoiceOver. Hints: "Pin a gas meter, door, window, AC unit, driveway or fence" on "Mark something"; "Marks the end of the wall at the circle in the middle of the screen" on "Wall ends here"; "Marks the point under the circle in the middle of the screen" on -"Mark"; "Skips this part of the wall. An installer will look at it instead." on "Can't get there". +"Mark"; "Skips this part of the wall. An installer will look at it instead." on "Can't get there"; +"A fence, gate, or your neighbor's yard" on "Something blocks it". The picker's heading is a header, its close button reads "Close", and each tile reads "Mark" and the kind in lower case. The saved close-up, while relocalizing, reads "Your photo of the meter from the start of the scan". **Haptics and motion.** A firm tap for each feature added and a warning for each refused mark. -No haptic for walk photos, for setting an end or for "Can't get there". Fog lifts over 0.7 s; +No haptic for walk photos, for setting an end, for answering the end question or for "Can't get +there". Fog lifts over 0.7 s; button rows change with a spring of 0.4 s; the target ring pulses. **Verification hooks.** `STATE=wallWalk` on arrival. Each instruction change logs `GUIDANCE=`: `walk.left`, `walk.right`, `markEnd.left`, `markEnd.right`, `aimAtGround`, `aimAtWall`, `stepBack`, `walkComplete`. The engine log (category `engine`) records each end -("end left at s=… (limit)" or "(unexplored)"), "cannot access area during ", and "spatial +("end left at s=… (unexplored)", or "(limit)" when the autopilot sets one directly), each answer +("end left is limit" or "end left is unexplored"), "cannot access area during ", and "spatial reset: relocalization timed out". Accessibility identifiers: `action.markSomething`, -`action.markEnd`, `action.finishWalk`, `action.cannotAccess`, `action.markPoint`, +`action.markEnd`, `action.endCorner`, `action.endBlocked`, `action.finishWalk`, `action.cannotAccess`, `action.markPoint`, `action.cancelMarking`, `action.closeTray`, `feature.`, `wallTape`, `photoCount`, `instruction`, `relocalize.meterPhoto`. ## Edge cases -- Asked for the left end while aiming right of the meter, "Wall ends here" sets the right end. The - question about the left end stays up. +- Asked for the left end while aiming right of the meter, "Wall ends here" sets the right end and + asks "What's at the right end?". After the answer, "Is this the left end of the wall?" comes back. - A wall end or a door, window, gas meter or AC unit mark lands wherever the aim meets the wall's line extended, with no limit: past a corner, above the roof line or below ground. A door's bottom is raised to ground level. @@ -281,44 +302,47 @@ reset: relocalization timed out". Accessibility identifiers: `action.markSomethi A side can read 100 percent while none of it is covered and its ground is unseen. - Standing at the meter, VoiceOver says "You are 0 inches right of your meter". - "About N ft to go" can never appear on the walk instruction: the engine always passes no - remaining distance (`Runtime/ScanEngine.swift:644`). + remaining distance (`Runtime/ScanEngine.swift:699`). ## Open questions and verification -- **Suspected bug: a real wall end closer than the reach cannot be recorded.** "Wall ends here" is - shown only while the instruction is "Is this the … end of the wall?" - (`UI/Screens/WallWalkScreen.swift:99`, `:141`), and the planner asks that only once coverage - runs unbroken to its reach (`HouseScanKit/Guidance/GuidancePlanner.swift:113`). A homeowner - whose wall turns a corner 10 ft from the meter is told "Walk slowly to your left" with only - "Can't get there" to answer, which records an unexplored end instead of a real one. +- **Suspected bug: a blocked end closer than the reach cannot be recorded.** The end question + makes the kind of a marked end truthful, but "Wall ends here" is still shown only while the + instruction is "Is this the … end of the wall?" (`UI/Screens/WallWalkScreen.swift:101`, `:165`), + and the planner asks that only once coverage runs unbroken to 6.1 m, about 20 ft + (`HouseScanKit/Guidance/GuidancePlanner.swift:27`, `:113`). A homeowner whose wall stops at a + fence 10 ft from the meter is told "Walk slowly to your left" with only "Can't get there" to + answer, which records an unexplored end instead of a real one. A corner at 10 ft comes out right + by accident, since a corner is unexplored anyway. - **Suspected bug: no "Can't get there" at "Is this the … end of the wall?".** The pill is limited - to walk and tilt instructions (`WallWalkScreen.swift:105-110`), although the engine handles it - for this question (`Runtime/ScanEngine+Actions.swift:177`). A wall that keeps going can only be - answered with "Wall ends here", a real end. This sharpens B-06 in [bug-triage.md](../bug-triage.md), - which assumes both answers are offered. + to walk and tilt instructions (`WallWalkScreen.swift:107-112`), although the engine handles it + for this question (`Runtime/ScanEngine+Actions.swift:190`). A wall that goes on straight past + 20 ft can be recorded as unexplored only by marking an end where there is none and answering "It + turns a corner". This sharpens B-06 in [bug-triage.md](../bug-triage.md). - **Suspected bug: "Wall ends here" aims at a circle that is not drawn.** The end goes where the middle of the screen meets the wall (`ScanEngine+Actions.swift:68`), but the center circle is drawn only while marking (`WallWalkScreen.swift:28-33`). The ring on screen is the planner's guess at the end (`GuidancePlanner.swift:187-188`), so a homeowner who lines the ring up with the corner sets the end elsewhere. The hint promises "the circle in the middle of the screen". -- **Suspected bug: "Wall ends here" is unchecked and silent.** It accepts a tap with limited - tracking, unlike marks (`ScanEngine+Actions.swift:66-70` against `:80`); it does nothing and says - nothing when the aim misses the wall (`:68`); and it gives no haptic when it succeeds. +- **Suspected bug: "Wall ends here" is unchecked and silent on a miss.** It accepts a tap with + limited tracking, unlike marks (`ScanEngine+Actions.swift:66-74` against `:93`); it does nothing + and says nothing when the aim misses the wall (`:68`). A tap that lands gives no haptic; the end + question is its only feedback. - **Suspected bug: the 20 s reset keeps a mark in progress.** `resetSpatialState` - (`Runtime/ScanEngine.swift:386-410`) clears the wall, marks and photos but not the marking prompt + (`Runtime/ScanEngine.swift:431-456`) clears the wall, marks and photos but not the marking prompt or its placed taps. After the meter is found again, the walk opens straight into the old prompt ("Now tap its top-right corner"), and the finished feature joins a point from the lost world frame to one from the new frame. - **Suspected bug: two taps on "Can't get there" finish an empty walk.** On arrival each tap sets an - end at the meter (`ScanEngine+Actions.swift:179-181`); with both ends at the same point the + end at the meter (`ScanEngine+Actions.swift:192-194`); with both ends at the same point the planner finds nothing to fill (`GuidancePlanner.swift:138`) and says "That's the whole wall". - **Suspected bug: the AC unit prompt reads "Tap the ac unit".** The prompt lower-cases the kind's name (`UI/Copy/ScanCopy.swift:122`, `:137`); the picker tile's VoiceOver label has the same - problem ("Mark ac unit", `WallWalkScreen.swift:258`). + problem ("Mark ac unit", `WallWalkScreen.swift:282`). - **Possible bug: "That spot is behind the wall" judges the phone, not the tap.** The check is on - where the phone stands (`ScanEngine+Actions.swift:88`), so the message blames the spot when it is + where the phone stands (`ScanEngine+Actions.swift:101`), so the message blames the spot when it is the homeowner who is behind the wall line. -- The kind picker slides in under Reduce Motion (`WallWalkScreen.swift:140`). +- The kind picker slides in under Reduce Motion (`WallWalkScreen.swift:164`). - The reset returns to "Find your electric meter" without telling the homeowner that the walk was lost. The old meter close-up stays in the scan (`Runtime/KeyframeStore.swift:79-81` clears only walk photos); if the second close-up is skipped, the first one is uploaded. @@ -329,7 +353,8 @@ reset: relocalization timed out". Accessibility identifiers: `action.markSomethi (`UI/Screens/GapRequestScreen.swift:26`). The glossary uses the second for both. - Everything on this screen that needs the live camera is read from code: the fog's look on a real wall, the ring and chevron placement, the overlays hiding and the saved close-up while the phone -finds its place, the 20 s reset, and interruption recovery. The first + finds its place, the end question after a real "Wall ends here", the 20 s reset, and + interruption recovery. The first Simulator pass showed this screen's chrome without text (B-05 in [bug-triage.md](../bug-triage.md)). -Verified against house-scanning commit `0876e03` (t3/ios-mvf). +Verified against house-scanning commit `525ea40` (t3/ios-mvf). diff --git a/verification/product/verification.md b/verification/product/verification.md index bc45f4fce..054067ce9 100644 --- a/verification/product/verification.md +++ b/verification/product/verification.md @@ -15,9 +15,10 @@ what each shows and speaks, what the app logged, and what the server answered. W cover: anything needing the live camera (tracking, the meter tap on a real wall, a close-up that succeeds), haptics, or timing a person feels. -## Current pass: `t3/ios-mvf` `0876e03`, server `6a623d8` +## Current pass: `t3/ios-mvf` `525ea40`, server `26d2870` -Report `sim/20260926-050058-t3-ios-mvf-0876e03c-replay`, plus the variant runs on the preview +Report `sim/20260926-052535-t3-ios-mvf-525ea401-replay` (real server), the audit at `24af434` and +`525ea40` (sample result), plus the variant runs on the preview `5520229` (the same UI before S3's last fixes) named in [the UX review](../ux/review.md#evidence). | ID | Claim | Result | @@ -32,12 +33,14 @@ Report `sim/20260926-050058-t3-ios-mvf-0876e03c-replay`, plus the variant runs o | WW-02 | Guidance moves from walking to tilting to stepping back | Pass: the audit saw "Walk slowly to your left", "Tilt down to show the ground", "Take a step back". | | MF-01 | The list shows each mark with its distance, the window question and "Add something" | Pass at default size; hidden behind "Looks complete" at the largest text size (UX-01). | | GAP-01 | A gap request states what to show and why, with a progress bar and "I can't get there" | Pass ("Show the ground around your meter", "0%"). | +| GAP-02 | New photos close the request and the flow moves on | Pass: logged "gap 1 satisfied"; the audit saw "Got it, thanks" / "That's the view we needed." | +| WW-03 | After "Wall ends here" the walk asks what is at that end, and a corner stays unexplored | Pass: logged "end left at s=-3.81 (unexplored)", then "answered the left end: turns a corner"; same on the right. | | UP-01 | Upload progress is shown in steps apart from coverage | Pass: "Photos ready", "Sending photos, 99%", "Check clearances". | -| RES-01 | The result shows the headline, placement line, the server's summary, the rules-not-final note and the panel notice | Pass: "An installer will take a look", "13 ft 3 in left of your meter, 12 ft of cable". | +| RES-01 | The result shows the headline, placement line, the server's summary, the rules-not-final note and the panel notice | Pass: "An installer will take a look" with the server's placement and summary (seen at `0876e03`, `24af434` and `525ea40`). | | RES-02 | A borderline check shows the measurement, the rule and the error | Pass on the sample result: "Measured 3 ft 1 in. The rule is 3 ft, and the measurement can be off by about 4 in." | | RES-03 | A maximum reads as a maximum | Fail: "Measured 3 ft. The rule is 20 ft" (B-14). | | RES-04 | The result is visible when the app logs `STATE=result` | Fail, minor: the upload screen was still showing 1.2 s later on first load, in both passes (UX-07). | -| AR-01 | The AR view draws the spot and cable with one headline and "Done" | Pass: "Your battery could go here". On the sample result it has no sample badge (B-15). | +| AR-01 | The AR view draws the spot and cable with one headline and "Done" | Pass: "The spot an installer will check" for a manual review (`24af434`). | Superseded: at `21a63e7` the upload went to `/v1/scenes` and failed with 404, and the camera screens rendered without text. Both are fixed on `t3/ios-mvf`. diff --git a/verification/scoreboard/metrics.yaml b/verification/scoreboard/metrics.yaml index fcb8b9bf1..b0c61d687 100644 --- a/verification/scoreboard/metrics.yaml +++ b/verification/scoreboard/metrics.yaml @@ -279,11 +279,11 @@ metrics: ref: origin/t3/verification text: Product description written from the code and checked against screenshots manual: - status: partial + status: met evidence: >- - verification/product: scope, glossary, both foundations and the pilot drafted from S3's - integration commit 21a63e7; one Simulator pass recorded (verification.md); 7 suspected - defects in bug-triage.md. Seven screen documents to go. + verification/product covers every screen of t3/ios-mvf at 525ea40, checked in the + Simulator there with the real server's result (verification.md); bug-triage.md has 16 + entries, 3 resolved; ux/review.md judges every state. - id: S4-4 workstream: S4 diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index 78190db2c..eb5293c52 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,24 +1,24 @@ { - "generated_at": "2026-09-26T05:09:23-05:00", + "generated_at": "2026-09-26T05:40:43-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": false, "prs": [ { "number": 16, - "title": "experiments: meter close-up readability eval (S5)", + "title": "experiments: measure on-device meter close-up reading and retake thresholds", "branch": "t3/meter-closeup", - "head": "60a912bc884dd6d6defa4f4a94297e6ee86352ea", + "head": "03e04b54290b6dedd220eac4e4c4054bdf3c1e0c", "draft": true, - "updated": "2026-09-26T09:49:58Z", + "updated": "2026-09-26T10:36:35Z", "ci": { "counts": { - "success": 4, + "success": 5, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "4 success" + "text": "5 success" } }, { @@ -43,9 +43,9 @@ "number": 14, "title": "docs: trim AGENTS.md and fix stale decisions in the team docs", "branch": "t3/docs-cleanup", - "head": "9c9ceb2a608e9762bd002f20f1b979694bb0c92f", + "head": "ab03d7fd735884346f9182e3cb4cbf4aee60994a", "draft": false, - "updated": "2026-09-26T08:15:28Z", + "updated": "2026-09-26T10:17:43Z", "ci": { "counts": { "success": 4, @@ -61,9 +61,9 @@ "number": 13, "title": "verification: independent end-to-end checks (S4)", "branch": "t3/verification", - "head": "7c2c9ab5a5f4fc937d10cd8de0fd7904e61664e8", + "head": "e18a2eb7a213a5bd9d752a4fadccf09c50bcebf2", "draft": true, - "updated": "2026-09-26T09:28:49Z", + "updated": "2026-09-26T10:11:53Z", "ci": { "counts": { "success": 4, @@ -79,27 +79,29 @@ "number": 12, "title": "feat(evals): real-data accuracy evals for ARKit drift and photo-based wall reconstruction", "branch": "t3/evals", - "head": "f6d081d20df1063491a169c9365f1e295452b069", + "head": "7c5318d6610a06e657a98b0c8c7d1cc2acc53738", "draft": true, - "updated": "2026-09-26T09:57:22Z", + "updated": "2026-09-26T10:38:42Z", "ci": { "counts": { - "success": 5, - "failure": 0, + "success": 3, + "failure": 1, "pending": 0, "skipped": 0 }, - "failing": [], - "text": "5 success" + "failing": [ + "Vercel" + ], + "text": "3 success, 1 failure; failing: Vercel" } }, { "number": 11, "title": "server: placement solver, rules and API (S2)", "branch": "t3/server", - "head": "26d28707473f25f7c7d1f3660fbbf8321704f609", - "draft": true, - "updated": "2026-09-26T10:06:21Z", + "head": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", + "draft": false, + "updated": "2026-09-26T10:35:26Z", "ci": { "counts": { "success": 5, @@ -115,18 +117,18 @@ "number": 10, "title": "ios: guided wall scan, from meter to battery spot", "branch": "t3/ios-mvf", - "head": "24af4342bf760c4af7ed3f942c330533bed1f804", + "head": "525ea401641553fad581925023ed1bb82c0f7352", "draft": true, - "updated": "2026-09-26T10:02:50Z", + "updated": "2026-09-26T10:21:08Z", "ci": { "counts": { - "success": 4, + "success": 5, "failure": 0, - "pending": 1, + "pending": 0, "skipped": 0 }, "failing": [], - "text": "4 success, 1 pending" + "text": "5 success" } }, { @@ -187,9 +189,9 @@ "number": 6, "title": "docs: add research notes on capture, lane reviews and scoring", "branch": "t3/research", - "head": "a7d91f162c43e5e364e68d8ade0ad1002dd6b203", + "head": "9737e3f0eefe90f2a12a190bf8750e7fed64413f", "draft": false, - "updated": "2026-09-26T07:11:53Z", + "updated": "2026-09-26T10:13:56Z", "ci": { "counts": { "success": 4, @@ -228,7 +230,7 @@ "refs": [ { "ref": "origin/t3/evals", - "sha": "f6d081d20df1063491a169c9365f1e295452b069" + "sha": "7c5318d6610a06e657a98b0c8c7d1cc2acc53738" } ], "metrics": [ @@ -236,7 +238,7 @@ "id": "S1-1", "text": "One command regenerates every number from a clean checkout", "ref": "origin/t3/evals", - "sha": "f6d081d20df1063491a169c9365f1e295452b069", + "sha": "7c5318d6610a06e657a98b0c8c7d1cc2acc53738", "status": "partial", "evidence": "make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac.", "links": [] @@ -245,13 +247,13 @@ "id": "S1-2", "text": "Dataset URLs and sha256 recorded", "ref": "origin/t3/evals", - "sha": "f6d081d20df1063491a169c9365f1e295452b069", + "sha": "7c5318d6610a06e657a98b0c8c7d1cc2acc53738", "status": "met", "evidence": "all 2 found", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/README.md" ] ] }, @@ -259,7 +261,7 @@ "id": "S1-3", "text": "Metric code tested on hand-computed cases", "ref": "origin/t3/evals", - "sha": "f6d081d20df1063491a169c9365f1e295452b069", + "sha": "7c5318d6610a06e657a98b0c8c7d1cc2acc53738", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -268,21 +270,21 @@ "id": "S1-4", "text": "ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft", "ref": "origin/t3/evals", - "sha": "f6d081d20df1063491a169c9365f1e295452b069", + "sha": "7c5318d6610a06e657a98b0c8c7d1cc2acc53738", "status": "met", "evidence": "all 5 found", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/README.md" ], [ "experiments/evals/results/advio_drift.md", - "https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/results/advio_drift.md" + "https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/advio_drift.md" ], [ "experiments/evals/results/eth3d_recon.md", - "https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/results/eth3d_recon.md" + "https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/eth3d_recon.md" ] ] }, @@ -290,21 +292,21 @@ "id": "S1-5", "text": "ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question", "ref": "origin/t3/evals", - "sha": "f6d081d20df1063491a169c9365f1e295452b069", + "sha": "7c5318d6610a06e657a98b0c8c7d1cc2acc53738", "status": "partial", "evidence": "found 1-3 m row, 3-10 m row, scale error; missing verdict", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/README.md" ], [ "experiments/evals/results/eth3d_recon.md", - "https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/results/eth3d_recon.md" + "https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/eth3d_recon.md" ], [ "experiments/evals/results/eth3d_visibility_sensitivity.md", - "https://github.com/SamGu-NRX/house-scanning/blob/f6d081d20df1063491a169c9365f1e295452b069/experiments/evals/results/eth3d_visibility_sensitivity.md" + "https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/eth3d_visibility_sensitivity.md" ] ] } @@ -316,7 +318,7 @@ "refs": [ { "ref": "origin/t3/server", - "sha": "26d28707473f25f7c7d1f3660fbbf8321704f609" + "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03" } ], "metrics": [ @@ -324,7 +326,7 @@ "id": "S2-1", "text": "pytest green", "ref": "origin/t3/server", - "sha": "26d28707473f25f7c7d1f3660fbbf8321704f609", + "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -333,13 +335,13 @@ "id": "S2-2", "text": "Every public golden test 01-14 of docs/research/t3-lane-c-review.md present", "ref": "origin/t3/server", - "sha": "26d28707473f25f7c7d1f3660fbbf8321704f609", + "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", "status": "met", "evidence": "all 14 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_golden_public.py" ] ] }, @@ -347,13 +349,13 @@ "id": "S2-3", "text": "Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail", "ref": "origin/t3/server", - "sha": "26d28707473f25f7c7d1f3660fbbf8321704f609", + "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", "status": "partial", "evidence": "found missing coverage, mirror; missing equality unsure, larger clearance", "links": [ [ "server/tests/test_properties.py", - "https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_properties.py" + "https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_properties.py" ] ] }, @@ -361,17 +363,17 @@ "id": "S2-4", "text": "Schemas published", "ref": "origin/t3/server", - "sha": "26d28707473f25f7c7d1f3660fbbf8321704f609", + "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", "status": "met", - "evidence": "present at 26d28707", + "evidence": "present at f2705dd5", "links": [ [ "server/schemas/scene.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/schemas/scene.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/schemas/scene.schema.json" ], [ "server/schemas/result.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/schemas/result.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/schemas/result.schema.json" ] ] }, @@ -379,21 +381,21 @@ "id": "S2-5", "text": "Schemas validated in tests", "ref": "origin/t3/server", - "sha": "26d28707473f25f7c7d1f3660fbbf8321704f609", + "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", "status": "met", "evidence": "all 2 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_golden_public.py" ], [ "server/tests/test_schemas.py", - "https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_schemas.py" + "https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_schemas.py" ], [ "server/tests/test_api.py", - "https://github.com/SamGu-NRX/house-scanning/blob/26d28707473f25f7c7d1f3660fbbf8321704f609/server/tests/test_api.py" + "https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_api.py" ] ] }, @@ -401,16 +403,16 @@ "id": "S2-6", "text": "A real-derived scene returns a result in under 1 s", "ref": "origin/t3/server", - "sha": "26d28707473f25f7c7d1f3660fbbf8321704f609", - "status": "met", - "evidence": "latency_ms = 970.4 (want < 1000): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json", + "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", + "status": "stale", + "evidence": "report for 26d28707, ref at f2705dd5: ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json", "links": [] }, { "id": "S2-7", "text": "ruff clean", "ref": "origin/t3/server", - "sha": "26d28707473f25f7c7d1f3660fbbf8321704f609", + "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -419,9 +421,9 @@ "id": "S2-8", "text": "Black-box (verification e2e): every answer valid under C2, consistent with its decision and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric when mirrored, and never passing without coverage", "ref": "origin/t3/server", - "sha": "26d28707473f25f7c7d1f3660fbbf8321704f609", - "status": "met", - "evidence": "contract_problem_count = 0 (want 0): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json", + "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", + "status": "stale", + "evidence": "report for 26d28707, ref at f2705dd5: ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json", "links": [] } ] @@ -432,7 +434,7 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "24af4342bf760c4af7ed3f942c330533bed1f804" + "sha": "525ea401641553fad581925023ed1bb82c0f7352" } ], "metrics": [ @@ -440,9 +442,9 @@ "id": "S3-1", "text": "CI build green", "ref": "origin/t3/ios-mvf", - "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", - "status": "not run", - "evidence": "iOS build: pending at 24af4342", + "sha": "525ea401641553fad581925023ed1bb82c0f7352", + "status": "met", + "evidence": "iOS build: success at 525ea401", "links": [ [ "#10", @@ -454,13 +456,13 @@ "id": "S3-2", "text": "Built as Swift 6 with strict concurrency", "ref": "origin/t3/ios-mvf", - "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", + "sha": "525ea401641553fad581925023ed1bb82c0f7352", "status": "met", "evidence": "all 2 found", "links": [ [ "ios/project.yml", - "https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/project.yml" + "https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/project.yml" ] ] }, @@ -468,30 +470,30 @@ "id": "S3-3", "text": "Zero build warnings", "ref": "origin/t3/ios-mvf", - "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", - "status": "stale", - "evidence": "report for 0876e03c, ref at 24af4342: ~/house-scanning-data/reports/sim/20260926-050058-t3-ios-mvf-0876e03c-replay/report.json", + "sha": "525ea401641553fad581925023ed1bb82c0f7352", + "status": "met", + "evidence": "build.warnings = length 0 (want 0): ~/house-scanning-data/reports/sim/20260926-052535-t3-ios-mvf-525ea401-replay/report.json", "links": [] }, { "id": "S3-4", "text": "Unit tests for coverage, auto-capture and guidance", "ref": "origin/t3/ios-mvf", - "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", + "sha": "525ea401641553fad581925023ed1bb82c0f7352", "status": "partial", "evidence": "found coverage; missing auto-capture, guidance", "links": [ [ "ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift" ] ] }, @@ -499,17 +501,17 @@ "id": "S3-5", "text": "UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop", "ref": "origin/t3/ios-mvf", - "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", + "sha": "525ea401641553fad581925023ed1bb82c0f7352", "status": "partial", "evidence": "all 3 found (the tests exist; passing is S3-1)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanUITests/FullFlowUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -517,17 +519,17 @@ "id": "S3-6", "text": "Accessibility audit on every screen", "ref": "origin/t3/ios-mvf", - "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", + "sha": "525ea401641553fad581925023ed1bb82c0f7352", "status": "partial", "evidence": "3 matches in 2 files (an audit exists; which screens it covers needs a read)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanUITests/FullFlowUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/24af4342bf760c4af7ed3f942c330533bed1f804/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -535,16 +537,16 @@ "id": "S3-6b", "text": "Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, plus symbol-name button labels) finds no issues; S4-2 shows how many screens the app reached", "ref": "origin/t3/ios-mvf", - "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", - "status": "no evidence yet", - "evidence": "no report matches ~/house-scanning-data/reports/a11y/*t3-ios-mvf*/report.json", + "sha": "525ea401641553fad581925023ed1bb82c0f7352", + "status": "gap", + "evidence": "issue_count = 32 (want 0): ~/house-scanning-data/reports/a11y/20260926-052756-t3-ios-mvf-525ea401-head-sample/report.json", "links": [] }, { "id": "S3-7", "text": "Screenshots of each state in the PR", "ref": "origin/t3/ios-mvf", - "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", + "sha": "525ea401641553fad581925023ed1bb82c0f7352", "status": "gap", "evidence": "0 of 9 matches in the PR body", "links": [ @@ -558,7 +560,7 @@ "id": "S3-8", "text": "Exported scene.json validates against C1", "ref": "origin/t3/ios-mvf", - "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", + "sha": "525ea401641553fad581925023ed1bb82c0f7352", "status": "met", "evidence": "app_export_scene_valid = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json", "links": [] @@ -567,7 +569,7 @@ "id": "S3-9", "text": "Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design", "ref": "origin/t3/ios-mvf", - "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", + "sha": "525ea401641553fad581925023ed1bb82c0f7352", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -580,11 +582,11 @@ "refs": [ { "ref": "origin/t3/verification", - "sha": "7c2c9ab5a5f4fc937d10cd8de0fd7904e61664e8" + "sha": "e18a2eb7a213a5bd9d752a4fadccf09c50bcebf2" }, { "ref": "origin/t3/ios-mvf", - "sha": "24af4342bf760c4af7ed3f942c330533bed1f804" + "sha": "525ea401641553fad581925023ed1bb82c0f7352" } ], "metrics": [ @@ -592,7 +594,7 @@ "id": "S4-1", "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/verification", - "sha": "7c2c9ab5a5f4fc937d10cd8de0fd7904e61664e8", + "sha": "e18a2eb7a213a5bd9d752a4fadccf09c50bcebf2", "status": "met", "evidence": "app_export_passed = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json", "links": [] @@ -601,31 +603,31 @@ "id": "S4-2", "text": "Screenshot report per app state", "ref": "origin/t3/ios-mvf", - "sha": "24af4342bf760c4af7ed3f942c330533bed1f804", - "status": "stale", - "evidence": "report for 0876e03c, ref at 24af4342: ~/house-scanning-data/reports/sim/20260926-050058-t3-ios-mvf-0876e03c-replay/report.json", + "sha": "525ea401641553fad581925023ed1bb82c0f7352", + "status": "met", + "evidence": "states = length 10 (want at least 9): ~/house-scanning-data/reports/sim/20260926-052535-t3-ios-mvf-525ea401-replay/report.json", "links": [] }, { "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "7c2c9ab5a5f4fc937d10cd8de0fd7904e61664e8", - "status": "partial", - "evidence": "verification/product: scope, glossary, both foundations and the pilot drafted from S3's integration commit 21a63e7; one Simulator pass recorded (verification.md); 7 suspected defects in bug-triage.md. Seven screen documents to go.", + "sha": "e18a2eb7a213a5bd9d752a4fadccf09c50bcebf2", + "status": "met", + "evidence": "verification/product covers every screen of t3/ios-mvf at 525ea40, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state.", "links": [] }, { "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "7c2c9ab5a5f4fc937d10cd8de0fd7904e61664e8", + "sha": "e18a2eb7a213a5bd9d752a4fadccf09c50bcebf2", "status": "met", - "evidence": "present at 7c2c9ab5", + "evidence": "present at e18a2eb7", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/7c2c9ab5a5f4fc937d10cd8de0fd7904e61664e8/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/e18a2eb7a213a5bd9d752a4fadccf09c50bcebf2/verification/SCOREBOARD.md" ] ] } @@ -637,7 +639,7 @@ "refs": [ { "ref": "origin/t3/docs-cleanup", - "sha": "9c9ceb2a608e9762bd002f20f1b979694bb0c92f" + "sha": "ab03d7fd735884346f9182e3cb4cbf4aee60994a" } ], "metrics": [ @@ -645,7 +647,7 @@ "id": "M-1", "text": "AGENTS.md shorter than on main", "ref": "origin/t3/docs-cleanup", - "sha": "9c9ceb2a608e9762bd002f20f1b979694bb0c92f", + "sha": "ab03d7fd735884346f9182e3cb4cbf4aee60994a", "status": "met", "evidence": "exit 0: AGENTS.md 39 lines, main 45", "links": [] @@ -654,7 +656,7 @@ "id": "M-2", "text": "AGENTS.md accurate, every path and command real", "ref": "origin/t3/docs-cleanup", - "sha": "9c9ceb2a608e9762bd002f20f1b979694bb0c92f", + "sha": "ab03d7fd735884346f9182e3cb4cbf4aee60994a", "status": "gap", "evidence": "not yet reviewed", "links": [] diff --git a/verification/ux/review.md b/verification/ux/review.md index 9fd34fd6f..6c8c2c8bb 100644 --- a/verification/ux/review.md +++ b/verification/ux/review.md @@ -7,7 +7,11 @@ audit reports stay in `~/house-scanning-data/reports/` (replay frames are non-co | Run | What it shows | | --- | --- | -| `sim/20260926-050058-t3-ios-mvf-0876e03c-replay` | Every state at `t3/ios-mvf` `0876e03` with the real server's result | +| `sim/20260926-051323-t3-ios-mvf-24af4342-replay` | Every state at `t3/ios-mvf` `24af434` with the real server's result | +| `sim/20260926-052535-t3-ios-mvf-525ea401-replay` | Every state at `525ea40`, the current head, with the real server's result and the end question answered | +| `a11y/20260926-052756-t3-ios-mvf-525ea401-head-sample` | Apple's audit at `525ea40` on 21 screens, including the end question and the sample wording | +| `a11y/20260926-051535-t3-ios-mvf-24af4342-head-sample` | Apple's audit at `24af434` on 17 screens; the gap loop closes on the replay | +| `sim/20260926-050058-t3-ios-mvf-0876e03c-replay` | Every state at `0876e03` | | `sim/20260926-043152-5520229-5520229d-preview-sample` | Every state of the flow, default text size, light appearance, sample result | | `sim/20260926-044328-5520229-5520229d-preview-ax5-dark` | The same at the largest accessibility text size (AX5), dark appearance | | `sim/20260926-044527-5520229-5520229d-preview-unsupported` | The unsupported-phone screen | @@ -33,12 +37,12 @@ missed feature. | `unsupported` | Pass: "This phone can't measure walls" with what to do instead. Camera denied never reaches this screen. | B-03 | | `findMeter` | Pass at default size: one instruction, the reticle, "This is my meter". At AX5 the button is pushed partly off screen. Refusal wording ("Step a little closer to the wall") also covers tracking refusals. | UX-01, B-11 | | `meterCloseUp` | Pass: one instruction, no shutter, a named fix ("Center the meter in the circle"). The problem pill fails contrast; the way out can fail to appear. | UX-03, B-04 | -| `wallWalk` | One instruction with its reply inside the card, as the checklist asks. The strip's states differ by colour alone, the counter fails contrast, and at AX5 the card covers most of the camera and pushes "Mark something" and the strip off screen. Wall ends can't be recorded truthfully. | UX-01, UX-02, UX-03, UX-05, B-06 | +| `wallWalk` | One instruction with its reply inside the card, as the checklist asks. The strip's states differ by colour alone, the counter fails contrast, and at AX5 the card covers most of the camera and pushes "Mark something" and the strip off screen. At `525ea40` a marked end is followed by "What's at the left end?", so its kind is truthful, but an end closer than 20 ft still can't be marked. | UX-01, UX-02, UX-03, UX-05, UX-08, B-06 | | `markFeatures` | Pass at default size: clear list, remove buttons labelled, window question with two full-width answers. At AX5 the list is hidden behind "Looks complete". "Add something" does nothing. | UX-01, B-09 | | `gapRequest` | Pass: one line with the reason and target, a progress bar, "I can't get there". The requested stretch reuses the amber that means "seen". | UX-05, UX-06, B-08 | -| `uploading` | Pass: progress in its own steps, apart from coverage (R7). Four text elements do not scale. A failure shows raw error text. | UX-04, B-02 | +| `uploading` | Pass: progress in its own steps, apart from coverage (R7); with no server it says "Making a sample result" (`525ea40`). Four text elements do not scale. A failure shows raw error text. | UX-04, B-02 | | `result` | Pass on honesty: "An installer will take a look", the rules-not-final note, the unseen side disclosed, the borderline window check with measurement, rule and error, and "An installer will check this". A maximum reads like a minimum; the first load leaves the upload screen up for over 1.2 s. | B-14, UX-07 | -| `resultAR` | The spot and cable over the camera with one headline and "Done". It says "Your battery could go here" without the sample badge. | B-15 | +| `resultAR` | Pass: the spot and cable over the camera with one headline and "Done". The headline says "The spot an installer will check" unless the result is an approved pass, and "Example spot, not your result" for a sample (seen at `525ea40`). | none | Checks that passed across the flow: one primary instruction per camera screen (I1); plain words, no jargon in any on-screen string (I3); no shutter where capture is automatic (I5); every control @@ -62,10 +66,12 @@ homeowner with red-green colour blindness cannot tell done from not done. Sugges height difference per state, and "Wall" and "Ground" row labels. **UX-03. Minor. Contrast over the camera (A5).** The audit fails contrast on the photo counter -during the walk and on the close-up's problem pill. +during the walk, the close-up's problem pill and, at `24af434`, the walk's "Can't get there" +reply. Failures it reported on text mid-transition (the upload steps, the result headline as it +fades in) are left out. -**UX-04. Minor. Upload text that does not scale (A4).** The audit reports four elements on the -upload screen whose font size cannot change. +**UX-04. Minor. Text that does not scale (A4).** The audit reports elements whose font size cannot +change on the upload screen (four) and, at `525ea40`, on the close-up (four) and the walk (two). **UX-05. Minor. Two names for one action (V8).** The walk's reply is "Can't get there"; the gap request's is "I can't get there". @@ -77,6 +83,10 @@ marks "seen but not enough" cells on the same strip. ("Checking your wall") was still showing 1.2 s after the app logged the result; after returning from AR the result appeared at once. The 3D model's first load is the likely cause. +**UX-08. Minor. Text VoiceOver cannot read (A2).** At `525ea40` the audit flags text on the walk +that is drawn rather than exposed to accessibility, probably labels drawn into the camera overlay +or the strip. + ## Fixed since the preview (`0876e03`) Seen: the window question answers "It opens" and "It stays shut", stacked (I4), and the result From 4b4425b5cf899c1c43d78b75f2172a77aaa24c82 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 05:43:49 -0500 Subject: [PATCH 27/59] verification: scoreboard with the slow probes run S2's pytest and ruff and S1's metric tests now run against each branch head instead of showing "not run". Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/SCOREBOARD.md | 18 +++++------ verification/scoreboard/scoreboard.json | 42 +++++++++++++------------ 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index d4c8cacc6..42fb8cd01 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,6 +1,6 @@ # Scoreboard -Generated 2026-09-26 05:40 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-26 05:43 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran. Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests @@ -9,7 +9,7 @@ Generated 2026-09-26 05:40 CDT by `make scoreboard` (`uv run python -m hsverify. | [#16](https://github.com/SamGu-NRX/house-scanning/pull/16) | `t3/meter-closeup` | `03e04b54` | yes | 5 success | 2026-09-26 05:36 CDT | | [#15](https://github.com/SamGu-NRX/house-scanning/pull/15) | `t3/web-review` | `c69f8880` | yes | 5 success | 2026-09-26 05:06 CDT | | [#14](https://github.com/SamGu-NRX/house-scanning/pull/14) | `t3/docs-cleanup` | `ab03d7fd` | no | 4 success | 2026-09-26 05:17 CDT | -| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `e18a2eb7` | yes | 4 success | 2026-09-26 05:11 CDT | +| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `60fe0d32` | yes | 2 success, 1 failure; failing: Vercel | 2026-09-26 05:41 CDT | | [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `7c5318d6` | yes | 3 success, 1 failure; failing: Vercel | 2026-09-26 05:38 CDT | | [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `f2705dd5` | no | 5 success | 2026-09-26 05:35 CDT | | [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `525ea401` | yes | 5 success | 2026-09-26 05:21 CDT | @@ -21,29 +21,29 @@ Generated 2026-09-26 05:40 CDT by `make scoreboard` (`uv run python -m hsverify. ## S1 Evals on real data -`origin/t3/evals` at [`7c5318d6`](https://github.com/SamGu-NRX/house-scanning/tree/7c5318d6610a06e657a98b0c8c7d1cc2acc53738). 2 met, 2 partial, 1 not run. +`origin/t3/evals` at [`7c5318d6`](https://github.com/SamGu-NRX/house-scanning/tree/7c5318d6610a06e657a98b0c8c7d1cc2acc53738). 2 met, 3 partial. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S1-1 | One command regenerates every number from a clean checkout | partial | make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac. | | S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/README.md) | -| S1-3 | Metric code tested on hand-computed cases | not run | slow; run with --slow | +| S1-3 | Metric code tested on hand-computed cases | partial | exit 0: Installed 20 packages in 60ms / .................................................. [100%] / 50 passed in 1.60s (tests pass; whether the cases are hand-computed needs a read) | | S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/README.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/advio_drift.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/eth3d_recon.md) | | S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/README.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/eth3d_visibility_sensitivity.md) | ## S2 Server and solver -`origin/t3/server` at [`f2705dd5`](https://github.com/SamGu-NRX/house-scanning/tree/f2705dd5580ccd8c18ebd34a743b448bcb59ae03). 3 met, 1 partial, 2 stale, 2 not run. +`origin/t3/server` at [`f2705dd5`](https://github.com/SamGu-NRX/house-scanning/tree/f2705dd5580ccd8c18ebd34a743b448bcb59ae03). 5 met, 1 partial, 2 stale. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S2-1 | pytest green | not run | slow; run with --slow | +| S2-1 | pytest green | met | exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 222 passed, 1 warning in 11.01s | | S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_golden_public.py) | | S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_properties.py) | | S2-4 | Schemas published | met | present at f2705dd5 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/schemas/result.schema.json) | | S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_schemas.py), [server/tests/test_api.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_api.py) | | S2-6 | A real-derived scene returns a result in under 1 s | stale | report for 26d28707, ref at f2705dd5: ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | -| S2-7 | ruff clean | not run | slow; run with --slow | +| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 37ms / All checks passed! / 17 files already formatted | | S2-8 | Black-box (verification e2e): every answer valid under C2, consistent with its decision and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric when mirrored, and never passing without coverage | stale | report for 26d28707, ref at f2705dd5: ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | ## S3 iOS capture app @@ -65,14 +65,14 @@ Generated 2026-09-26 05:40 CDT by `make scoreboard` (`uv run python -m hsverify. ## S4 Verification -`origin/t3/verification` at [`e18a2eb7`](https://github.com/SamGu-NRX/house-scanning/tree/e18a2eb7a213a5bd9d752a4fadccf09c50bcebf2), `origin/t3/ios-mvf` at [`525ea401`](https://github.com/SamGu-NRX/house-scanning/tree/525ea401641553fad581925023ed1bb82c0f7352). 4 met. +`origin/t3/verification` at [`60fe0d32`](https://github.com/SamGu-NRX/house-scanning/tree/60fe0d327dfd0ac56b5570ffdc9daf24d8760a39), `origin/t3/ios-mvf` at [`525ea401`](https://github.com/SamGu-NRX/house-scanning/tree/525ea401641553fad581925023ed1bb82c0f7352). 4 met. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S4-1 | End to end from a real replay to a placement result passes | met | app_export_passed = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | | S4-2 | Screenshot report per app state | met | states = length 10 (want at least 9): ~/house-scanning-data/reports/sim/20260926-052535-t3-ios-mvf-525ea401-replay/report.json | | S4-3 | Product description written from the code and checked against screenshots | met | verification/product covers every screen of t3/ios-mvf at 525ea40, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state. | -| S4-4 | Scoreboard of every PR | met | present at e18a2eb7 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/e18a2eb7a213a5bd9d752a4fadccf09c50bcebf2/verification/SCOREBOARD.md) | +| S4-4 | Scoreboard of every PR | met | present at 60fe0d32 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/60fe0d327dfd0ac56b5570ffdc9daf24d8760a39/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index eb5293c52..b0ab5601a 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,7 +1,7 @@ { - "generated_at": "2026-09-26T05:40:43-05:00", + "generated_at": "2026-09-26T05:43:41-05:00", "command": "uv run python -m hsverify.scoreboard", - "slow": false, + "slow": true, "prs": [ { "number": 16, @@ -61,18 +61,20 @@ "number": 13, "title": "verification: independent end-to-end checks (S4)", "branch": "t3/verification", - "head": "e18a2eb7a213a5bd9d752a4fadccf09c50bcebf2", + "head": "60fe0d327dfd0ac56b5570ffdc9daf24d8760a39", "draft": true, - "updated": "2026-09-26T10:11:53Z", + "updated": "2026-09-26T10:41:29Z", "ci": { "counts": { - "success": 4, - "failure": 0, + "success": 2, + "failure": 1, "pending": 0, "skipped": 0 }, - "failing": [], - "text": "4 success" + "failing": [ + "Vercel" + ], + "text": "2 success, 1 failure; failing: Vercel" } }, { @@ -262,8 +264,8 @@ "text": "Metric code tested on hand-computed cases", "ref": "origin/t3/evals", "sha": "7c5318d6610a06e657a98b0c8c7d1cc2acc53738", - "status": "not run", - "evidence": "slow; run with --slow", + "status": "partial", + "evidence": "exit 0: Installed 20 packages in 60ms / .................................................. [100%] / 50 passed in 1.60s (tests pass; whether the cases are hand-computed needs a read)", "links": [] }, { @@ -327,8 +329,8 @@ "text": "pytest green", "ref": "origin/t3/server", "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", - "status": "not run", - "evidence": "slow; run with --slow", + "status": "met", + "evidence": "exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 222 passed, 1 warning in 11.01s", "links": [] }, { @@ -413,8 +415,8 @@ "text": "ruff clean", "ref": "origin/t3/server", "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", - "status": "not run", - "evidence": "slow; run with --slow", + "status": "met", + "evidence": "exit 0: Installed 33 packages in 37ms / All checks passed! / 17 files already formatted", "links": [] }, { @@ -582,7 +584,7 @@ "refs": [ { "ref": "origin/t3/verification", - "sha": "e18a2eb7a213a5bd9d752a4fadccf09c50bcebf2" + "sha": "60fe0d327dfd0ac56b5570ffdc9daf24d8760a39" }, { "ref": "origin/t3/ios-mvf", @@ -594,7 +596,7 @@ "id": "S4-1", "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/verification", - "sha": "e18a2eb7a213a5bd9d752a4fadccf09c50bcebf2", + "sha": "60fe0d327dfd0ac56b5570ffdc9daf24d8760a39", "status": "met", "evidence": "app_export_passed = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json", "links": [] @@ -612,7 +614,7 @@ "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "e18a2eb7a213a5bd9d752a4fadccf09c50bcebf2", + "sha": "60fe0d327dfd0ac56b5570ffdc9daf24d8760a39", "status": "met", "evidence": "verification/product covers every screen of t3/ios-mvf at 525ea40, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state.", "links": [] @@ -621,13 +623,13 @@ "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "e18a2eb7a213a5bd9d752a4fadccf09c50bcebf2", + "sha": "60fe0d327dfd0ac56b5570ffdc9daf24d8760a39", "status": "met", - "evidence": "present at e18a2eb7", + "evidence": "present at 60fe0d32", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/e18a2eb7a213a5bd9d752a4fadccf09c50bcebf2/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/60fe0d327dfd0ac56b5570ffdc9daf24d8760a39/verification/SCOREBOARD.md" ] ] } From 5e60831dcc87ca0b854b4ce51c1e8b220438fc7a Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 06:13:39 -0500 Subject: [PATCH 28/59] verification: regenerate scoreboard Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/SCOREBOARD.md | 52 ++++---- verification/scoreboard/scoreboard.json | 164 ++++++++++++------------ 2 files changed, 111 insertions(+), 105 deletions(-) diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index 42fb8cd01..c1a09b378 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,18 +1,18 @@ # Scoreboard -Generated 2026-09-26 05:43 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran. Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-26 06:13 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | -| [#16](https://github.com/SamGu-NRX/house-scanning/pull/16) | `t3/meter-closeup` | `03e04b54` | yes | 5 success | 2026-09-26 05:36 CDT | -| [#15](https://github.com/SamGu-NRX/house-scanning/pull/15) | `t3/web-review` | `c69f8880` | yes | 5 success | 2026-09-26 05:06 CDT | +| [#16](https://github.com/SamGu-NRX/house-scanning/pull/16) | `t3/meter-closeup` | `c5cb30d7` | yes | 3 success, 1 failure; failing: Vercel | 2026-09-26 06:13 CDT | +| [#15](https://github.com/SamGu-NRX/house-scanning/pull/15) | `t3/web-review` | `a7c9b129` | yes | 3 success, 3 failure; failing: Label PR size, Web checks, Vercel | 2026-09-26 06:08 CDT | | [#14](https://github.com/SamGu-NRX/house-scanning/pull/14) | `t3/docs-cleanup` | `ab03d7fd` | no | 4 success | 2026-09-26 05:17 CDT | -| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `60fe0d32` | yes | 2 success, 1 failure; failing: Vercel | 2026-09-26 05:41 CDT | -| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `7c5318d6` | yes | 3 success, 1 failure; failing: Vercel | 2026-09-26 05:38 CDT | -| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `f2705dd5` | no | 5 success | 2026-09-26 05:35 CDT | -| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `525ea401` | yes | 5 success | 2026-09-26 05:21 CDT | +| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `4b4425b5` | yes | 2 success, 1 failure; failing: Vercel | 2026-09-26 05:53 CDT | +| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `779d3391` | yes | 3 success, 1 failure; failing: Vercel | 2026-09-26 06:03 CDT | +| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `f2705dd5` | no | 5 success | 2026-09-26 05:55 CDT | +| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `beede15f` | yes | 4 success, 1 pending | 2026-09-26 06:12 CDT | | [#9](https://github.com/SamGu-NRX/house-scanning/pull/9) | `codex/capture-contract-probe` | `4b6c4309` | yes | 4 success | 2026-09-26 00:08 CDT | | [#8](https://github.com/SamGu-NRX/house-scanning/pull/8) | `t3/testflight` | `b896d59f` | no | 4 success | 2026-09-26 03:18 CDT | | [#7](https://github.com/SamGu-NRX/house-scanning/pull/7) | `t3/measure-lab` | `45770a3b` | no | 6 success | 2026-09-26 03:19 CDT | @@ -21,58 +21,58 @@ Generated 2026-09-26 05:43 CDT by `make scoreboard` (`uv run python -m hsverify. ## S1 Evals on real data -`origin/t3/evals` at [`7c5318d6`](https://github.com/SamGu-NRX/house-scanning/tree/7c5318d6610a06e657a98b0c8c7d1cc2acc53738). 2 met, 3 partial. +`origin/t3/evals` at [`779d3391`](https://github.com/SamGu-NRX/house-scanning/tree/779d33918a4d4dd997364b5a9345392ac9cff45e). 2 met, 2 partial, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S1-1 | One command regenerates every number from a clean checkout | partial | make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac. | -| S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/README.md) | -| S1-3 | Metric code tested on hand-computed cases | partial | exit 0: Installed 20 packages in 60ms / .................................................. [100%] / 50 passed in 1.60s (tests pass; whether the cases are hand-computed needs a read) | -| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/README.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/advio_drift.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/eth3d_recon.md) | -| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/README.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/eth3d_visibility_sensitivity.md) | +| S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/README.md) | +| S1-3 | Metric code tested on hand-computed cases | not run | slow; run with --slow | +| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/README.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/results/advio_drift.md), [experiments/evals/results/modern_arkit.md](https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/results/modern_arkit.md) | +| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/README.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/results/eth3d_visibility_sensitivity.md) | ## S2 Server and solver -`origin/t3/server` at [`f2705dd5`](https://github.com/SamGu-NRX/house-scanning/tree/f2705dd5580ccd8c18ebd34a743b448bcb59ae03). 5 met, 1 partial, 2 stale. +`origin/t3/server` at [`f2705dd5`](https://github.com/SamGu-NRX/house-scanning/tree/f2705dd5580ccd8c18ebd34a743b448bcb59ae03). 3 met, 1 partial, 2 stale, 2 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S2-1 | pytest green | met | exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 222 passed, 1 warning in 11.01s | +| S2-1 | pytest green | not run | slow; last run 2026-09-26 05:43 CDT: met, exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 222 passed, 1 warning in 11.01s | | S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_golden_public.py) | | S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_properties.py) | | S2-4 | Schemas published | met | present at f2705dd5 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/schemas/result.schema.json) | | S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_schemas.py), [server/tests/test_api.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_api.py) | | S2-6 | A real-derived scene returns a result in under 1 s | stale | report for 26d28707, ref at f2705dd5: ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | -| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 37ms / All checks passed! / 17 files already formatted | +| S2-7 | ruff clean | not run | slow; last run 2026-09-26 05:43 CDT: met, exit 0: Installed 33 packages in 37ms / All checks passed! / 17 files already formatted | | S2-8 | Black-box (verification e2e): every answer valid under C2, consistent with its decision and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric when mirrored, and never passing without coverage | stale | report for 26d28707, ref at f2705dd5: ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | ## S3 iOS capture app -`origin/t3/ios-mvf` at [`525ea401`](https://github.com/SamGu-NRX/house-scanning/tree/525ea401641553fad581925023ed1bb82c0f7352). 4 met, 3 partial, 3 gap. +`origin/t3/ios-mvf` at [`beede15f`](https://github.com/SamGu-NRX/house-scanning/tree/beede15f568b3a4d694fb275caf9eaaa882c546b). 2 met, 3 partial, 2 gap, 2 stale, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S3-1 | CI build green | met | iOS build: success at 525ea401 [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/project.yml) | -| S3-3 | Zero build warnings | met | build.warnings = length 0 (want 0): ~/house-scanning-data/reports/sim/20260926-052535-t3-ios-mvf-525ea401-replay/report.json | -| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage; missing auto-capture, guidance [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift) | -| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, plus symbol-name button labels) finds no issues; S4-2 shows how many screens the app reached | gap | issue_count = 32 (want 0): ~/house-scanning-data/reports/a11y/20260926-052756-t3-ios-mvf-525ea401-head-sample/report.json | +| S3-1 | CI build green | not run | iOS build: pending at beede15f [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | +| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/project.yml) | +| S3-3 | Zero build warnings | stale | report for 525ea401, ref at beede15f: ~/house-scanning-data/reports/sim/20260926-052535-t3-ios-mvf-525ea401-replay/report.json | +| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage; missing auto-capture, guidance [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift) | +| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, plus symbol-name button labels) finds no issues; S4-2 shows how many screens the app reached | stale | report for 525ea401, ref at beede15f: ~/house-scanning-data/reports/a11y/20260926-052756-t3-ios-mvf-525ea401-head-sample/report.json | | S3-7 | Screenshots of each state in the PR | gap | 0 of 9 matches in the PR body [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | | S3-8 | Exported scene.json validates against C1 | met | app_export_scene_valid = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/verification` at [`60fe0d32`](https://github.com/SamGu-NRX/house-scanning/tree/60fe0d327dfd0ac56b5570ffdc9daf24d8760a39), `origin/t3/ios-mvf` at [`525ea401`](https://github.com/SamGu-NRX/house-scanning/tree/525ea401641553fad581925023ed1bb82c0f7352). 4 met. +`origin/t3/verification` at [`4b4425b5`](https://github.com/SamGu-NRX/house-scanning/tree/4b4425b5cf899c1c43d78b75f2172a77aaa24c82), `origin/t3/ios-mvf` at [`beede15f`](https://github.com/SamGu-NRX/house-scanning/tree/beede15f568b3a4d694fb275caf9eaaa882c546b). 3 met, 1 stale. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S4-1 | End to end from a real replay to a placement result passes | met | app_export_passed = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | -| S4-2 | Screenshot report per app state | met | states = length 10 (want at least 9): ~/house-scanning-data/reports/sim/20260926-052535-t3-ios-mvf-525ea401-replay/report.json | +| S4-2 | Screenshot report per app state | stale | report for 525ea401, ref at beede15f: ~/house-scanning-data/reports/sim/20260926-052535-t3-ios-mvf-525ea401-replay/report.json | | S4-3 | Product description written from the code and checked against screenshots | met | verification/product covers every screen of t3/ios-mvf at 525ea40, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state. | -| S4-4 | Scoreboard of every PR | met | present at 60fe0d32 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/60fe0d327dfd0ac56b5570ffdc9daf24d8760a39/verification/SCOREBOARD.md) | +| S4-4 | Scoreboard of every PR | met | present at 4b4425b5 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/4b4425b5cf899c1c43d78b75f2172a77aaa24c82/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index b0ab5601a..41ab0e8e2 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,42 +1,48 @@ { - "generated_at": "2026-09-26T05:43:41-05:00", + "generated_at": "2026-09-26T06:13:39-05:00", "command": "uv run python -m hsverify.scoreboard", - "slow": true, + "slow": false, "prs": [ { "number": 16, "title": "experiments: measure on-device meter close-up reading and retake thresholds", "branch": "t3/meter-closeup", - "head": "03e04b54290b6dedd220eac4e4c4054bdf3c1e0c", + "head": "c5cb30d782fa8bd9a91e30f6997b7239764db640", "draft": true, - "updated": "2026-09-26T10:36:35Z", + "updated": "2026-09-26T11:13:17Z", "ci": { "counts": { - "success": 5, - "failure": 0, + "success": 3, + "failure": 1, "pending": 0, "skipped": 0 }, - "failing": [], - "text": "5 success" + "failing": [ + "Vercel" + ], + "text": "3 success, 1 failure; failing: Vercel" } }, { "number": 15, "title": "web: placement review page", "branch": "t3/web-review", - "head": "c69f88809c06175373079d7ec3b6b414194a73cf", + "head": "a7c9b129a52d46a87a9542e665dffcff0fc4d167", "draft": true, - "updated": "2026-09-26T10:06:25Z", + "updated": "2026-09-26T11:08:48Z", "ci": { "counts": { - "success": 5, - "failure": 0, + "success": 3, + "failure": 3, "pending": 0, "skipped": 0 }, - "failing": [], - "text": "5 success" + "failing": [ + "Label PR size", + "Web checks", + "Vercel" + ], + "text": "3 success, 3 failure; failing: Label PR size, Web checks, Vercel" } }, { @@ -61,9 +67,9 @@ "number": 13, "title": "verification: independent end-to-end checks (S4)", "branch": "t3/verification", - "head": "60fe0d327dfd0ac56b5570ffdc9daf24d8760a39", + "head": "4b4425b5cf899c1c43d78b75f2172a77aaa24c82", "draft": true, - "updated": "2026-09-26T10:41:29Z", + "updated": "2026-09-26T10:53:26Z", "ci": { "counts": { "success": 2, @@ -81,9 +87,9 @@ "number": 12, "title": "feat(evals): real-data accuracy evals for ARKit drift and photo-based wall reconstruction", "branch": "t3/evals", - "head": "7c5318d6610a06e657a98b0c8c7d1cc2acc53738", + "head": "779d33918a4d4dd997364b5a9345392ac9cff45e", "draft": true, - "updated": "2026-09-26T10:38:42Z", + "updated": "2026-09-26T11:03:22Z", "ci": { "counts": { "success": 3, @@ -103,7 +109,7 @@ "branch": "t3/server", "head": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", "draft": false, - "updated": "2026-09-26T10:35:26Z", + "updated": "2026-09-26T10:55:25Z", "ci": { "counts": { "success": 5, @@ -119,18 +125,18 @@ "number": 10, "title": "ios: guided wall scan, from meter to battery spot", "branch": "t3/ios-mvf", - "head": "525ea401641553fad581925023ed1bb82c0f7352", + "head": "beede15f568b3a4d694fb275caf9eaaa882c546b", "draft": true, - "updated": "2026-09-26T10:21:08Z", + "updated": "2026-09-26T11:12:52Z", "ci": { "counts": { - "success": 5, + "success": 4, "failure": 0, - "pending": 0, + "pending": 1, "skipped": 0 }, "failing": [], - "text": "5 success" + "text": "4 success, 1 pending" } }, { @@ -232,7 +238,7 @@ "refs": [ { "ref": "origin/t3/evals", - "sha": "7c5318d6610a06e657a98b0c8c7d1cc2acc53738" + "sha": "779d33918a4d4dd997364b5a9345392ac9cff45e" } ], "metrics": [ @@ -240,7 +246,7 @@ "id": "S1-1", "text": "One command regenerates every number from a clean checkout", "ref": "origin/t3/evals", - "sha": "7c5318d6610a06e657a98b0c8c7d1cc2acc53738", + "sha": "779d33918a4d4dd997364b5a9345392ac9cff45e", "status": "partial", "evidence": "make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac.", "links": [] @@ -249,13 +255,13 @@ "id": "S1-2", "text": "Dataset URLs and sha256 recorded", "ref": "origin/t3/evals", - "sha": "7c5318d6610a06e657a98b0c8c7d1cc2acc53738", + "sha": "779d33918a4d4dd997364b5a9345392ac9cff45e", "status": "met", "evidence": "all 2 found", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/README.md" ] ] }, @@ -263,30 +269,30 @@ "id": "S1-3", "text": "Metric code tested on hand-computed cases", "ref": "origin/t3/evals", - "sha": "7c5318d6610a06e657a98b0c8c7d1cc2acc53738", - "status": "partial", - "evidence": "exit 0: Installed 20 packages in 60ms / .................................................. [100%] / 50 passed in 1.60s (tests pass; whether the cases are hand-computed needs a read)", + "sha": "779d33918a4d4dd997364b5a9345392ac9cff45e", + "status": "not run", + "evidence": "slow; run with --slow", "links": [] }, { "id": "S1-4", "text": "ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft", "ref": "origin/t3/evals", - "sha": "7c5318d6610a06e657a98b0c8c7d1cc2acc53738", + "sha": "779d33918a4d4dd997364b5a9345392ac9cff45e", "status": "met", "evidence": "all 5 found", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/README.md" ], [ "experiments/evals/results/advio_drift.md", - "https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/advio_drift.md" + "https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/results/advio_drift.md" ], [ - "experiments/evals/results/eth3d_recon.md", - "https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/eth3d_recon.md" + "experiments/evals/results/modern_arkit.md", + "https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/results/modern_arkit.md" ] ] }, @@ -294,21 +300,21 @@ "id": "S1-5", "text": "ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question", "ref": "origin/t3/evals", - "sha": "7c5318d6610a06e657a98b0c8c7d1cc2acc53738", + "sha": "779d33918a4d4dd997364b5a9345392ac9cff45e", "status": "partial", "evidence": "found 1-3 m row, 3-10 m row, scale error; missing verdict", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/README.md" ], [ "experiments/evals/results/eth3d_recon.md", - "https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/eth3d_recon.md" + "https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/results/eth3d_recon.md" ], [ "experiments/evals/results/eth3d_visibility_sensitivity.md", - "https://github.com/SamGu-NRX/house-scanning/blob/7c5318d6610a06e657a98b0c8c7d1cc2acc53738/experiments/evals/results/eth3d_visibility_sensitivity.md" + "https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/results/eth3d_visibility_sensitivity.md" ] ] } @@ -329,8 +335,8 @@ "text": "pytest green", "ref": "origin/t3/server", "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", - "status": "met", - "evidence": "exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 222 passed, 1 warning in 11.01s", + "status": "not run", + "evidence": "slow; last run 2026-09-26 05:43 CDT: met, exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 222 passed, 1 warning in 11.01s", "links": [] }, { @@ -415,8 +421,8 @@ "text": "ruff clean", "ref": "origin/t3/server", "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", - "status": "met", - "evidence": "exit 0: Installed 33 packages in 37ms / All checks passed! / 17 files already formatted", + "status": "not run", + "evidence": "slow; last run 2026-09-26 05:43 CDT: met, exit 0: Installed 33 packages in 37ms / All checks passed! / 17 files already formatted", "links": [] }, { @@ -436,7 +442,7 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "525ea401641553fad581925023ed1bb82c0f7352" + "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b" } ], "metrics": [ @@ -444,9 +450,9 @@ "id": "S3-1", "text": "CI build green", "ref": "origin/t3/ios-mvf", - "sha": "525ea401641553fad581925023ed1bb82c0f7352", - "status": "met", - "evidence": "iOS build: success at 525ea401", + "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", + "status": "not run", + "evidence": "iOS build: pending at beede15f", "links": [ [ "#10", @@ -458,13 +464,13 @@ "id": "S3-2", "text": "Built as Swift 6 with strict concurrency", "ref": "origin/t3/ios-mvf", - "sha": "525ea401641553fad581925023ed1bb82c0f7352", + "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", "status": "met", "evidence": "all 2 found", "links": [ [ "ios/project.yml", - "https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/project.yml" + "https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/project.yml" ] ] }, @@ -472,30 +478,30 @@ "id": "S3-3", "text": "Zero build warnings", "ref": "origin/t3/ios-mvf", - "sha": "525ea401641553fad581925023ed1bb82c0f7352", - "status": "met", - "evidence": "build.warnings = length 0 (want 0): ~/house-scanning-data/reports/sim/20260926-052535-t3-ios-mvf-525ea401-replay/report.json", + "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", + "status": "stale", + "evidence": "report for 525ea401, ref at beede15f: ~/house-scanning-data/reports/sim/20260926-052535-t3-ios-mvf-525ea401-replay/report.json", "links": [] }, { "id": "S3-4", "text": "Unit tests for coverage, auto-capture and guidance", "ref": "origin/t3/ios-mvf", - "sha": "525ea401641553fad581925023ed1bb82c0f7352", + "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", "status": "partial", "evidence": "found coverage; missing auto-capture, guidance", "links": [ [ "ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift" ] ] }, @@ -503,17 +509,17 @@ "id": "S3-5", "text": "UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop", "ref": "origin/t3/ios-mvf", - "sha": "525ea401641553fad581925023ed1bb82c0f7352", + "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", "status": "partial", "evidence": "all 3 found (the tests exist; passing is S3-1)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/FullFlowUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -521,17 +527,17 @@ "id": "S3-6", "text": "Accessibility audit on every screen", "ref": "origin/t3/ios-mvf", - "sha": "525ea401641553fad581925023ed1bb82c0f7352", + "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", "status": "partial", "evidence": "3 matches in 2 files (an audit exists; which screens it covers needs a read)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/FullFlowUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/525ea401641553fad581925023ed1bb82c0f7352/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -539,16 +545,16 @@ "id": "S3-6b", "text": "Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, plus symbol-name button labels) finds no issues; S4-2 shows how many screens the app reached", "ref": "origin/t3/ios-mvf", - "sha": "525ea401641553fad581925023ed1bb82c0f7352", - "status": "gap", - "evidence": "issue_count = 32 (want 0): ~/house-scanning-data/reports/a11y/20260926-052756-t3-ios-mvf-525ea401-head-sample/report.json", + "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", + "status": "stale", + "evidence": "report for 525ea401, ref at beede15f: ~/house-scanning-data/reports/a11y/20260926-052756-t3-ios-mvf-525ea401-head-sample/report.json", "links": [] }, { "id": "S3-7", "text": "Screenshots of each state in the PR", "ref": "origin/t3/ios-mvf", - "sha": "525ea401641553fad581925023ed1bb82c0f7352", + "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", "status": "gap", "evidence": "0 of 9 matches in the PR body", "links": [ @@ -562,7 +568,7 @@ "id": "S3-8", "text": "Exported scene.json validates against C1", "ref": "origin/t3/ios-mvf", - "sha": "525ea401641553fad581925023ed1bb82c0f7352", + "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", "status": "met", "evidence": "app_export_scene_valid = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json", "links": [] @@ -571,7 +577,7 @@ "id": "S3-9", "text": "Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design", "ref": "origin/t3/ios-mvf", - "sha": "525ea401641553fad581925023ed1bb82c0f7352", + "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -584,11 +590,11 @@ "refs": [ { "ref": "origin/t3/verification", - "sha": "60fe0d327dfd0ac56b5570ffdc9daf24d8760a39" + "sha": "4b4425b5cf899c1c43d78b75f2172a77aaa24c82" }, { "ref": "origin/t3/ios-mvf", - "sha": "525ea401641553fad581925023ed1bb82c0f7352" + "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b" } ], "metrics": [ @@ -596,7 +602,7 @@ "id": "S4-1", "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/verification", - "sha": "60fe0d327dfd0ac56b5570ffdc9daf24d8760a39", + "sha": "4b4425b5cf899c1c43d78b75f2172a77aaa24c82", "status": "met", "evidence": "app_export_passed = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json", "links": [] @@ -605,16 +611,16 @@ "id": "S4-2", "text": "Screenshot report per app state", "ref": "origin/t3/ios-mvf", - "sha": "525ea401641553fad581925023ed1bb82c0f7352", - "status": "met", - "evidence": "states = length 10 (want at least 9): ~/house-scanning-data/reports/sim/20260926-052535-t3-ios-mvf-525ea401-replay/report.json", + "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", + "status": "stale", + "evidence": "report for 525ea401, ref at beede15f: ~/house-scanning-data/reports/sim/20260926-052535-t3-ios-mvf-525ea401-replay/report.json", "links": [] }, { "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "60fe0d327dfd0ac56b5570ffdc9daf24d8760a39", + "sha": "4b4425b5cf899c1c43d78b75f2172a77aaa24c82", "status": "met", "evidence": "verification/product covers every screen of t3/ios-mvf at 525ea40, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state.", "links": [] @@ -623,13 +629,13 @@ "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "60fe0d327dfd0ac56b5570ffdc9daf24d8760a39", + "sha": "4b4425b5cf899c1c43d78b75f2172a77aaa24c82", "status": "met", - "evidence": "present at 60fe0d32", + "evidence": "present at 4b4425b5", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/60fe0d327dfd0ac56b5570ffdc9daf24d8760a39/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/4b4425b5cf899c1c43d78b75f2172a77aaa24c82/verification/SCOREBOARD.md" ] ] } From 30e866c6d0c835c025dfa050d339d7b6ff350e69 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 06:28:30 -0500 Subject: [PATCH 29/59] verification: recheck at t3/ios-mvf beede15 The description, triage and verification record follow S3's engine fixes (reset only while capturing, ground from arriving planes, past-end settling, stricter covered rule). At beede15 the gap loop no longer closes on the ADVIO replay; the autopilot finds no held-back frames. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/SCOREBOARD.md | 30 +++---- verification/product/README.md | 6 +- verification/product/bug-triage.md | 21 +++-- .../foundations/coverage-and-guidance.md | 44 ++++++---- verification/product/foundations/flow.md | 14 ++-- verification/product/glossary.md | 10 ++- verification/product/screens/find-meter.md | 33 ++++---- verification/product/screens/gap-request.md | 34 ++++---- verification/product/screens/mark-features.md | 28 ++++--- .../product/screens/meter-close-up.md | 2 +- verification/product/screens/onboarding.md | 10 +-- verification/product/screens/result.md | 53 ++++++------ verification/product/screens/uploading.md | 23 ++--- verification/product/screens/wall-walk.md | 40 ++++----- verification/product/verification.md | 9 +- verification/scoreboard/scoreboard.json | 84 +++++++++---------- 16 files changed, 232 insertions(+), 209 deletions(-) diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index c1a09b378..12a83b43f 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,6 +1,6 @@ # Scoreboard -Generated 2026-09-26 06:13 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-26 06:28 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests @@ -9,10 +9,10 @@ Generated 2026-09-26 06:13 CDT by `make scoreboard` (`uv run python -m hsverify. | [#16](https://github.com/SamGu-NRX/house-scanning/pull/16) | `t3/meter-closeup` | `c5cb30d7` | yes | 3 success, 1 failure; failing: Vercel | 2026-09-26 06:13 CDT | | [#15](https://github.com/SamGu-NRX/house-scanning/pull/15) | `t3/web-review` | `a7c9b129` | yes | 3 success, 3 failure; failing: Label PR size, Web checks, Vercel | 2026-09-26 06:08 CDT | | [#14](https://github.com/SamGu-NRX/house-scanning/pull/14) | `t3/docs-cleanup` | `ab03d7fd` | no | 4 success | 2026-09-26 05:17 CDT | -| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `4b4425b5` | yes | 2 success, 1 failure; failing: Vercel | 2026-09-26 05:53 CDT | -| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `779d3391` | yes | 3 success, 1 failure; failing: Vercel | 2026-09-26 06:03 CDT | -| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `f2705dd5` | no | 5 success | 2026-09-26 05:55 CDT | -| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `beede15f` | yes | 4 success, 1 pending | 2026-09-26 06:12 CDT | +| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `5e60831d` | yes | 2 success, 1 failure; failing: Vercel | 2026-09-26 06:13 CDT | +| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `27293c5d` | yes | 3 success | 2026-09-26 06:20 CDT | +| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `f2705dd5` | no | 5 success | 2026-09-26 06:22 CDT | +| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `beede15f` | yes | 4 success, 1 failure; failing: iOS build | 2026-09-26 06:12 CDT | | [#9](https://github.com/SamGu-NRX/house-scanning/pull/9) | `codex/capture-contract-probe` | `4b6c4309` | yes | 4 success | 2026-09-26 00:08 CDT | | [#8](https://github.com/SamGu-NRX/house-scanning/pull/8) | `t3/testflight` | `b896d59f` | no | 4 success | 2026-09-26 03:18 CDT | | [#7](https://github.com/SamGu-NRX/house-scanning/pull/7) | `t3/measure-lab` | `45770a3b` | no | 6 success | 2026-09-26 03:19 CDT | @@ -21,15 +21,15 @@ Generated 2026-09-26 06:13 CDT by `make scoreboard` (`uv run python -m hsverify. ## S1 Evals on real data -`origin/t3/evals` at [`779d3391`](https://github.com/SamGu-NRX/house-scanning/tree/779d33918a4d4dd997364b5a9345392ac9cff45e). 2 met, 2 partial, 1 not run. +`origin/t3/evals` at [`27293c5d`](https://github.com/SamGu-NRX/house-scanning/tree/27293c5d4d34a27af652116341639f1826d806ae). 2 met, 2 partial, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S1-1 | One command regenerates every number from a clean checkout | partial | make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac. | -| S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/README.md) | +| S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/README.md) | | S1-3 | Metric code tested on hand-computed cases | not run | slow; run with --slow | -| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/README.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/results/advio_drift.md), [experiments/evals/results/modern_arkit.md](https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/results/modern_arkit.md) | -| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/README.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/results/eth3d_visibility_sensitivity.md) | +| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/README.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/results/advio_drift.md), [experiments/evals/results/modern_arkit.md](https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/results/modern_arkit.md) | +| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/README.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/results/eth3d_visibility_sensitivity.md) | ## S2 Server and solver @@ -48,13 +48,13 @@ Generated 2026-09-26 06:13 CDT by `make scoreboard` (`uv run python -m hsverify. ## S3 iOS capture app -`origin/t3/ios-mvf` at [`beede15f`](https://github.com/SamGu-NRX/house-scanning/tree/beede15f568b3a4d694fb275caf9eaaa882c546b). 2 met, 3 partial, 2 gap, 2 stale, 1 not run. +`origin/t3/ios-mvf` at [`beede15f`](https://github.com/SamGu-NRX/house-scanning/tree/beede15f568b3a4d694fb275caf9eaaa882c546b). 3 met, 3 partial, 3 gap, 1 stale. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S3-1 | CI build green | not run | iOS build: pending at beede15f [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | +| S3-1 | CI build green | gap | iOS build: failure at beede15f [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | | S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/project.yml) | -| S3-3 | Zero build warnings | stale | report for 525ea401, ref at beede15f: ~/house-scanning-data/reports/sim/20260926-052535-t3-ios-mvf-525ea401-replay/report.json | +| S3-3 | Zero build warnings | met | build.warnings = length 0 (want 0): ~/house-scanning-data/reports/sim/20260926-061551-t3-ios-mvf-beede15f-replay/report.json | | S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage; missing auto-capture, guidance [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift) | | S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/ScreenStatesUITests.swift) | | S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/ScreenStatesUITests.swift) | @@ -65,14 +65,14 @@ Generated 2026-09-26 06:13 CDT by `make scoreboard` (`uv run python -m hsverify. ## S4 Verification -`origin/t3/verification` at [`4b4425b5`](https://github.com/SamGu-NRX/house-scanning/tree/4b4425b5cf899c1c43d78b75f2172a77aaa24c82), `origin/t3/ios-mvf` at [`beede15f`](https://github.com/SamGu-NRX/house-scanning/tree/beede15f568b3a4d694fb275caf9eaaa882c546b). 3 met, 1 stale. +`origin/t3/verification` at [`5e60831d`](https://github.com/SamGu-NRX/house-scanning/tree/5e60831dcc87ca0b854b4ce51c1e8b220438fc7a), `origin/t3/ios-mvf` at [`beede15f`](https://github.com/SamGu-NRX/house-scanning/tree/beede15f568b3a4d694fb275caf9eaaa882c546b). 4 met. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S4-1 | End to end from a real replay to a placement result passes | met | app_export_passed = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | -| S4-2 | Screenshot report per app state | stale | report for 525ea401, ref at beede15f: ~/house-scanning-data/reports/sim/20260926-052535-t3-ios-mvf-525ea401-replay/report.json | +| S4-2 | Screenshot report per app state | met | states = length 10 (want at least 9): ~/house-scanning-data/reports/sim/20260926-061551-t3-ios-mvf-beede15f-replay/report.json | | S4-3 | Product description written from the code and checked against screenshots | met | verification/product covers every screen of t3/ios-mvf at 525ea40, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state. | -| S4-4 | Scoreboard of every PR | met | present at 4b4425b5 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/4b4425b5cf899c1c43d78b75f2172a77aaa24c82/verification/SCOREBOARD.md) | +| S4-4 | Scoreboard of every PR | met | present at 5e60831d [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/5e60831dcc87ca0b854b4ce51c1e8b220438fc7a/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/product/README.md b/verification/product/README.md index 5f0ea16dc..2356bedf5 100644 --- a/verification/product/README.md +++ b/verification/product/README.md @@ -10,8 +10,8 @@ wrong is collected in [bug-triage.md](bug-triage.md). - **Surface.** The HouseScan app, one run of the homeowner flow from onboarding to the result, default settings. The ARKit camera cannot run in the Simulator, so behavior that needs the live camera is described from code and marked unverified. -- **Source.** `t3/ios-mvf`. Line references and footers are at `525ea40`, and the app was checked - in the Simulator at `525ea40`. +- **Source.** `t3/ios-mvf`. Line references, footers and the last Simulator check are at + `beede15`. - **How it is run.** `make sim-app REF=` from `verification/`: the real ADVIO replay `advio-20-0040-0075` (C3), the autopilot, and the server started from `t3/server`. - **Out of scope.** The `-uiDemo` screens (a design preview, not the product), TestFlight @@ -72,5 +72,5 @@ and Open questions and verification, ending with the commit it was verified agai | [screens/gap-request.md](screens/gap-request.md) | One targeted extra view, from the phone or the server | drafted; checked in the Simulator | | [screens/uploading.md](screens/uploading.md) | Sending the scan and waiting for the result, offline and failure | drafted; checked in the Simulator | | [screens/result.md](screens/result.md) | The result, its checks and the 3D and AR views | drafted; checked in the Simulator | -| [verification.md](verification.md) | Checks against the running app and their results | current pass at `525ea40` | +| [verification.md](verification.md) | Checks against the running app and their results | current pass at `beede15` | | [bug-triage.md](bug-triage.md) | Every suspected defect, deduplicated | 16 entries, 2 resolved | diff --git a/verification/product/bug-triage.md b/verification/product/bug-triage.md index b20efc10a..208091196 100644 --- a/verification/product/bug-triage.md +++ b/verification/product/bug-triage.md @@ -1,8 +1,8 @@ # Suspected defects Collected from every document's open questions and from [verification.md](verification.md), -deduplicated. Source `t3/ios-mvf` at `0876e03` unless an entry names another commit; paths are -under `ios/`. **Seen** means observed in the Simulator; **code** means read from the +deduplicated. Status checked at `t3/ios-mvf` `beede15`; line references are at `0876e03` unless an +entry names another commit; paths are under `ios/`. **Seen** means observed in the Simulator; **code** means read from the source and not run. Every blocker and major entry was re-read in the source by a second reader. | ID | Severity | Where the homeowner meets it | Evidence | Decision | @@ -12,9 +12,9 @@ source and not run. Every blocker and major entry was re-read in the source by a | B-09 | Blocker | Feature list: "Add something" does nothing | code | fix | | B-06 | Major | Walk: an end closer than 20 ft can't be marked (answer half fixed at `525ea40`) | code | fix | | B-08 | Major | "Can't get there" is never sent to the server | code | fix (with S2) | -| B-10 | Major | Result, upload or list: the scan resets after 20 s of relocalizing | code | fix | -| B-11 | Major | Meter tap can land on another wall's extended plane | code | fix | -| B-12 | Major | "Capture it now" past a wall end turns it unexplored; a request beyond a marked end can't finish | code | fix | +| B-10 | Major | Feature list: the scan resets after 20 s of relocalizing (result and upload fixed at `beede15`) | code | fix | +| B-11 | Major | Meter tap can land on another wall's extended plane (ground height fixed at `beede15`) | code | fix | +| B-12 | Major | A server request beyond a still-marked wall end can't finish (past-end case fixed at `beede15`) | code | fix | | B-02 | Major | Upload and camera failures show raw error text | seen at `21a63e7` | fix | | B-13 | Minor | 3D and AR tint zones stop one battery width short | code | fix | | B-14 | Minor | Result rule lines hide whether a limit is a minimum or maximum | seen | fix | @@ -62,19 +62,24 @@ so a stretch the homeowner can't reach looks to the server like one nobody looke screen's hint promises "An installer will look at this part instead"; the server may ask for the same view again. C1 has no field for it, so this needs S2 and S3 together. -**B-10. The scan resets under the result.** The 20 s relocalization reset runs on every screen while +**B-10. The scan resets after a lost world frame.** At `beede15` the reset spares the upload, +result and AR screens; it still fires on the feature list, which shows no coaching. Before: The 20 s relocalization reset runs on every screen while the camera session runs (`ScanEngine.swift:189, 357-364`), including the feature list, the upload and the result, where no coaching is shown. After a call, the homeowner can come back to "Find your electric meter" with every photo, mark and the result gone. The reset also keeps a half-finished mark and the first close-up (`ScanEngine.swift:386-410`, `KeyframeStore.swift:79`). -**B-11. The meter tap can guess a depth.** The raycast falls back to `existingPlaneInfinite` +**B-11. The meter tap can guess a depth.** At `beede15` the ground height comes from planes as +they arrive; the infinite-plane fallback remains (`LiveCapture.swift:76`). Before: The raycast falls back to `existingPlaneInfinite` (`HouseScan/Runtime/LiveCapture.swift:76`): a tap at a meter whose wall is not detected yet can pin it on another wall's plane extended past its edge. The code's own comment says a tap never guesses a depth. The ground height also stays a guess (1.4 m below the phone) if no ground plane was seen at the tap (`ScanEngine+Actions.swift:48-57`). -**B-12. Requests past a wall end.** "Capture it now" on a past-end request first clears that end +**B-12. Requests past a wall end.** At `beede15` marking the end again settles a past-end +request, but only the walk offers "Wall ends here"; the gap request screen shows neither it nor +the end question (`WallWalkScreen.swift:137-173`), so a homeowner can't use the fix. The +band-request case below is unchanged. Before: "Capture it now" on a past-end request first clears that end (`ScanEngine+Actions.swift:206-209`) and nothing sets it again, so the next upload reports the side as unexplored even after the homeowner looked past it, and the server can ask again. A server band request whose stretch lies beyond an end that is still marked can never fill: the coverage map diff --git a/verification/product/foundations/coverage-and-guidance.md b/verification/product/foundations/coverage-and-guidance.md index 4bbfeb11d..a2a4265ea 100644 --- a/verification/product/foundations/coverage-and-guidance.md +++ b/verification/product/foundations/coverage-and-guidance.md @@ -14,7 +14,7 @@ decides whether a battery fits; covered means evidence exists, not that a spot p The homeowner walks slowly along the wall with the phone pointed at it. Every half meter or so the phone keeps a photo; the counter at the top goes up and the fog over the wall clears where the camera has looked. The strip at the bottom fills in: gray where nothing has been seen, then -covered once a stretch has been seen from two places. The instruction at the top says where to +covered once a stretch has been seen from two places, top to bottom. The instruction at the top says where to go ("Walk slowly to your left") and changes only when that is done or after at least 3 s. ## The interaction, event by event @@ -22,8 +22,8 @@ go ("Walk slowly to your left") and changes only when that is done or after at l ```mermaid stateDiagram-v2 [*] --> unseen - unseen --> seen : a kept photo shows most of the cell - seen --> covered : a second kept photo from 0.25 m away, tracking normal + unseen --> seen : a kept photo shows one of the cell's three sample lines + seen --> covered : every sample line seen from two places 0.25 m apart unseen --> skipped : "I can't get there" over this cell seen --> skipped : "I can't get there" over this cell ``` @@ -52,21 +52,29 @@ A frame is kept only when all of these hold: - the frame is at least half as sharp as the recent frames, not too dark (mean brightness at least 40 of 255) and not glaring (under a quarter of the pixels clipped); - at least 0.33 s has passed since the last kept photo (at most about three per second); -- the phone has moved 0.5 m or turned 15° since the last kept photo, or the frame would show at - least three cells nobody has seen yet. +- the phone has moved 0.5 m or turned 15° since the last kept photo, or the frame would show a + sample line nobody has seen yet in at least three cells. A kept photo counts, and coverage moves, only once its file is written to the phone, so the strip never claims a view the upload lacks. Walk photos give no haptic; the counter flashes instead. ### While capturing -A cell counts as **seen** when a kept photo shows most of it (60 % of its sample points) from no -farther than 6 m and no more than 65° from straight on, away from the outer 3 % of the image. It -becomes **covered** when a second kept photo, taken from at least 0.25 m away from the first, -also sees it. Photos are kept only with normal tracking, so every kept photo counts. +Each cell is checked along three **sample lines** across its band, evenly spaced from edge to edge. +On the wall they sit at the foot, 0.99 m up and 1.98 m up (the top of the band); on the ground +at the foot of the wall, 0.6 m out and 1.2 m out. A photo sees a line when both of its points, a +quarter and three quarters of the way along the cell, are no farther than 6 m, no more than 65° +from straight on and away from the outer 3 % of the image, and the phone is in front of the +wall's line. A cell counts as **seen** once any kept photo sees any of its lines. It becomes +**covered** only when every line has been seen from two places at least 0.25 m apart; different +photos may supply different lines. A cell whose top line no photo has reached stays seen, however +often its lower lines are photographed. Photos are kept only with normal tracking, so every kept +photo counts. (Fixed in `beede15`: before, most of a cell's points from two places were enough, +so a band could read covered without its top ever being in view.) The strip shows what has been seen plus 2.5 m of fog ahead on each side, and stops at a marked -wall end. +wall end. When the phone refines the meter's position, the seen stretches and the ends move with +it, in whole cells. #### One instruction at a time @@ -90,7 +98,10 @@ The instruction is chosen in this order, first match wins: 4. **Aim at a lagging band**: within 1 m either side of the phone, if one band is done over at least 0.45 m (three cells) where the other is not, "Tilt down to show the ground" or "Tilt up to show more wall", with a ring on the spot. -5. **Walk to the left**, until coverage runs unbroken 6.1 m (20 ft) from the meter on the left; +5. **Walk to the left**, until coverage runs unbroken 6.1 m (20 ft) from the meter on the left. + The ring and dots lead 1 m past whichever is farther along: the unbroken coverage or the + phone itself, so a homeowner already past a thin patch is not sent back to it (fixed in + `beede15`; the aim instructions ask for such patches); then **"Is this the left end of the wall?"**, with "Aim where the wall stops or turns a corner, and tap Wall ends here.", the only moment "Wall ends here" is offered. Only after the left end is set: the same for the right. @@ -99,7 +110,7 @@ The instruction is chosen in this order, first match wins: 7. **"That's the whole wall"**, with "Tap Done when you're ready". A new instruction replaces the current one only when the current one is satisfied or has been up -for 3 s. An aim instruction is satisfied at 80 % of the cells within 0.3 m of its spot covered; a +for 3 s of screen time. An aim instruction is satisfied at 80 % of the cells within 0.3 m of its spot covered; a walk instruction when that side's end is set; an end question when it is answered. ### Advancing @@ -115,7 +126,7 @@ can't reach looks the same as one nobody looked at, and it treats both as unknow | Modifier | At arrival | While capturing | | --- | --- | --- | | Live camera or replay | No effect on the rules; a replay's frames go through the same checks. | Same. | -| Autopilot | No effect on the rules. | The replay runs at three times speed, so the time-based limits (0.33 s between photos, 3 s per instruction) bite more often. | +| Autopilot | No effect on the rules. | The replay runs at three times speed, so the 0.33 s between photos is measured on the recording's clock and bites more often. The 3 s per instruction is screen time, so instructions stay up as long as live (fixed in `beede15`). | | Server or sample result | No effect. | No effect. | | Larger text sizes | No effect on coverage. | No effect. | | Reduce Motion | The fog still fades as cells clear, without drifting. | Same. | @@ -153,14 +164,15 @@ hooks.** Each instruction change logs `GUIDANCE=`. - Coverage is on the one wall set by the meter tap; wall past a corner never becomes covered. - Something standing between the phone and the wall (a bush, a bin) does not stop a cell counting as seen: coverage records where the camera looked, not what it could see. The server re-checks - from the photos (`HouseScanKit/.../CoverageMap.swift:51`). + from the photos (`HouseScanKit/.../CoverageMap.swift:58-59`). The one hidden case the app does + model is the wall itself: a phone on the far side of the wall's line sees nothing. ## Open questions and verification - **Suspected bug: a blocked end nearer than 20 ft can't be recorded** (triage B-06, half resolved). The end question makes a marked end's kind truthful, but "Wall ends here" is still offered only when the walk asks for the end, at 6.1 m of coverage - (`HouseScanKit/.../GuidancePlanner.swift:27`, `:113`). A wall blocked by a fence sooner can only + (`HouseScanKit/.../GuidancePlanner.swift:27`, `:119`). A wall blocked by a fence sooner can only be recorded as unexplored, through "Can't get there". A wall that goes on straight past 20 ft is recorded as unexplored only by answering "It turns a corner", which is not what the homeowner sees. @@ -170,4 +182,4 @@ hooks.** Each instruction change logs `GUIDANCE=`. (`Runtime/ScanEngine.swift`, `step(_:)`). - All thresholds are the code's stated hypotheses (research note), not measured values. -Verified against house-scanning commit `525ea40` (t3/ios-mvf). +Verified against house-scanning commit `beede15` (t3/ios-mvf). diff --git a/verification/product/foundations/flow.md b/verification/product/foundations/flow.md index ac9ab05e6..daac09425 100644 --- a/verification/product/foundations/flow.md +++ b/verification/product/foundations/flow.md @@ -35,13 +35,13 @@ stateDiagram-v2 wallWalk --> markFeatures : "Done with this wall" (both ends set) markFeatures --> gapRequest : "Looks complete", and the phone's check finds a gap markFeatures --> uploading : "Looks complete", no gap - gapRequest --> uploading : view taken, or "I can't get there" + gapRequest --> uploading : view taken, end marked again, or "I can't get there" uploading --> result : the server answered result --> gapRequest : "Capture it now" on a missing view result --> resultAR : "See it on your wall" resultAR --> result : "Done" result --> onboarding : "Start over" - wallWalk --> findMeter : phone lost its place for over 20 s + wallWalk --> findMeter : phone lost its place for over 20 s (any capture screen) ``` The names are the screen names the app logs (see the [glossary](../glossary.md#verification-words)). @@ -94,7 +94,7 @@ after the server has answered; nothing on `uploading` is skipped when the networ | The screen's own way out | See each screen. | See each screen. | | Start over | Not offered. The failure screen has it for failures other than an unsupported phone, but those never reach it (Open questions). | Offered only on `result`. It forgets the wall, photos, marks and result and returns to `onboarding`. | | Tracking limited | Coaching replaces the instruction until it clears ([coverage and guidance](coverage-and-guidance.md#one-instruction-at-a-time)). | Same; no photo counts toward coverage meanwhile. | -| Tracking lost or relocalizing | "Point at the meter like this." (relocalizing) or "Your phone lost its place". | If the phone has not found its place within 20 s, the app forgets the wall, photos and marks and returns to `findMeter`, on any screen while the camera runs, including the result. | +| Tracking lost or relocalizing | "Point at the meter like this." (relocalizing) or "Your phone lost its place". | If the phone has not found its place within 20 s on a capture screen (`findMeter` through `gapRequest`), the app forgets the wall, photos and marks and returns to `findMeter`. On `uploading`, `result` and `resultAR` the scan and result stay (fixed in `beede15`). | | App backgrounded or a call | The screen stays; coaching says "Point at the meter like this." when the camera resumes. | Same; photos and marks are kept. | | Camera off or session failed | Suspected dead end: see Open questions. | Suspected dead end: see Open questions. | | Network lost or upload failing | No effect. | Only `uploading` is affected; see [the upload](../screens/uploading.md). | @@ -130,16 +130,16 @@ the autopilot wait on each screen until a file named after it appears, for UI te - **Suspected bug: camera access denied is a dead end.** The failure screen text exists ("House Scan needs your camera ... Turn on Camera for House Scan in Settings") but only the `unsupported` screen shows failures, and the app switches to it only when motion tracking is - unsupported (`Runtime/ScanEngine.swift:99`). A denied camera, a failed camera session and an - unreadable replay set the failure (`ScanEngine.swift:111, 424, 426`) without changing screen. The + unsupported (`Runtime/ScanEngine.swift:107`). A denied camera, a failed camera session and an + unreadable replay set the failure (`ScanEngine.swift:119, 489, 491`) without changing screen. The homeowner would stay on "Find your electric meter" with no camera and no Settings button. Checkable in the Simulator for the replay case: launch with `-replay /nonexistent`. - "Your scan is saved on this phone" is true only while the app runs: after a relaunch nothing reopens or retries it. - Photos of the home accumulate in `Caches/Scans/` across scans and Start over; the app never - deletes them (`Runtime/KeyframeStore.swift:25`). Worth deciding whether a finished or abandoned + deletes them (`Runtime/KeyframeStore.swift:27`). Worth deciding whether a finished or abandoned scan should be removed. - The 1.2 s pause after a finished step and the 20 s relocalization limit are read from code; neither has been timed on a device. -Verified against house-scanning commit `525ea40` (t3/ios-mvf). +Verified against house-scanning commit `beede15` (t3/ios-mvf). diff --git a/verification/product/glossary.md b/verification/product/glossary.md index 082a04a81..2a7c73af6 100644 --- a/verification/product/glossary.md +++ b/verification/product/glossary.md @@ -30,13 +30,15 @@ app; everything else is this description's own term. - **Coverage strip.** The map along the bottom of the walk screen: the wall unrolled flat, meter at zero, in two rows (bands). - **Band.** One row of the strip. The **wall band** is the wall face from the ground up to - 2.4 m (7 ft 10 in); the **ground band** is the ground from the foot of the wall out to 1.2 m + 1.98 m (6 ft 6 in, the electrical code's headroom height); the **ground band** is the ground from the foot of the wall out to 1.2 m (3 ft 11 in). - **Cell.** A 6 in (0.1524 m) stretch of one band. Each cell is in one of four states: - **unseen**: no kept photo shows it; drawn as fog. - - **seen**: shown in kept photos, but not yet from two positions. - - **covered**: shown from two positions at least 0.25 m apart with normal tracking. Evidence - exists; it is not a pass. + - **seen**: at least one of the cell's three **sample lines** (lines across the band, at its + two edges and its middle) is shown in a kept photo, but not every line from two positions + yet. + - **covered**: every sample line shown from two positions at least 0.25 m apart with normal + tracking. Evidence exists; it is not a pass. - **skipped**: the homeowner said they can't get there. Drawn differently from both fog and covered, never evidence, and not sent to the server. - **Fog.** The overlay on the camera image over wall and ground not yet seen. It clears where diff --git a/verification/product/screens/find-meter.md b/verification/product/screens/find-meter.md index 490a9bb36..d69023982 100644 --- a/verification/product/screens/find-meter.md +++ b/verification/product/screens/find-meter.md @@ -7,7 +7,7 @@ electric meter and either taps "This is my meter", which pins the meter at the c middle of the screen, or taps the meter itself on the camera image. The phone accepts the tap only where it has detected a real vertical surface; that surface becomes the wall for the rest of the scan. It is the first camera screen (screen name `findMeter`), reached from "Allow camera" on -the introduction, and again after the phone loses its place for over 20 s +the introduction, and again after the phone loses its place for over 20 s during capture ([the flow](../foundations/flow.md#the-interaction-event-by-event)). Nothing is photographed here. An accepted tap goes straight to [the meter close-up](meter-close-up.md). @@ -75,8 +75,12 @@ until a tap is accepted. > Technical note: when the tap is accepted, the wall's outside is the side the phone is on, and > the ground at the wall is the highest detected level surface at least 0.3 m (1 ft) below the > meter whose detected extent comes within 2 m (6 ft 7 in) of it. With no such surface yet, the -> ground is taken as 1.4 m (4 ft 7 in) below the phone. This happens once, at the tap -> (`Runtime/ScanEngine+Actions.swift:39`, `:51`); see Open questions. +> ground is taken as 1.4 m (4 ft 7 in) below the phone (`Runtime/ScanEngine+Actions.swift:41-42`, +> `:53`). The lookup runs again whenever the phone detects or grows a level surface, for the rest +> of the scan: the first match replaces the guess, and a later one replaces the ground when it +> differs by more than 1 cm. Every mark is re-measured against the new ground +> (`Runtime/ScanEngine.swift:227-246`). While the ground is still a guess, the upload marks the +> meter and each door, window, gas meter and AC unit as uncertain by ±0.3 m (`Runtime/ScanEngine+Export.swift:50-55`, `:72`). ### While capturing @@ -140,18 +144,16 @@ replay logs where its wall came from when it loads. - Tapping past the edge of the part of the wall the phone has detected is accepted, because the surface counts as continuing (see Open questions). - A replay started with `-replay` whose homeowner taps "Allow camera" before the recording has - loaded starts the live camera as well (`Runtime/ScanEngine.swift:174`). Read from code only. + loaded starts the live camera as well (`Runtime/ScanEngine.swift:183`). Read from code only. - A replay that could not be opened leaves this screen dark, with no way on: every tap is ignored. ## Open questions and verification -- **Suspected bug: the ground can be guessed for the whole scan.** If no level surface has been - detected near the meter when the tap is accepted, the ground is set 1.4 m below the phone and - never corrected, though the code's own description says "until one appears" - (`Runtime/ScanEngine+Actions.swift:48-57`; nothing else writes it). A homeowner who taps quickly, - or holds the phone low or high, gets every height and the ground strip offset for the rest of - the scan. +- The chest-height ground guess is now replaced as soon as a level surface appears, fixed in + `beede15`. A scan where none ever appears still carries the guess, flagged only by the ±0.3 m, + which the code calls a hypothesis with no measured spread behind it + (`Runtime/ScanEngine+Export.swift:70-72`). - **Suspected bug: the refusal message is misleading and silent on repeat.** Both refusals show "Step a little closer to the wall" (`Runtime/ScanEngine+Actions.swift:26`, `:33`), though neither is about distance: one is tracking, and the other is a surface not yet detected, which @@ -162,15 +164,14 @@ replay logs where its wall came from when it loads. continuation past what was detected (`Runtime/LiveCapture.swift:76`, `existingPlaneInfinite`). A tap at a meter on a surface that has not been detected yet can pin the meter on another detected wall's continuation, at the wrong depth. The code's comment says a tap "never guesses a depth". -- **Suspected bug: a 20 s loss of place anywhere returns here.** The check runs on every screen - while the camera session exists, including the upload and the result (`Runtime/ScanEngine.swift:402-409`). - A homeowner who leaves the app during the upload and comes back to a phone lying face down - could lose the whole scan and land here. The reset also leaves an unfinished feature mark in - place (`Runtime/ScanEngine.swift:431-456` does not clear it). +- A 20 s loss of place now returns here only from the capture screens (finding the meter, the + close-up, the walk, the list and a gap request); the upload and the result keep the scan + (`Runtime/ScanEngine.swift:469-476`), fixed in `beede15`. The reset still leaves an unfinished + feature mark in place (`Runtime/ScanEngine.swift:496-522` does not clear it). - **Question: "Point at the meter like this." with nothing to match.** The relocalizing coaching (`UI/Copy/ScanCopy.swift:65-66`) is written for the walk, which shows the saved close-up under it. Here no close-up exists yet and no picture is shown, so "like this" points at nothing. - The live tap, the refusals, the coaching and the camera prompt need the live camera and have not been seen running. The Simulator shows only the replay path, where every tap is accepted. -Verified against house-scanning commit `525ea40` (t3/ios-mvf). +Verified against house-scanning commit `beede15` (t3/ios-mvf). diff --git a/verification/product/screens/gap-request.md b/verification/product/screens/gap-request.md index 049b4e26e..bd154c5f3 100644 --- a/verification/product/screens/gap-request.md +++ b/verification/product/screens/gap-request.md @@ -82,8 +82,9 @@ loses its amber, and the bar moves. ### While capturing The bar shows the share of the stretch's cells that are covered, rounded to a whole percent. It -counts only covered cells, so a cell seen from one place adds nothing until a second photo from -0.25 m away sees it. The instruction does not change while the request is open; only +counts only covered cells, so a cell adds nothing until each of its three sample lines has been +seen from two places 0.25 m apart +([coverage and guidance](../foundations/coverage-and-guidance.md#while-capturing)). The instruction does not change while the request is open; only [coaching](../foundations/coverage-and-guidance.md#one-instruction-at-a-time) replaces it, until the problem clears. The ring and the dots follow the homeowner as they move. "I can't get there" stays available, coaching or not. @@ -158,34 +159,37 @@ skipping it". With `-autopilotGate`, the upload after a completed view waits for - **Suspected bug: "I can't get there" is not sent.** The button's hint promises "An installer will look at this part instead.", and the cells turn skipped, but the upload carries only covered - stretches and the end kinds (`Runtime/ScanEngine+Export.swift:52-58`, - `HouseScanKit/.../SceneExport.swift:273-293`). The server cannot tell a skipped stretch from one + stretches and the end kinds (`Runtime/ScanEngine+Export.swift:57-63`, + `HouseScanKit/.../SceneExport.swift:283-303`). The server cannot tell a skipped stretch from one never looked at, and a server request the homeowner skipped can come back on the next result. This also contradicts the Advancing section of [coverage and guidance](../foundations/coverage-and-guidance.md#advancing), which says skipped cells are sent. - **Suspected bug: a past-end request turns a real end into an unexplored one.** "Capture it now" - forgets the end before the request starts (`Runtime/ScanEngine+Actions.swift:219-222`, - `Runtime/ScanEngine.swift:516-521`), and nothing marks it again. The next upload reports that - side as unexplored (`ScanEngine+Export.swift:53-54`) even when the homeowner has now shown the - ground beyond it, so the server may ask for the same view again. + forgets the end before the request starts (`Runtime/ScanEngine+Actions.swift:239-244`, + `Runtime/ScanEngine.swift:583-588`). Since `beede15` the engine lets that end be marked again + during the request, and marking it and answering the end question settles the request + (`ScanEngine+Actions.swift:71`, `:75`, `:87`; `Runtime/ScanEngine.swift:625-638`). But this screen + has no "Wall ends here" button and never shows the end question (both exist only on the walk, + `UI/Screens/WallWalkScreen.swift:137-173`), so only code can use it. For the homeowner the next + upload still reports that side as unexplored (`ScanEngine+Export.swift:58-59`), and the server + may ask for the same view again. - **Suspected bug: a second past-end request asks for the wrong place.** With the end forgotten, the plan falls back to the meter (`HouseScanKit/.../GapPlanner.swift:127, 130`): the ground from the meter to 2 m toward that side, "about 3 ft left of your meter", which is usually covered already. Checkable in the Simulator with the sample result: tap "Capture it now" on the left-end card twice, completing or skipping in between. -- **Suspected bug: "Capture it now" can do nothing.** The card offers the button for any wall or - ground item and any past-end item (`ScanEngine+Export.swift:134`), but the request needs a span - or a side (`GapPlanner.swift:113, 124`). Without one, the tap is ignored with no message - (`ScanEngine+Actions.swift:218`). +- "Capture it now" is offered only when a request can be built from the item: a band item needs + its span, a past-end item its side (`ScanEngine+Export.swift:152`, `GapPlanner.swift:113, 124`). + Fixed in `beede15`; before, the button could do nothing. - The phone's wording "This might be a spot for the battery" is stronger than its choice: it asks for the nearest missing ground, not a place it has judged a candidate. -- The 20 s reset keeps the list of skipped requests (`ScanEngine.swift:431-456`); after - re-walking, a phone request identical to a skipped one would be passed over (`ScanEngine.swift:529`). +- The 20 s reset keeps the list of skipped requests (`ScanEngine.swift:496-522`); after + re-walking, a phone request identical to a skipped one would be passed over (`ScanEngine.swift:596`). - **Question: "Point at the meter like this." without the picture.** The walk shows the saved close-up under this coaching (`UI/Screens/WallWalkScreen.swift:23-27`); this screen shows the words alone, even when a close-up was taken. - Nothing here has run in the Simulator. The completion moment, the amber highlight and the dots need the replay with `-autopilot`, which reaches this screen only when held-back frames exist. -Verified against house-scanning commit `525ea40` (t3/ios-mvf). +Verified against house-scanning commit `beede15` (t3/ios-mvf). diff --git a/verification/product/screens/mark-features.md b/verification/product/screens/mark-features.md index 84f158442..debbeadd7 100644 --- a/verification/product/screens/mark-features.md +++ b/verification/product/screens/mark-features.md @@ -96,7 +96,7 @@ it. What each kind asks for and records: | Gas meter, AC unit | One: "Tap the gas meter" / "Tap the ac unit", with "Put the circle on it and tap Mark, or tap it on screen." | The tapped point on the wall. The list and strip treat it as 0.3 m wide, centered on the tap; that width is nominal, not measured. | | Door, window | Two: "Tap the door's bottom-left corner" (or "window's"), with the same second line; then "Now tap its top-right corner". | A rectangle on the wall: from the lower tap's height (never below the ground) to the higher, and between the two taps along the wall. Any two opposite corners give the same rectangle. | | Driveway | Two: "Tap one end of the driveway's edge", with "Use the edge closest to the wall."; then "Now tap the other end of that edge". | The two ground points: one edge. | -| Fence | Two: "Tap the bottom of the fence at one end", with "Where it meets the ground."; then "Now tap the bottom at the other end". | The two ground points where the fence meets the ground. | +| Fence | Two: "Tap the bottom of the fence at one end", with "Where it meets the ground."; then "Now tap the bottom at the other end". | The two ground points where the fence meets the ground; its distance from the wall is the nearer point's. | ### While capturing @@ -122,9 +122,14 @@ scene description with the scan: - a door or window as an opening with its span along the wall, bottom and top; for a window, whether it opens (yes for "It opens", no for "It stays shut", or left empty when unanswered); a door always left empty; - a gas meter or AC unit as the tapped point; -- a fence as its two foot points, a driveway as its two edge points. +- a fence as the stretch of wall it faces and its distance out from the wall, taken at the foot + nearer the wall, so a fence angled toward the wall is not placed farther out than its narrow + end (fixed in `beede15`; it was the average of the two feet); +- a driveway as a strip along its two edge points. -Heights are sent as height above the ground at the wall. +Heights are sent as height above the ground at the wall. When the phone first measures the ground +or the meter's position is refined, every mark's position along the wall, heights and distance out +are recomputed from its taps ([finding the meter](find-meter.md#first-capture)). ## Modifiers @@ -185,25 +190,26 @@ appears; marking itself logs nothing. Identifiers: `action.markSomething`, `acti - **Suspected bug: "Add something" does nothing visible.** The buttons start a mark (`UI/Screens/MarkFeaturesScreen.swift:64`, accepted on this screen by - `Runtime/ScanEngine+Actions.swift:86`), but the list screen never shows the circle, the prompt, + `Runtime/ScanEngine+Actions.swift:93`), but the list screen never shows the circle, the prompt, "Mark" or camera taps; those exist only on the walk (`UI/Screens/WallWalkScreen.swift:28-45, 121-135`). The homeowner who spots a missed gas meter here presses a button and nothing - happens. "Looks complete" does not end the stray mark either (`ScanEngine+Actions.swift:168-171`). + happens. "Looks complete" does not end the stray mark either (`ScanEngine+Actions.swift:187-190`). - **Suspected bug: "Tap the ac unit".** The prompt lowercases the kind's name (`UI/Copy/ScanCopy.swift:122`), so "AC unit" becomes "ac unit" on screen, and VoiceOver hears "Remove ac unit", "Add ac unit", "Mark ac unit" (`MarkFeaturesScreen.swift:121, 189`, `WallWalkScreen.swift:282`). -- **Suspected bug: the list can vanish without a word.** Relocalization is timed on every screen - (`Runtime/ScanEngine.swift:195, 402-413`), but the list shows no coaching. After a call, a +- **Suspected bug: the list can vanish without a word.** The 20 s relocalization reset still + applies on the list, a capture screen (`Runtime/ScanEngine.swift:207, 458-478`), but the list + shows no coaching. After a call, a homeowner reading the list with the phone pointed away for 20 s is sent back to finding the meter with every mark and photo gone, never having seen "Point at the meter like this.". - **Suspected bug: a half-done mark survives the 20 s reset.** `resetSpatialState` clears the - features but not the mark in progress or its taps (`ScanEngine.swift:431-456`, taps held at - `ScanEngine.swift:65`). After the meter is tapped again the walk opens mid-mark, and a first tap + features but not the mark in progress or its taps (`ScanEngine.swift:496-522`, taps held at + `ScanEngine.swift:73`). After the meter is tapped again the walk opens mid-mark, and a first tap from the old, discarded map can be joined to a new one. The same holds for a stray mark from "Add something". - **Wrong words for the wrong-side refusal.** The check is where the phone is, not where the tap - lands (`ScanEngine+Actions.swift:101`). "Tap something on this side" cannot help; stepping back + lands (`ScanEngine+Actions.swift:108`). "Tap something on this side" cannot help; stepping back out in front of the wall's line does. - The place in a row reads "ft" to VoiceOver (`MarkFeaturesScreen.swift:107`), though the app has a spelled-out form for this (`UI/Copy/Distance.swift:25`). @@ -211,4 +217,4 @@ appears; marking itself logs nothing. Identifiers: `action.markSomething`, `acti "Mark". MF-01 in [verification](../verification.md) was observed on an earlier commit and has not been rechecked here. -Verified against house-scanning commit `525ea40` (t3/ios-mvf). +Verified against house-scanning commit `beede15` (t3/ios-mvf). diff --git a/verification/product/screens/meter-close-up.md b/verification/product/screens/meter-close-up.md index ba21405a0..dc6209061 100644 --- a/verification/product/screens/meter-close-up.md +++ b/verification/product/screens/meter-close-up.md @@ -132,4 +132,4 @@ failed attempts" when the homeowner skips. its text in that screenshot; see [verification](../verification.md). - The circle-versus-image mismatch above is read from code; it needs a device to confirm. -Verified against house-scanning commit `525ea40` (t3/ios-mvf). +Verified against house-scanning commit `beede15` (t3/ios-mvf). diff --git a/verification/product/screens/onboarding.md b/verification/product/screens/onboarding.md index 7f16e2f58..10b89eb2e 100644 --- a/verification/product/screens/onboarding.md +++ b/verification/product/screens/onboarding.md @@ -149,9 +149,9 @@ change. Tests find the controls as `action.onboardingNext`, `action.onboardingSk - **Suspected bug: camera denied, a failed session and an unreadable replay never reach the screen** (triaged as [B-03](../bug-triage.md)). The engine switches to the failure screen only - for an unsupported phone (`Runtime/ScanEngine.swift:97-99`). A denied camera and a failed - session set the failure (`ScanEngine.swift:424, 426`) and an unreadable replay does too - (`ScanEngine.swift:111`), with no screen change. The homeowner who declines the prompt sits on + for an unsupported phone (`Runtime/ScanEngine.swift:105-107`). A denied camera and a failed + session set the failure (`ScanEngine.swift:489, 491`) and an unreadable replay does too + (`ScanEngine.swift:119`), with no screen change. The homeowner who declines the prompt sits on "Find your electric meter" with no camera image; the "Open Settings" and "Start over" buttons on `UI/Screens/UnsupportedScreen.swift:35-48` are unreachable outside the `-uiDemo` preview. - **Suspected bug: an unsupported phone has no way out.** `UnsupportedScreen.swift:44` hides @@ -159,7 +159,7 @@ change. Tests find the controls as `action.onboardingNext`, `action.onboardingSk it contradicts [the flow](../foundations/flow.md#summary), which says "Start over" is offered on the unsupported screen. One of the two needs updating. - **Suspected bug: "Allow camera" during replay loading starts the live camera.** On a replay, - `startSourceIfNeeded` (`ScanEngine.swift:175`) starts the camera whenever the replay has not + `startSourceIfNeeded` (`ScanEngine.swift:184`) starts the camera whenever the replay has not loaded yet and no failure is set. A homeowner who taps "Skip" and "Allow camera" before a slow recording loads gets a live session as well as the replay. Affects replays only; not observed. - **"The camera stopped" would show raw system error text** as its detail @@ -173,4 +173,4 @@ change. Tests find the controls as `action.onboardingNext`, `action.onboardingSk session error, and the Reduce Motion stills. The unsupported screen and the `-replay /nonexistent` case can be checked in the Simulator. -Verified against house-scanning commit `525ea40` (t3/ios-mvf). +Verified against house-scanning commit `beede15` (t3/ios-mvf). diff --git a/verification/product/screens/result.md b/verification/product/screens/result.md index f57c1f62c..7beefcabb 100644 --- a/verification/product/screens/result.md +++ b/verification/product/screens/result.md @@ -29,8 +29,6 @@ stateDiagram-v2 resultAR --> result : "Done" result --> gapRequest : "Capture it now" result --> onboarding : "Start over" - result --> findMeter : phone relocalizing for over 20 s - resultAR --> findMeter : phone relocalizing for over 20 s ``` ### Arriving @@ -57,7 +55,7 @@ A success haptic plays, except when coming back from the AR view. The 3D model f the server's rules are not approved for automatic decisions. 6. "A closer spot may exist on the left of your meter. The scan didn't reach that side." (or right). It shows when the server asks for a view past a wall end, or when either wall end is - an unexplored end. + an unexplored end that the server does not mark as beyond cable reach. 7. "Your electrical panel still needs an electrician's review. This scan only covers where the battery can go.", on every result: the server's answer covers where the battery goes, not the panel. @@ -86,8 +84,10 @@ ground; the first drag stops the opening sweep where it is. Pinching zooms betwe times. The model shows the wall (to the marked wall ends, or 1.5 m past everything the result mentions), the meter, every marked feature, the cable as a blue tube, and the battery as a white box with a blue light bar. The server also sends its sweep: the stretches of wall where it tried -the battery, each with one outcome. Each stretch is a translucent tint on the ground in front of -the wall, as deep as the battery: green where a battery passes, amber where it is unsure, red +the battery, each with one outcome. The server gives each stretch as the range of places the +battery's left edge can start; the app extends it by the chosen battery's width so the tint +covers the wall the battery would stand on (without a chosen spot it covers the starts alone). +Each stretch is a translucent tint on the ground in front of the wall, as deep as the battery: green where a battery passes, amber where it is unsure, red where it fails. **The AR view.** "See it on your wall" crossfades to the camera. The instruction reads "Your @@ -107,8 +107,8 @@ from the server, its reason, and a measurement line when the server measured som "Measured 3 ft 1 in. The rule is 3 ft, and the measurement can be off by about 4 in." The second sentence drops its error clause when the server gives no error, and disappears when there is no rule value. An unsure row adds one note: "An installer will check this" when the doubt is a -narrow margin, an unknown attribute or a rule that always needs review, or "One more photo would -settle this" when the area was not seen. The rows have no buttons of their own. +narrow margin, an unknown attribute, a rule that always needs review or no stated cause, or "One +more photo would settle this" when the area was not seen. The rows have no buttons of their own. On the AR view the drawing is redrawn from every camera frame. Whenever tracking is not normal (limited, lost or relocalizing), the drawing disappears at once and the instruction reads "Point @@ -122,8 +122,9 @@ No other coaching appears here: the homeowner is not told why tracking is limite "Done" returns to the result screen. The screen is built again, so the headline fades in again and the model replays its sweep and loses any turn or zoom. -Each "Still needed" card shows the server's message. Under it is "Capture it now" when the view -is of the wall, the ground or past a wall end, and "An installer will check this" otherwise. +Each "Still needed" card shows the server's message. Under it is "Capture it now" when the phone +can build a request from it (a wall or ground view with its stretch, or a view past a wall end +with its side), and "An installer will check this" otherwise. "Capture it now" opens a [gap request](gap-request.md) for that stretch, using the server's message as the second line. For a view past a wall end, that end is forgotten first, because the wall may go on. When the view is taken, or the homeowner taps "I can't get there", the scan is @@ -146,8 +147,8 @@ uploaded again and a new result replaces this one. The old result cannot be reop | The screen's own way out | "Start over"; "Capture it now" leaves for a gap request. | "Done", back to the result. | | Start over | Offered at the bottom; see [the flow](../foundations/flow.md#cancel-and-interrupt). | Not offered. | | Tracking limited | Nothing shown; the camera is hidden but keeps tracking. | The drawing disappears and the instruction reads "Point at your meter" with "The battery comes back once your phone finds its place." until tracking is normal. | -| Tracking lost or relocalizing | After 20 s of relocalizing the app leaves the result for finding the meter, with the wall, photos and marks gone. Suspected bug; see Open questions. | The drawing disappears and "Point at your meter" shows, as above. After 20 s of relocalizing, the same reset as on the result. | -| App backgrounded or a call | The screen stays. When the app returns the phone relocalizes, and the 20 s limit above applies. | Same, with the camera resuming behind the drawing. | +| Tracking lost or relocalizing | Nothing shown; the result stays however long the phone takes (fixed in `beede15`; it used to reset to finding the meter after 20 s). | The drawing disappears and "Point at your meter" shows, as above, until the phone finds its place. No reset. | +| App backgrounded or a call | The screen stays. When the app returns the phone relocalizes in the background; nothing is lost. | Same, with the camera resuming behind the drawing. | | Camera off or session failed | No visible effect. | Suspected dead end, as in [the flow](../foundations/flow.md#open-questions-and-verification): no message, no picture. | | Network lost or upload failing | No effect; the result is already on the phone. "Capture it now" leads to a new upload, which needs the network ([the upload](uploading.md)). | No effect. | | App killed | The result is lost; nothing reopens it. | The result is lost. | @@ -186,31 +187,25 @@ appears, not when returning from the AR view. **Verification hooks.** `STATE=res ## Open questions and verification -- **Suspected bug: the result can disappear after a call or a trip to another app.** The 20 s - relocalization limit runs on every screen (`Runtime/ScanEngine.swift:195`, `402-409`). The live - camera keeps running under the result, so a phone that fails to relocalize within 20 s sends a - homeowner who is reading their result back to "Find your electric meter". On the result - screen the camera is hidden, so nothing even tells them to aim at the meter; only the AR view - says "Point at your meter". The reset - (`ScanEngine.swift:431-455`) also leaves the old result in memory. The limit should apply only - to the capture screens. -- **Suspected bug: the tinted stretches are in the wrong place.** Each sweep run's `start_ft` is - the range of the battery's left edge (`HouseScanKit/.../PlacementResult.swift:450`; the result - schema says the same). The app tints exactly that range (`Runtime/ScanEngine+Export.swift:124`), - so every stretch stops one battery width short on the right. +- The 20 s relocalization reset no longer applies on the result or the AR view + (`Runtime/ScanEngine.swift:472-476`), fixed in `beede15`. +- The tinted stretches now reach one battery width past the last start + (`Runtime/ScanEngine+Export.swift:132-141`; `start_ft` at `HouseScanKit/.../PlacementResult.swift:450`), + fixed in `beede15`. A result with no spot still tints only the starts, which understates each + stretch; the code says so. - The AR view now names a sample spot as an example (`UI/Screens/ResultARScreen.swift:49-51`), fixed in `525ea40`. It still has no badge like the result screen's (`UI/Screens/ResultScreen.swift:96-105`). - **Question: an end answered "It turns a corner" may read as unseen.** The left-or-right note - shows when the server reports an end as unexplored (`Runtime/ScanEngine+Export.swift:141-144`), + shows when the server reports an end as unexplored and not beyond cable reach + (`Runtime/ScanEngine+Export.swift:157-163`), and a corner answer is sent as unexplored. A homeowner who walked to the corner would then read that "the scan didn't reach that side". Depends on what the server echoes back; not checked. - The measurement line says "The rule is 20 ft" for a limit that is a maximum and "The rule is 3 ft" for a minimum. The app decodes the server's `comparison` but never shows it, and ignores `review_threshold_ft` (`UI/Copy/ScanCopy.swift:233-237`). -- "Capture it now" does nothing when the server's item has no stretch - (`Runtime/ScanEngine+Actions.swift:218`, `HouseScanKit/.../GapPlanner.swift:113`), although - the card offers it (`ScanEngine+Export.swift:134`). +- "Capture it now" is offered only when the phone can build a request from the item + (`Runtime/ScanEngine+Export.swift:152`), fixed in `beede15`; before, it could do nothing. - After "Capture it now" the old result is gone from view. If the new upload fails, the homeowner has only "Try again" on the upload screen and cannot reread the earlier answer. - VoiceOver reads the model's value and the measurement lines with "ft" and "in", which it reads @@ -222,6 +217,6 @@ appears, not when returning from the AR view. **Verification hooks.** `STATE=res scrolls the page needs a device. - Both screens were seen at `0876e03`, `24af434` and `525ea40` with the real server's answer, and with the sample result ([verification](../verification.md), RES-01 to AR-01). Whether the AR drawing lines up with a - real wall, and the relocalization reset, need the live camera. + real wall, and hiding it while the phone relocalizes, need the live camera. -Verified against house-scanning commit `525ea40` (t3/ios-mvf). +Verified against house-scanning commit `beede15` (t3/ios-mvf). diff --git a/verification/product/screens/uploading.md b/verification/product/screens/uploading.md index 58aaaa388..eb835ae08 100644 --- a/verification/product/screens/uploading.md +++ b/verification/product/screens/uploading.md @@ -97,7 +97,7 @@ screen until "Try again" succeeds. | The screen's own way out | None. | "Try again": zips and sends the whole scan again. | | Start over | Not offered. | Not offered. | | Tracking limited | No effect; this screen shows no coaching. | No effect. | -| Tracking lost or relocalizing | Suspected bug: after 20 s of relocalizing on the live camera, the scan is forgotten and the app returns to finding the meter; the answer, when it comes, is dropped. | Same: the failure gives way to finding the meter. | +| Tracking lost or relocalizing | No effect: the 20 s reset does not apply on this screen, so the scan and its answer are kept (fixed in `beede15`). | No effect; the failure and "Try again" stay. | | App backgrounded or a call | The app asks for no background time, so iOS may suspend the transfer; a broken connection would then show "You're offline". Unverified. | The failure stays. | | Camera off or session failed | No effect; the upload does not need the camera. | No effect. | | Network lost or upload failing | The steps give way to the failure and "Try again" (table above). | "Try again" fails the same way until the network or server is back. | @@ -148,32 +148,27 @@ autopilot logs "upload failed; retrying once". Test identifiers: `screen.uploadi Simulator run at this commit ([verification](../verification.md), FLOW-02). - **Suspected bug: raw error text on screen.** The second line of "That didn't go through" is the error's developer description (`UI/Copy/ScanCopy.swift:196`, fed by - `Runtime/ScanEngine.swift:587, 614`): HTTP status codes, server JSON, decoder errors. B-02. -- **Suspected bug: "You're offline" for any connection error.** `Runtime/ScanEngine.swift:614` + `Runtime/ScanEngine.swift:672, 699`): HTTP status codes, server JSON, decoder errors. B-02. +- **Suspected bug: "You're offline" for any connection error.** `Runtime/ScanEngine.swift:699` treats every `URLError` as offline, including an unreachable host, refused connection and timeout. The copy then tells a homeowner with full signal to wait for signal. - **Suspected bug: "Checking your wall" is skipped with a real server.** Progress is capped at 99% while sending (`Runtime/ResultClient.swift:68`) and reaches 100% only after the server has - answered (`:40`); the step changes only then (`Runtime/ScanEngine.swift:596, 600`). The + answered (`:40`); the step changes only then (`Runtime/ScanEngine.swift:681, 685`). The homeowner watches "Sending photos, 99%" for the whole analysis, then the result replaces "Checking your wall" almost at once. Only the sample result lingers on the third step, where it reads "Show the example". -- **Suspected bug: a relocalization reset can wipe the scan during the upload.** The - relocalization check runs on every live frame whatever the screen (`Runtime/ScanEngine.swift:195`, - `409`). After 20 s it forgets the wall, marks and photo list and goes to `findMeter` (`:431–455`), - and the upload's answer is dropped (the `generation` guards at `:599, 606`). A homeowner who - locks the phone mid-upload and returns somewhere the phone can't recognise would lose the scan. - Unverified: needs a device, and whether the AR session keeps delivering frames while the camera - view is off screen is not settled by the code. +- A relocalization reset can no longer wipe the scan during the upload: the reset applies only + on the capture screens (`Runtime/ScanEngine.swift:469-476`), fixed in `beede15`. - The app asks for no background execution time (no `beginBackgroundTask`), so "Keep the app open" is the only guard for the transfer. What iOS does to the transfer when the phone locks has not been tested. - "It takes about a minute" has no measurement behind it; no upload has been timed with a real scan. -- The comment on `writeBundle` (`Runtime/KeyframeStore.swift:83–84`) says the zip stays so a - failed upload can be retried, but "Try again" rebuilds it (`Runtime/ScanEngine+Actions.swift:207–209`). +- The comment on `writeBundle` (`Runtime/KeyframeStore.swift:89–90`) says the zip stays so a + failed upload can be retried, but "Try again" rebuilds it (`Runtime/ScanEngine+Actions.swift:226–228`). Harmless, but the kept zip is never reused. - Pass 1 (`21a63e7`) saw "That didn't go through" with no button or symbol visible ([verification](../verification.md) UP-01, B-05). Not rechecked at this commit. -Verified against house-scanning commit `525ea40` (t3/ios-mvf). +Verified against house-scanning commit `beede15` (t3/ios-mvf). diff --git a/verification/product/screens/wall-walk.md b/verification/product/screens/wall-walk.md index 76ee457c4..e27ff2ba7 100644 --- a/verification/product/screens/wall-walk.md +++ b/verification/product/screens/wall-walk.md @@ -76,7 +76,7 @@ The instructions this screen can show, with their second lines: | "Take a step back" | "Your phone needs to see more of the wall at once." | None. | | "Tilt down to show the ground" | "The strip along the wall, 3 ft 4 in left of your meter." (the distance varies) | Ring on the ground, dots toward it. | | "Tilt up to show more wall" | "Around 3 ft 4 in left of your meter." (the distance varies) | Ring on the wall, dots toward it. | -| "Walk slowly to your left" (or right) | "Keep the wall and the ground in view." | Ring on the wall past the covered stretch, dots toward it. | +| "Walk slowly to your left" (or right) | "Keep the wall and the ground in view." | Ring on the wall 1 m past whichever is farther along that side, the unbroken covered stretch or the phone itself; dots toward it. | | "Is this the left end of the wall?" (or right) | "Aim where the wall stops or turns a corner, and tap Wall ends here." | Ring at the end of the covered stretch, no dots. | | "What's at the left end?" (or right) | "This tells the installer whether the wall keeps going." | The ring and dots of the previous instruction stay until the answer. | | "That's the whole wall" | "Tap Done when you're ready." | None. | @@ -105,10 +105,12 @@ the phone losing its place for over 20 s, which throws the walk away. The first photo is kept by the rules in [coverage and guidance](../foundations/coverage-and-guidance.md#when-a-photo-is-kept). When it is saved, the counter's camera icon flashes green and the number rolls up by one. There is no haptic -tap for walk photos in this build. Wall and ground cells the photo shows most of go from fog to a -thinner haze: their fog fades over 0.7 s while drifting upward, like mist lifting. On the strip -those cells turn from gray to amber. A cell seen a second time from a different position turns -green on the strip and its haze clears completely. +tap for walk photos in this build. Wall and ground cells the photo shows any sample line of go +from fog to a thinner haze: their fog fades over 0.7 s while drifting upward, like mist lifting. +On the strip those cells turn from gray to amber. A cell turns green on the strip, and its haze +clears completely, once each of its three sample lines has been seen from two places +([coverage and guidance](../foundations/coverage-and-guidance.md#while-capturing)); a wall cell +whose top line, 1.98 m up, never came into view stays amber. ### While capturing @@ -120,7 +122,7 @@ mark every foot, longer every fifth. A marked end is a tall white cap; a marked icon above the rows. The strip always spans at least 3.5 m (about 11 ft), and its scale changes as coverage, the ends or the homeowner move outward. -**The fog.** A frosted haze painted on the wall face (up to the top of the wall band) and on the +**The fog.** A frosted haze painted on the wall face (up to the top of the wall band, 1.98 m) and on the ground in front of it, over what has been seen plus the fog ahead of it on each side. It is a thin frosted blur under a light white wash (35 % opacity), both scaled per cell: unseen cells at 75 % strength, so the wall's shapes still show through, seen cells at 35 %, covered cells clear. @@ -302,7 +304,7 @@ reset: relocalization timed out". Accessibility identifiers: `action.markSomethi A side can read 100 percent while none of it is covered and its ground is unseen. - Standing at the meter, VoiceOver says "You are 0 inches right of your meter". - "About N ft to go" can never appear on the walk instruction: the engine always passes no - remaining distance (`Runtime/ScanEngine.swift:699`). + remaining distance (`Runtime/ScanEngine.swift:784`). ## Open questions and verification @@ -310,41 +312,41 @@ reset: relocalization timed out". Accessibility identifiers: `action.markSomethi makes the kind of a marked end truthful, but "Wall ends here" is still shown only while the instruction is "Is this the … end of the wall?" (`UI/Screens/WallWalkScreen.swift:101`, `:165`), and the planner asks that only once coverage runs unbroken to 6.1 m, about 20 ft - (`HouseScanKit/Guidance/GuidancePlanner.swift:27`, `:113`). A homeowner whose wall stops at a + (`HouseScanKit/Guidance/GuidancePlanner.swift:27`, `:119`). A homeowner whose wall stops at a fence 10 ft from the meter is told "Walk slowly to your left" with only "Can't get there" to answer, which records an unexplored end instead of a real one. A corner at 10 ft comes out right by accident, since a corner is unexplored anyway. - **Suspected bug: no "Can't get there" at "Is this the … end of the wall?".** The pill is limited to walk and tilt instructions (`WallWalkScreen.swift:107-112`), although the engine handles it - for this question (`Runtime/ScanEngine+Actions.swift:190`). A wall that goes on straight past + for this question (`Runtime/ScanEngine+Actions.swift:209`). A wall that goes on straight past 20 ft can be recorded as unexplored only by marking an end where there is none and answering "It turns a corner". This sharpens B-06 in [bug-triage.md](../bug-triage.md). - **Suspected bug: "Wall ends here" aims at a circle that is not drawn.** The end goes where the - middle of the screen meets the wall (`ScanEngine+Actions.swift:68`), but the center circle is + middle of the screen meets the wall (`ScanEngine+Actions.swift:73`), but the center circle is drawn only while marking (`WallWalkScreen.swift:28-33`). The ring on screen is the planner's - guess at the end (`GuidancePlanner.swift:187-188`), so a homeowner who lines the ring up with the + guess at the end (`GuidancePlanner.swift:202-203`), so a homeowner who lines the ring up with the corner sets the end elsewhere. The hint promises "the circle in the middle of the screen". - **Suspected bug: "Wall ends here" is unchecked and silent on a miss.** It accepts a tap with - limited tracking, unlike marks (`ScanEngine+Actions.swift:66-74` against `:93`); it does nothing - and says nothing when the aim misses the wall (`:68`). A tap that lands gives no haptic; the end + limited tracking, unlike marks (`ScanEngine+Actions.swift:70-80` against `:100`); it does nothing + and says nothing when the aim misses the wall (`:73`). A tap that lands gives no haptic; the end question is its only feedback. - **Suspected bug: the 20 s reset keeps a mark in progress.** `resetSpatialState` - (`Runtime/ScanEngine.swift:431-456`) clears the wall, marks and photos but not the marking prompt + (`Runtime/ScanEngine.swift:496-522`) clears the wall, marks and photos but not the marking prompt or its placed taps. After the meter is found again, the walk opens straight into the old prompt ("Now tap its top-right corner"), and the finished feature joins a point from the lost world frame to one from the new frame. - **Suspected bug: two taps on "Can't get there" finish an empty walk.** On arrival each tap sets an - end at the meter (`ScanEngine+Actions.swift:192-194`); with both ends at the same point the - planner finds nothing to fill (`GuidancePlanner.swift:138`) and says "That's the whole wall". + end at the meter (`ScanEngine+Actions.swift:211-213`); with both ends at the same point the + planner finds nothing to fill (`GuidancePlanner.swift:144`) and says "That's the whole wall". - **Suspected bug: the AC unit prompt reads "Tap the ac unit".** The prompt lower-cases the kind's name (`UI/Copy/ScanCopy.swift:122`, `:137`); the picker tile's VoiceOver label has the same problem ("Mark ac unit", `WallWalkScreen.swift:282`). - **Possible bug: "That spot is behind the wall" judges the phone, not the tap.** The check is on - where the phone stands (`ScanEngine+Actions.swift:101`), so the message blames the spot when it is + where the phone stands (`ScanEngine+Actions.swift:108`), so the message blames the spot when it is the homeowner who is behind the wall line. - The kind picker slides in under Reduce Motion (`WallWalkScreen.swift:164`). - The reset returns to "Find your electric meter" without telling the homeowner that the walk was - lost. The old meter close-up stays in the scan (`Runtime/KeyframeStore.swift:79-81` clears only + lost. The old meter close-up stays in the scan (`Runtime/KeyframeStore.swift:84-87` clears only walk photos); if the second close-up is skipped, the first one is uploaded. - The foundations say each kept photo gives a light haptic tap. In this build walk photos give none (`UI/ScanRootView.swift:91`, `:101-104`); [the flow](../foundations/flow.md) and @@ -357,4 +359,4 @@ reset: relocalization timed out". Accessibility identifiers: `action.markSomethi interruption recovery. The first Simulator pass showed this screen's chrome without text (B-05 in [bug-triage.md](../bug-triage.md)). -Verified against house-scanning commit `525ea40` (t3/ios-mvf). +Verified against house-scanning commit `beede15` (t3/ios-mvf). diff --git a/verification/product/verification.md b/verification/product/verification.md index 054067ce9..72a4a1689 100644 --- a/verification/product/verification.md +++ b/verification/product/verification.md @@ -15,9 +15,10 @@ what each shows and speaks, what the app logged, and what the server answered. W cover: anything needing the live camera (tracking, the meter tap on a real wall, a close-up that succeeds), haptics, or timing a person feels. -## Current pass: `t3/ios-mvf` `525ea40`, server `26d2870` +## Current pass: `t3/ios-mvf` `beede15`, server `f2705dd` -Report `sim/20260926-052535-t3-ios-mvf-525ea401-replay` (real server), the audit at `24af434` and +Report `sim/20260926-061551-t3-ios-mvf-beede15f-replay` (real server; the pass at `525ea40` is +`sim/20260926-052535-t3-ios-mvf-525ea401-replay`), the audit at `24af434` and `525ea40` (sample result), plus the variant runs on the preview `5520229` (the same UI before S3's last fixes) named in [the UX review](../ux/review.md#evidence). @@ -33,10 +34,10 @@ Report `sim/20260926-052535-t3-ios-mvf-525ea401-replay` (real server), the audit | WW-02 | Guidance moves from walking to tilting to stepping back | Pass: the audit saw "Walk slowly to your left", "Tilt down to show the ground", "Take a step back". | | MF-01 | The list shows each mark with its distance, the window question and "Add something" | Pass at default size; hidden behind "Looks complete" at the largest text size (UX-01). | | GAP-01 | A gap request states what to show and why, with a progress bar and "I can't get there" | Pass ("Show the ground around your meter", "0%"). | -| GAP-02 | New photos close the request and the flow moves on | Pass: logged "gap 1 satisfied"; the audit saw "Got it, thanks" / "That's the view we needed." | +| GAP-02 | New photos close the request and the flow moves on | Pass at `24af434` and `525ea40` ("gap 1 satisfied"; "Got it, thanks"). Not exercised at `beede15`: the autopilot finds no closable gap on this replay and skips it. | | WW-03 | After "Wall ends here" the walk asks what is at that end, and a corner stays unexplored | Pass: logged "end left at s=-3.81 (unexplored)", then "answered the left end: turns a corner"; same on the right. | | UP-01 | Upload progress is shown in steps apart from coverage | Pass: "Photos ready", "Sending photos, 99%", "Check clearances". | -| RES-01 | The result shows the headline, placement line, the server's summary, the rules-not-final note and the panel notice | Pass: "An installer will take a look" with the server's placement and summary (seen at `0876e03`, `24af434` and `525ea40`). | +| RES-01 | The result shows the headline, placement line, the server's summary, the rules-not-final note and the panel notice | Pass: "An installer will take a look" with the server's placement and summary (seen at `0876e03`, `24af434`, `525ea40` and `beede15`). | | RES-02 | A borderline check shows the measurement, the rule and the error | Pass on the sample result: "Measured 3 ft 1 in. The rule is 3 ft, and the measurement can be off by about 4 in." | | RES-03 | A maximum reads as a maximum | Fail: "Measured 3 ft. The rule is 20 ft" (B-14). | | RES-04 | The result is visible when the app logs `STATE=result` | Fail, minor: the upload screen was still showing 1.2 s later on first load, in both passes (UX-07). | diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index 41ab0e8e2..0cd02b865 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,15 +1,15 @@ { - "generated_at": "2026-09-26T06:13:39-05:00", + "generated_at": "2026-09-26T06:28:21-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": false, "prs": [ { "number": 16, - "title": "experiments: measure on-device meter close-up reading and retake thresholds", + "title": "experiments: meter close-up reading, retake thresholds and number finding", "branch": "t3/meter-closeup", "head": "c5cb30d782fa8bd9a91e30f6997b7239764db640", "draft": true, - "updated": "2026-09-26T11:13:17Z", + "updated": "2026-09-26T11:13:43Z", "ci": { "counts": { "success": 3, @@ -67,9 +67,9 @@ "number": 13, "title": "verification: independent end-to-end checks (S4)", "branch": "t3/verification", - "head": "4b4425b5cf899c1c43d78b75f2172a77aaa24c82", + "head": "5e60831dcc87ca0b854b4ce51c1e8b220438fc7a", "draft": true, - "updated": "2026-09-26T10:53:26Z", + "updated": "2026-09-26T11:13:41Z", "ci": { "counts": { "success": 2, @@ -87,20 +87,18 @@ "number": 12, "title": "feat(evals): real-data accuracy evals for ARKit drift and photo-based wall reconstruction", "branch": "t3/evals", - "head": "779d33918a4d4dd997364b5a9345392ac9cff45e", + "head": "27293c5d4d34a27af652116341639f1826d806ae", "draft": true, - "updated": "2026-09-26T11:03:22Z", + "updated": "2026-09-26T11:20:39Z", "ci": { "counts": { "success": 3, - "failure": 1, + "failure": 0, "pending": 0, "skipped": 0 }, - "failing": [ - "Vercel" - ], - "text": "3 success, 1 failure; failing: Vercel" + "failing": [], + "text": "3 success" } }, { @@ -109,7 +107,7 @@ "branch": "t3/server", "head": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", "draft": false, - "updated": "2026-09-26T10:55:25Z", + "updated": "2026-09-26T11:22:27Z", "ci": { "counts": { "success": 5, @@ -131,12 +129,14 @@ "ci": { "counts": { "success": 4, - "failure": 0, - "pending": 1, + "failure": 1, + "pending": 0, "skipped": 0 }, - "failing": [], - "text": "4 success, 1 pending" + "failing": [ + "iOS build" + ], + "text": "4 success, 1 failure; failing: iOS build" } }, { @@ -238,7 +238,7 @@ "refs": [ { "ref": "origin/t3/evals", - "sha": "779d33918a4d4dd997364b5a9345392ac9cff45e" + "sha": "27293c5d4d34a27af652116341639f1826d806ae" } ], "metrics": [ @@ -246,7 +246,7 @@ "id": "S1-1", "text": "One command regenerates every number from a clean checkout", "ref": "origin/t3/evals", - "sha": "779d33918a4d4dd997364b5a9345392ac9cff45e", + "sha": "27293c5d4d34a27af652116341639f1826d806ae", "status": "partial", "evidence": "make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac.", "links": [] @@ -255,13 +255,13 @@ "id": "S1-2", "text": "Dataset URLs and sha256 recorded", "ref": "origin/t3/evals", - "sha": "779d33918a4d4dd997364b5a9345392ac9cff45e", + "sha": "27293c5d4d34a27af652116341639f1826d806ae", "status": "met", "evidence": "all 2 found", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/README.md" ] ] }, @@ -269,7 +269,7 @@ "id": "S1-3", "text": "Metric code tested on hand-computed cases", "ref": "origin/t3/evals", - "sha": "779d33918a4d4dd997364b5a9345392ac9cff45e", + "sha": "27293c5d4d34a27af652116341639f1826d806ae", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -278,21 +278,21 @@ "id": "S1-4", "text": "ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft", "ref": "origin/t3/evals", - "sha": "779d33918a4d4dd997364b5a9345392ac9cff45e", + "sha": "27293c5d4d34a27af652116341639f1826d806ae", "status": "met", "evidence": "all 5 found", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/README.md" ], [ "experiments/evals/results/advio_drift.md", - "https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/results/advio_drift.md" + "https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/results/advio_drift.md" ], [ "experiments/evals/results/modern_arkit.md", - "https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/results/modern_arkit.md" + "https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/results/modern_arkit.md" ] ] }, @@ -300,21 +300,21 @@ "id": "S1-5", "text": "ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question", "ref": "origin/t3/evals", - "sha": "779d33918a4d4dd997364b5a9345392ac9cff45e", + "sha": "27293c5d4d34a27af652116341639f1826d806ae", "status": "partial", "evidence": "found 1-3 m row, 3-10 m row, scale error; missing verdict", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/README.md" ], [ "experiments/evals/results/eth3d_recon.md", - "https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/results/eth3d_recon.md" + "https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/results/eth3d_recon.md" ], [ "experiments/evals/results/eth3d_visibility_sensitivity.md", - "https://github.com/SamGu-NRX/house-scanning/blob/779d33918a4d4dd997364b5a9345392ac9cff45e/experiments/evals/results/eth3d_visibility_sensitivity.md" + "https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/results/eth3d_visibility_sensitivity.md" ] ] } @@ -451,8 +451,8 @@ "text": "CI build green", "ref": "origin/t3/ios-mvf", "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", - "status": "not run", - "evidence": "iOS build: pending at beede15f", + "status": "gap", + "evidence": "iOS build: failure at beede15f", "links": [ [ "#10", @@ -479,8 +479,8 @@ "text": "Zero build warnings", "ref": "origin/t3/ios-mvf", "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", - "status": "stale", - "evidence": "report for 525ea401, ref at beede15f: ~/house-scanning-data/reports/sim/20260926-052535-t3-ios-mvf-525ea401-replay/report.json", + "status": "met", + "evidence": "build.warnings = length 0 (want 0): ~/house-scanning-data/reports/sim/20260926-061551-t3-ios-mvf-beede15f-replay/report.json", "links": [] }, { @@ -590,7 +590,7 @@ "refs": [ { "ref": "origin/t3/verification", - "sha": "4b4425b5cf899c1c43d78b75f2172a77aaa24c82" + "sha": "5e60831dcc87ca0b854b4ce51c1e8b220438fc7a" }, { "ref": "origin/t3/ios-mvf", @@ -602,7 +602,7 @@ "id": "S4-1", "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/verification", - "sha": "4b4425b5cf899c1c43d78b75f2172a77aaa24c82", + "sha": "5e60831dcc87ca0b854b4ce51c1e8b220438fc7a", "status": "met", "evidence": "app_export_passed = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json", "links": [] @@ -612,15 +612,15 @@ "text": "Screenshot report per app state", "ref": "origin/t3/ios-mvf", "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", - "status": "stale", - "evidence": "report for 525ea401, ref at beede15f: ~/house-scanning-data/reports/sim/20260926-052535-t3-ios-mvf-525ea401-replay/report.json", + "status": "met", + "evidence": "states = length 10 (want at least 9): ~/house-scanning-data/reports/sim/20260926-061551-t3-ios-mvf-beede15f-replay/report.json", "links": [] }, { "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "4b4425b5cf899c1c43d78b75f2172a77aaa24c82", + "sha": "5e60831dcc87ca0b854b4ce51c1e8b220438fc7a", "status": "met", "evidence": "verification/product covers every screen of t3/ios-mvf at 525ea40, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state.", "links": [] @@ -629,13 +629,13 @@ "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "4b4425b5cf899c1c43d78b75f2172a77aaa24c82", + "sha": "5e60831dcc87ca0b854b4ce51c1e8b220438fc7a", "status": "met", - "evidence": "present at 4b4425b5", + "evidence": "present at 5e60831d", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/4b4425b5cf899c1c43d78b75f2172a77aaa24c82/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/5e60831dcc87ca0b854b4ce51c1e8b220438fc7a/verification/SCOREBOARD.md" ] ] } From 97cebbf9cbfb9a01a1bad652d61996664777ba75 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 06:42:09 -0500 Subject: [PATCH 30/59] verification: derive coverage radii from rules.yaml, add ordering, completeness and hostile-input checks The end-to-end check passed four S2 defects (unseen ground past the modeled reach, a short cable cutoff under drift, unbounded work on small inputs, and missing_evidence naming only one band). It now reads each clearance radius and error from rules.yaml at the tested ref, checks that no pass rests on unobserved wall or ground within that radius, that more error, tap for tape or less coverage never improves an outcome, that every unobserved check is named in missing_evidence, and that hostile inputs are refused or answered within 10 s. Adds drift cases, a CI workflow for these tools, the app SHA in e2e reports, and requires the result state in sim-app. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/verification.yml | 42 +++ verification/Makefile | 5 +- verification/e2e/cases/README.md | 96 +++++- verification/e2e/cases/d-gas-drift.json | 168 +++++++++++ .../e2e/cases/d-reach-drift-left.json | 160 ++++++++++ .../e2e/cases/d-reach-drift-right.json | 160 ++++++++++ verification/e2e/cases/generate.py | 98 ++++++- verification/hsverify/a11yaudit.py | 65 ++++- verification/hsverify/e2e.py | 249 +++++++++++++--- verification/hsverify/resultcheck.py | 274 ++++++++++++++++-- verification/hsverify/scoreboard.py | 49 +++- verification/hsverify/simrun.py | 144 +++++++-- verification/hsverify/statelog.py | 35 ++- verification/scoreboard/metrics.yaml | 27 +- verification/tests/test_a11yaudit.py | 49 +++- verification/tests/test_e2e.py | 96 +++++- verification/tests/test_resultcheck.py | 189 +++++++++++- verification/tests/test_scoreboard.py | 73 ++++- verification/tests/test_simrun.py | 124 +++++++- verification/tests/test_statelog.py | 22 ++ 20 files changed, 1976 insertions(+), 149 deletions(-) create mode 100644 .github/workflows/verification.yml create mode 100644 verification/e2e/cases/d-gas-drift.json create mode 100644 verification/e2e/cases/d-reach-drift-left.json create mode 100644 verification/e2e/cases/d-reach-drift-right.json diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml new file mode 100644 index 000000000..ad38024b2 --- /dev/null +++ b/.github/workflows/verification.yml @@ -0,0 +1,42 @@ +name: Verification + +on: + push: + branches: [main] + paths: + - "verification/**" + - ".github/workflows/verification.yml" + pull_request: + paths: + - "verification/**" + - ".github/workflows/verification.yml" + +permissions: + contents: read + +concurrency: + group: verification-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + verification-checks: + name: Verification checks + runs-on: ubuntu-24.04 + timeout-minutes: 10 + defaults: + run: + working-directory: verification + steps: + # The default depth of 1 is enough: tests/test_gitref.py only reads HEAD, and a detached + # worktree of HEAD works in a shallow clone. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # Reads tool.uv.required-version from verification/pyproject.toml to pick the uv version. + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + working-directory: verification + - run: uv sync --locked + - run: uv run ruff check . + - run: uv run ruff format --check . + - run: uv run pytest -q diff --git a/verification/Makefile b/verification/Makefile index ea4c6e888..910827cdb 100644 --- a/verification/Makefile +++ b/verification/Makefile @@ -19,11 +19,12 @@ sim: uv run python -m hsverify.simrun --ref $(REF) $(ARGS) # The app at REF on the real ADVIO replay (C3), driven by its autopilot, uploading to the server -# started from SERVER_REF: a screenshot per STATE in ~/house-scanning-data/reports/sim/. +# started from SERVER_REF: a screenshot per STATE in ~/house-scanning-data/reports/sim/. Fails +# unless the run reaches the result state; the scan bundle is copied into the report as scan.zip. sim-app: uv run python -m hsverify.simrun --ref $(REF) --replay $(REPLAY) --autopilot \ --server-ref $(SERVER_REF) --extra-arg=-autopilotHold --extra-arg=3 \ - --idle 60 --timeout 900 --label replay $(ARGS) + --idle 60 --timeout 900 --require result --label replay $(ARGS) # Every case in e2e/cases/ plus the ETH3D facade scene through the server's API at SERVER_REF; # each result is validated against server/schemas/result.schema.json at the same ref. diff --git a/verification/e2e/cases/README.md b/verification/e2e/cases/README.md index 06b6c9634..fa5396b6a 100644 --- a/verification/e2e/cases/README.md +++ b/verification/e2e/cases/README.md @@ -34,6 +34,8 @@ The keys follow the lead's case format. The cases use two additions and one conv gap the case built. With a different battery depth it measures a different gap, and these keys catch that before the outcome is compared. - `missing_evidence_empty: false` means the list must be non-empty. +- `start_outcomes` (d cases): `{"wall_id", "start_ft", "outcome", "why"}`. The sweep run that + contains that battery start must have that outcome. - `rules_assumed` maps a check-id substring to the `threshold_ft` the case depends on: `gas` 3 ft (`clearances.gas_ft`), `opening` 3 ft (`clearances.opening_ft`), `headroom` 6.5 ft (`headroom.min_ft`) and `route_length` 20 ft (`route.max_ft`). `route.confident_reach_ft` (15) @@ -42,7 +44,7 @@ The keys follow the lead's case format. The cases use two additions and one conv ## The rules these cases depend on -From `rules.yaml` at 2c9348f: +From `rules.yaml` at 2c9348f, rechecked at f2705dd: - Battery 31/12 × 11/6 ft (W ≈ 2.5833, D ≈ 1.8333), matching the review's table. - `meter_working_space` 2.5 ft wide, centred on the meter. A battery overlapping s = (−1.25, 1.25) @@ -51,8 +53,9 @@ From `rules.yaml` at 2c9348f: - `pool_ft` 10 and `drive_ft` 5, both placeholders. With no pool or driveway in the scene these checks pass only if the ground was seen 10 ft around the footprint. Otherwise they are unsure and ask for a photo. -- `openings.types` is `[door, window]`. A garage door is not an opening, so it has no 3 ft - clearance. It does fail the route (`route.crossing.garage_door: fail`). +- `openings.types` is `[door, garage_door, window]` since f2705dd (it was `[door, window]` at + 2c9348f). A garage door therefore has the 3 ft opening clearance, and it also fails the route + (`route.crossing.garage_door: fail`). g01 and g08a hold under either version. - `route.crossing.window: detour`, `gas_meter: detour` and `elec_box: detour`. `route.corner_allowance_ft` adds 0.5 ft per corner the cable turns. `route.height_ft` 1: the cable runs 1 ft above the ground. The route length has no vertical run from the meter down to @@ -60,7 +63,8 @@ From `rules.yaml` at 2c9348f: 2 × (top − 1): up to its top and back down. Going under is impossible when the object sits on the ground. A window whose bottom is above 1 ft is crossed with no extra (g12a: route 7.042 for start 7.042). -- `route.confident_reach_ft` 15: a longer route is unsure. `route.max_ft` 20 (`route_length`, +- `route.confident_reach_ft` 15: a longer route is unsure. Since f2705dd the `route_length` + check reports it as `review_threshold_ft`: pass needs length + error < 15. `route.max_ft` 20 (`route_length`, `at_most`): pass needs length + error < 20, fail needs length − error > 20. The route's error is the meter's and the wall's errors summed (at 2c9348f a wall at ±0.3 with an exact meter gives a route at ±0.3). @@ -92,7 +96,7 @@ From `rules.yaml` at 2c9348f: | Case | Golden | Asserts | Arithmetic | | --- | --- | --- | --- | -| g01-unseen-beyond-garage | 01 | `missing_evidence` empty; starts in (−19.9, −8.2) fail with `route` | Wall s = [−20, 10], garage [−5.5, −1], nothing observed left of s = −9. A battery with right edge b < −1 needs a cable across the garage (b ≤ −5.5 means a ≤ −8.083). A battery with b > −1 overlaps the working space. So every start left of the meter fails. The first start right of the meter that clears the working space is 1.25, and pool needs ground from 1.25 − 10 = −8.75 ≥ −9. | +| g01-unseen-beyond-garage | 01 | `missing_evidence` empty; starts in (−19.9, −8.2) fail with `route` | Wall s = [−20, 10], garage [−5.5, −1], nothing observed left of s = −9. A battery with right edge b < −1 needs a cable across the garage (b ≤ −5.5 means a ≤ −8.083). A battery with b > −1 overlaps the working space. So every start left of the meter fails. Right of the meter a start must clear the working space (a ≥ 1.25) and, since f2705dd, the garage's 3 ft opening clearance (a ≥ −1 + 3 = 2). Pool then needs ground from 2 − 10 = −8 (−8.75 at 2c9348f), both ≥ −9. | | g03-negative-s-near-edge | 03 | spot on w1 within [−9, −6]; starts in (−8.4, 6.9) fail; no photo request | Pads are straight 3 ft segments at s = [−9, −6] and [7, 10]. Between them sit 7 segments of 13/7 ≈ 1.857 ft, each shorter than the battery, with corners at s = −4.143, −2.286, −0.429, 1.429, 3.286, 5.143. Headings run 32°, 24°, …, −24°, −32°, so every corner is an outside corner. Left route = 6 + 3 corners × 0.5 = 7.5; right = 7 + 3 × 0.5 = 8.5; left wins. Measuring to the far edge gives 103/12 + 1.5 ≈ 10.08 > 8.5 and would pick the right pad. Pad starts are (−9, −8.583] and [7, 7.417]; every other start crosses a corner. | | g05-inside-corner-gas | 05 | starts on w1 in (6, 7.4) fail with `gas`; never pass | w1 is z = 0 from x = −7 to 4 (w1 extended 1 ft left of the golden, so the meter at x = −6 sits inside the wall rather than on its end); s = x + 6. w2 is x = 4 going +z; gas point [4, 4] at s = 14. For start s the battery is x ∈ [s − 6, s − 6 + W], z ∈ [0, 11/6]. At s = 6 the nearest corner is (31/12, 11/6): √((17/12)² + (13/6)²) = √965/12 ≈ 2.589 < 3. Fail needs the right edge past 4 − √(9 − (13/6)²) = 4 − √155/6 ≈ 1.925, so s > 5.342. The last start on w1 is 89/12 ≈ 7.417 (2.167). Unrolled gap 14 − 103/12 = 65/12 ≈ 5.417 would wrongly pass. | | g06-regulator-footprint | 06 | starts in (3, 9) fail with `gas`; never pass | Gas footprint: body x ∈ [6.5, 7.5], z ∈ [9.5, 10.5], with a spike reaching [7, 23/6]. Battery front at z = 11/6, so the gap to the tip is 23/6 − 11/6 = 2 whenever the battery covers x = 7, i.e. starts in [7 − W, 7] ≈ [4.417, 7]. Outside that the gap is √(dx² + 4) < 3 while dx < √5 ≈ 2.236, so starts in (2.18, 9.24) fail. The body alone gives 10 − 11/6 ≈ 8.17 and would pass. Facing depth is 12 here so the body stands inside it. | @@ -120,6 +124,88 @@ From `rules.yaml` at 2c9348f: | g09-vertical-run | 09 | `route_length` unsure, measured 16, ±0; `route_path` pass; spot at [11, 11 + W] | Pad at 15 − 4 = 11. An `elec_box` at s = [4, 5], 0 to 3.5 ft tall, sits across the 1 ft cable run, and elec boxes detour. Over: 2 × (3.5 − 1) = 5. Routed 11 + 5 = 16 = 15 + 1, past the confident reach. Measuring along the wall alone (11) would pass. | | c5-no-coverage | C5, scene schema | decision manual_review; `missing_evidence` non-empty; starts in (−6, −3.9) and (1.3, 7.4) unsure, starts in (−3.8, 1.2) fail with `meter_working_space` | Clean wall s = [−6, 10] with ground and facing data but no `coverage`. "Absent means nothing is known to be observed" makes every start unsure except where the working space, which needs no observation, fails it. Absent ends default to unexplored, which rules out reject. | +## Drift cases + +The cases above set every error explicitly, so they never exercise the error model's growth with +distance. The `d-` cases leave `plus_minus_ft` off the walls, the meter and the objects, and give +objects `source: "tap"`. Ground and facing keep explicit errors so only the checks under test +move. + +### The error model, from rules.yaml and server/README.md at f2705dd + +- `errors.meter_ft` 0.3, `errors.wall_ft` 0.3, `errors.tap_ft` 0.3. +- `errors.drift_per_ft` 0.16 is added per foot walked along the walls from the meter to the + default error of walls, tap and vlm objects and ground patches. It is not added to tape, the + meter itself or an explicit `plus_minus_ft`. +- A battery position comes from the wall, so a point of the battery at walked distance x carries + 0.3 + 0.16x. +- A gap or a route sums the errors of its two ends (linear, as the C5 margin rule and g10's + derived row use). +- A route's ends are the meter and the battery's near edge. A gas gap's ends are the gas meter + and the battery edge nearest it. + +### d-reach-drift-right: cable reach, `at_most` 20 with review line 15 + +Wall s = [−1, 29], meter at 0. For a start a > 0 the route is a (straight wall, no corners) and its +error is 0.3 (meter) + 0.3 + 0.16a (wall at the near edge) = 0.6 + 0.16a. + +| Line | Condition | Start | +| --- | --- | --- | +| pass / unsure | a + 0.6 + 0.16a < 15 | a < 14.4 / 1.16 = 12.4138 | +| unsure / fail | a − 0.6 − 0.16a > 20 | a > 20.6 / 0.84 = 24.5238 | + +Asserted starts: pass at 5 and 12.2138. Unsure at 12.6138, 18 and 24.3238. Fail at 24.7238 and +25.5. Each boundary is bracketed 0.2 ft inside on both sides, which is more than one 2 in sweep +step, so each point falls inside a run rather than between two. Using the far edge (a + W) for the +wall term would move both lines by 0.16W / 1.16 = 0.356 and 0.16W / 0.84 = 0.492, which the points +0.2 ft inside catch. So would any cutoff that is off by 0.16W = 0.413. + +### d-reach-drift-left: the same, left of the meter + +Wall s = [−29, 1]. Left of the meter the near edge is the battery's right edge b = a + W, and +the route is −b. The table above holds for −b, so each asserted start is a = −(right-case start) − +W: pass at −7.5833 and −14.7971, unsure at −15.1971, −20.5833 and −26.9071, fail at −27.3071 and +−28.0833. The lines sit at a = −12.4138 − W = −14.9971 and a = −24.5238 − W = −27.1071. + +### d-gas-drift: gas clearance, `at_least` 3 + +Wall s = [−1, 20]. A tapped gas meter on the wall line at s = 4 (span [4, 4], no footprint) +carries 0.3 + 0.16 × 4 = 0.94. For a start a > 4 the gap is d = a − 4 and its error is 0.94 + +(0.3 + 0.16a) = 1.24 + 0.16a. + +| Line | Condition | Start | +| --- | --- | --- | +| fail / unsure | (a − 4) + 1.24 + 0.16a < 3 | a < 5.76 / 1.16 = 4.9655 | +| unsure / pass | (a − 4) − 1.24 − 0.16a > 3 | a > 8.24 / 0.84 = 9.8095 | + +Asserted starts: + +- **Fail:** 3, where the battery [3, 5.583] covers the gas meter: gap 0, and 0 + 1.88 < 3. Also + 4.7655. +- **Unsure:** 5.1655, 7.5 and 9.6095. +- **Pass:** 10.0095 and 11. Both routes also pass: 11 + 0.6 + 1.76 = 13.36 < 15. + +All starts sit right of the working space. The route to them crosses the gas meter, which +detours; a gas meter with no heights added no extra length at 2c9348f. + +### What rules.yaml and the README leave open + +- **Which battery point carries the drift.** The model gives drift for "AR-placed positions". + The battery is placed from the wall, not tapped, and it spans W, over which drift changes by + 0.16W = 0.413 ft. These cases take the point the measurement ends at: the near edge for the + route, the edge nearest the gas for a gas gap. The far edge or the centre is an equally literal + reading and moves every line (see the reach arithmetic above). +- **Walked distance for an object with a span.** For a tapped object spanning [s0, s1], it could + be s0, s1 or the nearest end. d-gas-drift uses a zero-length span to avoid the question. +- **Whether the meter's error enters the route.** The README says drift is not added to the + meter. Whether its 0.3 base enters the route error is inferred from 2c9348f, where both + defaults gave a route at ±0.6. +- **How errors combine.** Linear sum is assumed, as elsewhere in this suite. Root-sum-square + would move every line. +- **Whether drift enters checks with explicit inputs.** Facing gaps and the working space involve + the wall's drifting error. These cases give facing 20 ft (18.17 from the battery front, far + above 3 + any error here) and keep every asserted start well clear of the working space. + ## Assumptions a failure should be checked against - A battery must sit flush on one straight segment (review item 4, golden 04; the result's diff --git a/verification/e2e/cases/d-gas-drift.json b/verification/e2e/cases/d-gas-drift.json new file mode 100644 index 000000000..bf8d3d875 --- /dev/null +++ b/verification/e2e/cases/d-gas-drift.json @@ -0,0 +1,168 @@ +{ + "id": "d-gas-drift", + "source": "rules.yaml errors.tap_ft, wall_ft, drift_per_ft and clearances.gas_ft at origin/t3/server f2705dd; C5 margin rule", + "description": "A tapped gas meter at s = 4 on a wall with default errors. For a start a right of it the gap is (a - 4) +/- (1.24 + 0.16a): fail below a = 4.966, pass above a = 9.810, unsure between.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1" + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -1, + 0.0 + ], + [ + 20, + 0.0 + ] + ] + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [ + 4.0, + 4.0 + ], + "source": "tap" + } + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -1, + 0 + ], + [ + 20, + 0 + ], + [ + 20, + 10 + ], + [ + -1, + 10 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -1, + 20 + ], + "depth_ft": 20.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -16.0, + 35.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -16.0, + 35.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -16.0, + 35.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -16.0, + 35.0 + ], + "out_ft": 15.0 + } + ] + } + }, + "rules_assumed": { + "gas": 3.0, + "route_length": 20.0 + }, + "expect": { + "start_outcomes": [ + { + "wall_id": "w1", + "start_ft": 3.0, + "outcome": "fail", + "why": "battery [3, 5.58] covers the gas meter: gap 0 + 1.88 < 3" + }, + { + "wall_id": "w1", + "start_ft": 4.7655, + "outcome": "fail", + "why": "just inside (a - 4) + 1.24 + 0.16a < 3" + }, + { + "wall_id": "w1", + "start_ft": 5.1655, + "outcome": "unsure", + "why": "just past the fail line" + }, + { + "wall_id": "w1", + "start_ft": 7.5, + "outcome": "unsure", + "why": "gap 3.5 +/- 2.44" + }, + { + "wall_id": "w1", + "start_ft": 9.6095, + "outcome": "unsure", + "why": "just inside (a - 4) - 1.24 - 0.16a <= 3" + }, + { + "wall_id": "w1", + "start_ft": 10.0095, + "outcome": "pass", + "why": "just past (a - 4) - 1.24 - 0.16a > 3" + }, + { + "wall_id": "w1", + "start_ft": 11.0, + "outcome": "pass", + "why": "gap 7 - 3.0 = 4 > 3; route 11 + 2.36 < 15" + } + ] + } +} diff --git a/verification/e2e/cases/d-reach-drift-left.json b/verification/e2e/cases/d-reach-drift-left.json new file mode 100644 index 000000000..6ba91abc0 --- /dev/null +++ b/verification/e2e/cases/d-reach-drift-left.json @@ -0,0 +1,160 @@ +{ + "id": "d-reach-drift-left", + "source": "rules.yaml errors.drift_per_ft and route.* at origin/t3/server f2705dd; C5 margin rule", + "description": "Mirror of d-reach-drift-right. Left of the meter the route runs to the battery's right edge b = a + W, so the boundaries sit at a = -12.414 - W and a = -24.524 - W.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1" + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -29, + 0.0 + ], + [ + 1, + 0.0 + ] + ] + } + ], + "objects": [], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -29, + 0 + ], + [ + 1, + 0 + ], + [ + 1, + 10 + ], + [ + -29, + 10 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -29, + 1 + ], + "depth_ft": 20.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -44.0, + 16.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -44.0, + 16.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -44.0, + 16.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -44.0, + 16.0 + ], + "out_ft": 15.0 + } + ] + } + }, + "rules_assumed": { + "route_length": 20.0 + }, + "expect": { + "start_outcomes": [ + { + "wall_id": "w1", + "start_ft": -7.5833, + "outcome": "pass", + "why": "route 5 + 1.4 = 6.4 < 15 (b = a + W)" + }, + { + "wall_id": "w1", + "start_ft": -14.7971, + "outcome": "pass", + "why": "just inside a + 0.6 + 0.16a < 15 (b = a + W)" + }, + { + "wall_id": "w1", + "start_ft": -15.1971, + "outcome": "unsure", + "why": "just past the 15 ft review line (b = a + W)" + }, + { + "wall_id": "w1", + "start_ft": -20.5833, + "outcome": "unsure", + "why": "18 + 3.48 > 15; 18 - 3.48 < 20 (b = a + W)" + }, + { + "wall_id": "w1", + "start_ft": -26.9071, + "outcome": "unsure", + "why": "just inside a - 0.6 - 0.16a <= 20 (b = a + W)" + }, + { + "wall_id": "w1", + "start_ft": -27.3071, + "outcome": "fail", + "why": "just past a - 0.6 - 0.16a > 20 (b = a + W)" + }, + { + "wall_id": "w1", + "start_ft": -28.0833, + "outcome": "fail", + "why": "25.5 - 4.68 = 20.82 > 20 (b = a + W)" + } + ], + "decision_not": [ + "pass" + ] + } +} diff --git a/verification/e2e/cases/d-reach-drift-right.json b/verification/e2e/cases/d-reach-drift-right.json new file mode 100644 index 000000000..84eeaae4a --- /dev/null +++ b/verification/e2e/cases/d-reach-drift-right.json @@ -0,0 +1,160 @@ +{ + "id": "d-reach-drift-right", + "source": "rules.yaml errors.drift_per_ft and route.* at origin/t3/server f2705dd; C5 margin rule", + "description": "Straight wall with default (drifting) errors. The route to a start a right of the meter is a +/- (0.6 + 0.16a): pass below a = 12.414, fail above a = 24.524, unsure between.", + "scene": { + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 4.0, + 0.0 + ], + "wall_id": "w1" + }, + "walls": [ + { + "id": "w1", + "baseline": [ + [ + -1, + 0.0 + ], + [ + 29, + 0.0 + ] + ] + } + ], + "objects": [], + "ground": [ + { + "type": "concrete", + "polygon": [ + [ + -1, + 0 + ], + [ + 29, + 0 + ], + [ + 29, + 10 + ], + [ + -1, + 10 + ] + ], + "plus_minus_ft": 0.0 + } + ], + "facing": [ + { + "wall_id": "w1", + "span_ft": [ + -1, + 29 + ], + "depth_ft": 20.0, + "plus_minus_ft": 0.0 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit" + }, + "right": { + "kind": "limit" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -16.0, + 44.0 + ] + }, + { + "band": "overhead", + "span_ft": [ + -16.0, + 44.0 + ] + }, + { + "band": "facing", + "span_ft": [ + -16.0, + 44.0 + ] + }, + { + "band": "ground", + "span_ft": [ + -16.0, + 44.0 + ], + "out_ft": 15.0 + } + ] + } + }, + "rules_assumed": { + "route_length": 20.0 + }, + "expect": { + "start_outcomes": [ + { + "wall_id": "w1", + "start_ft": 5.0, + "outcome": "pass", + "why": "route 5 + 1.4 = 6.4 < 15" + }, + { + "wall_id": "w1", + "start_ft": 12.2138, + "outcome": "pass", + "why": "just inside a + 0.6 + 0.16a < 15" + }, + { + "wall_id": "w1", + "start_ft": 12.6138, + "outcome": "unsure", + "why": "just past the 15 ft review line" + }, + { + "wall_id": "w1", + "start_ft": 18.0, + "outcome": "unsure", + "why": "18 + 3.48 > 15; 18 - 3.48 < 20" + }, + { + "wall_id": "w1", + "start_ft": 24.3238, + "outcome": "unsure", + "why": "just inside a - 0.6 - 0.16a <= 20" + }, + { + "wall_id": "w1", + "start_ft": 24.7238, + "outcome": "fail", + "why": "just past a - 0.6 - 0.16a > 20" + }, + { + "wall_id": "w1", + "start_ft": 25.5, + "outcome": "fail", + "why": "25.5 - 4.68 = 20.82 > 20" + } + ], + "decision_not": [ + "pass" + ] + } +} diff --git a/verification/e2e/cases/generate.py b/verification/e2e/cases/generate.py index ed3717ddb..7abbcb81c 100644 --- a/verification/e2e/cases/generate.py +++ b/verification/e2e/cases/generate.py @@ -181,7 +181,7 @@ def straight_scene(x0, x1, z=0.0, meter_x=0.0, objects=(), ground=None, cov=None # g01: an unseen stretch behind a garage door needs no photo, since no cable can reach it. # The garage ends at s = -1, inside the meter's working space, so no battery can sit between # garage and meter; the unseen stretch starts at s = -9, more than pool_ft (10) left of the -# first start right of the meter (1.25). +# first start right of the meter (1.25; 2 once garage doors count as openings, f2705dd). case( "g01-unseen-beyond-garage", golden("01"), @@ -864,6 +864,102 @@ def reach_case(cid, L, route_ft, e, outcome, description, objects=(), extra_chec extra_checks=[{"match": "route_path", "outcome": "pass"}], ) +# Drift cases. Walls, meter and objects carry no plus_minus_ft, so rules.yaml's defaults apply: +# errors.meter_ft 0.3 (no drift), errors.wall_ft 0.3 and errors.tap_ft 0.3, each plus +# errors.drift_per_ft 0.16 per foot walked along the walls from the meter. Ground and facing keep +# explicit errors so that only the checks under test move. README.md ("Drift cases") has the +# derivation; the model assumed is: a measured point's error is its default plus drift times its +# walked distance, and a gap or route sums the errors of its two ends. +TAP, WALL_E, METER_E, DRIFT = 0.3, 0.3, 0.3, 0.16 +PT = 0.2 # sample points sit 0.2 ft inside a boundary: more than one 2 in sweep step + + +def drift_scene(x0, x1, objects=()): + return { + "schema_version": "1.0", + "meter": {"pos": [0.0, METER_Y, 0.0], "wall_id": "w1"}, + "walls": [{"id": "w1", "baseline": [[x0, 0.0], [x1, 0.0]]}], + "objects": list(objects), + "ground": [concrete(rect(x0, x1, 0, 10))], + "facing": [facing("w1", (x0, x1), depth=20.0)], + "coverage": full_coverage((x0, x1)), + } + + +def starts(points): + return [ + {"wall_id": "w1", "start_ft": round(s, 4), "outcome": o, "why": why} for s, o, why in points + ] + + +# Cable reach, right of the meter. Route = a (near edge), error = meter 0.3 + wall at a +# (0.3 + 0.16a) = 0.6 + 0.16a. Pass: a + e < 15 (review line) -> a < 14.4/1.16. Fail: a - e > 20 +# -> a > 20.6/0.84. +R_PASS = (CONFIDENT - METER_E - WALL_E) / (1 + DRIFT) +R_FAIL = (MAX_ROUTE + METER_E + WALL_E) / (1 - DRIFT) +reach_points = [ + (5.0, "pass", "route 5 + 1.4 = 6.4 < 15"), + (R_PASS - PT, "pass", "just inside a + 0.6 + 0.16a < 15"), + (R_PASS + PT, "unsure", "just past the 15 ft review line"), + (18.0, "unsure", "18 + 3.48 > 15; 18 - 3.48 < 20"), + (R_FAIL - PT, "unsure", "just inside a - 0.6 - 0.16a <= 20"), + (R_FAIL + PT, "fail", "just past a - 0.6 - 0.16a > 20"), + (25.5, "fail", "25.5 - 4.68 = 20.82 > 20"), +] +case( + "d-reach-drift-right", + "rules.yaml errors.drift_per_ft and route.* at origin/t3/server f2705dd; C5 margin rule", + "Straight wall with default (drifting) errors. The route to a start a right of the meter is " + "a +/- (0.6 + 0.16a): pass below a = 12.414, fail above a = 24.524, unsure between.", + drift_scene(-1, 29), + {"start_outcomes": starts(reach_points), "decision_not": ["pass"]}, + {"route_length": MAX_ROUTE}, +) +# Mirror: left of the meter the near edge is the battery's right edge b = a + W, route = -b. +case( + "d-reach-drift-left", + "rules.yaml errors.drift_per_ft and route.* at origin/t3/server f2705dd; C5 margin rule", + "Mirror of d-reach-drift-right. Left of the meter the route runs to the battery's right edge " + "b = a + W, so the boundaries sit at a = -12.414 - W and a = -24.524 - W.", + drift_scene(-29, 1), + { + "start_outcomes": starts([(-s - W, o, why + " (b = a + W)") for s, o, why in reach_points]), + "decision_not": ["pass"], + }, + {"route_length": MAX_ROUTE}, +) + +# Gas clearance. A tapped gas meter on the wall line at s = 4 (error 0.3 + 0.16 x 4 = 0.94). A +# battery right of it has gap d = a - 4 and error 0.94 + (0.3 + 0.16a) = 1.24 + 0.16a. +# Fail: d + e < 3 -> 1.16a < 5.76. Pass: d - e > 3 -> 0.84a > 8.24. +G_AT = 4.0 +G_E = TAP + DRIFT * G_AT +G_FAIL = (3.0 + G_AT - G_E - WALL_E) / (1 + DRIFT) +G_PASS = (3.0 + G_AT + G_E + WALL_E) / (1 - DRIFT) +tap_gas = {"type": "gas_meter", "wall_id": "w1", "span_ft": [G_AT, G_AT], "source": "tap"} +case( + "d-gas-drift", + "rules.yaml errors.tap_ft, wall_ft, drift_per_ft and clearances.gas_ft at origin/t3/server " + "f2705dd; C5 margin rule", + "A tapped gas meter at s = 4 on a wall with default errors. For a start a right of it the gap " + "is (a - 4) +/- (1.24 + 0.16a): fail below a = 4.966, pass above a = 9.810, unsure between.", + drift_scene(-1, 20, objects=[tap_gas]), + { + "start_outcomes": starts( + [ + (3.0, "fail", "battery [3, 5.58] covers the gas meter: gap 0 + 1.88 < 3"), + (G_FAIL - PT, "fail", "just inside (a - 4) + 1.24 + 0.16a < 3"), + (G_FAIL + PT, "unsure", "just past the fail line"), + (7.5, "unsure", "gap 3.5 +/- 2.44"), + (G_PASS - PT, "unsure", "just inside (a - 4) - 1.24 - 0.16a <= 3"), + (G_PASS + PT, "pass", "just past (a - 4) - 1.24 - 0.16a > 3"), + (11.0, "pass", "gap 7 - 3.0 = 4 > 3; route 11 + 2.36 < 15"), + ] + ), + }, + {"gas": 3.0, "route_length": MAX_ROUTE}, +) + def main(): for old in HERE.glob("*.json"): diff --git a/verification/hsverify/a11yaudit.py b/verification/hsverify/a11yaudit.py index cb4e8ede6..48abb4681 100644 --- a/verification/hsverify/a11yaudit.py +++ b/verification/hsverify/a11yaudit.py @@ -15,7 +15,12 @@ Reports go to ~/house-scanning-data/reports/a11y// with a screenshot per audited screen. The probe's middle screen carries a deliberately undersized, unlabelled button, so `--ref HEAD --project verification/fixtures/state-probe/StateProbe.xcodeproj --scheme -StateProbe` must report at least one issue; a clean result there means the audit is blind. +StateProbe --min-screens 1` must report at least one issue; a clean result there means the +audit is blind. + +A run proves nothing when the harness did not run, the audit threw on a screen, or fewer than +`--min-screens` screens were audited. The report then says `ok: false` with the reasons in +`problems`, and the exit code is 1 whatever the issue count. """ from __future__ import annotations @@ -70,6 +75,31 @@ def symbol_name_labels(screens: list[dict]) -> list[dict]: return found +def audited_screens(screens: list[dict]) -> set: + """Indexes of screens the audit completed on. + + A screen entry carrying "error" means the audit threw on that screen or the app stopped. + After a throw the harness still prints a normal entry for the same index, so a screen + counts only if no entry for its index carries an error. + """ + errored = {s.get("index") for s in screens if "error" in s} + return {s.get("index") for s in screens if "error" not in s} - errored + + +def audit_problems(screens: list[dict], harness_ran: bool, min_screens: int) -> list[str]: + """Why this run is not evidence of an audit; empty when it is.""" + problems = [] + if not harness_ran: + problems.append("the audit harness did not run (no TEST SUCCEEDED or TEST FAILED line)") + problems += [f"screen {s.get('index')}: {s['error']}" for s in screens if "error" in s] + audited = audited_screens(screens) + if not audited: + problems.append("no screen was audited") + elif len(audited) < min_screens: + problems.append(f"{len(audited)} screens audited, fewer than --min-screens {min_screens}") + return problems + + def run_harness( udid: str, bundle_id: str, args: argparse.Namespace, launch: list[str], out: Path ) -> str: @@ -103,6 +133,12 @@ def write_report(out: Path, data: dict) -> None: (out / "report.json").write_text(json.dumps(data, indent=2) + "\n") rows = [] for screen in data["screens"]: + if "error" in screen: + rows.append( + f"

    Screen {screen['index']}: audit error

    " + f"

    {html.escape(str(screen['error']))}

    " + ) + continue issues = [i for i in data["issues"] if i.get("screen") == screen["index"]] symbols = [s for s in data["symbol_name_labels"] if s["screen"] == screen["index"]] items = "".join( @@ -120,6 +156,7 @@ def write_report(out: Path, data: dict) -> None: f"

    {html.escape(' · '.join(texts[:8]))}

    " f"
      {items or '
    • No issues
      • '}
    " ) + verdict = "".join(f"

    Not valid: {html.escape(p)}

    " for p in data["problems"]) page = ( "Accessibility audit" f"

    Accessibility audit: {html.escape(data['ref'])} at " - f"{data['sha'][:12]}

    {len(data['screens'])} screens, " + f"{data['sha'][:12]}{verdict}

    {data['screens_audited']} screens, " f"{data['issue_count']} issues. Text size {html.escape(str(data['content_size']))}, " f"{html.escape(data['appearance'])}.

    " + "".join(rows) ) @@ -149,6 +186,9 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--content-size") parser.add_argument("--label", default="") parser.add_argument("--build-wait", type=float, default=1200) + # Nine: the app's ten states less `unsupported`, which needs a device without LiDAR (the + # same count as metric S3-7). The state-probe fixture has fewer screens; pass a lower value. + parser.add_argument("--min-screens", type=int, default=9) args = parser.parse_args(argv) gitref.fetch() @@ -199,7 +239,12 @@ def main(argv: list[str] | None = None) -> int: screens, issues = parse_output(output) symbols = symbol_name_labels(screens) + harness_ran = "** TEST SUCCEEDED **" in output or "** TEST FAILED **" in output + problems = audit_problems(screens, harness_ran, args.min_screens) data = { + # ok: the run is evidence of an audit. Issues found are counted separately below. + "ok": not problems, + "problems": problems, "ref": args.ref, "build": build, "sha": sha, @@ -210,20 +255,24 @@ def main(argv: list[str] | None = None) -> int: "issues": issues, "symbol_name_labels": symbols, "issue_count": len(issues) + len(symbols), - "harness_ran": "** TEST SUCCEEDED **" in output or "** TEST FAILED **" in output, + "screens_audited": len(audited_screens(screens)), + "min_screens": args.min_screens, + "harness_ran": harness_ran, } write_report(out, data) - print(f"{len(screens)} screens, {data['issue_count']} issues. Open {out / 'index.html'}") + print( + f"{data['screens_audited']} screens, {data['issue_count']} issues. " + f"Open {out / 'index.html'}" + ) for issue in issues[:20]: print( f" screen {issue.get('screen')}: {issue.get('description')} {issue.get('element', '')}" ) for s in symbols: print(f" screen {s['screen']}: button label is a symbol name: {s['label']}") - if not data["harness_ran"]: - print(f"The audit harness did not run; see {out / 'xcodebuild-test.log'}") - return 1 - return 0 if data["issue_count"] == 0 else 1 + for problem in problems: + print(f"PROBLEM: {problem}; see {out / 'xcodebuild-test.log'}") + return 0 if not problems and data["issue_count"] == 0 else 1 if __name__ == "__main__": diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index 13e821726..c2beaa9a5 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -44,13 +44,22 @@ from hsverify import gitref from hsverify.memory import peak_rss_mb from hsverify.resultcheck import ( + RuleSet, assumption_mismatches, comparable, expectation_problems, invariant_problems, + less_coverage_problems, mirror_scene, + more_error_problems, outcome_lengths, schema_errors, + tape_to_tap, + unobserved_checks, + with_ground_short_of, + with_less_coverage, + with_more_error, + with_requests_captured, ) HERE = Path(__file__).resolve().parents[1] @@ -78,6 +87,24 @@ class SceneInput: source: str = "" skip_reason: str | None = None app_export: bool = False # exported by the iOS app (from a replay), the S4 end-to-end path + app_sha: str | None = None # the app commit that exported it, when known + hostile: bool = False # judged by judge_hostile: refused or answered within a budget + + +def load_app_export(path: Path) -> SceneInput: + """A scene the iOS app exported: a scan.zip, or a Simulator report folder holding one (whose + report.json names the app commit that produced it).""" + app_sha = None + if path.is_dir(): + run = json.loads((path / "report.json").read_text()) + if not run.get("app_export"): + raise SystemExit(f"{path} has no app export; the run never reached the upload") + app_sha = run["sha"] + path = path / run["app_export"] + item = load_input(path, real=True) + item.app_export, item.app_sha = True, app_sha + item.source = str(path) + return item def load_input(path: Path, real: bool = False) -> SceneInput: @@ -258,7 +285,10 @@ def bundle_zip(item: SceneInput) -> bytes: return buf.getvalue() -def post(url: str, endpoint: Endpoint, item: SceneInput) -> tuple[int, bytes, float]: +def post( + url: str, endpoint: Endpoint, item: SceneInput, timeout_s: float = 60 +) -> tuple[int | None, bytes, float]: + """Send one scene. A timeout or a dropped connection comes back as status None.""" if endpoint.content_type == "application/json": body, ctype = item.raw, "application/json" else: @@ -277,10 +307,12 @@ def post(url: str, endpoint: Endpoint, item: SceneInput) -> tuple[int, bytes, fl ) start = time.perf_counter() try: - with urllib.request.urlopen(request, timeout=60) as response: + with urllib.request.urlopen(request, timeout=timeout_s) as response: status, payload = response.status, response.read() except urllib.error.HTTPError as exc: status, payload = exc.code, exc.read() + except (urllib.error.URLError, TimeoutError, ConnectionError) as exc: + status, payload = None, str(exc).encode() return status, payload, (time.perf_counter() - start) * 1000 @@ -295,15 +327,28 @@ def variant(item: SceneInput, name: str, scene: dict) -> SceneInput: ) -def judge(url: str, endpoint: Endpoint, item: SceneInput, schemas: dict) -> dict: +def answer(url: str, endpoint: Endpoint, item: SceneInput) -> tuple[dict | None, str | None]: + """The server's result for a scene, or why there is none.""" + status, payload, _ = post(url, endpoint, item) + if status is None: + return None, f"no answer: {payload.decode(errors='replace')[:200]}" + if status != 200: + return None, f"HTTP {status}: {payload[:300]!r}" + return json.loads(payload), None + + +def judge(url: str, endpoint: Endpoint, item: SceneInput, schemas: dict, rules: RuleSet) -> dict: record: dict = { "name": item.name, "source": item.source, "real": item.real, "app_export": item.app_export, + "app_sha": item.app_sha, } if item.skip_reason: return record | {"status": "skipped", "problems": [], "skip_reason": item.skip_reason} + if item.hostile: + return record | judge_hostile(url, endpoint, item, schemas) input_errors = schema_errors(item.scene, schemas["scene"]) if input_errors: return record | {"status": "bad input", "problems": input_errors[:20]} @@ -311,7 +356,11 @@ def judge(url: str, endpoint: Endpoint, item: SceneInput, schemas: dict) -> dict status, payload, ms = post(url, endpoint, item) record |= {"http_status": status, "latency_ms": round(ms, 1)} if status != 200: - failure = [f"HTTP {status} for a scene that validates: {payload[:500]!r}"] + failure = [ + f"HTTP {status} for a scene that validates: {payload[:500]!r}" + if status is not None + else f"no answer within 60 s: {payload[:200]!r}" + ] return record | {"status": "fail", "problems": failure, "contract_problems": failure} result = json.loads(payload) record["result"] = result @@ -320,8 +369,8 @@ def judge(url: str, endpoint: Endpoint, item: SceneInput, schemas: dict) -> dict return record | {"status": "fail", "problems": problems[:20], "contract_problems": problems} # Contract problems hold for any scene; expectation problems depend on one case's geometry. - contract = problems + invariant_problems(item.scene, result, sent=item.raw) - contract += property_problems(url, endpoint, item, result) + contract = invariant_problems(item.scene, result, sent=item.raw, rules=rules) + contract += property_problems(url, endpoint, item, result, schemas, rules) if item.real and ms > LATENCY_BUDGET_MS: contract.append(f"real-derived scene took {ms:.0f} ms, budget {LATENCY_BUDGET_MS:.0f}") mismatches = assumption_mismatches(item.rules_assumed, result) @@ -339,45 +388,166 @@ def judge(url: str, endpoint: Endpoint, item: SceneInput, schemas: dict) -> dict return record -def property_problems(url: str, endpoint: Endpoint, item: SceneInput, result: dict) -> list[str]: - problems = [] +def property_problems( + url: str, endpoint: Endpoint, item: SceneInput, result: dict, schemas: dict, rules: RuleSet +) -> list[str]: + """The same scene resent unchanged and transformed. Every answer to a transformed scene must + also satisfy the invariants, and the transforms that only lose information must not make + the server surer.""" + problems: list[str] = [] + + def resend(name: str, scene: dict) -> dict | None: + other, failure = answer(url, endpoint, variant(item, name, scene)) + if failure: + problems.append(f"{name}: {failure}") + return None + errors = schema_errors(other, schemas["result"]) + if errors: + problems.append(f"{name}: result schema: {errors[0]}") + return None + problems.extend(f"{name}: {p}" for p in invariant_problems(scene, other, rules=rules)) + return other + # 1. Same input, same answer. status, payload, _ = post(url, endpoint, item) if status != 200 or comparable(json.loads(payload)) != comparable(result): problems.append("sending the same scene twice gave different results") + # 2. Mirrored left to right: same decision, same amount of each outcome along the wall. - mirrored = variant(item, "mirror", mirror_scene(item.scene)) - status, payload, _ = post(url, endpoint, mirrored) - if status != 200: - problems.append(f"mirrored scene: HTTP {status}") - else: - other = json.loads(payload) - if other["decision"] != result["decision"]: + mirrored = resend("mirrored", mirror_scene(item.scene)) + if mirrored is not None: + if mirrored["decision"] != result["decision"]: problems.append( - f"mirrored scene decided {other['decision']}, original {result['decision']}" + f"mirrored: decided {mirrored['decision']}, original {result['decision']}" ) - a, b = outcome_lengths(result), outcome_lengths(other) + a, b = outcome_lengths(result), outcome_lengths(mirrored) for outcome in a: if abs(a[outcome] - b[outcome]) > MIRROR_LENGTH_TOL_FT: problems.append( - f"mirrored scene: {outcome} starts cover {b[outcome]:.2f} ft, " + f"mirrored: {outcome} starts cover {b[outcome]:.2f} ft, " f"original {a[outcome]:.2f} ft" ) - # 3. Without coverage nothing was observed, so nothing may pass. + + # 3. Less information never makes the server surer. if "coverage" in item.scene: bare = {k: v for k, v in item.scene.items() if k != "coverage"} - status, payload, _ = post(url, endpoint, variant(item, "no-coverage", bare)) - if status != 200: - problems.append(f"scene without coverage: HTTP {status}") - else: - other = json.loads(payload) - if other["decision"] == "pass" or any( - r["outcome"] == "pass" for r in other.get("sweep", []) - ): - problems.append("scene without coverage still produced a pass") + if (other := resend("no coverage", bare)) is not None: + problems += less_coverage_problems(result, other, "no coverage") + for name, scene in ( + ("less coverage", with_less_coverage(item.scene)), + ("ground short of the largest clearance", with_ground_short_of(item.scene, rules)), + ): + if scene is not None and (other := resend(name, scene)) is not None: + problems += less_coverage_problems(result, other, name) + for name, scene in ( + ("more error", with_more_error(item.scene, rules)), + ("tape re-measured by tap", tape_to_tap(item.scene)), + ): + if scene is not None and (other := resend(name, scene)) is not None: + problems += more_error_problems(result, other, name) + + # 4. Showing everything the result asked for settles every check that was unsure only + # because an area was unobserved (the chosen spot may move; three rounds at most). + scene, current = item.scene, result + for round_ in range(1, 4): + if not unobserved_checks(current): + break + scene = with_requests_captured(scene, current) + if scene is None: + break # what remains needs a walk past an end, not another view + current = resend(f"requests captured ({round_})", scene) + if current is None: + break + else: + if unobserved_checks(current): + problems.append( + "after capturing every requested view three times, checks are still unsure " + f"for coverage: {unobserved_checks(current)}" + ) return problems +# --- Hostile inputs --------------------------------------------------------------------------- + +# A refusal or an answer must come within this; an input that keeps the server busy longer is a +# denial-of-service risk. Generous for a laptop-hosted server; the real-scene budget is 1 s. +HOSTILE_BUDGET_MS = 10_000.0 + + +def hostile_inputs() -> list[SceneInput]: + """Small requests that cost a careless server a lot: many objects, huge numbers, deep + nesting and a very long wall. Built here, not stored, so the repository carries no blobs.""" + wall = {"id": "w1", "baseline": [[-15.0, 0.0], [15.0, 0.0]]} + base = {"meter": {"pos": [0.0, 4.5, 0.0], "wall_id": "w1"}, "walls": [wall]} + many = base | { + "objects": [ + { + "type": "window", + "wall_id": "w1", + "span_ft": [i * 0.001, i * 0.001 + 0.5], + "source": "tap", + } + for i in range(20_000) + ] + } + huge = { + "meter": {"pos": [1e300, 4.5, 0.0], "wall_id": "w1"}, + "walls": [{"id": "w1", "baseline": [[-1e300, 0.0], [1e300, 0.0]]}], + } + long_wall = { + "meter": {"pos": [0.0, 4.5, 0.0], "wall_id": "w1"}, + "walls": [{"id": "w1", "baseline": [[-2500.0, 0.0], [2500.0, 0.0]]}], + "coverage": { + "observed": [ + {"band": "wall", "span_ft": [-2500, 2500]}, + {"band": "ground", "span_ft": [-2500, 2500], "out_ft": 20}, + ] + }, + } + depth = 20_000 + nested = ( + json.dumps(base)[:-1] + + ', "objects": [{"type": "door", "wall_id": "w1", "span_ft": [1, 2], "source": "tap", ' + + '"attrs": {"note": ' + + "[" * depth + + "]" * depth + + "}}]}" + ).encode() + items = [ + SceneInput("hostile: 20 000 objects", many, json.dumps(many).encode()), + SceneInput("hostile: coordinates of 1e300 ft", huge, json.dumps(huge).encode()), + SceneInput("hostile: a 5000 ft wall", long_wall, json.dumps(long_wall).encode()), + SceneInput("hostile: attrs nested 20 000 deep", {}, nested), + ] + for item in items: + item.hostile = True + return items + + +def judge_hostile(url: str, endpoint: Endpoint, item: SceneInput, schemas: dict) -> dict: + """A refusal (400, 413, 422) or a valid result, either within the budget. A crash, a hang or + a slow answer is a failure.""" + status, payload, ms = post(url, endpoint, item, timeout_s=HOSTILE_BUDGET_MS / 1000 + 5) + record = {"http_status": status, "latency_ms": round(ms, 1)} + if status is None: + failure = f"no answer within {HOSTILE_BUDGET_MS / 1000 + 5:.0f} s" + elif ms > HOSTILE_BUDGET_MS: + failure = f"answered HTTP {status} after {ms:.0f} ms, budget {HOSTILE_BUDGET_MS:.0f}" + elif status in (400, 413, 422): + failure = None + elif status == 200: + errors = schema_errors(json.loads(payload), schemas["result"]) + failure = f"result schema: {errors[0]}" if errors else None + else: + failure = f"HTTP {status}: {payload[:200]!r}" + problems = [failure] if failure else [] + return record | { + "status": "fail" if problems else "pass", + "problems": problems, + "contract_problems": problems, + } + + # --- Report ---------------------------------------------------------------------------------- @@ -402,6 +572,12 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: if exported else None, "app_export_passed": all(r["status"] == "pass" for r in exported) if exported else None, + # The app commit the export came from (the scoreboard ties app metrics to it); null when + # the run had none or several, or the export's origin is unknown. + "app_sha": app_shas.pop() + if len(app_shas := {r.get("app_sha") for r in exported}) == 1 + else None, + "app_export_source": [r["source"] for r in exported], "records": records, } (out / "report.json").write_text(json.dumps(report, indent=2) + "\n") @@ -453,7 +629,10 @@ def main(argv: list[str] | None = None) -> int: type=Path, action="append", default=[], - help="scene.json or bundle the iOS app exported from a replay (real, latency budget)", + help="scan.zip the app exported, or a sim report folder with one (records the app SHA)", + ) + parser.add_argument( + "--no-hostile", action="store_true", help="skip the hostile inputs (quicker local runs)" ) parser.add_argument("--out", type=Path) args = parser.parse_args(argv) @@ -468,6 +647,10 @@ def main(argv: list[str] | None = None) -> int: if text is None: raise SystemExit(f"{path} does not exist at {schema_ref}") schemas[key] = json.loads(text) + rules_text = gitref.show(schema_sha, "server/rules.yaml") + if rules_text is None: + raise SystemExit(f"server/rules.yaml does not exist at {schema_ref}") + rules = RuleSet.from_yaml(rules_text) items = ( [load_input(p) for p in sorted(args.cases.glob("*.json"))] if args.cases.exists() else [] @@ -475,9 +658,9 @@ def main(argv: list[str] | None = None) -> int: items += [load_input(p) for p in args.scene] items += [load_input(p, real=True) for p in args.real] for path in args.app_export: - exported = load_input(path, real=True) - exported.app_export = True - items.append(exported) + items.append(load_app_export(path.expanduser())) + if not args.no_hostile: + items += hostile_inputs() if not items: raise SystemExit("No scenes: add case files to e2e/cases or pass --scene/--real.") @@ -495,7 +678,7 @@ def main(argv: list[str] | None = None) -> int: print(f"Server {url}, endpoint POST {endpoint.path} ({endpoint.content_type})") records = [] for item in items: - record = judge(url, endpoint, item, schemas) + record = judge(url, endpoint, item, schemas, rules) records.append(record) print( f" {record['status']:<20} {item.name}" diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index ea8493566..f79ad0a8e 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -21,17 +21,60 @@ import copy import hashlib +from dataclasses import dataclass from typing import Any +import yaml from jsonschema import Draft202012Validator EPS = 1e-6 NUMBER_TOL_FT = 1e-5 # Spot centres and offsets are rounded by the server; a hundredth of a foot is 1/8 inch. OFFSET_TOL_FT = 0.01 -# Sweep start ranges are sampled; a pass run may reach one 2 in step past a covered edge. -SWEEP_STEP_FT = 2 / 12 -DEFAULT_BATTERY_WIDTH_FT = 31 / 12 +# Check ids whose PASS depends on seeing an area around the footprint, the band that area lies +# in, and the rules.yaml clearance that sets its radius. ground_surface needs only the ground +# under the battery. +CLEARANCE_CHECKS = { + "gas_clearance": ("ground", "gas_ft"), + "ac_clearance": ("ground", "ac_ft"), + "drive_clearance": ("ground", "drive_ft"), + "pool_clearance": ("ground", "pool_ft"), + "opening_clearance": ("wall", "opening_ft"), +} + + +@dataclass(frozen=True) +class RuleSet: + """What the checks here need from the server's rules.yaml at the tested ref.""" + + width_ft: float + depth_ft: float + # check id -> (band, radius in feet around the battery footprint that must be observed) + radii: dict[str, tuple[str, float]] + # default error by object source, and for walls and the meter + errors: dict[str, float] + + @classmethod + def from_yaml(cls, text: str) -> RuleSet: + data = yaml.safe_load(text) + + def value(*path: str) -> float: + node: Any = data + for key in path: + if not isinstance(node, dict) or key not in node: + raise ValueError(f"rules.yaml has no {'.'.join(path)}") + node = node[key] + return float(node["value"] if isinstance(node, dict) else node) + + radii = { + check: (band, value("clearances", key)) + for check, (band, key) in CLEARANCE_CHECKS.items() + } + radii["ground_surface"] = ("ground", 0.0) + errors = { + name: value("errors", f"{name}_ft") for name in ("tap", "vlm", "tape", "wall", "meter") + } + return cls(value("battery", "width_ft"), value("battery", "depth_ft"), radii, errors) def schema_errors(instance: Any, schema: dict) -> list[str]: @@ -111,7 +154,9 @@ def margin_problem(check: dict) -> str | None: ) -def invariant_problems(scene: dict, result: dict, sent: bytes | None = None) -> list[str]: +def invariant_problems( + scene: dict, result: dict, sent: bytes | None = None, rules: RuleSet | None = None +) -> list[str]: problems: list[str] = [] decision = result["decision"] spot = result.get("spot") @@ -163,18 +208,41 @@ def invariant_problems(scene: dict, result: dict, sent: bytes | None = None) -> if max(abs(want[0] - got[0]), abs(want[1] - got[1])) > OFFSET_TOL_FT: problems.append(f"spot.meter_offset_ft {got} != centre - meter {want}") - problems += coverage_problems(scene, result) + problems += missing_evidence_problems(result) + if rules is not None: + problems += coverage_problems(scene, result, rules) return problems -def coverage_problems(scene: dict, result: dict) -> list[str]: - """Missing coverage is never a pass (C5); photo requests only for unobserved areas.""" +def required_span(lo: float, hi: float, radius: float) -> tuple[float, float]: + """Along-wall stretch within `radius` of a battery covering [lo, hi]. + + Along the wall chain, distance is never shorter than straight-line distance, so every point + in this stretch lies within the radius even where the chain bends: observing it is necessary + (not sufficient) for a PASS, and demanding it cannot wrongly flag a correct server. + """ + return lo - radius, hi + radius + + +def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: + """Missing coverage is never a pass (C5), to the radius each check's rule looks out to. + + A PASS for a check at a battery covering [lo, hi] along the wall needs that check's band + observed over [lo - R, hi + R], and for the ground band out to the battery depth plus R. A + sweep run that passes needs every check's area. The wall and cable route back to the meter + must be observed too. No slack: runs list exactly the starts that were evaluated. + """ problems: list[str] = [] - spot = result.get("spot") - width = spot["width_ft"] if spot else DEFAULT_BATTERY_WIDTH_FT - depth = spot["depth_ft"] if spot else 22 / 12 + width, depth = rules.width_ft, rules.depth_ft wall = observed(scene, "wall") - ground = observed(scene, "ground", min_out_ft=depth) + + def unobserved(band: str, radius: float, lo: float, hi: float) -> str | None: + a, b = required_span(lo, hi, radius) + seen = observed(scene, band, min_out_ft=depth + radius if band == "ground" else 0.0) + if covers(seen, a, b): + return None + out = f", {depth + radius:.2f} ft out" if band == "ground" else "" + return f"{band} [{a:.2f}, {b:.2f}]{out}" if "coverage" not in scene and result["decision"] == "pass": problems.append("decision pass for a scene with no coverage at all") @@ -184,16 +252,28 @@ def coverage_problems(scene: dict, result: dict) -> list[str]: continue lo, hi = run["start_ft"][0], run["start_ft"][1] + width route_lo, route_hi = min(0.0, lo), max(0.0, hi) - if not covers(wall, route_lo, route_hi, slack=SWEEP_STEP_FT): + if not covers(wall, route_lo, route_hi): problems.append( f"sweep pass for starts {run['start_ft']} but the wall and cable route " f"[{route_lo:.2f}, {route_hi:.2f}] were not all observed" ) - if not covers(ground, lo, hi, slack=SWEEP_STEP_FT): - problems.append( - f"sweep pass for starts {run['start_ft']} but the ground under " - f"[{lo:.2f}, {hi:.2f}] was not observed {depth:.2f} ft out" - ) + for check, (band, radius) in sorted(rules.radii.items()): + gap = unobserved(band, radius, lo, hi) + if gap: + problems.append( + f"sweep pass for starts {run['start_ft']} but {check} needs {gap} observed" + ) + + spot = result.get("spot") + if spot is not None: + lo, hi = spot["span_ft"] + for check in result.get("checks", []): + if check["outcome"] != "pass" or check["id"] not in rules.radii: + continue + band, radius = rules.radii[check["id"]] + gap = unobserved(band, radius, lo, hi) + if gap: + problems.append(f"check {check['id']} passes but needs {gap} observed") for request in result.get("missing_evidence", []): if request["kind"] != "band" or "span_ft" not in request or "band" not in request: @@ -207,6 +287,24 @@ def coverage_problems(scene: dict, result: dict) -> list[str]: return problems +def missing_evidence_problems(result: dict) -> list[str]: + """Every check left unsure because an area was unobserved has a request naming it.""" + listed = { + c for request in result.get("missing_evidence", []) for c in request.get("checks", []) + } + problems = [ + f"check {c['id']} is unsure (unobserved) but no missing_evidence entry names it" + for c in result.get("checks", []) + if c["outcome"] == "unsure" + and c.get("unsure_cause") == "unobserved" + and c["id"] not in listed + ] + codes = {r["code"] for r in result.get("reasons", [])} + if "unobserved_area" in codes and not result.get("missing_evidence"): + problems.append("reason unobserved_area but missing_evidence is empty") + return problems + + # --- Case expectations ----------------------------------------------------------------------- @@ -293,6 +391,15 @@ def expectation_problems(expect: dict, result: dict) -> list[str]: if field in rule and (got is None or abs(got - rule[field]) > NUMBER_TOL_FT): problems.append(f"check {check['id']}: {field} {got}, expected {rule[field]}") + for rule in expect.get("start_outcomes", []): + got = outcome_at(result, rule["wall_id"], rule["start_ft"]) + if got != rule["outcome"]: + why = f" ({rule['why']})" if "why" in rule else "" + problems.append( + f"start {rule['wall_id']} {rule['start_ft']} is {got or 'not evaluated'}, " + f"expected {rule['outcome']}{why}" + ) + if "missing_evidence_empty" in expect: empty = not result.get("missing_evidence") if empty != expect["missing_evidence_empty"]: @@ -363,3 +470,136 @@ def comparable(result: dict) -> dict: r = copy.deepcopy(result) r["stats"].pop("elapsed_ms", None) return r + + +# --- Ordering properties --------------------------------------------------------------------- +# +# Comparisons between the answers to a scene and to a changed copy of it. They need no expected +# numbers: more measurement error or less coverage can only make the server less sure. + + +def outcome_at(result: dict, wall_id: str, start_ft: float) -> str | None: + """Outcome of the sweep run containing this battery start, or None if none contains it.""" + for run in result.get("sweep", []): + a, b = run["start_ft"] + if run["wall_id"] == wall_id and a - EPS <= start_ft <= b + EPS: + return run["outcome"] + return None + + +def probe_starts(*results: dict) -> list[tuple[str, float]]: + """Every run's ends and middle, from all results: where the outcomes can be compared.""" + points = set() + for result in results: + for run in result.get("sweep", []): + a, b = run["start_ft"] + points |= {(run["wall_id"], a), (run["wall_id"], b), (run["wall_id"], (a + b) / 2)} + return sorted(points) + + +def more_error_problems(before: dict, after: dict, change: str) -> list[str]: + """With more error, a start may only move toward UNSURE: UNSURE stays, PASS and FAIL stay + or become UNSURE.""" + problems = [] + for wall_id, s in probe_starts(before, after): + a, b = outcome_at(before, wall_id, s), outcome_at(after, wall_id, s) + if a is None or b is None or a == b or b == "unsure": + continue + problems.append(f"{change}: start {wall_id} {s:.3f} went from {a} to {b}") + if before["decision"] != "pass" and after["decision"] == "pass": + problems.append(f"{change}: the decision became pass") + return problems + + +def less_coverage_problems(before: dict, after: dict, change: str) -> list[str]: + """With less observed, no start may newly PASS.""" + problems = [ + f"{change}: start {wall_id} {s:.3f} went from {outcome_at(before, wall_id, s)} to pass" + for wall_id, s in probe_starts(before, after) + if outcome_at(after, wall_id, s) == "pass" + and outcome_at(before, wall_id, s) not in (None, "pass") + ] + if before["decision"] != "pass" and after["decision"] == "pass": + problems.append(f"{change}: the decision became pass") + return problems + + +def with_more_error(scene: dict, rules: RuleSet, extra_ft: float = 0.5) -> dict: + """Every object, wall and the meter measured `extra_ft` less precisely than stated (or than + its source's default).""" + m = copy.deepcopy(scene) + for obj in m.get("objects", []): + base = obj.get("plus_minus_ft", rules.errors[obj["source"]]) + obj["plus_minus_ft"] = base + extra_ft + for wall in m["walls"]: + wall["plus_minus_ft"] = wall.get("plus_minus_ft", rules.errors["wall"]) + extra_ft + meter = m["meter"] + meter["plus_minus_ft"] = meter.get("plus_minus_ft", rules.errors["meter"]) + extra_ft + return m + + +def tape_to_tap(scene: dict) -> dict | None: + """Objects measured by tape (and not given their own error) re-measured by AR tap.""" + m = copy.deepcopy(scene) + changed = False + for obj in m.get("objects", []): + if obj["source"] == "tape" and "plus_minus_ft" not in obj: + obj["source"] = "tap" + changed = True + return m if changed else None + + +def with_less_coverage(scene: dict, trim_ft: float = 0.5) -> dict | None: + """Every observed span shortened by `trim_ft` at each end and the ground seen less far out.""" + if not scene.get("coverage", {}).get("observed"): + return None + m = copy.deepcopy(scene) + kept = [] + for entry in m["coverage"]["observed"]: + a, b = sorted(entry["span_ft"]) + if b - a <= 2 * trim_ft: + continue + entry["span_ft"] = [a + trim_ft, b - trim_ft] + if "out_ft" in entry: + entry["out_ft"] = max(0.0, entry["out_ft"] - trim_ft) + kept.append(entry) + m["coverage"]["observed"] = kept + return m + + +def with_ground_short_of(scene: dict, rules: RuleSet, margin_ft: float = 0.1) -> dict | None: + """Ground seen out to just short of the largest clearance radius, where any correct server + must stop passing the check that needs it.""" + radius = max(r for band, r in rules.radii.values() if band == "ground") + reach = rules.depth_ft + radius - margin_ft + grounds = [e for e in scene.get("coverage", {}).get("observed", []) if e["band"] == "ground"] + if not any(e.get("out_ft", 0.0) > reach for e in grounds): + return None + m = copy.deepcopy(scene) + for entry in m["coverage"]["observed"]: + if entry["band"] == "ground": + entry["out_ft"] = min(entry["out_ft"], reach) + return m + + +def with_requests_captured(scene: dict, result: dict, out_ft: float = 40.0) -> dict | None: + """The scene as if the homeowner had shown every band the result asked for.""" + added = [ + {"band": r["band"], "span_ft": sorted(r["span_ft"])} + | ({"out_ft": out_ft} if r["band"] == "ground" else {}) + for r in result.get("missing_evidence", []) + if r["kind"] == "band" and "band" in r and "span_ft" in r + ] + if not added: + return None + m = copy.deepcopy(scene) + m.setdefault("coverage", {}).setdefault("observed", []).extend(added) + return m + + +def unobserved_checks(result: dict) -> list[str]: + return [ + c["id"] + for c in result.get("checks", []) + if c["outcome"] == "unsure" and c.get("unsure_cause") == "unobserved" + ] diff --git a/verification/hsverify/scoreboard.py b/verification/hsverify/scoreboard.py index afa1fd06a..5dff080a9 100644 --- a/verification/hsverify/scoreboard.py +++ b/verification/hsverify/scoreboard.py @@ -20,9 +20,13 @@ met on exit 0. Slow probes run only with --slow; otherwise the last cached result for the same SHA is shown with status "not run". - report {glob, key, equals | less_than | length_equals | length_at_least, - require_current_sha}: judges the newest matching report.json (by mtime). With - require_current_sha only a report whose `sha` equals the ref's SHA counts; an older one - makes the metric "stale". + require_current_sha, sha_key, valid_key}: judges the newest matching report.json (by + mtime). `sha_key` (default "sha") names the report field holding the SHA the report was + measured at; an e2e report carries the server's as `sha` and the app's as `app_sha`. With + require_current_sha only a report whose SHA equals the ref's counts; an older one makes the + metric "stale". With `valid_key`, only a report whose value there is true counts, for + reports that record whether the run itself was evidence (a11yaudit's `ok`). The evidence + names the SHA the judged report was measured at. - pr_check {name}: the named CI check on the ref's open PR. - pr_body {pattern, min_matches}: the ref's open PR description. - manual {status, evidence}: a judgment the lead records by editing the YAML. @@ -83,6 +87,8 @@ "key": None, **dict.fromkeys(COMPARATORS), "require_current_sha": False, + "sha_key": "sha", + "valid_key": None, }, "pr_check": {"name": None}, "pr_body": {"pattern": None, "min_matches": 1}, @@ -161,7 +167,7 @@ def parse_metric(raw: dict, workstreams: dict) -> Metric: if unknown: raise MetricsError(f"{mid}: {kind} does not take {', '.join(sorted(unknown))}") required = {k for k, v in PROBE_KEYS[kind].items() if v is None and k not in COMPARATORS} - required -= {"each"} + required -= {"each", "valid_key"} absent = required - given.keys() if absent: raise MetricsError(f"{mid}: {kind} needs {', '.join(sorted(absent))}") @@ -472,15 +478,35 @@ def probe_report(probe: dict, sha: str | None) -> Outcome: if not paths: return Outcome("no evidence yet", f"no report matches {probe['glob']}") reports = [(p, _read_report(p)) for p in paths] + sha_key = probe["sha_key"] + + def measured_at(data: dict | None) -> str: + value = _value_or_none(data or {}, sha_key) + return f"{sha_key} {str(value)[:8]}" if value else f"no {sha_key}" + if probe["require_current_sha"]: if sha is None: raise ValueError("require_current_sha needs the ref's SHA") - current = [(p, d) for p, d in reports if d is not None and d.get("sha") == sha] + current = [ + (p, d) for p, d in reports if d is not None and _value_or_none(d, sha_key) == sha + ] if not current: newest, data = reports[0] - got = str((data or {}).get("sha") or "no sha")[:8] - return Outcome("stale", f"report for {got}, ref at {sha[:8]}: {short_home(newest)}") + return Outcome( + "stale", + f"report for {measured_at(data)}, ref at {sha[:8]}: {short_home(newest)}", + ) reports = current + if probe.get("valid_key"): + valid = [(p, d) for p, d in reports if d and _value_or_none(d, probe["valid_key"]) is True] + if not valid: + newest, data = reports[0] + return Outcome( + "gap", + f"{probe['valid_key']} is not true in the newest report ({measured_at(data)}): " + f"{short_home(newest)}", + ) + reports = valid # A run that did not exercise this metric leaves its key null; use the newest that did. with_value = [ (p, d) for p, d in reports if d is not None and _value_or_none(d, probe["key"]) is not None @@ -489,11 +515,16 @@ def probe_report(probe: dict, sha: str | None) -> Outcome: path, data = reports[0] if data is None: return Outcome("gap", f"unreadable report {short_home(path)}") - return Outcome("no evidence yet", f"no {probe['key']} in {short_home(path)}") + return Outcome( + "no evidence yet", f"no {probe['key']} in {short_home(path)} ({measured_at(data)})" + ) path, data = with_value[0] value = lookup(data, probe["key"]) ok, described = compare(value, probe) - return Outcome("met" if ok else "gap", f"{probe['key']} = {described}: {short_home(path)}") + return Outcome( + "met" if ok else "gap", + f"{probe['key']} = {described} at {measured_at(data)}: {short_home(path)}", + ) def _pr_for(ref: str, ctx: Context) -> tuple[dict | None, str]: diff --git a/verification/hsverify/simrun.py b/verification/hsverify/simrun.py index 5d989a33a..6bcb70d6c 100644 --- a/verification/hsverify/simrun.py +++ b/verification/hsverify/simrun.py @@ -8,8 +8,10 @@ a screenshot once it has been on screen for `--settle` seconds; a state replaced sooner is captured immediately and flagged `transient`, because its image may already show the next state. The run ends at `--until`, after `--idle` seconds without a new state, or at -`--timeout`. Reports go outside git (default ~/house-scanning-data/reports/sim/) because a -replay can put dataset frames on screen. +`--timeout`. A state named with `--require` that never appeared makes the run fail. If the app +logged a scan bundle, it is copied into the report as scan.zip and named in report.json's +`app_export`, so the report folder can be replayed as an app export. Reports go outside git +(default ~/house-scanning-data/reports/sim/) because a replay can put dataset frames on screen. """ from __future__ import annotations @@ -24,17 +26,25 @@ import queue import re import shlex +import shutil import subprocess import sys import threading import time +from collections.abc import Iterable from dataclasses import asdict, dataclass, field from pathlib import Path +from typing import TextIO from hsverify import gitref, report from hsverify.e2e import server_from_ref from hsverify.memory import peak_rss_mb -from hsverify.statelog import LOG_PREDICATE, RedactedStateError, parse_ndjson_line +from hsverify.statelog import ( + LOG_PREDICATE, + RedactedStateError, + parse_bundle_line, + parse_ndjson_line, +) DEVICE_NAME = "HouseScan Verify" DEVICE_TYPE = "com.apple.CoreSimulator.SimDeviceType.iPhone-17" @@ -70,6 +80,7 @@ class RunReport: final_screenshot: str | None = None server: dict | None = None peak_memory: dict | None = None + app_export: str | None = None # the app's scan bundle, copied into the report folder def run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess: @@ -239,8 +250,8 @@ def static_c4_check(tree: Path) -> dict: # --- Run ------------------------------------------------------------------------------------- -def stream_states(udid: str, events: queue.Queue, stop: threading.Event, raw: Path) -> None: - cmd = [ +def log_stream_command(udid: str) -> list[str]: + return [ "xcrun", "simctl", "spawn", @@ -254,23 +265,65 @@ def stream_states(udid: str, events: queue.Queue, stop: threading.Event, raw: Pa "--predicate", LOG_PREDICATE, ] - with raw.open("w") as sink: - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) - events.put(("ready", None)) - assert proc.stdout is not None - for line in proc.stdout: - sink.write(line) - sink.flush() - try: - event = parse_ndjson_line(line) - except RedactedStateError as exc: - events.put(("redacted", str(exc))) - continue - if event is not None: - events.put(("state", event)) - if stop.is_set(): - break - proc.terminate() + + +def pump_log(lines: Iterable[str], events: queue.Queue, sink: TextIO) -> None: + """Copy `log stream` output to `sink` and queue what the runner acts on. + + The first line is the stream's "Filtering the log data" banner, printed once it is attached; + it is queued as ("ready", line) so the app is launched only after that, and no early STATE + is lost. Starting the process is not enough: it prints nothing until attached. + """ + ready = False + for line in lines: + sink.write(line) + sink.flush() + if not ready: + events.put(("ready", line.strip())) + ready = True + try: + event = parse_ndjson_line(line) + except RedactedStateError as exc: + events.put(("redacted", str(exc))) + continue + if event is not None: + events.put(("state", event)) + + +def wait_for_log_stream(events: queue.Queue, timeout_s: float) -> str | None: + """None once the stream's first line arrived; otherwise the problem to report.""" + try: + kind, _ = events.get(timeout=timeout_s) + except queue.Empty: + return f"log stream printed nothing within {timeout_s:.0f} s; the app was not launched" + if kind != "ready": + return f"log stream sent {kind!r} before its first line" + return None + + +def missing_required(required: list[str], states: list[ShotRecord]) -> list[str]: + seen = {s.state for s in states} + return [f"required state {name} never appeared" for name in required if name not in seen] + + +def copy_app_export(out: Path) -> tuple[str | None, list[str]]: + """Copy the scan bundle the app last logged into `out` as scan.zip. + + Returns (file name or None, problems). A run whose app never logged a bundle has no export + and no problem; a logged bundle that cannot be copied is a problem. + """ + log = out / "state.ndjson" + bundles = [b for line in log.open() if (b := parse_bundle_line(line))] if log.exists() else [] + if not bundles: + return None, [] + path, keyframes = bundles[-1] + try: + shutil.copyfile(path, out / "scan.zip") + except OSError as exc: + return None, [ + f"the app logged bundle {path} with {keyframes} keyframes; copy failed: {exc}" + ] + return "scan.zip", [] def app_running(udid: str, bundle_id: str) -> bool: @@ -303,14 +356,32 @@ def slug(text: str) -> str: def follow(args, udid: str, bundle_id: str, out: Path, rep: RunReport) -> None: events: queue.Queue = queue.Queue() - stop = threading.Event() - reader = threading.Thread( - target=stream_states, args=(udid, events, stop, out / "state.ndjson"), daemon=True + proc = subprocess.Popen( + log_stream_command(udid), stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True ) - reader.start() - events.get(timeout=30) # log stream is attached before launch, so no early state is lost - time.sleep(1.0) + assert proc.stdout is not None + with (out / "state.ndjson").open("w") as sink: + reader = threading.Thread(target=pump_log, args=(proc.stdout, events, sink), daemon=True) + reader.start() + try: + problem = wait_for_log_stream(events, args.log_ready_timeout) + if problem: + rep.end_reason = "log stream did not start" + rep.problems.append(problem) + return + follow_launched(args, udid, bundle_id, out, rep, events) + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + reader.join(timeout=10) + +def follow_launched( + args, udid: str, bundle_id: str, out: Path, rep: RunReport, events: queue.Queue +) -> None: launch = [ "xcrun", "simctl", @@ -392,7 +463,6 @@ def shoot(event, arrived: float, transient: bool) -> None: if pending is not None: shoot(pending[0], pending[1], transient=False) - stop.set() final = out / "final.png" screenshot(udid, final) rep.final_screenshot = final.name @@ -428,6 +498,19 @@ def main(argv: list[str] | None = None) -> int: ) parser.add_argument("--extra-arg", action="append", default=[], help="more launch args") parser.add_argument("--until", action="append", default=[], help="state that ends the run") + parser.add_argument( + "--require", + action="append", + default=[], + metavar="STATE", + help="a state the run must reach; one that never appears is a problem (exit 1)", + ) + parser.add_argument( + "--log-ready-timeout", + type=float, + default=30.0, + help="seconds to wait for `log stream` to print its first line before launching", + ) parser.add_argument("--settle", type=float, default=1.2, help="seconds before a screenshot") parser.add_argument("--idle", type=float, default=25.0, help="stop after this long quiet") parser.add_argument("--timeout", type=float, default=300.0) @@ -512,6 +595,9 @@ def main(argv: list[str] | None = None) -> int: ) print("Running...", flush=True) follow(args, udid, bundle_id, out, rep) + rep.problems += missing_required(args.require, rep.states) + rep.app_export, export_problems = copy_app_export(out) + rep.problems += export_problems rep.crash_reports = collect_crashes(started, out) if rep.crash_reports: rep.problems.append(f"Crash reports: {', '.join(rep.crash_reports)}") diff --git a/verification/hsverify/statelog.py b/verification/hsverify/statelog.py index 8efe9974a..08dce3ff3 100644 --- a/verification/hsverify/statelog.py +++ b/verification/hsverify/statelog.py @@ -16,6 +16,11 @@ # happened); only `state` lines are parsed as screens. LOG_PREDICATE = f'subsystem == "{SUBSYSTEM}"' +ENGINE_CATEGORY = "engine" +# The engine logs `bundle with keyframes` once it has written a scan bundle; the path +# is on the Mac's disk, inside the Simulator's app container. +_BUNDLE_RE = re.compile(r"^bundle (.+) with (\d+) keyframes$") + # A state name is an identifier-like token. Anything after it on the line is detail. _STATE_RE = re.compile(r"STATE=([A-Za-z0-9_.\-]+)") _REDACTED = "" @@ -35,6 +40,26 @@ class RedactedStateError(ValueError): """ +def _app_entry(line: str, category: str) -> dict | None: + line = line.strip() + if not line.startswith("{"): + return None + try: + entry = json.loads(line) + except json.JSONDecodeError: + return None + if entry.get("subsystem") != SUBSYSTEM or entry.get("category") != category: + return None + return entry + + +def parse_bundle_line(line: str) -> tuple[str, int] | None: + """(zip path, keyframe count) from the engine's bundle line, or None for any other line.""" + entry = _app_entry(line, ENGINE_CATEGORY) + match = _BUNDLE_RE.match((entry or {}).get("eventMessage") or "") + return (match.group(1), int(match.group(2))) if match else None + + def parse_ndjson_line(line: str) -> StateEvent | None: """Return the STATE event on one `log stream --style ndjson` line, or None. @@ -43,14 +68,8 @@ def parse_ndjson_line(line: str) -> StateEvent | None: Raises RedactedStateError when the message is the redaction placeholder, because silently dropping it would make a working app look like it logs nothing. """ - line = line.strip() - if not line.startswith("{"): - return None - try: - entry = json.loads(line) - except json.JSONDecodeError: - return None - if entry.get("subsystem") != SUBSYSTEM or entry.get("category") != CATEGORY: + entry = _app_entry(line, CATEGORY) + if entry is None: return None message = entry.get("eventMessage") or "" match = _STATE_RE.search(message) diff --git a/verification/scoreboard/metrics.yaml b/verification/scoreboard/metrics.yaml index b0c61d687..e53278dc5 100644 --- a/verification/scoreboard/metrics.yaml +++ b/verification/scoreboard/metrics.yaml @@ -2,10 +2,11 @@ # A probe that only shows something exists is capped at partial with a cap_reason. # Where only a person can judge, the probe is `manual`; the lead edits status and evidence. # -# e2e report keys (hsverify/e2e.py): `sha` is the server SHA; `latency_ms` is the slowest real -# scene's round trip; `app_export_scene_valid` and `app_export_passed` cover scenes the iOS app -# exported from a replay. A key is null when the run did not exercise it; the probe then uses -# the newest report where it is set. +# e2e report keys (hsverify/e2e.py): `sha` is the server SHA and `app_sha` the SHA of the app +# build whose export was replayed; `latency_ms` is the slowest real scene's round trip; +# `app_export_scene_valid` and `app_export_passed` cover scenes the iOS app exported from a +# replay, so their metrics compare `app_sha` (sha_key) with the app branch. A key is null when +# the run did not exercise it; the probe then uses the newest report where it is set. workstreams: S1: {title: Evals on real data} @@ -149,9 +150,9 @@ metrics: workstream: S2 ref: origin/t3/server text: >- - Black-box (verification e2e): every answer valid under C2, consistent with its decision - and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric - when mirrored, and never passing without coverage + Black-box (verification e2e at this SHA): results valid under C2, consistent with the + decision and C5 margins, coverage to each check's required radius, missing evidence + complete, ordering properties, hostile inputs refused in time report: glob: ~/house-scanning-data/reports/e2e/*/report.json key: contract_problem_count @@ -223,13 +224,14 @@ metrics: ref: origin/t3/ios-mvf text: >- Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the - run reached, plus symbol-name button labels) finds no issues; S4-2 shows how many screens - the app reached + run reached, at least nine, plus symbol-name button labels) finds no issues report: glob: ~/house-scanning-data/reports/a11y/*t3-ios-mvf*/report.json key: issue_count equals: 0 require_current_sha: true + # ok is false when the harness did not run, a screen errored or too few were audited. + valid_key: ok - id: S3-7 workstream: S3 @@ -246,6 +248,8 @@ metrics: glob: ~/house-scanning-data/reports/e2e/*/report.json key: app_export_scene_valid equals: true + require_current_sha: true + sha_key: app_sha - id: S3-9 workstream: S3 @@ -256,12 +260,15 @@ metrics: # --- S4 -------------------------------------------------------------------------------------- - id: S4-1 workstream: S4 - ref: origin/t3/verification + # The export comes from the app, so the report must be for the app branch's head. + ref: origin/t3/ios-mvf text: End to end from a real replay to a placement result passes report: glob: ~/house-scanning-data/reports/e2e/*/report.json key: app_export_passed equals: true + require_current_sha: true + sha_key: app_sha - id: S4-2 workstream: S4 diff --git a/verification/tests/test_a11yaudit.py b/verification/tests/test_a11yaudit.py index afdf911dc..6013a20e3 100644 --- a/verification/tests/test_a11yaudit.py +++ b/verification/tests/test_a11yaudit.py @@ -1,4 +1,4 @@ -from hsverify.a11yaudit import parse_output, symbol_name_labels +from hsverify.a11yaudit import audit_problems, audited_screens, parse_output, symbol_name_labels def test_parse_output_reads_screens_and_issues_and_skips_noise(): @@ -32,3 +32,50 @@ def test_symbol_name_labels(): {"screen": 1, "label": "gearshape"}, {"screen": 1, "label": "arrow.left.circle.fill"}, ] + + +def screen(index, **extra): + return {"index": index, "labels": ["text:x"], "issues": 0} | extra + + +NINE = [screen(i) for i in range(1, 10)] + + +def test_nine_clean_screens_are_valid(): + assert audit_problems(NINE, harness_ran=True, min_screens=9) == [] + + +def test_zero_screens_is_not_evidence(): + assert audit_problems([], harness_ran=True, min_screens=9) == ["no screen was audited"] + + +def test_harness_that_did_not_run_is_not_evidence(): + problems = audit_problems(NINE, harness_ran=False, min_screens=9) + assert problems == ["the audit harness did not run (no TEST SUCCEEDED or TEST FAILED line)"] + + +def test_too_few_distinct_screens(): + repeated = [screen(1), screen(1), screen(2)] + assert audit_problems(repeated, harness_ran=True, min_screens=9) == [ + "2 screens audited, fewer than --min-screens 9" + ] + + +def test_screen_error_fails_and_that_screen_is_not_counted(): + # After a throw the harness prints an error entry and then a normal one for the same index. + screens = [*NINE, screen(9, error="audit threw"), {"index": 10, "error": "app not running"}] + assert audit_problems(screens, harness_ran=True, min_screens=9) == [ + "screen 9: audit threw", + "screen 10: app not running", + "8 screens audited, fewer than --min-screens 9", + ] + assert audited_screens(screens) == set(range(1, 9)) + + +def test_error_lines_from_the_harness_are_parsed_as_screens(): + text = 'A11Y_SCREEN={"error":"app not running","index":1}' + screens, _ = parse_output(text) + assert audit_problems(screens, harness_ran=True, min_screens=1) == [ + "screen 1: app not running", + "no screen was audited", + ] diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index fdf0c3340..4c411adf4 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -2,11 +2,30 @@ import json import re import threading +import time from http.server import BaseHTTPRequestHandler, HTTPServer import pytest -from hsverify.e2e import Endpoint, SceneInput, discover_endpoint, judge, load_input +from hsverify import e2e +from hsverify.e2e import ( + Endpoint, + SceneInput, + discover_endpoint, + hostile_inputs, + judge, + judge_hostile, + load_app_export, + load_input, +) +from hsverify.resultcheck import RuleSet + +RULES = RuleSet( + width_ft=31 / 12, + depth_ft=22 / 12, + radii={"ground_surface": ("ground", 0.0), "gas_clearance": ("ground", 1.0)}, + errors={"tap": 0.3, "vlm": 1.5, "tape": 0.05, "wall": 0.3, "meter": 0.3}, +) SCENE = { "meter": {"pos": [0.0, 4.0, 0.0], "wall_id": "w1"}, @@ -219,29 +238,92 @@ def item() -> SceneInput: [Endpoint("/place", "application/json"), Endpoint("/place", "multipart/form-data", "file")], ) def test_consistent_server_passes(fake_url, endpoint): - record = judge(fake_url, endpoint, item(), SCHEMAS) + record = judge(fake_url, endpoint, item(), SCHEMAS, RULES) assert record["status"] == "pass", record["problems"] assert record["decision"] == "manual_review" def test_server_breaking_coverage_rule_fails(fake_url): FakeServer.decide_pass = True - record = judge(fake_url, Endpoint("/place", "application/json"), item(), SCHEMAS) + record = judge(fake_url, Endpoint("/place", "application/json"), item(), SCHEMAS, RULES) assert record["status"] == "fail" joined = "\n".join(record["problems"]) assert "ruled out for this case" in joined - assert "without coverage still produced a pass" in joined - assert re.search(r"ground under .* was not observed", joined) + assert "no coverage: decision pass for a scene with no coverage at all" in joined + assert re.search(r"needs ground \[.*\], 2\.83 ft out observed", joined) def test_a_result_breaking_its_schema_counts_against_the_contract(fake_url): strict = {"scene": {}, "result": {"required": ["nothing_has_this"]}} - record = judge(fake_url, Endpoint("/place", "application/json"), item(), strict) + record = judge(fake_url, Endpoint("/place", "application/json"), item(), strict, RULES) assert record["status"] == "fail" assert record["contract_problems"] and record["contract_problems"] == record["problems"] def test_bad_input_is_not_blamed_on_the_server(fake_url): strict = {"scene": {"required": ["nothing_has_this"]}, "result": {}} - record = judge(fake_url, Endpoint("/place", "application/json"), item(), strict) + record = judge(fake_url, Endpoint("/place", "application/json"), item(), strict, RULES) assert record["status"] == "bad input" + + +class RefusingServer(BaseHTTPRequestHandler): + """Refuses every request with 422 after `delay` seconds.""" + + delay = 0.0 + + def do_POST(self): + self.rfile.read(int(self.headers["Content-Length"])) + time.sleep(self.delay) + self.send_response(422) + self.send_header("Content-Length", "2") + self.end_headers() + self.wfile.write(b"{}") + + def log_message(self, *args): + pass + + +@pytest.fixture +def refusing_url(): + server = HTTPServer(("127.0.0.1", 0), RefusingServer) + threading.Thread(target=server.serve_forever, daemon=True).start() + yield f"http://127.0.0.1:{server.server_port}" + server.shutdown() + RefusingServer.delay = 0.0 + + +def test_a_quick_refusal_of_a_hostile_input_passes(refusing_url): + hostile = hostile_inputs()[1] + record = judge_hostile(refusing_url, Endpoint("/p", "application/json"), hostile, SCHEMAS) + assert record["status"] == "pass" and record["http_status"] == 422 + + +def test_a_slow_refusal_or_a_hang_fails(refusing_url, monkeypatch): + monkeypatch.setattr(e2e, "HOSTILE_BUDGET_MS", 100.0) + RefusingServer.delay = 0.3 + hostile = hostile_inputs()[1] + record = judge_hostile(refusing_url, Endpoint("/p", "application/json"), hostile, SCHEMAS) + assert record["status"] == "fail" and "budget 100" in record["problems"][0] + RefusingServer.delay = 6.0 # longer than budget + 5 s: the request times out + record = judge_hostile(refusing_url, Endpoint("/p", "application/json"), hostile, SCHEMAS) + assert record["status"] == "fail" and record["problems"][0].startswith("no answer within") + + +def test_hostile_inputs_are_small_to_send(): + items = hostile_inputs() + assert all(i.hostile for i in items) + assert max(len(i.raw) for i in items) < 5_000_000 # the harness stays light + + +def test_a_sim_report_folder_is_an_app_export_with_its_sha(tmp_path): + (tmp_path / "scan.zip").write_bytes(b"") + import zipfile + + with zipfile.ZipFile(tmp_path / "scan.zip", "w") as z: + z.writestr("scene.json", json.dumps(SCENE)) + (tmp_path / "report.json").write_text(json.dumps({"sha": "abc123", "app_export": "scan.zip"})) + item = load_app_export(tmp_path) + assert (item.app_export, item.app_sha, item.real) == (True, "abc123", True) + (tmp_path / "report.json").write_text(json.dumps({"sha": "abc123"})) + with pytest.raises(SystemExit, match="never reached the upload"): + load_app_export(tmp_path) diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 29b29ce6d..0d6ea07bf 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -5,14 +5,36 @@ import pytest from hsverify.resultcheck import ( + RuleSet, assumption_mismatches, comparable, expectation_problems, invariant_problems, + less_coverage_problems, margin_problem, mirror_scene, + missing_evidence_problems, + more_error_problems, observed, + outcome_at, outcome_lengths, + tape_to_tap, + with_ground_short_of, + with_less_coverage, + with_more_error, + with_requests_captured, +) + +# Small radii keep the geometry checkable by hand: a battery 31 x 22 in, gas and openings 1 ft. +RULES = RuleSet( + width_ft=31 / 12, + depth_ft=22 / 12, + radii={ + "ground_surface": ("ground", 0.0), + "gas_clearance": ("ground", 1.0), + "opening_clearance": ("wall", 1.0), + }, + errors={"tap": 0.3, "vlm": 1.5, "tape": 0.05, "wall": 0.3, "meter": 0.3}, ) SCENE = { @@ -31,8 +53,8 @@ "ends": {"left": {"kind": "limit"}, "right": {"kind": "unexplored"}}, "observed": [ {"band": "wall", "span_ft": [-12.0, 10.0]}, - {"band": "ground", "span_ft": [-12.0, 4.0], "out_ft": 3.0}, - {"band": "ground", "span_ft": [4.0, 10.0], "out_ft": 1.0}, + {"band": "ground", "span_ft": [-12.0, 10.0], "out_ft": 3.0}, + {"band": "ground", "span_ft": [10.0, 12.0], "out_ft": 1.0}, ], }, } @@ -113,7 +135,7 @@ def result(decision="manual_review", spot=True, checks=None, sweep=None, sent=b" def test_consistent_result_has_no_problems(): - assert invariant_problems(SCENE, result(), sent=b"{}") == [] + assert invariant_problems(SCENE, result(), sent=b"{}", rules=RULES) == [] @pytest.mark.parametrize( @@ -197,29 +219,53 @@ def test_counts_hash_and_offset(): def test_observed_merges_and_filters_ground_depth(): assert observed(SCENE, "wall") == [(-12.0, 10.0)] - assert observed(SCENE, "ground", min_out_ft=22 / 12) == [(-12.0, 4.0)] + assert observed(SCENE, "ground", min_out_ft=2.0) == [(-12.0, 10.0)] + assert observed(SCENE, "ground", min_out_ft=0.5) == [(-12.0, 12.0)] assert observed({"meter": {}}, "wall") == [] -def test_pass_over_unobserved_ground_is_flagged(): - # Starts [3, 3] put the battery over s = [3, 5.58]; ground past s = 4 was seen only 1 ft out. - sweep = [ - {"wall_id": "w1", "start_ft": [3.0, 3.0], "outcome": "pass", "failing": [], "unsure": []} - ] - msgs = invariant_problems(SCENE, result(sweep=sweep)) - assert any("ground under" in m for m in msgs) +def with_ground(out_ft: float, span: list[float]) -> dict: + scene = copy.deepcopy(SCENE) + scene["coverage"]["observed"][1:] = [{"band": "ground", "span_ft": span, "out_ft": out_ft}] + return scene + + +def test_a_pass_needs_ground_out_to_depth_plus_each_radius(): + # Start 1 covers [1, 3.58]. Gas (radius 1) needs ground out to 1.83 + 1 = 2.83 ft. + msgs = invariant_problems(with_ground(2.5, [-12.0, 10.0]), result(), rules=RULES) + assert any("gas_clearance needs ground [0.00, 4.58], 2.83 ft out observed" in m for m in msgs) + assert invariant_problems(with_ground(2.84, [-12.0, 10.0]), result(), rules=RULES) == [] + + +def test_a_pass_needs_ground_along_the_wall_to_each_radius_without_slack(): + # Ground from 0.1: the 1 ft gas radius around [1, 3.58] reaches back to 0. + msgs = invariant_problems(with_ground(3.0, [0.1, 10.0]), result(), rules=RULES) + assert any("gas_clearance needs ground [0.00, 4.58]" in m for m in msgs) + + +def test_a_passing_opening_check_needs_the_wall_to_its_radius(): + scene = copy.deepcopy(SCENE) + scene["coverage"]["observed"][0]["span_ft"] = [0.0, 4.0] # radius needs [0, 4.58] + r = result( + checks=[ + check(), + {**check(), "id": "opening_clearance", "measured_ft": None, "plus_minus_ft": None}, + ] + ) + msgs = invariant_problems(scene, r, rules=RULES) + assert any("check opening_clearance passes but needs wall [0.00, 4.58]" in m for m in msgs) def test_pass_whose_route_crosses_unobserved_wall_is_flagged(): scene = copy.deepcopy(SCENE) scene["coverage"]["observed"][0]["span_ft"] = [0.5, 10.0] # the wall at s = [0, 0.5] unseen - msgs = invariant_problems(scene, result()) + msgs = invariant_problems(scene, result(), rules=RULES) assert any("wall and cable route" in m for m in msgs) def test_no_coverage_never_passes(): scene = {k: v for k, v in SCENE.items() if k != "coverage"} - msgs = invariant_problems(scene, result("pass")) + msgs = invariant_problems(scene, result("pass"), rules=RULES) assert any("no coverage at all" in m for m in msgs) @@ -228,7 +274,43 @@ def test_photo_request_for_an_observed_area_is_flagged(): r["missing_evidence"] = [ {"kind": "band", "band": "wall", "span_ft": [-3.0, -1.0], "message": ""} ] - assert any("lists as observed" in m for m in invariant_problems(SCENE, r)) + assert any("lists as observed" in m for m in invariant_problems(SCENE, r, rules=RULES)) + + +def test_every_unobserved_check_needs_a_request_naming_it(): + unseen = check("unsure", None, cause="unobserved") + r = result(checks=[unseen]) + r["reasons"] = [{"code": "unobserved_area", "message": ""}] + assert missing_evidence_problems(r) == [ + "check gas_clearance is unsure (unobserved) but no missing_evidence entry names it", + "reason unobserved_area but missing_evidence is empty", + ] + r["missing_evidence"] = [ + { + "kind": "band", + "band": "ground", + "span_ft": [11, 12], + "checks": ["gas_clearance"], + "message": "", + } + ] + assert missing_evidence_problems(r) == [] + + +def test_rules_come_from_rules_yaml_and_a_missing_one_is_loud(): + text = """ +battery: {width_ft: {value: 2.5}, depth_ft: {value: 1.5}} +errors: {tap_ft: {value: 0.3}, vlm_ft: {value: 1.5}, tape_ft: {value: 0.05}, + wall_ft: {value: 0.3}, meter_ft: {value: 0.3}} +clearances: {gas_ft: {value: 3}, ac_ft: {value: 3}, drive_ft: {value: 5}, + pool_ft: {value: 10}, opening_ft: {value: 3}} +""" + rules = RuleSet.from_yaml(text) + assert rules.radii["pool_clearance"] == ("ground", 10.0) + assert rules.radii["opening_clearance"] == ("wall", 3.0) + assert (rules.width_ft, rules.errors["tape"]) == (2.5, 0.05) + with pytest.raises(ValueError, match=r"rules\.yaml has no clearances\.pool_ft"): + RuleSet.from_yaml(text.replace("pool_ft: {value: 10}, ", "")) def test_expectations(): @@ -368,3 +450,82 @@ def test_outcome_lengths_and_comparable(): other = copy.deepcopy(r) other["stats"]["elapsed_ms"] = 99.0 assert comparable(r) == comparable(other) + + +def run(a, b, outcome, wall="w1"): + return {"wall_id": wall, "start_ft": [a, b], "outcome": outcome, "failing": [], "unsure": []} + + +def swept(*runs, decision="manual_review"): + r = result(decision=decision, sweep=list(runs)) + return r + + +def test_start_outcomes_expectation(): + r = swept(run(0.0, 2.0, "fail"), run(2.1667, 5.0, "unsure")) + assert outcome_at(r, "w1", 1.0) == "fail" and outcome_at(r, "w1", 2.1) is None + ok = {"start_outcomes": [{"wall_id": "w1", "start_ft": 3.0, "outcome": "unsure"}]} + assert expectation_problems(ok, r) == [] + bad = { + "start_outcomes": [ + {"wall_id": "w1", "start_ft": 1.0, "outcome": "unsure", "why": "inside the error band"} + ] + } + assert expectation_problems(bad, r) == [ + "start w1 1.0 is fail, expected unsure (inside the error band)" + ] + + +@pytest.mark.parametrize( + ("before", "after", "allowed"), + [ + ("pass", "unsure", True), + ("fail", "unsure", True), + ("unsure", "unsure", True), + ("unsure", "fail", False), + ("unsure", "pass", False), + ("fail", "pass", False), + ("pass", "fail", False), + ], +) +def test_more_error_only_moves_toward_unsure(before, after, allowed): + msgs = more_error_problems(swept(run(0, 4, before)), swept(run(0, 4, after)), "more error") + assert (msgs == []) == allowed, msgs + + +def test_less_coverage_never_creates_a_pass(): + assert less_coverage_problems(swept(run(0, 4, "pass")), swept(run(0, 4, "unsure")), "x") == [] + msgs = less_coverage_problems(swept(run(0, 4, "unsure")), swept(run(0, 4, "pass")), "x") + assert msgs and all("went from unsure to pass" in m for m in msgs) + + +def test_transforms(): + scene = copy.deepcopy(SCENE) + scene["objects"].append({"type": "ac", "wall_id": "w1", "span_ft": [5, 6], "source": "tape"}) + more = with_more_error(scene, RULES) + assert [o["plus_minus_ft"] for o in more["objects"]] == [0.8, 0.55] + assert more["walls"][0]["plus_minus_ft"] == 0.8 and more["meter"]["plus_minus_ft"] == 0.8 + assert [o["source"] for o in tape_to_tap(scene)["objects"]] == ["tap", "tap"] + assert tape_to_tap(SCENE) is None + less = with_less_coverage(scene) + assert less["coverage"]["observed"][0]["span_ft"] == [-11.5, 9.5] + assert less["coverage"]["observed"][1]["out_ft"] == 2.5 + short = with_ground_short_of(scene, RULES) # largest ground radius 1: out to 2.73 + assert [e.get("out_ft") for e in short["coverage"]["observed"]] == [ + None, + pytest.approx(22 / 12 + 1 - 0.1), + 1.0, + ] + assert with_ground_short_of(short, RULES) is None # nothing left beyond the radius + r = result() + r["missing_evidence"] = [ + {"kind": "band", "band": "ground", "span_ft": [12, 11], "message": ""}, + {"kind": "past_end", "side": "left", "message": ""}, + ] + captured = with_requests_captured(scene, r) + assert captured["coverage"]["observed"][-1] == { + "band": "ground", + "span_ft": [11, 12], + "out_ft": 40.0, + } + assert with_requests_captured(scene, result()) is None diff --git a/verification/tests/test_scoreboard.py b/verification/tests/test_scoreboard.py index 87fd2f07d..fd7d12a3e 100644 --- a/verification/tests/test_scoreboard.py +++ b/verification/tests/test_scoreboard.py @@ -378,7 +378,7 @@ def test_report_uses_newest_and_compares(tmp_path): write_report(tmp_path / "old" / "report.json", {"latency_ms": 2000}, 100) new = write_report(tmp_path / "new" / "report.json", {"latency_ms": 420}, 200) out = evaluate(report_metric(tmp_path, key="latency_ms", less_than=1000), SHA, ctx()) - assert out == Outcome("met", f"latency_ms = 420 (want < 1000): {new}") + assert out == Outcome("met", f"latency_ms = 420 (want < 1000) at no sha: {new}") @pytest.mark.parametrize( @@ -404,7 +404,7 @@ def test_report_missing_key_and_no_reports(tmp_path): m = report_metric(tmp_path, key="build.ok", equals=True) assert evaluate(m, SHA, ctx()).status == "no evidence yet" path = write_report(tmp_path / "r" / "report.json", {"build": {}}, 100) - assert evaluate(m, SHA, ctx()) == Outcome("no evidence yet", f"no build.ok in {path}") + assert evaluate(m, SHA, ctx()) == Outcome("no evidence yet", f"no build.ok in {path} (no sha)") def test_report_skips_newer_runs_that_did_not_set_the_key(tmp_path): @@ -421,10 +421,75 @@ def test_report_requiring_current_sha_marks_stale_or_picks_matching(tmp_path): newest = write_report(tmp_path / "b" / "report.json", {"sha": OLD, "states": []}, 200) m = report_metric(tmp_path, key="states", length_at_least=1, require_current_sha=True) out = evaluate(m, SHA, ctx()) - assert out == Outcome("met", f"states = length 1 (want at least 1): {older}") + assert out == Outcome("met", f"states = length 1 (want at least 1) at sha aaaaaaaa: {older}") out = evaluate(m, "c" * 40, ctx()) - assert out == Outcome("stale", f"report for bbbbbbbb, ref at cccccccc: {newest}") + assert out == Outcome("stale", f"report for sha bbbbbbbb, ref at cccccccc: {newest}") + + +def test_report_sha_key_compares_the_named_field_not_sha(tmp_path): + # An e2e report: `sha` is the server's, `app_sha` the app's the export came from. + path = write_report( + tmp_path / "r" / "report.json", + {"sha": OLD, "app_sha": SHA, "app_export_passed": True}, + 100, + ) + m = report_metric( + tmp_path, key="app_export_passed", equals=True, require_current_sha=True, sha_key="app_sha" + ) + assert evaluate(m, SHA, ctx()) == Outcome( + "met", f"app_export_passed = true (want True) at app_sha aaaaaaaa: {path}" + ) + # The server SHA matching the ref is not enough. + assert evaluate(m, OLD, ctx()) == Outcome( + "stale", f"report for app_sha aaaaaaaa, ref at bbbbbbbb: {path}" + ) + + +def test_report_sha_key_missing_from_the_report_is_stale(tmp_path): + path = write_report(tmp_path / "r" / "report.json", {"sha": SHA, "passed": True}, 100) + m = report_metric( + tmp_path, key="passed", equals=True, require_current_sha=True, sha_key="app_sha" + ) + assert evaluate(m, SHA, ctx()) == Outcome( + "stale", f"report for no app_sha, ref at aaaaaaaa: {path}" + ) + + +def test_report_sha_key_defaults_to_sha(): + assert metric("report", glob="x", key="k", equals=1).probe["sha_key"] == "sha" + + +def test_report_valid_key_ignores_runs_that_were_not_evidence(tmp_path): + m = report_metric( + tmp_path, key="issue_count", equals=0, require_current_sha=True, valid_key="ok" + ) + good = write_report( + tmp_path / "a" / "report.json", {"sha": SHA, "ok": True, "issue_count": 0}, 100 + ) + # Newer, but zero screens audited: issue_count 0 must not count. + blind = write_report( + tmp_path / "b" / "report.json", {"sha": SHA, "ok": False, "issue_count": 0}, 200 + ) + assert evaluate(m, SHA, ctx()) == Outcome( + "met", f"issue_count = 0 (want 0) at sha aaaaaaaa: {good}" + ) + good.unlink() + assert evaluate(m, SHA, ctx()) == Outcome( + "gap", f"ok is not true in the newest report (sha aaaaaaaa): {blind}" + ) + # A report from before `ok` existed is not evidence either. + write_report(tmp_path / "b" / "report.json", {"sha": SHA, "issue_count": 0}, 300) + assert evaluate(m, SHA, ctx()).status == "gap" + + +def test_real_app_metrics_compare_the_app_sha(): + for mid in ("S3-8", "S4-1"): + m = real_metric(mid) + assert m.ref == "origin/t3/ios-mvf" + assert m.probe["require_current_sha"] is True + assert m.probe["sha_key"] == "app_sha" + assert real_metric("S3-6b").probe["valid_key"] == "ok" def test_report_path_shortens_home(monkeypatch, tmp_path): diff --git a/verification/tests/test_simrun.py b/verification/tests/test_simrun.py index 279ee6b71..9b43331f5 100644 --- a/verification/tests/test_simrun.py +++ b/verification/tests/test_simrun.py @@ -1,6 +1,36 @@ import datetime as dt +import io +import json +import queue +from pathlib import Path -from hsverify.simrun import compiling, seconds_since, slug +from hsverify.simrun import ( + ShotRecord, + compiling, + copy_app_export, + missing_required, + pump_log, + seconds_since, + slug, + wait_for_log_stream, +) +from hsverify.statelog import SUBSYSTEM + +BANNER = 'Filtering the log data using "subsystem == "dev.housescanning.housescan""\n' + + +def log_line(message: str, category: str = "state") -> str: + return ( + json.dumps( + { + "timestamp": "2026-09-26 02:50:00.123456-0500", + "subsystem": SUBSYSTEM, + "category": category, + "eventMessage": message, + } + ) + + "\n" + ) def test_state_time_uses_the_log_clock(): @@ -22,3 +52,95 @@ def test_only_compiling_xcodebuilds_block_a_build(): assert compiling("/Applications/Xcode.app/.../xcodebuild -project a.xcodeproj build") assert compiling("xcodebuild test -project a.xcodeproj -scheme A") assert not compiling("xcodebuild -project a.xcodeproj test-without-building") + + +# --- log stream readiness -------------------------------------------------------------------- + + +def drain(events: queue.Queue) -> list: + items = [] + while not events.empty(): + items.append(events.get_nowait()) + return items + + +def test_ready_is_signalled_by_the_first_line_then_states_follow(): + events: queue.Queue = queue.Queue() + sink = io.StringIO() + pump_log(iter([BANNER, log_line("STATE=onboarding")]), events, sink) + kinds = [(kind, getattr(payload, "name", payload)) for kind, payload in drain(events)] + assert kinds == [("ready", BANNER.strip()), ("state", "onboarding")] + assert sink.getvalue() == BANNER + log_line("STATE=onboarding") + + +def test_a_silent_log_stream_never_signals_ready_and_times_out(): + events: queue.Queue = queue.Queue() + pump_log(iter([]), events, io.StringIO()) + assert events.empty() + assert wait_for_log_stream(events, 0.01) == ( + "log stream printed nothing within 0 s; the app was not launched" + ) + + +def test_wait_for_log_stream_passes_once_ready(): + events: queue.Queue = queue.Queue() + pump_log(iter([BANNER]), events, io.StringIO()) + assert wait_for_log_stream(events, 1.0) is None + + +# --- required states ------------------------------------------------------------------------- + + +def shot(state: str) -> ShotRecord: + return ShotRecord(1, state, 0.0, "01.png", False, f"STATE={state}") + + +def test_missing_required_state_is_a_problem(): + states = [shot("onboarding"), shot("uploading")] + assert missing_required(["result"], states) == ["required state result never appeared"] + assert missing_required(["result"], [*states, shot("result")]) == [] + assert missing_required([], []) == [] + + +def test_sim_app_requires_the_result_state(): + makefile = (Path(__file__).resolve().parents[1] / "Makefile").read_text() + recipe = makefile.split("\nsim-app:\n", 1)[1].split("\n\n", 1)[0] + assert "--require result" in recipe + + +# --- app export ------------------------------------------------------------------------------ + + +def write_log(out: Path, *lines: str) -> None: + (out / "state.ndjson").write_text(BANNER + "".join(lines)) + + +def test_last_logged_bundle_is_copied_as_scan_zip(tmp_path): + first, last = tmp_path / "first.zip", tmp_path / "last.zip" + first.write_bytes(b"old") + last.write_bytes(b"PK new") + out = tmp_path / "report" + out.mkdir() + write_log( + out, + log_line(f"bundle {first} with 3 keyframes", category="engine"), + log_line("STATE=result"), + log_line(f"bundle {last} with 77 keyframes", category="engine"), + ) + assert copy_app_export(out) == ("scan.zip", []) + assert (out / "scan.zip").read_bytes() == b"PK new" + + +def test_no_bundle_logged_means_no_export_and_no_problem(tmp_path): + write_log(tmp_path, log_line("STATE=onboarding")) + assert copy_app_export(tmp_path) == (None, []) + assert not (tmp_path / "scan.zip").exists() + + +def test_logged_bundle_that_is_gone_is_a_problem(tmp_path): + gone = tmp_path / "gone.zip" + write_log(tmp_path, log_line(f"bundle {gone} with 5 keyframes", category="engine")) + export, problems = copy_app_export(tmp_path) + assert export is None + assert len(problems) == 1 + assert problems[0].startswith(f"the app logged bundle {gone} with 5 keyframes; copy failed") diff --git a/verification/tests/test_statelog.py b/verification/tests/test_statelog.py index eed95836f..ee59be7d9 100644 --- a/verification/tests/test_statelog.py +++ b/verification/tests/test_statelog.py @@ -6,6 +6,7 @@ CATEGORY, SUBSYSTEM, RedactedStateError, + parse_bundle_line, parse_ndjson_line, parse_plain_line, ) @@ -59,3 +60,24 @@ def test_redacted_marker_is_loud(message): def test_plain_line(): assert parse_plain_line("2026 app[12] STATE=result_reveal") == "result_reveal" assert parse_plain_line("no marker here") is None + + +BUNDLE_PATH = "/Users/x/Library/Developer/CoreSimulator/Devices/D/data/Caches/Scans/S/scan.zip" + + +def test_reads_the_engine_bundle_line(): + line = entry(f"bundle {BUNDLE_PATH} with 77 keyframes", category="engine") + assert parse_bundle_line(line) == (BUNDLE_PATH, 77) + + +@pytest.mark.parametrize( + "line", + [ + entry(f"bundle {BUNDLE_PATH} with 77 keyframes"), # state category + entry(f"bundle {BUNDLE_PATH} with 77 keyframes", subsystem="other", category="engine"), + entry("bundle written", category="engine"), + "Filtering the log data using ...", + ], +) +def test_bundle_line_ignores_everything_else(line): + assert parse_bundle_line(line) is None From 3b6459cd2cc2fb9bc20ac79049efd6c41aa6468e Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 07:08:38 -0500 Subject: [PATCH 31/59] verification: widen each clearance's reach by the battery's error, check requests per band, bound hostile inputs by memory Findings from running the hardened check against 26d2870 and the head: - A ground request carries no distance out, and when every position fails C2 lets missing_evidence stay empty; both had been flagged wrongly. - An explicit plus_minus_ft replaces the drift-grown default, so the more-error variant now adds drift over the whole chain before adding. - The ground a clearance needs tapers past the battery's ends (R - d) and widens by the battery's position error, which is what catches (a). - gas_clearance needs the wall band as well as the ground; checking each band's request names it catches (d). - Drift cases take drift at the battery's far edge (S2 70ab0b0), so they bracket the corrected reach cutoff and catch (b). - Hostile inputs now include a crowded scene, a compressed 400 MB scene.json and 5000-deep nesting; the server under test is capped at 3.5 GB and killed after SIGTERM if still busy, which catches (c). Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/README.md | 34 ++- verification/e2e/cases/README.md | 50 ++-- verification/e2e/cases/d-gas-drift.json | 22 +- .../e2e/cases/d-reach-drift-left.json | 22 +- .../e2e/cases/d-reach-drift-right.json | 22 +- verification/e2e/cases/generate.py | 55 ++--- verification/hsverify/e2e.py | 167 +++++++++---- verification/hsverify/memory.py | 58 +++++ verification/hsverify/resultcheck.py | 221 +++++++++++++----- verification/tests/test_e2e.py | 24 +- verification/tests/test_memory.py | 40 +++- verification/tests/test_resultcheck.py | 95 ++++++-- 12 files changed, 583 insertions(+), 227 deletions(-) diff --git a/verification/README.md b/verification/README.md index 67c7856c1..2563c9686 100644 --- a/verification/README.md +++ b/verification/README.md @@ -8,7 +8,7 @@ datasets. Requirements: uv, and Xcode 26 or newer with an iOS Simulator runtime. | Check | Command, from `verification/` | Output | | --- | --- | --- | | The app at `t3/ios-mvf` on the real ADVIO replay, uploading to a server started from `t3/server`; a screenshot per `STATE` | `make sim-app` (`REF=`, `SERVER_REF=`, `ARGS=` to vary) | `reports/sim//index.html` | -| Every case, the ETH3D facade scene and optional app exports through the server's API | `make e2e` (`ARGS="--app-export "`) | `reports/e2e//report.md` | +| Every case, the ETH3D facade scene and optional app exports through the server's API | `make e2e` (`ARGS="--app-export "`, `SERVER_REF=`, or `ARGS=--server-url `) | `reports/e2e//report.md` | | Apple's accessibility audit on every screen the app reaches | `uv run python -m hsverify.a11yaudit --ref --replay --autopilot` | `reports/a11y//index.html` | | A replay session is well formed; its poses against ground truth | `uv run python -m hsverify.replaycheck ` | printed | | Every open PR and plan metric with its evidence | `make scoreboard` (`ARGS=--slow` also runs test suites) | `SCOREBOARD.md` | @@ -25,31 +25,47 @@ its own Simulator ("HouseScan Verify") and launches it with the C4 arguments. It `STATE=` from subsystem `dev.housescanning.housescan` and screenshots each state once it has settled for 1.2 s; a state replaced sooner is captured at once and marked *transient*. The report lists a failed build, a crash, markers logged as ``, and a run with no markers as -problems, and keeps every app log line. It waits while another `xcodebuild` compiles, because +problems, and keeps every app log line. `make sim-app` also fails unless the app reaches +`result`, and keeps the scan it uploaded as `scan.zip`. Log capture starts only once the log +stream has attached, so early states are not missed. It waits while another `xcodebuild` compiles, because several workers share the Mac. **End-to-end check** (`hsverify/e2e.py`, `hsverify/resultcheck.py`). Starts the server from a ref and finds its placement endpoint in the OpenAPI document, refusing to guess between candidates. -Each scene is validated against `server/schemas/scene.schema.json` at that ref first, so bad input -is not blamed on the server. Each answer must validate against `server/schemas/result.schema.json` -and satisfy, for any policy: +The server is killed if it passes 3.5 GB, and the report says so. Each scene is validated against +`server/schemas/scene.schema.json` at that ref first, so bad input is not blamed on the server. +Battery size, clearance radii and measurement errors are read from `server/rules.yaml` at the same +ref. Each answer must validate against `server/schemas/result.schema.json` and satisfy, for any +policy: - `pass` has a passing spot, all checks passing, no photo request and an approved policy; `reject` has no spot; `manual_review` gives reasons. - Each check's outcome follows from its numbers (C5): a minimum T passes only when measured − error > T and fails only when measured + error < T, a maximum mirrored, a review line also cleared for a pass; an unsure labelled `margin` lies within its error of a line. -- No battery start passes over wall, cable route or ground that was not observed, and no photo +- No battery position passes unless the wall and cable route under it were observed, and each + clearance's band was observed for its full radius on both sides (ground out to the battery's + depth plus the radius). Distance along the wall is never shorter than straight-line distance, + so this needs no model of the solver's geometry and cannot flag a correct pass. +- Every check that is unsure for lack of coverage is named in `missing_evidence`, and no photo request covers an observed area. - Counts add up, `input_sha256` is the hash of the bytes sent, the spot's offset equals its centre minus the meter. -- The same scene sent again answers the same; mirrored left to right it gives the same decision - and the same length of each outcome; without coverage nothing passes. -The 40 cases in [`e2e/cases/`](e2e/cases/README.md) were written from the public goldens and C5 +Each scene is also resent changed, and the pairs must stay ordered: the same scene answers the +same; mirrored it gives the same decision and outcome lengths; with no coverage, less coverage, or +ground trimmed just short of the largest clearance nothing new passes; with more error, or tape +measurements re-taken by tap, no outcome moves except to unsure; adding every requested photo, up +to three rounds, leaves no check unsure for lack of coverage. Four hostile inputs (20 000 objects, +coordinates of 1e300, a 5000 ft wall, 20 000 levels of nesting) must be refused with 400, 413 or +422, or answered validly, within 10 s; a timeout is a failure. `--no-hostile` skips them. + +The 43 cases in [`e2e/cases/`](e2e/cases/README.md) were written from the public goldens and C5 without reading the server's tests. Each names the thresholds it assumes; a mismatch is reported as such rather than as a pass or fail. [`scenes/eth3d-facade`](scenes/eth3d-facade/README.md) builds a scene from real laser-scanned geometry; real scenes must answer within 1 s. +`--app-export` takes a `make sim-app` report folder and sends the scan the app uploaded, recording +the app's commit. **Accessibility audit** (`hsverify/a11yaudit.py`). Runs the UI test bundle in `fixtures/a11y-audit`, which launches the installed app by bundle id and calls diff --git a/verification/e2e/cases/README.md b/verification/e2e/cases/README.md index fa5396b6a..c74b5f290 100644 --- a/verification/e2e/cases/README.md +++ b/verification/e2e/cases/README.md @@ -137,64 +137,60 @@ move. - `errors.drift_per_ft` 0.16 is added per foot walked along the walls from the meter to the default error of walls, tap and vlm objects and ground patches. It is not added to tape, the meter itself or an explicit `plus_minus_ft`. -- A battery position comes from the wall, so a point of the battery at walked distance x carries - 0.3 + 0.16x. +- A battery position comes from the wall and carries the drift of its edge further from the + meter: 0.3 + 0.16x, where x is that edge's walked distance. S2's 70ab0b0 states this ("checks + take it at the far edge"), and 26d2870's reach cutoff, which took the near edge instead, is the + defect these cases were written to catch. - A gap or a route sums the errors of its two ends (linear, as the C5 margin rule and g10's derived row use). -- A route's ends are the meter and the battery's near edge. A gas gap's ends are the gas meter - and the battery edge nearest it. +- A route runs from the meter to the battery's near edge. A gas gap runs from the gas meter to + the battery edge nearest it. ### d-reach-drift-right: cable reach, `at_most` 20 with review line 15 Wall s = [−1, 29], meter at 0. For a start a > 0 the route is a (straight wall, no corners) and its -error is 0.3 (meter) + 0.3 + 0.16a (wall at the near edge) = 0.6 + 0.16a. +error is 0.3 (meter) + 0.3 + 0.16(a + W) (wall at the far edge) = 0.6 + 0.16(a + W). | Line | Condition | Start | | --- | --- | --- | -| pass / unsure | a + 0.6 + 0.16a < 15 | a < 14.4 / 1.16 = 12.4138 | -| unsure / fail | a − 0.6 − 0.16a > 20 | a > 20.6 / 0.84 = 24.5238 | +| pass / unsure | a + 0.6 + 0.16(a + W) < 15 | a < (14.4 − 0.16W) / 1.16 = 12.0575 | +| unsure / fail | a − 0.6 − 0.16(a + W) > 20 | a > (20.6 + 0.16W) / 0.84 = 25.0159 | -Asserted starts: pass at 5 and 12.2138. Unsure at 12.6138, 18 and 24.3238. Fail at 24.7238 and +Asserted starts: pass at 5 and 11.8575. Unsure at 12.2575, 18 and 24.8159. Fail at 25.2159 and 25.5. Each boundary is bracketed 0.2 ft inside on both sides, which is more than one 2 in sweep -step, so each point falls inside a run rather than between two. Using the far edge (a + W) for the -wall term would move both lines by 0.16W / 1.16 = 0.356 and 0.16W / 0.84 = 0.492, which the points -0.2 ft inside catch. So would any cutoff that is off by 0.16W = 0.413. +step, so each point falls inside a run rather than between two. A cutoff that takes drift at the +near edge sits 0.16W / 0.84 = 0.492 ft lower, at 24.5238, and fails the start at 24.8159. ### d-reach-drift-left: the same, left of the meter -Wall s = [−29, 1]. Left of the meter the near edge is the battery's right edge b = a + W, and -the route is −b. The table above holds for −b, so each asserted start is a = −(right-case start) − -W: pass at −7.5833 and −14.7971, unsure at −15.1971, −20.5833 and −26.9071, fail at −27.3071 and -−28.0833. The lines sit at a = −12.4138 − W = −14.9971 and a = −24.5238 − W = −27.1071. +Wall s = [−29, 1]. Left of the meter the near edge is the battery's right edge b = a + W and the +far edge is a. The table above holds for −b, so each asserted start is a = −(right-case start) − +W: pass at −7.5833 and −14.4408, unsure at −14.8408, −20.5833 and −27.3992, fail at −27.7992 and +−28.0833. The lines sit at a = −12.0575 − W = −14.6408 and a = −25.0159 − W = −27.5992. ### d-gas-drift: gas clearance, `at_least` 3 Wall s = [−1, 20]. A tapped gas meter on the wall line at s = 4 (span [4, 4], no footprint) carries 0.3 + 0.16 × 4 = 0.94. For a start a > 4 the gap is d = a − 4 and its error is 0.94 + -(0.3 + 0.16a) = 1.24 + 0.16a. +0.3 + 0.16(a + W) = 1.24 + 0.16(a + W). | Line | Condition | Start | | --- | --- | --- | -| fail / unsure | (a − 4) + 1.24 + 0.16a < 3 | a < 5.76 / 1.16 = 4.9655 | -| unsure / pass | (a − 4) − 1.24 − 0.16a > 3 | a > 8.24 / 0.84 = 9.8095 | +| fail / unsure | (a − 4) + 1.24 + 0.16(a + W) < 3 | a < (5.76 − 0.16W) / 1.16 = 4.6092 | +| unsure / pass | (a − 4) − 1.24 − 0.16(a + W) > 3 | a > (8.24 + 0.16W) / 0.84 = 10.3016 | Asserted starts: -- **Fail:** 3, where the battery [3, 5.583] covers the gas meter: gap 0, and 0 + 1.88 < 3. Also - 4.7655. -- **Unsure:** 5.1655, 7.5 and 9.6095. -- **Pass:** 10.0095 and 11. Both routes also pass: 11 + 0.6 + 1.76 = 13.36 < 15. +- **Fail:** 3, where the battery [3, 5.583] covers the gas meter: gap 0, and 0 + 2.13 < 3. Also + 4.4092. +- **Unsure:** 4.8092, 7.5 and 10.1016. +- **Pass:** 10.5016 and 11. Both routes also pass: 11 + 0.6 + 2.17 = 13.77 < 15. All starts sit right of the working space. The route to them crosses the gas meter, which detours; a gas meter with no heights added no extra length at 2c9348f. ### What rules.yaml and the README leave open -- **Which battery point carries the drift.** The model gives drift for "AR-placed positions". - The battery is placed from the wall, not tapped, and it spans W, over which drift changes by - 0.16W = 0.413 ft. These cases take the point the measurement ends at: the near edge for the - route, the edge nearest the gas for a gas gap. The far edge or the centre is an equally literal - reading and moves every line (see the reach arithmetic above). - **Walked distance for an object with a span.** For a tapped object spanning [s0, s1], it could be s0, s1 or the nearest end. d-gas-drift uses a zero-length span to avoid the question. - **Whether the meter's error enters the route.** The README says drift is not added to the diff --git a/verification/e2e/cases/d-gas-drift.json b/verification/e2e/cases/d-gas-drift.json index bf8d3d875..cfc2290ba 100644 --- a/verification/e2e/cases/d-gas-drift.json +++ b/verification/e2e/cases/d-gas-drift.json @@ -1,7 +1,7 @@ { "id": "d-gas-drift", "source": "rules.yaml errors.tap_ft, wall_ft, drift_per_ft and clearances.gas_ft at origin/t3/server f2705dd; C5 margin rule", - "description": "A tapped gas meter at s = 4 on a wall with default errors. For a start a right of it the gap is (a - 4) +/- (1.24 + 0.16a): fail below a = 4.966, pass above a = 9.810, unsure between.", + "description": "A tapped gas meter at s = 4 on a wall with default errors. For a start a right of it the gap is (a - 4) +/- (1.24 + 0.16(a + W)): fail below a = 4.609, pass above a = 10.302, unsure between.", "scene": { "schema_version": "1.0", "meter": { @@ -125,17 +125,17 @@ "wall_id": "w1", "start_ft": 3.0, "outcome": "fail", - "why": "battery [3, 5.58] covers the gas meter: gap 0 + 1.88 < 3" + "why": "battery [3, 5.58] covers the gas meter: gap 0 + 2.13 < 3" }, { "wall_id": "w1", - "start_ft": 4.7655, + "start_ft": 4.4092, "outcome": "fail", - "why": "just inside (a - 4) + 1.24 + 0.16a < 3" + "why": "just inside (a - 4) + 1.24 + 0.16(a + W) < 3" }, { "wall_id": "w1", - "start_ft": 5.1655, + "start_ft": 4.8092, "outcome": "unsure", "why": "just past the fail line" }, @@ -143,25 +143,25 @@ "wall_id": "w1", "start_ft": 7.5, "outcome": "unsure", - "why": "gap 3.5 +/- 2.44" + "why": "gap 3.5 +/- 2.85" }, { "wall_id": "w1", - "start_ft": 9.6095, + "start_ft": 10.1016, "outcome": "unsure", - "why": "just inside (a - 4) - 1.24 - 0.16a <= 3" + "why": "just inside (a - 4) - 1.24 - 0.16(a + W) <= 3" }, { "wall_id": "w1", - "start_ft": 10.0095, + "start_ft": 10.5016, "outcome": "pass", - "why": "just past (a - 4) - 1.24 - 0.16a > 3" + "why": "just past (a - 4) - 1.24 - 0.16(a + W) > 3" }, { "wall_id": "w1", "start_ft": 11.0, "outcome": "pass", - "why": "gap 7 - 3.0 = 4 > 3; route 11 + 2.36 < 15" + "why": "gap 7 - 3.41 = 3.59 > 3; route 11 + 2.77 < 15" } ] } diff --git a/verification/e2e/cases/d-reach-drift-left.json b/verification/e2e/cases/d-reach-drift-left.json index 6ba91abc0..674cb4611 100644 --- a/verification/e2e/cases/d-reach-drift-left.json +++ b/verification/e2e/cases/d-reach-drift-left.json @@ -1,7 +1,7 @@ { "id": "d-reach-drift-left", "source": "rules.yaml errors.drift_per_ft and route.* at origin/t3/server f2705dd; C5 margin rule", - "description": "Mirror of d-reach-drift-right. Left of the meter the route runs to the battery's right edge b = a + W, so the boundaries sit at a = -12.414 - W and a = -24.524 - W.", + "description": "Mirror of d-reach-drift-right. Left of the meter the route runs to the battery's right edge b = a + W, so the boundaries sit at a = -12.058 - W and a = -25.016 - W.", "scene": { "schema_version": "1.0", "meter": { @@ -114,17 +114,17 @@ "wall_id": "w1", "start_ft": -7.5833, "outcome": "pass", - "why": "route 5 + 1.4 = 6.4 < 15 (b = a + W)" + "why": "route 5 + 1.81 = 6.81 < 15 (b = a + W)" }, { "wall_id": "w1", - "start_ft": -14.7971, + "start_ft": -14.4408, "outcome": "pass", - "why": "just inside a + 0.6 + 0.16a < 15 (b = a + W)" + "why": "just inside a + 0.6 + 0.16(a + W) < 15 (b = a + W)" }, { "wall_id": "w1", - "start_ft": -15.1971, + "start_ft": -14.8408, "outcome": "unsure", "why": "just past the 15 ft review line (b = a + W)" }, @@ -132,25 +132,25 @@ "wall_id": "w1", "start_ft": -20.5833, "outcome": "unsure", - "why": "18 + 3.48 > 15; 18 - 3.48 < 20 (b = a + W)" + "why": "18 + 3.89 > 15; 18 - 3.89 < 20 (b = a + W)" }, { "wall_id": "w1", - "start_ft": -26.9071, + "start_ft": -27.3992, "outcome": "unsure", - "why": "just inside a - 0.6 - 0.16a <= 20 (b = a + W)" + "why": "just inside a - 0.6 - 0.16(a + W) <= 20 (b = a + W)" }, { "wall_id": "w1", - "start_ft": -27.3071, + "start_ft": -27.7992, "outcome": "fail", - "why": "just past a - 0.6 - 0.16a > 20 (b = a + W)" + "why": "just past a - 0.6 - 0.16(a + W) > 20 (b = a + W)" }, { "wall_id": "w1", "start_ft": -28.0833, "outcome": "fail", - "why": "25.5 - 4.68 = 20.82 > 20 (b = a + W)" + "why": "25.5 - 5.09 = 20.41 > 20 (b = a + W)" } ], "decision_not": [ diff --git a/verification/e2e/cases/d-reach-drift-right.json b/verification/e2e/cases/d-reach-drift-right.json index 84eeaae4a..daafc01fc 100644 --- a/verification/e2e/cases/d-reach-drift-right.json +++ b/verification/e2e/cases/d-reach-drift-right.json @@ -1,7 +1,7 @@ { "id": "d-reach-drift-right", "source": "rules.yaml errors.drift_per_ft and route.* at origin/t3/server f2705dd; C5 margin rule", - "description": "Straight wall with default (drifting) errors. The route to a start a right of the meter is a +/- (0.6 + 0.16a): pass below a = 12.414, fail above a = 24.524, unsure between.", + "description": "Straight wall with default (drifting) errors. The route to a start a right of the meter is a +/- (0.6 + 0.16(a + W)): pass below a = 12.058, fail above a = 25.016, unsure between.", "scene": { "schema_version": "1.0", "meter": { @@ -114,17 +114,17 @@ "wall_id": "w1", "start_ft": 5.0, "outcome": "pass", - "why": "route 5 + 1.4 = 6.4 < 15" + "why": "route 5 + 1.81 = 6.81 < 15" }, { "wall_id": "w1", - "start_ft": 12.2138, + "start_ft": 11.8575, "outcome": "pass", - "why": "just inside a + 0.6 + 0.16a < 15" + "why": "just inside a + 0.6 + 0.16(a + W) < 15" }, { "wall_id": "w1", - "start_ft": 12.6138, + "start_ft": 12.2575, "outcome": "unsure", "why": "just past the 15 ft review line" }, @@ -132,25 +132,25 @@ "wall_id": "w1", "start_ft": 18.0, "outcome": "unsure", - "why": "18 + 3.48 > 15; 18 - 3.48 < 20" + "why": "18 + 3.89 > 15; 18 - 3.89 < 20" }, { "wall_id": "w1", - "start_ft": 24.3238, + "start_ft": 24.8159, "outcome": "unsure", - "why": "just inside a - 0.6 - 0.16a <= 20" + "why": "just inside a - 0.6 - 0.16(a + W) <= 20" }, { "wall_id": "w1", - "start_ft": 24.7238, + "start_ft": 25.2159, "outcome": "fail", - "why": "just past a - 0.6 - 0.16a > 20" + "why": "just past a - 0.6 - 0.16(a + W) > 20" }, { "wall_id": "w1", "start_ft": 25.5, "outcome": "fail", - "why": "25.5 - 4.68 = 20.82 > 20" + "why": "25.5 - 5.09 = 20.41 > 20" } ], "decision_not": [ diff --git a/verification/e2e/cases/generate.py b/verification/e2e/cases/generate.py index 7abbcb81c..d385284b9 100644 --- a/verification/e2e/cases/generate.py +++ b/verification/e2e/cases/generate.py @@ -869,7 +869,8 @@ def reach_case(cid, L, route_ft, e, outcome, description, objects=(), extra_chec # errors.drift_per_ft 0.16 per foot walked along the walls from the meter. Ground and facing keep # explicit errors so that only the checks under test move. README.md ("Drift cases") has the # derivation; the model assumed is: a measured point's error is its default plus drift times its -# walked distance, and a gap or route sums the errors of its two ends. +# walked distance, the battery carries the drift of its edge further from the meter (S2 70ab0b0, +# "checks take it at the far edge"), and a gap or route sums the errors of its two ends. TAP, WALL_E, METER_E, DRIFT = 0.3, 0.3, 0.3, 0.16 PT = 0.2 # sample points sit 0.2 ft inside a boundary: more than one 2 in sweep step @@ -892,35 +893,36 @@ def starts(points): ] -# Cable reach, right of the meter. Route = a (near edge), error = meter 0.3 + wall at a -# (0.3 + 0.16a) = 0.6 + 0.16a. Pass: a + e < 15 (review line) -> a < 14.4/1.16. Fail: a - e > 20 -# -> a > 20.6/0.84. -R_PASS = (CONFIDENT - METER_E - WALL_E) / (1 + DRIFT) -R_FAIL = (MAX_ROUTE + METER_E + WALL_E) / (1 - DRIFT) +# Cable reach, right of the meter. Route = a (near edge), error = meter 0.3 + wall at the far +# edge (0.3 + 0.16(a + W)) = 0.6 + 0.16(a + W). Pass: a + e < 15 (review line) -> +# a < (14.4 - 0.16W)/1.16. Fail: a - e > 20 -> a > (20.6 + 0.16W)/0.84. +R_PASS = (CONFIDENT - METER_E - WALL_E - DRIFT * W) / (1 + DRIFT) +R_FAIL = (MAX_ROUTE + METER_E + WALL_E + DRIFT * W) / (1 - DRIFT) reach_points = [ - (5.0, "pass", "route 5 + 1.4 = 6.4 < 15"), - (R_PASS - PT, "pass", "just inside a + 0.6 + 0.16a < 15"), + (5.0, "pass", "route 5 + 1.81 = 6.81 < 15"), + (R_PASS - PT, "pass", "just inside a + 0.6 + 0.16(a + W) < 15"), (R_PASS + PT, "unsure", "just past the 15 ft review line"), - (18.0, "unsure", "18 + 3.48 > 15; 18 - 3.48 < 20"), - (R_FAIL - PT, "unsure", "just inside a - 0.6 - 0.16a <= 20"), - (R_FAIL + PT, "fail", "just past a - 0.6 - 0.16a > 20"), - (25.5, "fail", "25.5 - 4.68 = 20.82 > 20"), + (18.0, "unsure", "18 + 3.89 > 15; 18 - 3.89 < 20"), + (R_FAIL - PT, "unsure", "just inside a - 0.6 - 0.16(a + W) <= 20"), + (R_FAIL + PT, "fail", "just past a - 0.6 - 0.16(a + W) > 20"), + (25.5, "fail", "25.5 - 5.09 = 20.41 > 20"), ] case( "d-reach-drift-right", "rules.yaml errors.drift_per_ft and route.* at origin/t3/server f2705dd; C5 margin rule", "Straight wall with default (drifting) errors. The route to a start a right of the meter is " - "a +/- (0.6 + 0.16a): pass below a = 12.414, fail above a = 24.524, unsure between.", + "a +/- (0.6 + 0.16(a + W)): pass below a = 12.058, fail above a = 25.016, unsure between.", drift_scene(-1, 29), {"start_outcomes": starts(reach_points), "decision_not": ["pass"]}, {"route_length": MAX_ROUTE}, ) -# Mirror: left of the meter the near edge is the battery's right edge b = a + W, route = -b. +# Mirror: left of the meter the near edge is the battery's right edge b = a + W, route = -b, and +# the far edge is a. case( "d-reach-drift-left", "rules.yaml errors.drift_per_ft and route.* at origin/t3/server f2705dd; C5 margin rule", "Mirror of d-reach-drift-right. Left of the meter the route runs to the battery's right edge " - "b = a + W, so the boundaries sit at a = -12.414 - W and a = -24.524 - W.", + "b = a + W, so the boundaries sit at a = -12.058 - W and a = -25.016 - W.", drift_scene(-29, 1), { "start_outcomes": starts([(-s - W, o, why + " (b = a + W)") for s, o, why in reach_points]), @@ -930,30 +932,31 @@ def starts(points): ) # Gas clearance. A tapped gas meter on the wall line at s = 4 (error 0.3 + 0.16 x 4 = 0.94). A -# battery right of it has gap d = a - 4 and error 0.94 + (0.3 + 0.16a) = 1.24 + 0.16a. -# Fail: d + e < 3 -> 1.16a < 5.76. Pass: d - e > 3 -> 0.84a > 8.24. +# battery right of it has gap d = a - 4 and error 0.94 + (0.3 + 0.16(a + W)) = 1.24 + 0.16(a + W). +# Fail: d + e < 3 -> 1.16a < 5.76 - 0.16W. Pass: d - e > 3 -> 0.84a > 8.24 + 0.16W. G_AT = 4.0 G_E = TAP + DRIFT * G_AT -G_FAIL = (3.0 + G_AT - G_E - WALL_E) / (1 + DRIFT) -G_PASS = (3.0 + G_AT + G_E + WALL_E) / (1 - DRIFT) +G_FAIL = (3.0 + G_AT - G_E - WALL_E - DRIFT * W) / (1 + DRIFT) +G_PASS = (3.0 + G_AT + G_E + WALL_E + DRIFT * W) / (1 - DRIFT) tap_gas = {"type": "gas_meter", "wall_id": "w1", "span_ft": [G_AT, G_AT], "source": "tap"} case( "d-gas-drift", "rules.yaml errors.tap_ft, wall_ft, drift_per_ft and clearances.gas_ft at origin/t3/server " "f2705dd; C5 margin rule", "A tapped gas meter at s = 4 on a wall with default errors. For a start a right of it the gap " - "is (a - 4) +/- (1.24 + 0.16a): fail below a = 4.966, pass above a = 9.810, unsure between.", + "is (a - 4) +/- (1.24 + 0.16(a + W)): fail below a = 4.609, pass above a = 10.302, unsure " + "between.", drift_scene(-1, 20, objects=[tap_gas]), { "start_outcomes": starts( [ - (3.0, "fail", "battery [3, 5.58] covers the gas meter: gap 0 + 1.88 < 3"), - (G_FAIL - PT, "fail", "just inside (a - 4) + 1.24 + 0.16a < 3"), + (3.0, "fail", "battery [3, 5.58] covers the gas meter: gap 0 + 2.13 < 3"), + (G_FAIL - PT, "fail", "just inside (a - 4) + 1.24 + 0.16(a + W) < 3"), (G_FAIL + PT, "unsure", "just past the fail line"), - (7.5, "unsure", "gap 3.5 +/- 2.44"), - (G_PASS - PT, "unsure", "just inside (a - 4) - 1.24 - 0.16a <= 3"), - (G_PASS + PT, "pass", "just past (a - 4) - 1.24 - 0.16a > 3"), - (11.0, "pass", "gap 7 - 3.0 = 4 > 3; route 11 + 2.36 < 15"), + (7.5, "unsure", "gap 3.5 +/- 2.85"), + (G_PASS - PT, "unsure", "just inside (a - 4) - 1.24 - 0.16(a + W) <= 3"), + (G_PASS + PT, "pass", "just past (a - 4) - 1.24 - 0.16(a + W) > 3"), + (11.0, "pass", "gap 7 - 3.41 = 3.59 > 3; route 11 + 2.77 < 15"), ] ), }, diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index c2beaa9a5..14916e0e1 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -29,11 +29,13 @@ import json import os import re +import signal import socket import subprocess import sys import time import urllib.error +import urllib.parse import urllib.request import uuid import zipfile @@ -41,12 +43,13 @@ from dataclasses import dataclass, field from pathlib import Path -from hsverify import gitref +from hsverify import gitref, memory from hsverify.memory import peak_rss_mb from hsverify.resultcheck import ( RuleSet, assumption_mismatches, comparable, + coverage_blocks, expectation_problems, invariant_problems, less_coverage_problems, @@ -89,6 +92,7 @@ class SceneInput: app_export: bool = False # exported by the iOS app (from a replay), the S4 end-to-end path app_sha: str | None = None # the app commit that exported it, when known hostile: bool = False # judged by judge_hostile: refused or answered within a budget + bundle: bytes | None = None # a prebuilt upload, sent as is (the compressed-scene input) def load_app_export(path: Path) -> SceneInput: @@ -173,9 +177,14 @@ def find_app(server_dir: Path) -> str: return found[0] +# A server under test that passes this is killed and the run says so: the Mac is shared, and a +# careless server can parse a small hostile input into gigabytes. +SERVER_LIMIT_MB = 3500 + + @contextlib.contextmanager -def server_from_ref(sha: str, log_path: Path) -> Iterator[str]: - """The server at `sha` running its own FastAPI app on a free port.""" +def server_from_ref(sha: str, log_path: Path) -> Iterator[tuple[str, memory.TreeLimit]]: + """The server at `sha` running its own FastAPI app on a free port, under a memory limit.""" with gitref.detached_worktree(sha) as tree: server_dir = tree / "server" subprocess.run(["uv", "sync", "--locked", "--quiet"], cwd=server_dir, check=True) @@ -188,13 +197,22 @@ def server_from_ref(sha: str, log_path: Path) -> Iterator[str]: cwd=server_dir, stdout=log, stderr=subprocess.STDOUT, + start_new_session=True, ) try: - wait_until_up(url, proc) - yield url + with memory.TreeLimit(proc, SERVER_LIMIT_MB, interval_s=0.2) as limit: + wait_until_up(url, proc) + yield url, limit finally: - proc.terminate() - proc.wait(timeout=10) + # SIGTERM lets uvicorn finish in-flight requests, and a server still solving a + # hostile input would run on for minutes: kill it after 10 s. + if proc.poll() is None: + os.killpg(proc.pid, signal.SIGTERM) + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() def wait_until_up(url: str, proc: subprocess.Popen, timeout_s: float = 60) -> None: @@ -277,6 +295,8 @@ def resolve_ref(openapi: dict, schema: dict) -> dict: def bundle_zip(item: SceneInput) -> bytes: + if item.bundle is not None: + return item.bundle buf = io.BytesIO() with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z: z.writestr("scene.json", item.raw) @@ -337,7 +357,14 @@ def answer(url: str, endpoint: Endpoint, item: SceneInput) -> tuple[dict | None, return json.loads(payload), None -def judge(url: str, endpoint: Endpoint, item: SceneInput, schemas: dict, rules: RuleSet) -> dict: +def judge( + url: str, + endpoint: Endpoint, + item: SceneInput, + schemas: dict, + rules: RuleSet, + limit: memory.TreeLimit | None = None, +) -> dict: record: dict = { "name": item.name, "source": item.source, @@ -348,7 +375,7 @@ def judge(url: str, endpoint: Endpoint, item: SceneInput, schemas: dict, rules: if item.skip_reason: return record | {"status": "skipped", "problems": [], "skip_reason": item.skip_reason} if item.hostile: - return record | judge_hostile(url, endpoint, item, schemas) + return record | judge_hostile(url, endpoint, item, schemas, limit) input_errors = schema_errors(item.scene, schemas["scene"]) if input_errors: return record | {"status": "bad input", "problems": input_errors[:20]} @@ -450,7 +477,7 @@ def resend(name: str, scene: dict) -> dict | None: # because an area was unobserved (the chosen spot may move; three rounds at most). scene, current = item.scene, result for round_ in range(1, 4): - if not unobserved_checks(current): + if not coverage_blocks(current): break scene = with_requests_captured(scene, current) if scene is None: @@ -459,7 +486,7 @@ def resend(name: str, scene: dict) -> dict | None: if current is None: break else: - if unobserved_checks(current): + if coverage_blocks(current): problems.append( "after capturing every requested view three times, checks are still unsure " f"for coverage: {unobserved_checks(current)}" @@ -475,27 +502,38 @@ def resend(name: str, scene: dict) -> dict | None: def hostile_inputs() -> list[SceneInput]: - """Small requests that cost a careless server a lot: many objects, huge numbers, deep - nesting and a very long wall. Built here, not stored, so the repository carries no blobs.""" - wall = {"id": "w1", "baseline": [[-15.0, 0.0], [15.0, 0.0]]} - base = {"meter": {"pos": [0.0, 4.5, 0.0], "wall_id": "w1"}, "walls": [wall]} - many = base | { + """Small requests that cost a careless server a lot. Built here, not stored, so the + repository carries no blobs; none takes the harness more than a few MB to build.""" + base = {"meter": {"pos": [0.0, 4.5, 0.0], "wall_id": "w1"}} + # Every object and coverage fragment adds boundaries, and each object its own error: a + # solver that pairs every boundary with every error offset grows quadratically. + crowded = base | { + "walls": [{"id": "w1", "baseline": [[-60.0, 0.0], [60.0, 0.0]]}], "objects": [ { - "type": "window", + "type": "elec_box", "wall_id": "w1", - "span_ft": [i * 0.001, i * 0.001 + 0.5], + "span_ft": [-55 + 0.75 * i, -54.85 + 0.75 * i], + "bottom_ft": 3.5, + "top_ft": 4.5, "source": "tap", + "plus_minus_ft": round(0.02 + 0.0013 * i, 4), } - for i in range(20_000) - ] + for i in range(150) + ], + "coverage": { + "observed": [ + {"band": "wall", "span_ft": [-55.6 + 0.75 * i, -55.2 + 0.75 * i]} + for i in range(150) + ] + + [{"band": "ground", "span_ft": [-60, 60], "out_ft": 15}] + }, } huge = { "meter": {"pos": [1e300, 4.5, 0.0], "wall_id": "w1"}, "walls": [{"id": "w1", "baseline": [[-1e300, 0.0], [1e300, 0.0]]}], } - long_wall = { - "meter": {"pos": [0.0, 4.5, 0.0], "wall_id": "w1"}, + long_wall = base | { "walls": [{"id": "w1", "baseline": [[-2500.0, 0.0], [2500.0, 0.0]]}], "coverage": { "observed": [ @@ -504,32 +542,59 @@ def hostile_inputs() -> list[SceneInput]: ] }, } - depth = 20_000 - nested = ( - json.dumps(base)[:-1] - + ', "objects": [{"type": "door", "wall_id": "w1", "span_ft": [1, 2], "source": "tap", ' - + '"attrs": {"note": ' - + "[" * depth - + "]" * depth - + "}}]}" - ).encode() + nested = ('{"meter": ' + "[" * 5000 + "]" * 5000 + "}").encode() items = [ - SceneInput("hostile: 20 000 objects", many, json.dumps(many).encode()), + SceneInput( + "hostile: 150 objects, each its own error", crowded, json.dumps(crowded).encode() + ), SceneInput("hostile: coordinates of 1e300 ft", huge, json.dumps(huge).encode()), SceneInput("hostile: a 5000 ft wall", long_wall, json.dumps(long_wall).encode()), - SceneInput("hostile: attrs nested 20 000 deep", {}, nested), + SceneInput("hostile: nested 5000 deep", {}, nested), + SceneInput("hostile: 400 MB scene.json in a small zip", {}, b"", bundle=zip_bomb(400)), ] for item in items: item.hostile = True return items -def judge_hostile(url: str, endpoint: Endpoint, item: SceneInput, schemas: dict) -> dict: - """A refusal (400, 413, 422) or a valid result, either within the budget. A crash, a hang or - a slow answer is a failure.""" +def zip_bomb(megabytes: int) -> bytes: + """A zip whose scene.json is `megabytes` of spaces around a number: under 1 MB to send, + written in chunks so the harness never holds the expanded text.""" + buf = io.BytesIO() + chunk = b" " * 2**20 + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z, z.open("scene.json", "w") as f: + f.write(b'{"meter": ') + for _ in range(megabytes): + f.write(chunk) + f.write(b"1}") + return buf.getvalue() + + +# A hostile input may cost the server this much memory above what it held before the request. +HOSTILE_MEMORY_MB = 500 + + +def judge_hostile( + url: str, + endpoint: Endpoint, + item: SceneInput, + schemas: dict, + limit: memory.TreeLimit | None = None, +) -> dict: + """A refusal (400, 413, 422) or a valid result, either within the time budget and, for a + server started here, the memory budget. A crash, a hang, a slow answer or a large + allocation is a failure.""" + if item.bundle is not None and endpoint.content_type == "application/json": + return {"status": "skipped", "skip_reason": "the endpoint takes JSON, not a zip"} + before = limit.reset() if limit else 0.0 status, payload, ms = post(url, endpoint, item, timeout_s=HOSTILE_BUDGET_MS / 1000 + 5) record = {"http_status": status, "latency_ms": round(ms, 1)} - if status is None: + grew = limit.peak_mb - before if limit else 0.0 + if limit: + record["server_growth_mb"] = round(grew, 1) + if grew > HOSTILE_MEMORY_MB: + failure = f"the server grew by {grew:.0f} MB, budget {HOSTILE_MEMORY_MB} MB" + elif status is None: failure = f"no answer within {HOSTILE_BUDGET_MS / 1000 + 5:.0f} s" elif ms > HOSTILE_BUDGET_MS: failure = f"answered HTTP {status} after {ms:.0f} ms, budget {HOSTILE_BUDGET_MS:.0f}" @@ -559,7 +624,9 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: exported = [r for r in records if r.get("app_export")] report = meta | { "counts": counts, - "all_passed": all(r["status"] in ("pass", "skipped") for r in records) and bool(records), + "all_passed": all(r["status"] in ("pass", "skipped") for r in records) + and bool(records) + and not meta["peak_memory"].get("server_killed"), "latency_ms": max(real) if real else None, "scenes_answered": sum(1 for r in records if "decision" in r), # Invariants and properties broken, summed over every answered scene. Case @@ -588,6 +655,14 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: f"Load average {meta['load_average_1_5_15']} on {os.cpu_count()} cores. " + ", ".join(f"{v} {k}" for k, v in sorted(counts.items())), "", + ] + if meta["peak_memory"].get("server_killed"): + lines += [ + f"**The server passed {SERVER_LIMIT_MB} MB and was killed**; every scene after that " + "failed to connect.", + "", + ] + lines += [ "| Scene | Status | Decision | ms | Problems |", "| --- | --- | --- | --- | --- |", ] @@ -666,19 +741,22 @@ def main(argv: list[str] | None = None) -> int: server_sha = "(running server)" if args.server_url else gitref.resolve(args.server_ref) stamp = dt.datetime.now().strftime("%Y%m%d-%H%M%S") - out = (args.out or DEFAULT_REPORTS / f"{stamp}-{server_sha[:8]}").expanduser() + label = urllib.parse.urlsplit(args.server_url).hostname if args.server_url else server_sha[:8] + out = (args.out or DEFAULT_REPORTS / f"{stamp}-{label}").expanduser() out.mkdir(parents=True, exist_ok=True) + limit = None with contextlib.ExitStack() as stack: - url = args.server_url or stack.enter_context( - server_from_ref(server_sha, out / "server.log") - ) + if args.server_url: + url = args.server_url + else: + url, limit = stack.enter_context(server_from_ref(server_sha, out / "server.log")) openapi = json.loads(urllib.request.urlopen(f"{url}/openapi.json", timeout=10).read()) endpoint = discover_endpoint(openapi, args.endpoint) print(f"Server {url}, endpoint POST {endpoint.path} ({endpoint.content_type})") records = [] for item in items: - record = judge(url, endpoint, item, schemas, rules) + record = judge(url, endpoint, item, schemas, rules, limit) records.append(record) print( f" {record['status']:<20} {item.name}" @@ -702,7 +780,8 @@ def main(argv: list[str] | None = None) -> int: "command": " ".join([sys.executable, "-m", "hsverify.e2e", *(argv or sys.argv[1:])]), # Latency on this shared Mac depends on what else runs; keep the load with the numbers. "load_average_1_5_15": [round(x, 1) for x in os.getloadavg()], - "peak_memory": peak_rss_mb(), + "peak_memory": peak_rss_mb() + | ({"server_mb": round(limit.peak_mb, 1), "server_killed": limit.killed} if limit else {}), "cases_sha256": hashlib.sha256(b"".join(i.raw for i in items)).hexdigest(), } report = write_report(out, meta, records) diff --git a/verification/hsverify/memory.py b/verification/hsverify/memory.py index 9b8e0e314..7ba411f8c 100644 --- a/verification/hsverify/memory.py +++ b/verification/hsverify/memory.py @@ -6,10 +6,68 @@ from __future__ import annotations +import os import resource +import signal +import subprocess +import threading def peak_rss_mb() -> dict[str, float]: own = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss children = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss return {"harness_mb": round(own / 2**20, 1), "largest_child_mb": round(children / 2**20, 1)} + + +def tree_rss_mb(root: int) -> float: + """Resident memory of `root` and all its descendants, from one `ps` listing.""" + listing = subprocess.run( + ["ps", "-A", "-o", "pid=,ppid=,rss="], capture_output=True, text=True, check=True + ).stdout + children: dict[int, list[int]] = {} + rss: dict[int, int] = {} + for line in listing.splitlines(): + pid, ppid, kb = (int(field) for field in line.split()) + children.setdefault(ppid, []).append(pid) + rss[pid] = kb + total, stack = 0, [root] + while stack: + pid = stack.pop() + total += rss.get(pid, 0) + stack.extend(children.get(pid, [])) + return total / 1024 + + +class TreeLimit: + """Kills a process group once its resident memory passes `limit_mb`. + + A server under test is started in its own process group, so one signal stops `uv` and the + Python it started. `peak_mb` and `killed` go into the report.""" + + def __init__(self, proc: subprocess.Popen, limit_mb: float, interval_s: float = 0.5): + self.proc, self.limit_mb, self.interval_s = proc, limit_mb, interval_s + self.peak_mb, self.killed = 0.0, False + self._stop = threading.Event() + self._thread = threading.Thread(target=self._watch, daemon=True) + + def __enter__(self) -> TreeLimit: + self._thread.start() + return self + + def __exit__(self, *exc) -> None: + self._stop.set() + self._thread.join() + + def reset(self) -> float: + """Restart the peak from the group's current memory, and return that.""" + self.peak_mb = tree_rss_mb(self.proc.pid) + return self.peak_mb + + def _watch(self) -> None: + while not self._stop.wait(self.interval_s) and self.proc.poll() is None: + used = tree_rss_mb(self.proc.pid) + self.peak_mb = max(self.peak_mb, used) + if used > self.limit_mb: + self.killed = True + os.killpg(self.proc.pid, signal.SIGKILL) + return diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index f79ad0a8e..3e75f7be1 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -21,6 +21,8 @@ import copy import hashlib +import itertools +import math from dataclasses import dataclass from typing import Any @@ -34,12 +36,15 @@ # Check ids whose PASS depends on seeing an area around the footprint, the band that area lies # in, and the rules.yaml clearance that sets its radius. ground_surface needs only the ground # under the battery. +# Clearance check -> the bands an unseen hazard could hide in, and its rules.yaml radius. C1: a +# gas meter hangs on the wall face (wall band) and its regulator stands on the ground (ground +# band); openings are on the wall face; AC units, drives and pools are on the ground. CLEARANCE_CHECKS = { - "gas_clearance": ("ground", "gas_ft"), - "ac_clearance": ("ground", "ac_ft"), - "drive_clearance": ("ground", "drive_ft"), - "pool_clearance": ("ground", "pool_ft"), - "opening_clearance": ("wall", "opening_ft"), + "gas_clearance": (("ground", "wall"), "gas_ft"), + "ac_clearance": (("ground",), "ac_ft"), + "drive_clearance": (("ground",), "drive_ft"), + "pool_clearance": (("ground",), "pool_ft"), + "opening_clearance": (("wall",), "opening_ft"), } @@ -49,9 +54,9 @@ class RuleSet: width_ft: float depth_ft: float - # check id -> (band, radius in feet around the battery footprint that must be observed) - radii: dict[str, tuple[str, float]] - # default error by object source, and for walls and the meter + # check id -> (bands, radius in feet around the battery footprint that must be observed) + radii: dict[str, tuple[tuple[str, ...], float]] + # default error by object source, for walls and the meter, and drift_per_ft errors: dict[str, float] @classmethod @@ -67,13 +72,14 @@ def value(*path: str) -> float: return float(node["value"] if isinstance(node, dict) else node) radii = { - check: (band, value("clearances", key)) - for check, (band, key) in CLEARANCE_CHECKS.items() + check: (bands, value("clearances", key)) + for check, (bands, key) in CLEARANCE_CHECKS.items() } - radii["ground_surface"] = ("ground", 0.0) + radii["ground_surface"] = (("ground",), 0.0) errors = { name: value("errors", f"{name}_ft") for name in ("tap", "vlm", "tape", "wall", "meter") } + errors["drift_per_ft"] = value("errors", "drift_per_ft") return cls(value("battery", "width_ft"), value("battery", "depth_ft"), radii, errors) @@ -208,7 +214,7 @@ def invariant_problems( if max(abs(want[0] - got[0]), abs(want[1] - got[1])) > OFFSET_TOL_FT: problems.append(f"spot.meter_offset_ft {got} != centre - meter {want}") - problems += missing_evidence_problems(result) + problems += missing_evidence_problems(scene, result, rules) if rules is not None: problems += coverage_problems(scene, result, rules) return problems @@ -224,45 +230,72 @@ def required_span(lo: float, hi: float, radius: float) -> tuple[float, float]: return lo - radius, hi + radius -def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: - """Missing coverage is never a pass (C5), to the radius each check's rule looks out to. - - A PASS for a check at a battery covering [lo, hi] along the wall needs that check's band - observed over [lo - R, hi + R], and for the ground band out to the battery depth plus R. A - sweep run that passes needs every check's area. The wall and cable route back to the meter - must be observed too. No slack: runs list exactly the starts that were evaluated. +def battery_error(scene: dict, rules: RuleSet, wall_id: str, lo: float, hi: float) -> float: + """Position error of a battery over [lo, hi] on `wall_id`: the wall's explicit error, or its + default plus drift at the battery's edge further from the meter (S2 70ab0b0).""" + wall = next((w for w in scene["walls"] if w["id"] == wall_id), None) + if wall is None: + raise ValueError(f"the result names wall {wall_id!r}, which the scene does not have") + if "plus_minus_ft" in wall: + return wall["plus_minus_ft"] + return rules.errors["wall"] + rules.errors["drift_per_ft"] * max(abs(lo), abs(hi)) + + +def reach_gaps( + scene: dict, rules: RuleSet, check: str, wall_id: str, lo: float, hi: float +) -> list[tuple[str, str]]: + """(band, what is missing) for each band `check` needs observed around a battery over + [lo, hi] and does not have. + + Only points within the check's radius R of the battery whatever the wall's shape are + required, so a correct server is never flagged. R is widened by the battery's position + error e (C5: a pass needs the margin to exceed the error, and an unseen hazard just past the + seen area is a margin of zero). + + - wall band: [lo - R - e, hi + R + e] along the wall, since distance along the wall is + never shorter than straight-line distance; + - ground band: out to depth + R + e in front of the battery (a battery sits flush on one + straight segment), and at distance d past either end out to R + e - d. The ground point + there is at most d along the wall plus R + e - d out from the battery's end. """ - problems: list[str] = [] - width, depth = rules.width_ft, rules.depth_ft - wall = observed(scene, "wall") + bands, radius = rules.radii[check] + reach = radius + battery_error(scene, rules, wall_id, lo, hi) + gaps = [] + for band in bands: + if band == "ground": + gap = ground_gap(scene, lo, hi, rules.depth_ft, reach) + else: + a, b = required_span(lo, hi, reach) + seen = covers(observed(scene, band), a, b) + gap = None if seen else f"{band} [{a:.2f}, {b:.2f}] observed" + if gap: + gaps.append((band, gap)) + return gaps - def unobserved(band: str, radius: float, lo: float, hi: float) -> str | None: - a, b = required_span(lo, hi, radius) - seen = observed(scene, band, min_out_ft=depth + radius if band == "ground" else 0.0) - if covers(seen, a, b): - return None - out = f", {depth + radius:.2f} ft out" if band == "ground" else "" - return f"{band} [{a:.2f}, {b:.2f}]{out}" +def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: + """Missing coverage is never a pass (C5), to the reach each check's rule looks out to (see + reach_gaps). A sweep run that passes needs every check's area for every start in it, and the + wall and cable route back to the meter. No slack: runs list exactly the starts that were + evaluated.""" + problems: list[str] = [] + wall = observed(scene, "wall") if "coverage" not in scene and result["decision"] == "pass": problems.append("decision pass for a scene with no coverage at all") for run in result.get("sweep", []): if run["outcome"] != "pass": continue - lo, hi = run["start_ft"][0], run["start_ft"][1] + width + lo, hi = run["start_ft"][0], run["start_ft"][1] + rules.width_ft route_lo, route_hi = min(0.0, lo), max(0.0, hi) if not covers(wall, route_lo, route_hi): problems.append( f"sweep pass for starts {run['start_ft']} but the wall and cable route " f"[{route_lo:.2f}, {route_hi:.2f}] were not all observed" ) - for check, (band, radius) in sorted(rules.radii.items()): - gap = unobserved(band, radius, lo, hi) - if gap: - problems.append( - f"sweep pass for starts {run['start_ft']} but {check} needs {gap} observed" - ) + for check in sorted(rules.radii): + for _, gap in reach_gaps(scene, rules, check, run["wall_id"], lo, hi): + problems.append(f"sweep pass for starts {run['start_ft']} but {check} needs {gap}") spot = result.get("spot") if spot is not None: @@ -270,16 +303,17 @@ def unobserved(band: str, radius: float, lo: float, hi: float) -> str | None: for check in result.get("checks", []): if check["outcome"] != "pass" or check["id"] not in rules.radii: continue - band, radius = rules.radii[check["id"]] - gap = unobserved(band, radius, lo, hi) - if gap: - problems.append(f"check {check['id']} passes but needs {gap} observed") + for _, gap in reach_gaps(scene, rules, check["id"], spot["wall_id"], lo, hi): + problems.append(f"check {check['id']} passes but needs {gap}") for request in result.get("missing_evidence", []): if request["kind"] != "band" or "span_ft" not in request or "band" not in request: continue a, b = sorted(request["span_ft"]) - if b - a > EPS and covers(observed(scene, request["band"]), a, b): + # A ground request carries no distance out, so it may want deeper ground over a span + # already seen; only ground seen out to GROUND_FAR_FT makes it redundant. + far = GROUND_FAR_FT if request["band"] == "ground" else 0.0 + if b - a > EPS and covers(observed(scene, request["band"], min_out_ft=far), a, b): problems.append( f"missing_evidence asks for {request['band']} {request['span_ft']}, " "which the scene lists as observed" @@ -287,21 +321,60 @@ def unobserved(band: str, radius: float, lo: float, hi: float) -> str | None: return problems -def missing_evidence_problems(result: dict) -> list[str]: - """Every check left unsure because an area was unobserved has a request naming it.""" - listed = { - c for request in result.get("missing_evidence", []) for c in request.get("checks", []) - } - problems = [ - f"check {c['id']} is unsure (unobserved) but no missing_evidence entry names it" - for c in result.get("checks", []) - if c["outcome"] == "unsure" - and c.get("unsure_cause") == "unobserved" - and c["id"] not in listed +def ground_gap(scene: dict, lo: float, hi: float, depth: float, radius: float) -> str | None: + """The first stretch of ground seen less far out than a battery over [lo, hi] needs (see + coverage_problems), or None.""" + entries = [ + (*sorted(e["span_ft"]), e.get("out_ft", 0.0)) + for e in scene.get("coverage", {}).get("observed", []) + if e["band"] == "ground" ] + a, b = lo - radius, hi + radius + cuts = sorted({a, lo, hi, b} | {x for e in entries for x in e[:2] if a < x < b}) + for p, q in itertools.pairwise(cuts): + if q - p <= EPS: + continue + inside = lo - EPS <= p and q <= hi + EPS + need = depth + radius if inside else radius - min(abs(lo - q), abs(p - hi)) + seen = max((out for x, y, out in entries if x <= p + EPS and q - EPS <= y), default=None) + if seen is None or seen + EPS < need: + where = f"ground [{p:.2f}, {q:.2f}] observed" + if seen is None: + return f"{where}, none seen" + return f"{where} out to {need:.2f} ft, seen {seen:.2f} ft" + return None + + +def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = None) -> list[str]: + """When coverage is what stands in the way (reason unobserved_area), every check left unsure + for an unobserved area is named by a request, and a clearance check by a request of each + band it lacks around the chosen spot: a check that forgets one band leaves the next capture + still unsure. Otherwise no photo would change the answer, and C2 lets missing_evidence stay + empty.""" codes = {r["code"] for r in result.get("reasons", [])} - if "unobserved_area" in codes and not result.get("missing_evidence"): - problems.append("reason unobserved_area but missing_evidence is empty") + if "unobserved_area" not in codes: + return [] + requests = result.get("missing_evidence", []) + if not requests: + return ["reason unobserved_area but missing_evidence is empty"] + named = {(r.get("band"), c) for r in requests for c in r.get("checks", [])} + problems = [ + f"check {c} is unsure (unobserved) but no missing_evidence entry names it" + for c in unobserved_checks(result) + if not any(check == c for _, check in named) + ] + spot = result.get("spot") + if rules is None or spot is None: + return problems + lo, hi = spot["span_ft"] + for c in unobserved_checks(result): + if c not in rules.radii: + continue + for band, gap in reach_gaps(scene, rules, c, spot["wall_id"], lo, hi): + if (band, c) not in named: + problems.append( + f"check {c} is unsure and needs {gap}, but no {band} request names it" + ) return problems @@ -524,15 +597,30 @@ def less_coverage_problems(before: dict, after: dict, change: str) -> list[str]: return problems +def chain_length_ft(scene: dict) -> float: + """Length of the wall chain, gaps between walls included: no point on it is further than + this from the meter along the walls.""" + points = [tuple(p) for wall in scene["walls"] for p in wall["baseline"]] + return sum(math.dist(p, q) for p, q in itertools.pairwise(points)) + + def with_more_error(scene: dict, rules: RuleSet, extra_ft: float = 0.5) -> dict: - """Every object, wall and the meter measured `extra_ft` less precisely than stated (or than - its source's default).""" + """Every object, wall and the meter measured `extra_ft` less precisely than before. + + An explicit error replaces the default, and the default for walls and for tap and vlm + objects grows by drift_per_ft with distance from the meter. So an item left on its default + gets that default at the far end of the chain, plus `extra_ft`: at least as much as before + everywhere. + """ m = copy.deepcopy(scene) + drift = rules.errors["drift_per_ft"] * chain_length_ft(scene) for obj in m.get("objects", []): - base = obj.get("plus_minus_ft", rules.errors[obj["source"]]) - obj["plus_minus_ft"] = base + extra_ft + if "plus_minus_ft" not in obj: + grows = obj["source"] in ("tap", "vlm") + obj["plus_minus_ft"] = rules.errors[obj["source"]] + (drift if grows else 0.0) + obj["plus_minus_ft"] += extra_ft for wall in m["walls"]: - wall["plus_minus_ft"] = wall.get("plus_minus_ft", rules.errors["wall"]) + extra_ft + wall["plus_minus_ft"] = wall.get("plus_minus_ft", rules.errors["wall"] + drift) + extra_ft meter = m["meter"] meter["plus_minus_ft"] = meter.get("plus_minus_ft", rules.errors["meter"]) + extra_ft return m @@ -570,7 +658,7 @@ def with_less_coverage(scene: dict, trim_ft: float = 0.5) -> dict | None: def with_ground_short_of(scene: dict, rules: RuleSet, margin_ft: float = 0.1) -> dict | None: """Ground seen out to just short of the largest clearance radius, where any correct server must stop passing the check that needs it.""" - radius = max(r for band, r in rules.radii.values() if band == "ground") + radius = max(r for bands, r in rules.radii.values() if "ground" in bands) reach = rules.depth_ft + radius - margin_ft grounds = [e for e in scene.get("coverage", {}).get("observed", []) if e["band"] == "ground"] if not any(e.get("out_ft", 0.0) > reach for e in grounds): @@ -582,7 +670,13 @@ def with_ground_short_of(scene: dict, rules: RuleSet, margin_ft: float = 0.1) -> return m -def with_requests_captured(scene: dict, result: dict, out_ft: float = 40.0) -> dict | None: +# Further out than any clearance looks: the largest in rules.yaml is 10 ft, plus a 1.8 ft +# battery and its position error. Requested ground is added out to here, and ground seen this +# far never needs asking for again. +GROUND_FAR_FT = 40.0 + + +def with_requests_captured(scene: dict, result: dict, out_ft: float = GROUND_FAR_FT) -> dict | None: """The scene as if the homeowner had shown every band the result asked for.""" added = [ {"band": r["band"], "span_ft": sorted(r["span_ft"])} @@ -597,6 +691,11 @@ def with_requests_captured(scene: dict, result: dict, out_ft: float = 40.0) -> d return m +def coverage_blocks(result: dict) -> bool: + """The result names an unobserved area as a reason, so photos could change it.""" + return any(r["code"] == "unobserved_area" for r in result.get("reasons", [])) + + def unobserved_checks(result: dict) -> list[str]: return [ c["id"] diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index 4c411adf4..29c3507d7 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -1,8 +1,10 @@ import hashlib +import io import json import re import threading import time +import zipfile from http.server import BaseHTTPRequestHandler, HTTPServer import pytest @@ -11,20 +13,22 @@ from hsverify.e2e import ( Endpoint, SceneInput, + bundle_zip, discover_endpoint, hostile_inputs, judge, judge_hostile, load_app_export, load_input, + zip_bomb, ) from hsverify.resultcheck import RuleSet RULES = RuleSet( width_ft=31 / 12, depth_ft=22 / 12, - radii={"ground_surface": ("ground", 0.0), "gas_clearance": ("ground", 1.0)}, - errors={"tap": 0.3, "vlm": 1.5, "tape": 0.05, "wall": 0.3, "meter": 0.3}, + radii={"ground_surface": (("ground",), 0.0), "gas_clearance": (("ground",), 1.0)}, + errors={"tap": 0.3, "vlm": 1.5, "tape": 0.05, "wall": 0.3, "meter": 0.3, "drift_per_ft": 0.16}, ) SCENE = { @@ -250,7 +254,7 @@ def test_server_breaking_coverage_rule_fails(fake_url): joined = "\n".join(record["problems"]) assert "ruled out for this case" in joined assert "no coverage: decision pass for a scene with no coverage at all" in joined - assert re.search(r"needs ground \[.*\], 2\.83 ft out observed", joined) + assert re.search(r"needs ground \[.*\] observed, none seen", joined) def test_a_result_breaking_its_schema_counts_against_the_contract(fake_url): @@ -293,7 +297,7 @@ def refusing_url(): def test_a_quick_refusal_of_a_hostile_input_passes(refusing_url): - hostile = hostile_inputs()[1] + hostile = hostile_inputs()[3] record = judge_hostile(refusing_url, Endpoint("/p", "application/json"), hostile, SCHEMAS) assert record["status"] == "pass" and record["http_status"] == 422 @@ -301,7 +305,7 @@ def test_a_quick_refusal_of_a_hostile_input_passes(refusing_url): def test_a_slow_refusal_or_a_hang_fails(refusing_url, monkeypatch): monkeypatch.setattr(e2e, "HOSTILE_BUDGET_MS", 100.0) RefusingServer.delay = 0.3 - hostile = hostile_inputs()[1] + hostile = hostile_inputs()[3] record = judge_hostile(refusing_url, Endpoint("/p", "application/json"), hostile, SCHEMAS) assert record["status"] == "fail" and "budget 100" in record["problems"][0] RefusingServer.delay = 6.0 # longer than budget + 5 s: the request times out @@ -312,13 +316,15 @@ def test_a_slow_refusal_or_a_hang_fails(refusing_url, monkeypatch): def test_hostile_inputs_are_small_to_send(): items = hostile_inputs() assert all(i.hostile for i in items) - assert max(len(i.raw) for i in items) < 5_000_000 # the harness stays light + assert max(len(bundle_zip(i)) for i in items) < 2_000_000 # the harness stays light -def test_a_sim_report_folder_is_an_app_export_with_its_sha(tmp_path): - (tmp_path / "scan.zip").write_bytes(b"") - import zipfile +def test_the_compressed_scene_expands_to_its_size(): + with zipfile.ZipFile(io.BytesIO(zip_bomb(3))) as z: + assert z.getinfo("scene.json").file_size == 3 * 2**20 + len(b'{"meter": 1}') + +def test_a_sim_report_folder_is_an_app_export_with_its_sha(tmp_path): with zipfile.ZipFile(tmp_path / "scan.zip", "w") as z: z.writestr("scene.json", json.dumps(SCENE)) (tmp_path / "report.json").write_text(json.dumps({"sha": "abc123", "app_export": "scan.zip"})) diff --git a/verification/tests/test_memory.py b/verification/tests/test_memory.py index c162b65ff..c23f5d2fe 100644 --- a/verification/tests/test_memory.py +++ b/verification/tests/test_memory.py @@ -1,4 +1,11 @@ -from hsverify.memory import peak_rss_mb +import subprocess +import sys +import time + +from hsverify.memory import TreeLimit, peak_rss_mb, tree_rss_mb + +# A child that writes `mb` megabytes, so they are resident, then waits. +HOLD_MB = "import time; b = b'x' * ({mb} * 2**20); time.sleep(10)" def test_peak_memory_is_reported_in_megabytes(): @@ -6,3 +13,34 @@ def test_peak_memory_is_reported_in_megabytes(): # A Python test process uses tens of MB, not bytes or gigabytes. assert 5 < peak["harness_mb"] < 2000 assert peak["largest_child_mb"] >= 0 + + +def test_tree_rss_counts_descendants(): + child = subprocess.Popen( + [ + sys.executable, + "-c", + HOLD_MB.format(mb=60), + ], + start_new_session=True, + ) + try: + time.sleep(1.0) + assert tree_rss_mb(child.pid) > 50 + finally: + child.kill() + child.wait() + + +def test_a_process_group_over_the_limit_is_killed(): + child = subprocess.Popen( + [ + sys.executable, + "-c", + HOLD_MB.format(mb=80), + ], + start_new_session=True, + ) + with TreeLimit(child, limit_mb=40, interval_s=0.1) as limit: + child.wait(timeout=5) + assert limit.killed and limit.peak_mb > 40 diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 0d6ea07bf..226d8941b 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -30,16 +30,17 @@ width_ft=31 / 12, depth_ft=22 / 12, radii={ - "ground_surface": ("ground", 0.0), - "gas_clearance": ("ground", 1.0), - "opening_clearance": ("wall", 1.0), + "ground_surface": (("ground",), 0.0), + "gas_clearance": (("ground",), 1.0), + "opening_clearance": (("wall",), 1.0), }, - errors={"tap": 0.3, "vlm": 1.5, "tape": 0.05, "wall": 0.3, "meter": 0.3}, + errors={"tap": 0.3, "vlm": 1.5, "tape": 0.05, "wall": 0.3, "meter": 0.3, "drift_per_ft": 0.16}, ) SCENE = { "meter": {"pos": [0.0, 4.0, 0.0], "wall_id": "w1"}, - "walls": [{"id": "w1", "baseline": [[-12.0, 0.0], [12.0, 0.0]]}], + # An explicit zero error keeps the reach arithmetic in these tests to the radii alone. + "walls": [{"id": "w1", "baseline": [[-12.0, 0.0], [12.0, 0.0]], "plus_minus_ft": 0.0}], "objects": [ { "type": "gas_meter", @@ -233,14 +234,28 @@ def with_ground(out_ft: float, span: list[float]) -> dict: def test_a_pass_needs_ground_out_to_depth_plus_each_radius(): # Start 1 covers [1, 3.58]. Gas (radius 1) needs ground out to 1.83 + 1 = 2.83 ft. msgs = invariant_problems(with_ground(2.5, [-12.0, 10.0]), result(), rules=RULES) - assert any("gas_clearance needs ground [0.00, 4.58], 2.83 ft out observed" in m for m in msgs) + needs = "gas_clearance needs ground [1.00, 3.58] observed out to 2.83 ft, seen 2.50 ft" + assert any(needs in m for m in msgs) assert invariant_problems(with_ground(2.84, [-12.0, 10.0]), result(), rules=RULES) == [] +def test_past_the_battery_the_ground_needed_shrinks_with_distance(): + # d past the battery's left end at 1, gas needs ground out to 1 - d: 0.5 for [0, 0.5]. + scene = with_ground(2.84, [1.0, 10.0]) + scene["coverage"]["observed"] += [ + {"band": "ground", "span_ft": [0.0, 0.5], "out_ft": 0.55}, + {"band": "ground", "span_ft": [0.5, 1.0], "out_ft": 1.0}, + ] + assert invariant_problems(scene, result(), rules=RULES) == [] + scene["coverage"]["observed"][-1]["out_ft"] = 0.9 + msgs = invariant_problems(scene, result(), rules=RULES) + assert any("ground [0.50, 1.00] observed out to 1.00 ft, seen 0.90 ft" in m for m in msgs) + + def test_a_pass_needs_ground_along_the_wall_to_each_radius_without_slack(): # Ground from 0.1: the 1 ft gas radius around [1, 3.58] reaches back to 0. msgs = invariant_problems(with_ground(3.0, [0.1, 10.0]), result(), rules=RULES) - assert any("gas_clearance needs ground [0.00, 4.58]" in m for m in msgs) + assert any("gas_clearance needs ground [0.00, 0.10] observed, none seen" in m for m in msgs) def test_a_passing_opening_check_needs_the_wall_to_its_radius(): @@ -253,7 +268,8 @@ def test_a_passing_opening_check_needs_the_wall_to_its_radius(): ] ) msgs = invariant_problems(scene, r, rules=RULES) - assert any("check opening_clearance passes but needs wall [0.00, 4.58]" in m for m in msgs) + needs = "check opening_clearance passes but needs wall [0.00, 4.58] observed" + assert any(needs in m for m in msgs) def test_pass_whose_route_crosses_unobserved_wall_is_flagged(): @@ -280,10 +296,17 @@ def test_photo_request_for_an_observed_area_is_flagged(): def test_every_unobserved_check_needs_a_request_naming_it(): unseen = check("unsure", None, cause="unobserved") r = result(checks=[unseen]) + # No unobserved_area reason: every position fails anyway, so no photo is owed. + assert missing_evidence_problems(SCENE, r) == [] r["reasons"] = [{"code": "unobserved_area", "message": ""}] - assert missing_evidence_problems(r) == [ - "check gas_clearance is unsure (unobserved) but no missing_evidence entry names it", - "reason unobserved_area but missing_evidence is empty", + assert missing_evidence_problems(SCENE, r) == [ + "reason unobserved_area but missing_evidence is empty" + ] + r["missing_evidence"] = [ + {"kind": "band", "band": "wall", "span_ft": [11, 12], "checks": ["x"], "message": ""} + ] + assert missing_evidence_problems(SCENE, r) == [ + "check gas_clearance is unsure (unobserved) but no missing_evidence entry names it" ] r["missing_evidence"] = [ { @@ -294,20 +317,53 @@ def test_every_unobserved_check_needs_a_request_naming_it(): "message": "", } ] - assert missing_evidence_problems(r) == [] + assert missing_evidence_problems(SCENE, r) == [] + + +def test_the_reach_widens_by_the_battery_position_error(): + # Default wall error at the far edge 3.58: 0.3 + 0.16 x 3.58 = 0.87; gas reach 1.87 ft, so + # ground out to 1.83 + 1.87 = 3.71 ft in front. + scene = with_ground(3.6, [-12.0, 10.0]) + del scene["walls"][0]["plus_minus_ft"] + msgs = invariant_problems(scene, result(), rules=RULES) + assert any("gas_clearance needs ground [1.00, 3.58] observed out to 3.71 ft" in m for m in msgs) + scene["coverage"]["observed"][1]["out_ft"] = 3.72 + assert invariant_problems(scene, result(), rules=RULES) == [] + + +def test_a_clearance_is_named_under_each_band_it_lacks(): + two_band = RuleSet( + RULES.width_ft, + RULES.depth_ft, + RULES.radii | {"gas_clearance": (("ground", "wall"), 1.0)}, + RULES.errors, + ) + scene = copy.deepcopy(SCENE) + scene["coverage"]["observed"] = [] # nothing seen: both bands missing around the spot + r = result(checks=[check("unsure", None, cause="unobserved")]) + r["reasons"] = [{"code": "unobserved_area", "message": ""}] + ground_only = {"kind": "band", "band": "ground", "span_ft": [0, 5], "checks": ["gas_clearance"]} + r["missing_evidence"] = [ground_only | {"message": ""}] + assert missing_evidence_problems(scene, r, two_band) == [ + "check gas_clearance is unsure and needs wall [0.00, 4.58] observed, but no wall request " + "names it" + ] + r["missing_evidence"].append(ground_only | {"band": "wall", "message": ""}) + assert missing_evidence_problems(scene, r, two_band) == [] def test_rules_come_from_rules_yaml_and_a_missing_one_is_loud(): text = """ battery: {width_ft: {value: 2.5}, depth_ft: {value: 1.5}} errors: {tap_ft: {value: 0.3}, vlm_ft: {value: 1.5}, tape_ft: {value: 0.05}, - wall_ft: {value: 0.3}, meter_ft: {value: 0.3}} + wall_ft: {value: 0.3}, meter_ft: {value: 0.3}, drift_per_ft: {value: 0.16}} clearances: {gas_ft: {value: 3}, ac_ft: {value: 3}, drive_ft: {value: 5}, pool_ft: {value: 10}, opening_ft: {value: 3}} """ rules = RuleSet.from_yaml(text) - assert rules.radii["pool_clearance"] == ("ground", 10.0) - assert rules.radii["opening_clearance"] == ("wall", 3.0) + assert rules.radii["pool_clearance"] == (("ground",), 10.0) + assert rules.radii["gas_clearance"] == (("ground", "wall"), 3.0) + assert rules.radii["opening_clearance"] == (("wall",), 3.0) assert (rules.width_ft, rules.errors["tape"]) == (2.5, 0.05) with pytest.raises(ValueError, match=r"rules\.yaml has no clearances\.pool_ft"): RuleSet.from_yaml(text.replace("pool_ft: {value: 10}, ", "")) @@ -503,8 +559,13 @@ def test_transforms(): scene = copy.deepcopy(SCENE) scene["objects"].append({"type": "ac", "wall_id": "w1", "span_ft": [5, 6], "source": "tape"}) more = with_more_error(scene, RULES) - assert [o["plus_minus_ft"] for o in more["objects"]] == [0.8, 0.55] - assert more["walls"][0]["plus_minus_ft"] == 0.8 and more["meter"]["plus_minus_ft"] == 0.8 + # A 24 ft chain: a tap default can drift to 0.3 + 0.16 * 24 = 4.14; tape does not drift. + assert [o["plus_minus_ft"] for o in more["objects"]] == pytest.approx([4.64, 0.55]) + assert more["walls"][0]["plus_minus_ft"] == 0.5 + default_wall = copy.deepcopy(scene) + del default_wall["walls"][0]["plus_minus_ft"] + assert with_more_error(default_wall, RULES)["walls"][0]["plus_minus_ft"] == pytest.approx(4.64) + assert more["meter"]["plus_minus_ft"] == 0.8 assert [o["source"] for o in tape_to_tap(scene)["objects"]] == ["tap", "tap"] assert tape_to_tap(SCENE) is None less = with_less_coverage(scene) From 56bdde65b11bacd88883ff63c61cb6f2f76aa953 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 07:20:02 -0500 Subject: [PATCH 32/59] verification: recheck at t3/ios-mvf a39d0a5 and t3/server 739fb6f The Simulator run reaches all ten states at a39d0a5 and uploads to the server; Apple's audit finds 10 issues on 21 screens, down from 32. The product description, UX review and scoreboard point at those runs. Also: simrun unpacks the server URL from server_from_ref's new (url, limit) pair, and the hostile-input budgets state the measurements behind them. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/README.md | 33 ++- verification/SCOREBOARD.md | 69 ++--- verification/hsverify/e2e.py | 5 +- verification/hsverify/resultcheck.py | 4 +- verification/hsverify/simrun.py | 2 +- verification/product/README.md | 8 +- verification/product/bug-triage.md | 2 +- .../foundations/coverage-and-guidance.md | 2 +- verification/product/foundations/flow.md | 2 +- verification/product/screens/find-meter.md | 2 +- verification/product/screens/gap-request.md | 2 +- verification/product/screens/mark-features.md | 2 +- .../product/screens/meter-close-up.md | 2 +- verification/product/screens/onboarding.md | 2 +- verification/product/screens/result.md | 2 +- verification/product/screens/uploading.md | 2 +- verification/product/screens/wall-walk.md | 2 +- verification/product/verification.md | 12 +- verification/scoreboard/metrics.yaml | 2 +- verification/scoreboard/scoreboard.json | 242 ++++++++++-------- verification/ux/review.md | 24 +- 21 files changed, 226 insertions(+), 197 deletions(-) diff --git a/verification/README.md b/verification/README.md index 2563c9686..01a0f2863 100644 --- a/verification/README.md +++ b/verification/README.md @@ -26,9 +26,9 @@ its own Simulator ("HouseScan Verify") and launches it with the C4 arguments. It settled for 1.2 s; a state replaced sooner is captured at once and marked *transient*. The report lists a failed build, a crash, markers logged as ``, and a run with no markers as problems, and keeps every app log line. `make sim-app` also fails unless the app reaches -`result`, and keeps the scan it uploaded as `scan.zip`. Log capture starts only once the log -stream has attached, so early states are not missed. It waits while another `xcodebuild` compiles, because -several workers share the Mac. +`result`, and keeps the scan it uploaded as `scan.zip`. The app is launched only once the log +stream has attached, so early states are not missed. The runner waits while another +`xcodebuild` compiles, because several workers share the Mac. **End-to-end check** (`hsverify/e2e.py`, `hsverify/resultcheck.py`). Starts the server from a ref and finds its placement endpoint in the OpenAPI document, refusing to guess between candidates. @@ -43,12 +43,14 @@ policy: - Each check's outcome follows from its numbers (C5): a minimum T passes only when measured − error > T and fails only when measured + error < T, a maximum mirrored, a review line also cleared for a pass; an unsure labelled `margin` lies within its error of a line. -- No battery position passes unless the wall and cable route under it were observed, and each - clearance's band was observed for its full radius on both sides (ground out to the battery's - depth plus the radius). Distance along the wall is never shorter than straight-line distance, - so this needs no model of the solver's geometry and cannot flag a correct pass. -- Every check that is unsure for lack of coverage is named in `missing_evidence`, and no photo - request covers an observed area. +- No battery position passes unless the wall and cable route under it were observed, and every + point within each clearance's radius, widened by the battery's position error, was observed + in the bands a hazard could hide in (gas: wall and ground; openings: wall; AC, drives, pools: + ground). Only points within that reach whatever the wall's shape are required: along the wall + to the radius, and on the ground out to the battery's depth plus the radius in front, tapering + past its ends. So a correct server is never flagged. +- When the result gives an unobserved area as a reason, each check left unsure for that reason + is named by a request of every band it lacks, and no request covers an area already observed. - Counts add up, `input_sha256` is the hash of the bytes sent, the spot's offset equals its centre minus the meter. @@ -56,13 +58,16 @@ Each scene is also resent changed, and the pairs must stay ordered: the same sce same; mirrored it gives the same decision and outcome lengths; with no coverage, less coverage, or ground trimmed just short of the largest clearance nothing new passes; with more error, or tape measurements re-taken by tap, no outcome moves except to unsure; adding every requested photo, up -to three rounds, leaves no check unsure for lack of coverage. Four hostile inputs (20 000 objects, -coordinates of 1e300, a 5000 ft wall, 20 000 levels of nesting) must be refused with 400, 413 or -422, or answered validly, within 10 s; a timeout is a failure. `--no-hostile` skips them. +to three rounds, leaves no check unsure for lack of coverage. Five hostile inputs (150 objects +with their own errors, coordinates of 1e300, a 5000 ft wall, nesting 5000 deep, and a small zip +holding a 400 MB scene.json) must each be refused with 400, 413 or 422, or answered validly, +within 10 s and, for a server started here, 500 MB of extra memory. `--no-hostile` skips them. The 43 cases in [`e2e/cases/`](e2e/cases/README.md) were written from the public goldens and C5 -without reading the server's tests. Each names the thresholds it assumes; a mismatch is reported -as such rather than as a pass or fail. [`scenes/eth3d-facade`](scenes/eth3d-facade/README.md) +without reading the server's tests, except that the three drift cases take drift at the +battery's far edge as S2's 70ab0b0 states. The hostile inputs were chosen after reading S2's +hardening tests. Each case names the thresholds it assumes; a mismatch is reported as such +rather than as a pass or fail. [`scenes/eth3d-facade`](scenes/eth3d-facade/README.md) builds a scene from real laser-scanned geometry; real scenes must answer within 1 s. `--app-export` takes a `make sim-app` report folder and sends the scan the app uploaded, recording the app's commit. diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index 12a83b43f..535ee4f36 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,18 +1,19 @@ # Scoreboard -Generated 2026-09-26 06:28 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-26 07:19 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | -| [#16](https://github.com/SamGu-NRX/house-scanning/pull/16) | `t3/meter-closeup` | `c5cb30d7` | yes | 3 success, 1 failure; failing: Vercel | 2026-09-26 06:13 CDT | +| [#17](https://github.com/SamGu-NRX/house-scanning/pull/17) | `t3/panel-label` | `2dc7a4f1` | yes | 2 success | 2026-09-26 07:02 CDT | +| [#16](https://github.com/SamGu-NRX/house-scanning/pull/16) | `t3/meter-closeup` | `944cbe17` | yes | 3 success | 2026-09-26 06:45 CDT | | [#15](https://github.com/SamGu-NRX/house-scanning/pull/15) | `t3/web-review` | `a7c9b129` | yes | 3 success, 3 failure; failing: Label PR size, Web checks, Vercel | 2026-09-26 06:08 CDT | | [#14](https://github.com/SamGu-NRX/house-scanning/pull/14) | `t3/docs-cleanup` | `ab03d7fd` | no | 4 success | 2026-09-26 05:17 CDT | -| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `5e60831d` | yes | 2 success, 1 failure; failing: Vercel | 2026-09-26 06:13 CDT | -| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `27293c5d` | yes | 3 success | 2026-09-26 06:20 CDT | -| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `f2705dd5` | no | 5 success | 2026-09-26 06:22 CDT | -| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `beede15f` | yes | 4 success, 1 failure; failing: iOS build | 2026-09-26 06:12 CDT | +| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `30e866c6` | yes | 2 success | 2026-09-26 06:28 CDT | +| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `d64be59f` | yes | 3 success | 2026-09-26 07:14 CDT | +| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `739fb6f7` | no | 3 success | 2026-09-26 06:35 CDT | +| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `a39d0a50` | no | 2 success, 1 pending | 2026-09-26 07:11 CDT | | [#9](https://github.com/SamGu-NRX/house-scanning/pull/9) | `codex/capture-contract-probe` | `4b6c4309` | yes | 4 success | 2026-09-26 00:08 CDT | | [#8](https://github.com/SamGu-NRX/house-scanning/pull/8) | `t3/testflight` | `b896d59f` | no | 4 success | 2026-09-26 03:18 CDT | | [#7](https://github.com/SamGu-NRX/house-scanning/pull/7) | `t3/measure-lab` | `45770a3b` | no | 6 success | 2026-09-26 03:19 CDT | @@ -21,58 +22,58 @@ Generated 2026-09-26 06:28 CDT by `make scoreboard` (`uv run python -m hsverify. ## S1 Evals on real data -`origin/t3/evals` at [`27293c5d`](https://github.com/SamGu-NRX/house-scanning/tree/27293c5d4d34a27af652116341639f1826d806ae). 2 met, 2 partial, 1 not run. +`origin/t3/evals` at [`d64be59f`](https://github.com/SamGu-NRX/house-scanning/tree/d64be59f7850fabaa86b0c08cf6a5003e66ab007). 2 met, 2 partial, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S1-1 | One command regenerates every number from a clean checkout | partial | make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac. | -| S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/README.md) | +| S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/README.md) | | S1-3 | Metric code tested on hand-computed cases | not run | slow; run with --slow | -| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/README.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/results/advio_drift.md), [experiments/evals/results/modern_arkit.md](https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/results/modern_arkit.md) | -| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/README.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/results/eth3d_visibility_sensitivity.md) | +| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/README.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/results/advio_drift.md), [experiments/evals/results/modern_arkit.md](https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/results/modern_arkit.md) | +| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/README.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/results/eth3d_visibility_sensitivity.md) | ## S2 Server and solver -`origin/t3/server` at [`f2705dd5`](https://github.com/SamGu-NRX/house-scanning/tree/f2705dd5580ccd8c18ebd34a743b448bcb59ae03). 3 met, 1 partial, 2 stale, 2 not run. +`origin/t3/server` at [`739fb6f7`](https://github.com/SamGu-NRX/house-scanning/tree/739fb6f747294b7253eb7d8936315392a80d178a). 4 met, 1 partial, 1 gap, 2 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S2-1 | pytest green | not run | slow; last run 2026-09-26 05:43 CDT: met, exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 222 passed, 1 warning in 11.01s | -| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_golden_public.py) | -| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_properties.py) | -| S2-4 | Schemas published | met | present at f2705dd5 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/schemas/result.schema.json) | -| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_schemas.py), [server/tests/test_api.py](https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_api.py) | -| S2-6 | A real-derived scene returns a result in under 1 s | stale | report for 26d28707, ref at f2705dd5: ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | -| S2-7 | ruff clean | not run | slow; last run 2026-09-26 05:43 CDT: met, exit 0: Installed 33 packages in 37ms / All checks passed! / 17 files already formatted | -| S2-8 | Black-box (verification e2e): every answer valid under C2, consistent with its decision and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric when mirrored, and never passing without coverage | stale | report for 26d28707, ref at f2705dd5: ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | +| S2-1 | pytest green | not run | slow; run with --slow | +| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_golden_public.py) | +| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_properties.py) | +| S2-4 | Schemas published | met | present at 739fb6f7 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/schemas/result.schema.json) | +| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_schemas.py), [server/tests/test_api.py](https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_api.py) | +| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 364.2 (want < 1000) at sha 739fb6f7: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json | +| S2-7 | ruff clean | not run | slow; run with --slow | +| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | gap | contract_problem_count = 12 (want 0) at sha 739fb6f7: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json | ## S3 iOS capture app -`origin/t3/ios-mvf` at [`beede15f`](https://github.com/SamGu-NRX/house-scanning/tree/beede15f568b3a4d694fb275caf9eaaa882c546b). 3 met, 3 partial, 3 gap, 1 stale. +`origin/t3/ios-mvf` at [`a39d0a50`](https://github.com/SamGu-NRX/house-scanning/tree/a39d0a503412d688dd151f628cdeebc00f8221b6). 4 met, 3 partial, 2 gap, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S3-1 | CI build green | gap | iOS build: failure at beede15f [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/project.yml) | -| S3-3 | Zero build warnings | met | build.warnings = length 0 (want 0): ~/house-scanning-data/reports/sim/20260926-061551-t3-ios-mvf-beede15f-replay/report.json | -| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage; missing auto-capture, guidance [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift) | -| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, plus symbol-name button labels) finds no issues; S4-2 shows how many screens the app reached | stale | report for 525ea401, ref at beede15f: ~/house-scanning-data/reports/a11y/20260926-052756-t3-ios-mvf-525ea401-head-sample/report.json | -| S3-7 | Screenshots of each state in the PR | gap | 0 of 9 matches in the PR body [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-8 | Exported scene.json validates against C1 | met | app_export_scene_valid = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | +| S3-1 | CI build green | not run | iOS build: pending at a39d0a50 [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | +| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/project.yml) | +| S3-3 | Zero build warnings | met | build.warnings = length 0 (want 0) at sha a39d0a50: ~/house-scanning-data/reports/sim/20260926-071459-t3-ios-mvf-a39d0a50-replay/report.json | +| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage; missing auto-capture, guidance [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift) | +| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | gap | issue_count = 10 (want 0) at sha a39d0a50: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | +| S3-7 | Screenshots of each state in the PR | met | 18 of 9 matches in the PR body [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | +| S3-8 | Exported scene.json validates against C1 | met | app_export_scene_valid = true (want True) at app_sha a39d0a50: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/verification` at [`5e60831d`](https://github.com/SamGu-NRX/house-scanning/tree/5e60831dcc87ca0b854b4ce51c1e8b220438fc7a), `origin/t3/ios-mvf` at [`beede15f`](https://github.com/SamGu-NRX/house-scanning/tree/beede15f568b3a4d694fb275caf9eaaa882c546b). 4 met. +`origin/t3/ios-mvf` at [`a39d0a50`](https://github.com/SamGu-NRX/house-scanning/tree/a39d0a503412d688dd151f628cdeebc00f8221b6), `origin/t3/verification` at [`30e866c6`](https://github.com/SamGu-NRX/house-scanning/tree/30e866c6d0c835c025dfa050d339d7b6ff350e69). 3 met, 1 gap. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S4-1 | End to end from a real replay to a placement result passes | met | app_export_passed = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json | -| S4-2 | Screenshot report per app state | met | states = length 10 (want at least 9): ~/house-scanning-data/reports/sim/20260926-061551-t3-ios-mvf-beede15f-replay/report.json | -| S4-3 | Product description written from the code and checked against screenshots | met | verification/product covers every screen of t3/ios-mvf at 525ea40, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state. | -| S4-4 | Scoreboard of every PR | met | present at 5e60831d [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/5e60831dcc87ca0b854b4ce51c1e8b220438fc7a/verification/SCOREBOARD.md) | +| S4-1 | End to end from a real replay to a placement result passes | gap | app_export_passed = false (want True) at app_sha a39d0a50: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json | +| S4-2 | Screenshot report per app state | met | states = length 10 (want at least 9) at sha a39d0a50: ~/house-scanning-data/reports/sim/20260926-071459-t3-ios-mvf-a39d0a50-replay/report.json | +| S4-3 | Product description written from the code and checked against screenshots | met | verification/product covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state. | +| S4-4 | Scoreboard of every PR | met | present at 30e866c6 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/30e866c6d0c835c025dfa050d339d7b6ff350e69/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index 14916e0e1..d2ae13f89 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -497,7 +497,8 @@ def resend(name: str, scene: dict) -> dict | None: # --- Hostile inputs --------------------------------------------------------------------------- # A refusal or an answer must come within this; an input that keeps the server busy longer is a -# denial-of-service risk. Generous for a laptop-hosted server; the real-scene budget is 1 s. +# denial-of-service risk. Not derived from any requirement: it is ten times the real-scene +# budget. S2 at 739fb6f answered the crowded input in 8.2 s on this Mac, close to the line. HOSTILE_BUDGET_MS = 10_000.0 @@ -571,6 +572,8 @@ def zip_bomb(megabytes: int) -> bytes: # A hostile input may cost the server this much memory above what it held before the request. +# Not derived from any requirement: S2 at 739fb6f grew by at most 186 MB on these inputs, and at +# 26d2870, before its input caps, by 557 to 1176 MB. HOSTILE_MEMORY_MB = 500 diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 3e75f7be1..076ef6ed8 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -33,12 +33,10 @@ NUMBER_TOL_FT = 1e-5 # Spot centres and offsets are rounded by the server; a hundredth of a foot is 1/8 inch. OFFSET_TOL_FT = 0.01 -# Check ids whose PASS depends on seeing an area around the footprint, the band that area lies -# in, and the rules.yaml clearance that sets its radius. ground_surface needs only the ground -# under the battery. # Clearance check -> the bands an unseen hazard could hide in, and its rules.yaml radius. C1: a # gas meter hangs on the wall face (wall band) and its regulator stands on the ground (ground # band); openings are on the wall face; AC units, drives and pools are on the ground. +# ground_surface (radius 0, added in RuleSet.from_yaml) needs the ground under the battery. CLEARANCE_CHECKS = { "gas_clearance": (("ground", "wall"), "gas_ft"), "ac_clearance": (("ground",), "ac_ft"), diff --git a/verification/hsverify/simrun.py b/verification/hsverify/simrun.py index 6bcb70d6c..128d9f9f7 100644 --- a/verification/hsverify/simrun.py +++ b/verification/hsverify/simrun.py @@ -577,7 +577,7 @@ def main(argv: list[str] | None = None) -> int: else: if args.server_ref: server_sha = gitref.resolve(args.server_ref) - url = services.enter_context(server_from_ref(server_sha, out / "server.log")) + url, _ = services.enter_context(server_from_ref(server_sha, out / "server.log")) launch_args += ["-serverURL", url] rep.server = {"ref": args.server_ref, "sha": server_sha, "url": url} print(f"Server {args.server_ref} at {server_sha[:12]} on {url}", flush=True) diff --git a/verification/product/README.md b/verification/product/README.md index 2356bedf5..9ace0b1f6 100644 --- a/verification/product/README.md +++ b/verification/product/README.md @@ -10,8 +10,10 @@ wrong is collected in [bug-triage.md](bug-triage.md). - **Surface.** The HouseScan app, one run of the homeowner flow from onboarding to the result, default settings. The ARKit camera cannot run in the Simulator, so behavior that needs the live camera is described from code and marked unverified. -- **Source.** `t3/ios-mvf`. Line references, footers and the last Simulator check are at - `beede15`. +- **Source.** `t3/ios-mvf`. Footers and the last Simulator check are at `a39d0a5`; line + references are at `beede15`. Between the two, the only change a homeowner can reach is that the + strip's marks scale with text size and stay inside it; the upload refusal and meter-number + states added to the contract are not wired to anything yet. - **How it is run.** `make sim-app REF=` from `verification/`: the real ADVIO replay `advio-20-0040-0075` (C3), the autopilot, and the server started from `t3/server`. - **Out of scope.** The `-uiDemo` screens (a design preview, not the product), TestFlight @@ -72,5 +74,5 @@ and Open questions and verification, ending with the commit it was verified agai | [screens/gap-request.md](screens/gap-request.md) | One targeted extra view, from the phone or the server | drafted; checked in the Simulator | | [screens/uploading.md](screens/uploading.md) | Sending the scan and waiting for the result, offline and failure | drafted; checked in the Simulator | | [screens/result.md](screens/result.md) | The result, its checks and the 3D and AR views | drafted; checked in the Simulator | -| [verification.md](verification.md) | Checks against the running app and their results | current pass at `beede15` | +| [verification.md](verification.md) | Checks against the running app and their results | current pass at `a39d0a5` | | [bug-triage.md](bug-triage.md) | Every suspected defect, deduplicated | 16 entries, 2 resolved | diff --git a/verification/product/bug-triage.md b/verification/product/bug-triage.md index 208091196..c50fbdb4e 100644 --- a/verification/product/bug-triage.md +++ b/verification/product/bug-triage.md @@ -1,7 +1,7 @@ # Suspected defects Collected from every document's open questions and from [verification.md](verification.md), -deduplicated. Status checked at `t3/ios-mvf` `beede15`; line references are at `0876e03` unless an +deduplicated. Status checked at `t3/ios-mvf` `a39d0a5`; line references are at `0876e03` unless an entry names another commit; paths are under `ios/`. **Seen** means observed in the Simulator; **code** means read from the source and not run. Every blocker and major entry was re-read in the source by a second reader. diff --git a/verification/product/foundations/coverage-and-guidance.md b/verification/product/foundations/coverage-and-guidance.md index a2a4265ea..451a366d7 100644 --- a/verification/product/foundations/coverage-and-guidance.md +++ b/verification/product/foundations/coverage-and-guidance.md @@ -182,4 +182,4 @@ hooks.** Each instruction change logs `GUIDANCE=`. (`Runtime/ScanEngine.swift`, `step(_:)`). - All thresholds are the code's stated hypotheses (research note), not measured values. -Verified against house-scanning commit `beede15` (t3/ios-mvf). +Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/foundations/flow.md b/verification/product/foundations/flow.md index daac09425..e945590b2 100644 --- a/verification/product/foundations/flow.md +++ b/verification/product/foundations/flow.md @@ -142,4 +142,4 @@ the autopilot wait on each screen until a file named after it appears, for UI te - The 1.2 s pause after a finished step and the 20 s relocalization limit are read from code; neither has been timed on a device. -Verified against house-scanning commit `beede15` (t3/ios-mvf). +Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/screens/find-meter.md b/verification/product/screens/find-meter.md index d69023982..fae5aa084 100644 --- a/verification/product/screens/find-meter.md +++ b/verification/product/screens/find-meter.md @@ -174,4 +174,4 @@ replay logs where its wall came from when it loads. - The live tap, the refusals, the coaching and the camera prompt need the live camera and have not been seen running. The Simulator shows only the replay path, where every tap is accepted. -Verified against house-scanning commit `beede15` (t3/ios-mvf). +Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/screens/gap-request.md b/verification/product/screens/gap-request.md index bd154c5f3..b5872e596 100644 --- a/verification/product/screens/gap-request.md +++ b/verification/product/screens/gap-request.md @@ -192,4 +192,4 @@ skipping it". With `-autopilotGate`, the upload after a completed view waits for - Nothing here has run in the Simulator. The completion moment, the amber highlight and the dots need the replay with `-autopilot`, which reaches this screen only when held-back frames exist. -Verified against house-scanning commit `beede15` (t3/ios-mvf). +Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/screens/mark-features.md b/verification/product/screens/mark-features.md index debbeadd7..ba8af0cf3 100644 --- a/verification/product/screens/mark-features.md +++ b/verification/product/screens/mark-features.md @@ -217,4 +217,4 @@ appears; marking itself logs nothing. Identifiers: `action.markSomething`, `acti "Mark". MF-01 in [verification](../verification.md) was observed on an earlier commit and has not been rechecked here. -Verified against house-scanning commit `beede15` (t3/ios-mvf). +Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/screens/meter-close-up.md b/verification/product/screens/meter-close-up.md index dc6209061..c5a086ddc 100644 --- a/verification/product/screens/meter-close-up.md +++ b/verification/product/screens/meter-close-up.md @@ -132,4 +132,4 @@ failed attempts" when the homeowner skips. its text in that screenshot; see [verification](../verification.md). - The circle-versus-image mismatch above is read from code; it needs a device to confirm. -Verified against house-scanning commit `beede15` (t3/ios-mvf). +Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/screens/onboarding.md b/verification/product/screens/onboarding.md index 10b89eb2e..27775d5a8 100644 --- a/verification/product/screens/onboarding.md +++ b/verification/product/screens/onboarding.md @@ -173,4 +173,4 @@ change. Tests find the controls as `action.onboardingNext`, `action.onboardingSk session error, and the Reduce Motion stills. The unsupported screen and the `-replay /nonexistent` case can be checked in the Simulator. -Verified against house-scanning commit `beede15` (t3/ios-mvf). +Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/screens/result.md b/verification/product/screens/result.md index 7beefcabb..bf51f53b4 100644 --- a/verification/product/screens/result.md +++ b/verification/product/screens/result.md @@ -219,4 +219,4 @@ appears, not when returning from the AR view. **Verification hooks.** `STATE=res ([verification](../verification.md), RES-01 to AR-01). Whether the AR drawing lines up with a real wall, and hiding it while the phone relocalizes, need the live camera. -Verified against house-scanning commit `beede15` (t3/ios-mvf). +Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/screens/uploading.md b/verification/product/screens/uploading.md index eb835ae08..3429be51d 100644 --- a/verification/product/screens/uploading.md +++ b/verification/product/screens/uploading.md @@ -171,4 +171,4 @@ autopilot logs "upload failed; retrying once". Test identifiers: `screen.uploadi - Pass 1 (`21a63e7`) saw "That didn't go through" with no button or symbol visible ([verification](../verification.md) UP-01, B-05). Not rechecked at this commit. -Verified against house-scanning commit `beede15` (t3/ios-mvf). +Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/screens/wall-walk.md b/verification/product/screens/wall-walk.md index e27ff2ba7..6f23a6a8a 100644 --- a/verification/product/screens/wall-walk.md +++ b/verification/product/screens/wall-walk.md @@ -359,4 +359,4 @@ reset: relocalization timed out". Accessibility identifiers: `action.markSomethi interruption recovery. The first Simulator pass showed this screen's chrome without text (B-05 in [bug-triage.md](../bug-triage.md)). -Verified against house-scanning commit `beede15` (t3/ios-mvf). +Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/verification.md b/verification/product/verification.md index 72a4a1689..be83efaec 100644 --- a/verification/product/verification.md +++ b/verification/product/verification.md @@ -15,11 +15,11 @@ what each shows and speaks, what the app logged, and what the server answered. W cover: anything needing the live camera (tracking, the meter tap on a real wall, a close-up that succeeds), haptics, or timing a person feels. -## Current pass: `t3/ios-mvf` `beede15`, server `f2705dd` +## Current pass: `t3/ios-mvf` `a39d0a5`, server `739fb6f` -Report `sim/20260926-061551-t3-ios-mvf-beede15f-replay` (real server; the pass at `525ea40` is -`sim/20260926-052535-t3-ios-mvf-525ea401-replay`), the audit at `24af434` and -`525ea40` (sample result), plus the variant runs on the preview +Report `sim/20260926-071459-t3-ios-mvf-a39d0a50-replay` (real server; the pass at `525ea40` is +`sim/20260926-052535-t3-ios-mvf-525ea401-replay`), the audit at `a39d0a5` (`a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample`, sample result), plus +the variant runs on the preview `5520229` (the same UI before S3's last fixes) named in [the UX review](../ux/review.md#evidence). | ID | Claim | Result | @@ -34,10 +34,10 @@ Report `sim/20260926-061551-t3-ios-mvf-beede15f-replay` (real server; the pass a | WW-02 | Guidance moves from walking to tilting to stepping back | Pass: the audit saw "Walk slowly to your left", "Tilt down to show the ground", "Take a step back". | | MF-01 | The list shows each mark with its distance, the window question and "Add something" | Pass at default size; hidden behind "Looks complete" at the largest text size (UX-01). | | GAP-01 | A gap request states what to show and why, with a progress bar and "I can't get there" | Pass ("Show the ground around your meter", "0%"). | -| GAP-02 | New photos close the request and the flow moves on | Pass at `24af434` and `525ea40` ("gap 1 satisfied"; "Got it, thanks"). Not exercised at `beede15`: the autopilot finds no closable gap on this replay and skips it. | +| GAP-02 | New photos close the request and the flow moves on | Pass at `24af434` and `525ea40` ("gap 1 satisfied"; "Got it, thanks"). Not exercised since `beede15`: at `a39d0a5` the request appears, but the autopilot holds back no frames that could fill it and answers "I can't get there" ("gap 1 skipped"). | | WW-03 | After "Wall ends here" the walk asks what is at that end, and a corner stays unexplored | Pass: logged "end left at s=-3.81 (unexplored)", then "answered the left end: turns a corner"; same on the right. | | UP-01 | Upload progress is shown in steps apart from coverage | Pass: "Photos ready", "Sending photos, 99%", "Check clearances". | -| RES-01 | The result shows the headline, placement line, the server's summary, the rules-not-final note and the panel notice | Pass: "An installer will take a look" with the server's placement and summary (seen at `0876e03`, `24af434`, `525ea40` and `beede15`). | +| RES-01 | The result shows the headline, placement line, the server's summary, the rules-not-final note and the panel notice | Pass: "An installer will take a look" with the server's placement and summary (seen at `0876e03`, `24af434`, `525ea40`, `beede15` and `a39d0a5`). | | RES-02 | A borderline check shows the measurement, the rule and the error | Pass on the sample result: "Measured 3 ft 1 in. The rule is 3 ft, and the measurement can be off by about 4 in." | | RES-03 | A maximum reads as a maximum | Fail: "Measured 3 ft. The rule is 20 ft" (B-14). | | RES-04 | The result is visible when the app logs `STATE=result` | Fail, minor: the upload screen was still showing 1.2 s later on first load, in both passes (UX-07). | diff --git a/verification/scoreboard/metrics.yaml b/verification/scoreboard/metrics.yaml index e53278dc5..42288a71e 100644 --- a/verification/scoreboard/metrics.yaml +++ b/verification/scoreboard/metrics.yaml @@ -288,7 +288,7 @@ metrics: manual: status: met evidence: >- - verification/product covers every screen of t3/ios-mvf at 525ea40, checked in the + verification/product covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state. diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index 0cd02b865..d9b719424 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,26 +1,42 @@ { - "generated_at": "2026-09-26T06:28:21-05:00", + "generated_at": "2026-09-26T07:19:11-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": false, "prs": [ + { + "number": 17, + "title": "experiments: can the phone read the main panel's label?", + "branch": "t3/panel-label", + "head": "2dc7a4f1a53cf801156f09fd84e6a6e4d1c27baf", + "draft": true, + "updated": "2026-09-26T12:02:53Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "2 success" + } + }, { "number": 16, - "title": "experiments: meter close-up reading, retake thresholds and number finding", + "title": "experiments: meter close-up reading, number finding and retake checks", "branch": "t3/meter-closeup", - "head": "c5cb30d782fa8bd9a91e30f6997b7239764db640", + "head": "944cbe176c684f45ac2e1caccda2bc4b15737e19", "draft": true, - "updated": "2026-09-26T11:13:43Z", + "updated": "2026-09-26T11:45:06Z", "ci": { "counts": { "success": 3, - "failure": 1, + "failure": 0, "pending": 0, "skipped": 0 }, - "failing": [ - "Vercel" - ], - "text": "3 success, 1 failure; failing: Vercel" + "failing": [], + "text": "3 success" } }, { @@ -67,29 +83,27 @@ "number": 13, "title": "verification: independent end-to-end checks (S4)", "branch": "t3/verification", - "head": "5e60831dcc87ca0b854b4ce51c1e8b220438fc7a", + "head": "30e866c6d0c835c025dfa050d339d7b6ff350e69", "draft": true, - "updated": "2026-09-26T11:13:41Z", + "updated": "2026-09-26T11:28:43Z", "ci": { "counts": { "success": 2, - "failure": 1, + "failure": 0, "pending": 0, "skipped": 0 }, - "failing": [ - "Vercel" - ], - "text": "2 success, 1 failure; failing: Vercel" + "failing": [], + "text": "2 success" } }, { "number": 12, "title": "feat(evals): real-data accuracy evals for ARKit drift and photo-based wall reconstruction", "branch": "t3/evals", - "head": "27293c5d4d34a27af652116341639f1826d806ae", + "head": "d64be59f7850fabaa86b0c08cf6a5003e66ab007", "draft": true, - "updated": "2026-09-26T11:20:39Z", + "updated": "2026-09-26T12:14:06Z", "ci": { "counts": { "success": 3, @@ -105,38 +119,36 @@ "number": 11, "title": "server: placement solver, rules and API (S2)", "branch": "t3/server", - "head": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", + "head": "739fb6f747294b7253eb7d8936315392a80d178a", "draft": false, - "updated": "2026-09-26T11:22:27Z", + "updated": "2026-09-26T11:35:35Z", "ci": { "counts": { - "success": 5, + "success": 3, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "5 success" + "text": "3 success" } }, { "number": 10, - "title": "ios: guided wall scan, from meter to battery spot", + "title": "ios: guided wall scan from the meter to a battery spot", "branch": "t3/ios-mvf", - "head": "beede15f568b3a4d694fb275caf9eaaa882c546b", - "draft": true, - "updated": "2026-09-26T11:12:52Z", + "head": "a39d0a503412d688dd151f628cdeebc00f8221b6", + "draft": false, + "updated": "2026-09-26T12:11:15Z", "ci": { "counts": { - "success": 4, - "failure": 1, - "pending": 0, + "success": 2, + "failure": 0, + "pending": 1, "skipped": 0 }, - "failing": [ - "iOS build" - ], - "text": "4 success, 1 failure; failing: iOS build" + "failing": [], + "text": "2 success, 1 pending" } }, { @@ -238,7 +250,7 @@ "refs": [ { "ref": "origin/t3/evals", - "sha": "27293c5d4d34a27af652116341639f1826d806ae" + "sha": "d64be59f7850fabaa86b0c08cf6a5003e66ab007" } ], "metrics": [ @@ -246,7 +258,7 @@ "id": "S1-1", "text": "One command regenerates every number from a clean checkout", "ref": "origin/t3/evals", - "sha": "27293c5d4d34a27af652116341639f1826d806ae", + "sha": "d64be59f7850fabaa86b0c08cf6a5003e66ab007", "status": "partial", "evidence": "make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac.", "links": [] @@ -255,13 +267,13 @@ "id": "S1-2", "text": "Dataset URLs and sha256 recorded", "ref": "origin/t3/evals", - "sha": "27293c5d4d34a27af652116341639f1826d806ae", + "sha": "d64be59f7850fabaa86b0c08cf6a5003e66ab007", "status": "met", "evidence": "all 2 found", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/README.md" ] ] }, @@ -269,7 +281,7 @@ "id": "S1-3", "text": "Metric code tested on hand-computed cases", "ref": "origin/t3/evals", - "sha": "27293c5d4d34a27af652116341639f1826d806ae", + "sha": "d64be59f7850fabaa86b0c08cf6a5003e66ab007", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -278,21 +290,21 @@ "id": "S1-4", "text": "ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft", "ref": "origin/t3/evals", - "sha": "27293c5d4d34a27af652116341639f1826d806ae", + "sha": "d64be59f7850fabaa86b0c08cf6a5003e66ab007", "status": "met", "evidence": "all 5 found", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/README.md" ], [ "experiments/evals/results/advio_drift.md", - "https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/results/advio_drift.md" + "https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/results/advio_drift.md" ], [ "experiments/evals/results/modern_arkit.md", - "https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/results/modern_arkit.md" + "https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/results/modern_arkit.md" ] ] }, @@ -300,21 +312,21 @@ "id": "S1-5", "text": "ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question", "ref": "origin/t3/evals", - "sha": "27293c5d4d34a27af652116341639f1826d806ae", + "sha": "d64be59f7850fabaa86b0c08cf6a5003e66ab007", "status": "partial", "evidence": "found 1-3 m row, 3-10 m row, scale error; missing verdict", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/README.md" ], [ "experiments/evals/results/eth3d_recon.md", - "https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/results/eth3d_recon.md" + "https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/results/eth3d_recon.md" ], [ "experiments/evals/results/eth3d_visibility_sensitivity.md", - "https://github.com/SamGu-NRX/house-scanning/blob/27293c5d4d34a27af652116341639f1826d806ae/experiments/evals/results/eth3d_visibility_sensitivity.md" + "https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/results/eth3d_visibility_sensitivity.md" ] ] } @@ -326,7 +338,7 @@ "refs": [ { "ref": "origin/t3/server", - "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03" + "sha": "739fb6f747294b7253eb7d8936315392a80d178a" } ], "metrics": [ @@ -334,22 +346,22 @@ "id": "S2-1", "text": "pytest green", "ref": "origin/t3/server", - "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", + "sha": "739fb6f747294b7253eb7d8936315392a80d178a", "status": "not run", - "evidence": "slow; last run 2026-09-26 05:43 CDT: met, exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 222 passed, 1 warning in 11.01s", + "evidence": "slow; run with --slow", "links": [] }, { "id": "S2-2", "text": "Every public golden test 01-14 of docs/research/t3-lane-c-review.md present", "ref": "origin/t3/server", - "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", + "sha": "739fb6f747294b7253eb7d8936315392a80d178a", "status": "met", "evidence": "all 14 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_golden_public.py" ] ] }, @@ -357,13 +369,17 @@ "id": "S2-3", "text": "Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail", "ref": "origin/t3/server", - "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", + "sha": "739fb6f747294b7253eb7d8936315392a80d178a", "status": "partial", "evidence": "found missing coverage, mirror; missing equality unsure, larger clearance", "links": [ + [ + "server/tests/test_coderabbit_review.py", + "https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_coderabbit_review.py" + ], [ "server/tests/test_properties.py", - "https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_properties.py" + "https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_properties.py" ] ] }, @@ -371,17 +387,17 @@ "id": "S2-4", "text": "Schemas published", "ref": "origin/t3/server", - "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", + "sha": "739fb6f747294b7253eb7d8936315392a80d178a", "status": "met", - "evidence": "present at f2705dd5", + "evidence": "present at 739fb6f7", "links": [ [ "server/schemas/scene.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/schemas/scene.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/schemas/scene.schema.json" ], [ "server/schemas/result.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/schemas/result.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/schemas/result.schema.json" ] ] }, @@ -389,21 +405,21 @@ "id": "S2-5", "text": "Schemas validated in tests", "ref": "origin/t3/server", - "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", + "sha": "739fb6f747294b7253eb7d8936315392a80d178a", "status": "met", "evidence": "all 2 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_golden_public.py" ], [ "server/tests/test_schemas.py", - "https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_schemas.py" + "https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_schemas.py" ], [ "server/tests/test_api.py", - "https://github.com/SamGu-NRX/house-scanning/blob/f2705dd5580ccd8c18ebd34a743b448bcb59ae03/server/tests/test_api.py" + "https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_api.py" ] ] }, @@ -411,27 +427,27 @@ "id": "S2-6", "text": "A real-derived scene returns a result in under 1 s", "ref": "origin/t3/server", - "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", - "status": "stale", - "evidence": "report for 26d28707, ref at f2705dd5: ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json", + "sha": "739fb6f747294b7253eb7d8936315392a80d178a", + "status": "met", + "evidence": "latency_ms = 364.2 (want < 1000) at sha 739fb6f7: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json", "links": [] }, { "id": "S2-7", "text": "ruff clean", "ref": "origin/t3/server", - "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", + "sha": "739fb6f747294b7253eb7d8936315392a80d178a", "status": "not run", - "evidence": "slow; last run 2026-09-26 05:43 CDT: met, exit 0: Installed 33 packages in 37ms / All checks passed! / 17 files already formatted", + "evidence": "slow; run with --slow", "links": [] }, { "id": "S2-8", - "text": "Black-box (verification e2e): every answer valid under C2, consistent with its decision and the C5 margin rule, never passing unobserved ground or wall, deterministic, symmetric when mirrored, and never passing without coverage", + "text": "Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time", "ref": "origin/t3/server", - "sha": "f2705dd5580ccd8c18ebd34a743b448bcb59ae03", - "status": "stale", - "evidence": "report for 26d28707, ref at f2705dd5: ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json", + "sha": "739fb6f747294b7253eb7d8936315392a80d178a", + "status": "gap", + "evidence": "contract_problem_count = 12 (want 0) at sha 739fb6f7: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json", "links": [] } ] @@ -442,7 +458,7 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b" + "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6" } ], "metrics": [ @@ -450,9 +466,9 @@ "id": "S3-1", "text": "CI build green", "ref": "origin/t3/ios-mvf", - "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", - "status": "gap", - "evidence": "iOS build: failure at beede15f", + "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", + "status": "not run", + "evidence": "iOS build: pending at a39d0a50", "links": [ [ "#10", @@ -464,13 +480,13 @@ "id": "S3-2", "text": "Built as Swift 6 with strict concurrency", "ref": "origin/t3/ios-mvf", - "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", + "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", "status": "met", "evidence": "all 2 found", "links": [ [ "ios/project.yml", - "https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/project.yml" + "https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/project.yml" ] ] }, @@ -478,30 +494,30 @@ "id": "S3-3", "text": "Zero build warnings", "ref": "origin/t3/ios-mvf", - "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", + "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", "status": "met", - "evidence": "build.warnings = length 0 (want 0): ~/house-scanning-data/reports/sim/20260926-061551-t3-ios-mvf-beede15f-replay/report.json", + "evidence": "build.warnings = length 0 (want 0) at sha a39d0a50: ~/house-scanning-data/reports/sim/20260926-071459-t3-ios-mvf-a39d0a50-replay/report.json", "links": [] }, { "id": "S3-4", "text": "Unit tests for coverage, auto-capture and guidance", "ref": "origin/t3/ios-mvf", - "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", + "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", "status": "partial", "evidence": "found coverage; missing auto-capture, guidance", "links": [ [ "ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift" ] ] }, @@ -509,17 +525,17 @@ "id": "S3-5", "text": "UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop", "ref": "origin/t3/ios-mvf", - "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", + "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", "status": "partial", "evidence": "all 3 found (the tests exist; passing is S3-1)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanUITests/FullFlowUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -527,36 +543,36 @@ "id": "S3-6", "text": "Accessibility audit on every screen", "ref": "origin/t3/ios-mvf", - "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", + "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", "status": "partial", "evidence": "3 matches in 2 files (an audit exists; which screens it covers needs a read)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanUITests/FullFlowUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/beede15f568b3a4d694fb275caf9eaaa882c546b/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, { "id": "S3-6b", - "text": "Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, plus symbol-name button labels) finds no issues; S4-2 shows how many screens the app reached", + "text": "Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues", "ref": "origin/t3/ios-mvf", - "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", - "status": "stale", - "evidence": "report for 525ea401, ref at beede15f: ~/house-scanning-data/reports/a11y/20260926-052756-t3-ios-mvf-525ea401-head-sample/report.json", + "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", + "status": "gap", + "evidence": "issue_count = 10 (want 0) at sha a39d0a50: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", "links": [] }, { "id": "S3-7", "text": "Screenshots of each state in the PR", "ref": "origin/t3/ios-mvf", - "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", - "status": "gap", - "evidence": "0 of 9 matches in the PR body", + "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", + "status": "met", + "evidence": "18 of 9 matches in the PR body", "links": [ [ "#10", @@ -568,16 +584,16 @@ "id": "S3-8", "text": "Exported scene.json validates against C1", "ref": "origin/t3/ios-mvf", - "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", + "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", "status": "met", - "evidence": "app_export_scene_valid = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json", + "evidence": "app_export_scene_valid = true (want True) at app_sha a39d0a50: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json", "links": [] }, { "id": "S3-9", "text": "Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design", "ref": "origin/t3/ios-mvf", - "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", + "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -589,53 +605,53 @@ "title": "Verification", "refs": [ { - "ref": "origin/t3/verification", - "sha": "5e60831dcc87ca0b854b4ce51c1e8b220438fc7a" + "ref": "origin/t3/ios-mvf", + "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6" }, { - "ref": "origin/t3/ios-mvf", - "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b" + "ref": "origin/t3/verification", + "sha": "30e866c6d0c835c025dfa050d339d7b6ff350e69" } ], "metrics": [ { "id": "S4-1", "text": "End to end from a real replay to a placement result passes", - "ref": "origin/t3/verification", - "sha": "5e60831dcc87ca0b854b4ce51c1e8b220438fc7a", - "status": "met", - "evidence": "app_export_passed = true (want True): ~/house-scanning-data/reports/e2e/20260926-050517-26d28707/report.json", + "ref": "origin/t3/ios-mvf", + "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", + "status": "gap", + "evidence": "app_export_passed = false (want True) at app_sha a39d0a50: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json", "links": [] }, { "id": "S4-2", "text": "Screenshot report per app state", "ref": "origin/t3/ios-mvf", - "sha": "beede15f568b3a4d694fb275caf9eaaa882c546b", + "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", "status": "met", - "evidence": "states = length 10 (want at least 9): ~/house-scanning-data/reports/sim/20260926-061551-t3-ios-mvf-beede15f-replay/report.json", + "evidence": "states = length 10 (want at least 9) at sha a39d0a50: ~/house-scanning-data/reports/sim/20260926-071459-t3-ios-mvf-a39d0a50-replay/report.json", "links": [] }, { "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "5e60831dcc87ca0b854b4ce51c1e8b220438fc7a", + "sha": "30e866c6d0c835c025dfa050d339d7b6ff350e69", "status": "met", - "evidence": "verification/product covers every screen of t3/ios-mvf at 525ea40, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state.", + "evidence": "verification/product covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state.", "links": [] }, { "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "5e60831dcc87ca0b854b4ce51c1e8b220438fc7a", + "sha": "30e866c6d0c835c025dfa050d339d7b6ff350e69", "status": "met", - "evidence": "present at 5e60831d", + "evidence": "present at 30e866c6", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/5e60831dcc87ca0b854b4ce51c1e8b220438fc7a/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/30e866c6d0c835c025dfa050d339d7b6ff350e69/verification/SCOREBOARD.md" ] ] } diff --git a/verification/ux/review.md b/verification/ux/review.md index 6c8c2c8bb..bf9eaa6a6 100644 --- a/verification/ux/review.md +++ b/verification/ux/review.md @@ -7,8 +7,10 @@ audit reports stay in `~/house-scanning-data/reports/` (replay frames are non-co | Run | What it shows | | --- | --- | +| `sim/20260926-071459-t3-ios-mvf-a39d0a50-replay` | Every state at `a39d0a5`, the current head, with the real server's result; the gap request appears and is answered "I can't get there" | +| `a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample` | Apple's audit at `a39d0a5` on 21 screens: 10 issues, down from 32 at `525ea40` | | `sim/20260926-051323-t3-ios-mvf-24af4342-replay` | Every state at `t3/ios-mvf` `24af434` with the real server's result | -| `sim/20260926-052535-t3-ios-mvf-525ea401-replay` | Every state at `525ea40`, the current head, with the real server's result and the end question answered | +| `sim/20260926-052535-t3-ios-mvf-525ea401-replay` | Every state at `525ea40` with the real server's result and the end question answered | | `a11y/20260926-052756-t3-ios-mvf-525ea401-head-sample` | Apple's audit at `525ea40` on 21 screens, including the end question and the sample wording | | `a11y/20260926-051535-t3-ios-mvf-24af4342-head-sample` | Apple's audit at `24af434` on 17 screens; the gap loop closes on the replay | | `sim/20260926-050058-t3-ios-mvf-0876e03c-replay` | Every state at `0876e03` | @@ -65,13 +67,15 @@ amber; in grayscale both are the same light gray, and the strip has no legend or homeowner with red-green colour blindness cannot tell done from not done. Suggested: a pattern or height difference per state, and "Wall" and "Ground" row labels. -**UX-03. Minor. Contrast over the camera (A5).** The audit fails contrast on the photo counter -during the walk, the close-up's problem pill and, at `24af434`, the walk's "Can't get there" -reply. Failures it reported on text mid-transition (the upload steps, the result headline as it -fades in) are left out. +**UX-03. Minor. Contrast over the camera (A5).** At `a39d0a5` the audit still fails contrast on +the photo counter during the walk; the close-up's problem pill and the walk's "Can't get there" +reply, which failed at `24af434` and `525ea40`, now pass. Failures on text mid-transition (the +result headline and the sample note as they fade in) are left out. -**UX-04. Minor. Text that does not scale (A4).** The audit reports elements whose font size cannot -change on the upload screen (four) and, at `525ea40`, on the close-up (four) and the walk (two). +**UX-04. Minor. Text that does not scale or clips (A4).** At `a39d0a5` one element's font size +cannot change ("Mark something" on the walk), down from ten at `525ea40`; the strip's marks now +scale. Text may clip at large sizes on the close-up, the mark prompt ("Now tap its top-right +corner") and the end question ("What's at the right end?" and its explanation). **UX-05. Minor. Two names for one action (V8).** The walk's reply is "Can't get there"; the gap request's is "I can't get there". @@ -83,9 +87,9 @@ marks "seen but not enough" cells on the same strip. ("Checking your wall") was still showing 1.2 s after the app logged the result; after returning from AR the result appeared at once. The 3D model's first load is the likely cause. -**UX-08. Minor. Text VoiceOver cannot read (A2).** At `525ea40` the audit flags text on the walk -that is drawn rather than exposed to accessibility, probably labels drawn into the camera overlay -or the strip. +**UX-08. Minor, not seen at `a39d0a5`. Text VoiceOver cannot read (A2).** At `525ea40` the audit +flagged text on the walk that is drawn rather than exposed to accessibility; the audit at +`a39d0a5` does not report it. ## Fixed since the preview (`0876e03`) From 683a386844392145ce2d838c77c6f76a6fb1b770 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 07:21:38 -0500 Subject: [PATCH 33/59] verification: read ru_maxrss in kilobytes on Linux CI runs on Linux, where ru_maxrss is in kilobytes, so peak memory read 1024 times too small there. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/hsverify/memory.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/verification/hsverify/memory.py b/verification/hsverify/memory.py index 7ba411f8c..8fcec4ec9 100644 --- a/verification/hsverify/memory.py +++ b/verification/hsverify/memory.py @@ -1,7 +1,7 @@ """Peak memory of this harness process and of the child processes it waited for. The Mac is shared by several agents and one oversized process froze it once, so every report -records these two numbers. macOS reports `ru_maxrss` in bytes. +records these two numbers. `ru_maxrss` is in bytes on macOS and in kilobytes on Linux (CI). """ from __future__ import annotations @@ -10,12 +10,15 @@ import resource import signal import subprocess +import sys import threading +MAXRSS_UNIT = 1 if sys.platform == "darwin" else 1024 + def peak_rss_mb() -> dict[str, float]: - own = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - children = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + own = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * MAXRSS_UNIT + children = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss * MAXRSS_UNIT return {"harness_mb": round(own / 2**20, 1), "largest_child_mb": round(children / 2**20, 1)} From eb903b7074f330b89802e58dcb9aef64d42e018c Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 07:34:14 -0500 Subject: [PATCH 34/59] verification: friction audit of the homeowner's path at t3/ios-mvf a39d0a5 Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/README.md | 3 +- verification/ux/friction.md | 94 +++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 verification/ux/friction.md diff --git a/verification/README.md b/verification/README.md index 01a0f2863..a1184188d 100644 --- a/verification/README.md +++ b/verification/README.md @@ -15,7 +15,8 @@ datasets. Requirements: uv, and Xcode 26 or newer with an iOS Simulator runtime. | These tools' own tests | `make test` | pass or fail | | The Simulator runner against a probe app with a known sequence | `make sim-probe` | `reports/sim//` | -Written results: [the UX review](ux/review.md) against [the checklist](ux/checklist.md), and +Written results: [the UX review](ux/review.md) against [the checklist](ux/checklist.md), [the +friction audit](ux/friction.md) of the homeowner's path, and [the product description](product/README.md) with its [bug triage](product/bug-triage.md). ## What the checks guarantee diff --git a/verification/ux/friction.md b/verification/ux/friction.md new file mode 100644 index 000000000..60cfe30f5 --- /dev/null +++ b/verification/ux/friction.md @@ -0,0 +1,94 @@ +# Friction on the homeowner's path + +Where a first-time homeowner can stall, redo something or have to decide, and what would most +raise the share of scans that finish on the first try. Written for the owner of `t3/ios-mvf`, at +`a39d0a5`. Evidence: + +- the Simulator run `sim/20260926-071459-t3-ios-mvf-a39d0a50-replay` on the ADVIO replay; +- the server's answer to that run's upload, `e2e/20260926-071717-739fb6f7` (server `739fb6f`); +- the source at `a39d0a5`. `Kit/` below is `ios/HouseScanKit/Sources/HouseScanKit/`; other paths are under `ios/HouseScan/`. + +## The run in numbers + +The replay is 34.7 s of walking along a 39.7 m path, 79 frames, with tracking normal throughout; +the app turned it into 26 ft of wall (s −11 to +15 ft). With the autopilot holding each step 3 s, +launch to result took 44 s: + +| Stage | Seconds | +| --- | --- | +| Onboarding | 4.6 | +| Find the meter | 3.2 | +| Meter close-up | 4.0, skipped after 2 failed tries (the replay has no meter) | +| Wall walk | 24.7 | +| Feature list | 3.2 | +| Gap request | 3.4, answered "I can't get there" | +| Upload | 0.7 | + +Without failures, reaching the result takes 10 taps plus iOS's camera dialog: "Next", "Next", +"Allow camera", "This is my meter", "Wall ends here" and its end answer on each side, "Done with +this wall", "Looks complete". Five of them are decisions: where the meter is, where each end is, +and what is at each end. Seeing the spot in AR adds "See it on your wall" and "Done". + +The first try did not finish. The result said "An installer will take a look … 8 checks depend on +areas the scan did not see". The server asked for four more views (ground, wall, facing, overhead) +and for walking past both ends. + +## Where a homeowner can stall, in flow order + +| Stage | Trigger | What they see | Way forward | Kind | +| --- | --- | --- | --- | --- | +| Find meter | Camera denied, camera failure, unreadable replay (`Runtime/ScanEngine.swift:119, 489, 491` set the failure without changing screen) | "Find your electric meter" with no camera; taps do nothing | None; close the app | Stuck (B-03) | +| Find meter | Tap refused: tracking not normal, or no wall under the tap | "Step a little closer to the wall" for both causes | Tap again | Redo | +| Find meter to gap request | Relocalizing for over 20 s (`Runtime/ScanEngine.swift:466-471`) | Back to "Find your electric meter"; wall, marks and photos gone, no explanation | Start the scan again | Redo | +| Close-up | One problem held for 4 s is a failed try; a photo needs 0.6 s of good frames (`Kit/Capture/CloseUpGate.swift:13, 15`) | One instruction and a named fix, "Can't get a clear shot" from the second failed try | Skip; if frames alternate good and blurry, no try fails and the skip never appears | Redo, can stick (B-04) | +| Walk | A side's end is closer than 6.1 m (20 ft); "Wall ends here" is offered only after that much is covered (`Kit/Guidance/GuidancePlanner.swift:27, 119`) | "Walk slowly to your left" with no way to say the wall ends | "Can't get there", which records the end as unexplored | Decision, recorded wrong (B-06) | +| Walk | After "Wall ends here" | "What's at the left end?": "It turns a corner" / "Something blocks it" | Answer; a corner exports as unexplored (`Runtime/ScanEngine+Actions.swift:85`) | Decision, twice | +| Walk | Ground not seen | "Tilt down to show the ground" | Tilt; on the replay ground was seen under 4 ft of the 26 ft | Redo | +| Walk | Marking a door, window, drive or fence takes two taps; each can be refused | Warning haptic and a reason | Tap again or "Cancel" | Redo | +| Feature list | A chip under "Add something" (`UI/Screens/MarkFeaturesScreen.swift:64`) | Nothing visible; the marking circle exists only on the walk | None; "Looks complete" uploads without the feature | Stuck (B-09) | +| Feature list | A marked window | "Does this window open?" | Answer | Decision | +| Gap request | The phone's own check finds a gap after "Looks complete" | "Show the ground around your meter", a progress bar | Fill 80 %, or "I can't get there", which the upload does not carry (B-08) | Redo | +| Upload | Any failure | "That didn't go through" and raw error text (B-02) | "Try again" resends everything | Redo | +| Result | A check depends on an area not seen: every scan at `a39d0a5` (below) | "An installer will take a look", "Still needed" cards with "Capture it now" | Capture and upload again; ground, facing and overhead requests cannot be settled | Redo, cannot finish | +| Result | An end within reach is unexplored | "Keep walking past the left end of the scan (11 ft 0 in left of the meter)" | Walk past it; the server then asks again for ground there (S2 finding, routed) | Redo | + +The autopilot presses buttons directly. It skips the close-up after 4 s and answers every end "It +turns a corner". It never taps "Add something", "Capture it now" or "Try again", and never +triggers a camera denial or the 20 s reset. Those rows come from the source and were not seen +running. + +## Three changes, ranked + +**1. Report how far out the ground was seen.** Every upload says ground was seen out to exactly +1.2 m (3.9 ft). `Runtime/ScanEngine+Export.swift:62` exports the fixed `groundBandDepth` from +`Kit/Coverage/CoverageMap.swift:36`. The clearance checks need ground out to about 6 ft for gas +and AC, 8 ft for a drive and 13 ft for a pool: the battery's 1.83 ft depth, plus the radius, plus +its position error. So these four checks are unsure on every scan, and no "Show the ground" +request can be met however far the homeowner points. This alone guarantees a re-request. + +The fix: record each stretch's out distance from where the camera actually saw the ground, and +have the walk's tilt instruction ask for the distance the rules need. The drive (5 ft) and pool +(10 ft) radii are placeholders in `server/rules.yaml`, so the rules owner can also shrink what +is asked; that is S2's call. + +**2. Report the space above and in front of the wall, or stop requiring it.** The upload lists +only wall and ground coverage (`Kit/Scene/SceneExport.swift:284-288`), so the facing-gap and +headroom checks are unsure on every scan. The phone's gap planner drops overhead and facing +requests (`Kit/Guidance/GapPlanner.swift:118`), so "Capture it now" cannot settle them either. +The walk already counts the wall face up to the 6.5 ft headroom line +(`Kit/Coverage/CoverageMap.swift:34`), so the frames that saw it may also have seen the space in +front. The fix needs one decision shared with S2: which walk frames count as seeing overhead and +facing. The app then exports those bands. + +**3. Let the walk follow a wall round its corners.** The app exports a single wall +(`Kit/Scene/SceneExport.swift:295`), and "It turns a corner" exports the end as unexplored. So +any corner within the 20 ft cable reach becomes a server request to keep walking past it: two of +the six requests on the replay. Separately, a wall ending closer than 20 ft can only be recorded +through "Can't get there" (row 5 above). + +The fix: offer "Wall ends here" whenever the end is in view (`Kit/Guidance/GuidancePlanner.swift:119`, +`UI/Screens/WallWalkScreen.swift`). After "It turns a corner", continue the walk onto the next +wall, which C1's wall chain already supports. + +The blockers B-03 and B-09 stop a homeowner outright, but they occur on uncommon paths. The three +changes above affect every scan. From 52aa87f79e8dad281e96a6ef51dfe791a48b7779 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 08:00:33 -0500 Subject: [PATCH 35/59] verification: recheck t3/server 779f0c3, hosted and 0bad4de; stale rows cite the run that measured the SHA Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/SCOREBOARD.md | 60 ++++---- verification/hsverify/scoreboard.py | 5 +- verification/scoreboard/scoreboard.json | 192 ++++++++++++------------ verification/tests/test_scoreboard.py | 11 ++ 4 files changed, 142 insertions(+), 126 deletions(-) diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index 535ee4f36..6f7272980 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,19 +1,19 @@ # Scoreboard -Generated 2026-09-26 07:19 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-26 08:00 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | -| [#17](https://github.com/SamGu-NRX/house-scanning/pull/17) | `t3/panel-label` | `2dc7a4f1` | yes | 2 success | 2026-09-26 07:02 CDT | +| [#17](https://github.com/SamGu-NRX/house-scanning/pull/17) | `t3/panel-label` | `c70ebf37` | yes | 3 success | 2026-09-26 07:35 CDT | | [#16](https://github.com/SamGu-NRX/house-scanning/pull/16) | `t3/meter-closeup` | `944cbe17` | yes | 3 success | 2026-09-26 06:45 CDT | | [#15](https://github.com/SamGu-NRX/house-scanning/pull/15) | `t3/web-review` | `a7c9b129` | yes | 3 success, 3 failure; failing: Label PR size, Web checks, Vercel | 2026-09-26 06:08 CDT | | [#14](https://github.com/SamGu-NRX/house-scanning/pull/14) | `t3/docs-cleanup` | `ab03d7fd` | no | 4 success | 2026-09-26 05:17 CDT | -| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `30e866c6` | yes | 2 success | 2026-09-26 06:28 CDT | -| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `d64be59f` | yes | 3 success | 2026-09-26 07:14 CDT | -| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `739fb6f7` | no | 3 success | 2026-09-26 06:35 CDT | -| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `a39d0a50` | no | 2 success, 1 pending | 2026-09-26 07:11 CDT | +| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `eb903b70` | yes | 3 success | 2026-09-26 07:34 CDT | +| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `24793c32` | yes | 3 success | 2026-09-26 07:30 CDT | +| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `779f0c3c` | no | 3 success | 2026-09-26 07:59 CDT | +| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `e8bdfb0e` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 07:46 CDT | | [#9](https://github.com/SamGu-NRX/house-scanning/pull/9) | `codex/capture-contract-probe` | `4b6c4309` | yes | 4 success | 2026-09-26 00:08 CDT | | [#8](https://github.com/SamGu-NRX/house-scanning/pull/8) | `t3/testflight` | `b896d59f` | no | 4 success | 2026-09-26 03:18 CDT | | [#7](https://github.com/SamGu-NRX/house-scanning/pull/7) | `t3/measure-lab` | `45770a3b` | no | 6 success | 2026-09-26 03:19 CDT | @@ -22,58 +22,58 @@ Generated 2026-09-26 07:19 CDT by `make scoreboard` (`uv run python -m hsverify. ## S1 Evals on real data -`origin/t3/evals` at [`d64be59f`](https://github.com/SamGu-NRX/house-scanning/tree/d64be59f7850fabaa86b0c08cf6a5003e66ab007). 2 met, 2 partial, 1 not run. +`origin/t3/evals` at [`24793c32`](https://github.com/SamGu-NRX/house-scanning/tree/24793c328850fba2018ea1f8d951c45ac30ac281). 2 met, 2 partial, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S1-1 | One command regenerates every number from a clean checkout | partial | make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac. | -| S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/README.md) | +| S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/README.md) | | S1-3 | Metric code tested on hand-computed cases | not run | slow; run with --slow | -| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/README.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/results/advio_drift.md), [experiments/evals/results/modern_arkit.md](https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/results/modern_arkit.md) | -| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/README.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/results/eth3d_visibility_sensitivity.md) | +| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/README.md), [experiments/evals/field/FIELD_SHEET.md](https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/field/FIELD_SHEET.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/results/advio_drift.md) | +| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/README.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/results/eth3d_visibility_sensitivity.md) | ## S2 Server and solver -`origin/t3/server` at [`739fb6f7`](https://github.com/SamGu-NRX/house-scanning/tree/739fb6f747294b7253eb7d8936315392a80d178a). 4 met, 1 partial, 1 gap, 2 not run. +`origin/t3/server` at [`779f0c3c`](https://github.com/SamGu-NRX/house-scanning/tree/779f0c3c7d2e460b9228444e42104f67d69650e6). 5 met, 1 partial, 2 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S2-1 | pytest green | not run | slow; run with --slow | -| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_golden_public.py) | -| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_properties.py) | -| S2-4 | Schemas published | met | present at 739fb6f7 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/schemas/result.schema.json) | -| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_schemas.py), [server/tests/test_api.py](https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_api.py) | -| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 364.2 (want < 1000) at sha 739fb6f7: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json | +| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_golden_public.py) | +| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_properties.py) | +| S2-4 | Schemas published | met | present at 779f0c3c [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/schemas/result.schema.json) | +| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_schemas.py), [server/tests/test_api.py](https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_api.py) | +| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 159.7 (want < 1000) at sha 779f0c3c: ~/house-scanning-data/reports/e2e/20260926-075623-779f0c3c/report.json | | S2-7 | ruff clean | not run | slow; run with --slow | -| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | gap | contract_problem_count = 12 (want 0) at sha 739fb6f7: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json | +| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | met | contract_problem_count = 0 (want 0) at sha 779f0c3c: ~/house-scanning-data/reports/e2e/20260926-075623-779f0c3c/report.json | ## S3 iOS capture app -`origin/t3/ios-mvf` at [`a39d0a50`](https://github.com/SamGu-NRX/house-scanning/tree/a39d0a503412d688dd151f628cdeebc00f8221b6). 4 met, 3 partial, 2 gap, 1 not run. +`origin/t3/ios-mvf` at [`e8bdfb0e`](https://github.com/SamGu-NRX/house-scanning/tree/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a). 2 met, 3 partial, 2 gap, 3 stale. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S3-1 | CI build green | not run | iOS build: pending at a39d0a50 [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/project.yml) | -| S3-3 | Zero build warnings | met | build.warnings = length 0 (want 0) at sha a39d0a50: ~/house-scanning-data/reports/sim/20260926-071459-t3-ios-mvf-a39d0a50-replay/report.json | -| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage; missing auto-capture, guidance [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift) | -| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | gap | issue_count = 10 (want 0) at sha a39d0a50: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | +| S3-1 | CI build green | gap | iOS build: failure at e8bdfb0e [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | +| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/project.yml) | +| S3-3 | Zero build warnings | stale | report for sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/sim/20260926-071459-t3-ios-mvf-a39d0a50-replay/report.json | +| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage; missing auto-capture, guidance [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift) | +| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | | S3-7 | Screenshots of each state in the PR | met | 18 of 9 matches in the PR body [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-8 | Exported scene.json validates against C1 | met | app_export_scene_valid = true (want True) at app_sha a39d0a50: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json | +| S3-8 | Exported scene.json validates against C1 | stale | report for app_sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/ios-mvf` at [`a39d0a50`](https://github.com/SamGu-NRX/house-scanning/tree/a39d0a503412d688dd151f628cdeebc00f8221b6), `origin/t3/verification` at [`30e866c6`](https://github.com/SamGu-NRX/house-scanning/tree/30e866c6d0c835c025dfa050d339d7b6ff350e69). 3 met, 1 gap. +`origin/t3/ios-mvf` at [`e8bdfb0e`](https://github.com/SamGu-NRX/house-scanning/tree/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a), `origin/t3/verification` at [`eb903b70`](https://github.com/SamGu-NRX/house-scanning/tree/eb903b7074f330b89802e58dcb9aef64d42e018c). 2 met, 2 stale. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S4-1 | End to end from a real replay to a placement result passes | gap | app_export_passed = false (want True) at app_sha a39d0a50: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json | -| S4-2 | Screenshot report per app state | met | states = length 10 (want at least 9) at sha a39d0a50: ~/house-scanning-data/reports/sim/20260926-071459-t3-ios-mvf-a39d0a50-replay/report.json | +| S4-1 | End to end from a real replay to a placement result passes | stale | report for app_sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json | +| S4-2 | Screenshot report per app state | stale | report for sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/sim/20260926-071459-t3-ios-mvf-a39d0a50-replay/report.json | | S4-3 | Product description written from the code and checked against screenshots | met | verification/product covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state. | -| S4-4 | Scoreboard of every PR | met | present at 30e866c6 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/30e866c6d0c835c025dfa050d339d7b6ff350e69/verification/SCOREBOARD.md) | +| S4-4 | Scoreboard of every PR | met | present at eb903b70 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/eb903b7074f330b89802e58dcb9aef64d42e018c/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/hsverify/scoreboard.py b/verification/hsverify/scoreboard.py index 5dff080a9..078c87c15 100644 --- a/verification/hsverify/scoreboard.py +++ b/verification/hsverify/scoreboard.py @@ -491,7 +491,10 @@ def measured_at(data: dict | None) -> str: (p, d) for p, d in reports if d is not None and _value_or_none(d, sha_key) == sha ] if not current: - newest, data = reports[0] + # Cite the newest run that measured this SHA at all: a later run that did not (an + # e2e run without an app export) says nothing about which commit was checked. + measured = [(p, d) for p, d in reports if d and _value_or_none(d, sha_key)] + newest, data = (measured or reports)[0] return Outcome( "stale", f"report for {measured_at(data)}, ref at {sha[:8]}: {short_home(newest)}", diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index d9b719424..83c74e149 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,24 +1,24 @@ { - "generated_at": "2026-09-26T07:19:11-05:00", + "generated_at": "2026-09-26T08:00:08-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": false, "prs": [ { "number": 17, - "title": "experiments: can the phone read the main panel's label?", + "title": "experiments: panel label reading \u2014 stopped, too few openly licensed photos", "branch": "t3/panel-label", - "head": "2dc7a4f1a53cf801156f09fd84e6a6e4d1c27baf", + "head": "c70ebf3798f588c9b8d63fd8955b377fbe465ca7", "draft": true, - "updated": "2026-09-26T12:02:53Z", + "updated": "2026-09-26T12:35:06Z", "ci": { "counts": { - "success": 2, + "success": 3, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "2 success" + "text": "3 success" } }, { @@ -83,27 +83,27 @@ "number": 13, "title": "verification: independent end-to-end checks (S4)", "branch": "t3/verification", - "head": "30e866c6d0c835c025dfa050d339d7b6ff350e69", + "head": "eb903b7074f330b89802e58dcb9aef64d42e018c", "draft": true, - "updated": "2026-09-26T11:28:43Z", + "updated": "2026-09-26T12:34:24Z", "ci": { "counts": { - "success": 2, + "success": 3, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "2 success" + "text": "3 success" } }, { "number": 12, "title": "feat(evals): real-data accuracy evals for ARKit drift and photo-based wall reconstruction", "branch": "t3/evals", - "head": "d64be59f7850fabaa86b0c08cf6a5003e66ab007", + "head": "24793c328850fba2018ea1f8d951c45ac30ac281", "draft": true, - "updated": "2026-09-26T12:14:06Z", + "updated": "2026-09-26T12:30:50Z", "ci": { "counts": { "success": 3, @@ -119,9 +119,9 @@ "number": 11, "title": "server: placement solver, rules and API (S2)", "branch": "t3/server", - "head": "739fb6f747294b7253eb7d8936315392a80d178a", + "head": "779f0c3c7d2e460b9228444e42104f67d69650e6", "draft": false, - "updated": "2026-09-26T11:35:35Z", + "updated": "2026-09-26T12:59:27Z", "ci": { "counts": { "success": 3, @@ -137,18 +137,20 @@ "number": 10, "title": "ios: guided wall scan from the meter to a battery spot", "branch": "t3/ios-mvf", - "head": "a39d0a503412d688dd151f628cdeebc00f8221b6", + "head": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", "draft": false, - "updated": "2026-09-26T12:11:15Z", + "updated": "2026-09-26T12:46:16Z", "ci": { "counts": { "success": 2, - "failure": 0, - "pending": 1, + "failure": 1, + "pending": 0, "skipped": 0 }, - "failing": [], - "text": "2 success, 1 pending" + "failing": [ + "iOS build" + ], + "text": "2 success, 1 failure; failing: iOS build" } }, { @@ -250,7 +252,7 @@ "refs": [ { "ref": "origin/t3/evals", - "sha": "d64be59f7850fabaa86b0c08cf6a5003e66ab007" + "sha": "24793c328850fba2018ea1f8d951c45ac30ac281" } ], "metrics": [ @@ -258,7 +260,7 @@ "id": "S1-1", "text": "One command regenerates every number from a clean checkout", "ref": "origin/t3/evals", - "sha": "d64be59f7850fabaa86b0c08cf6a5003e66ab007", + "sha": "24793c328850fba2018ea1f8d951c45ac30ac281", "status": "partial", "evidence": "make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac.", "links": [] @@ -267,13 +269,13 @@ "id": "S1-2", "text": "Dataset URLs and sha256 recorded", "ref": "origin/t3/evals", - "sha": "d64be59f7850fabaa86b0c08cf6a5003e66ab007", + "sha": "24793c328850fba2018ea1f8d951c45ac30ac281", "status": "met", "evidence": "all 2 found", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/README.md" ] ] }, @@ -281,7 +283,7 @@ "id": "S1-3", "text": "Metric code tested on hand-computed cases", "ref": "origin/t3/evals", - "sha": "d64be59f7850fabaa86b0c08cf6a5003e66ab007", + "sha": "24793c328850fba2018ea1f8d951c45ac30ac281", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -290,21 +292,21 @@ "id": "S1-4", "text": "ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft", "ref": "origin/t3/evals", - "sha": "d64be59f7850fabaa86b0c08cf6a5003e66ab007", + "sha": "24793c328850fba2018ea1f8d951c45ac30ac281", "status": "met", "evidence": "all 5 found", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/README.md" ], [ - "experiments/evals/results/advio_drift.md", - "https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/results/advio_drift.md" + "experiments/evals/field/FIELD_SHEET.md", + "https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/field/FIELD_SHEET.md" ], [ - "experiments/evals/results/modern_arkit.md", - "https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/results/modern_arkit.md" + "experiments/evals/results/advio_drift.md", + "https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/results/advio_drift.md" ] ] }, @@ -312,21 +314,21 @@ "id": "S1-5", "text": "ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question", "ref": "origin/t3/evals", - "sha": "d64be59f7850fabaa86b0c08cf6a5003e66ab007", + "sha": "24793c328850fba2018ea1f8d951c45ac30ac281", "status": "partial", "evidence": "found 1-3 m row, 3-10 m row, scale error; missing verdict", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/README.md" ], [ "experiments/evals/results/eth3d_recon.md", - "https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/results/eth3d_recon.md" + "https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/results/eth3d_recon.md" ], [ "experiments/evals/results/eth3d_visibility_sensitivity.md", - "https://github.com/SamGu-NRX/house-scanning/blob/d64be59f7850fabaa86b0c08cf6a5003e66ab007/experiments/evals/results/eth3d_visibility_sensitivity.md" + "https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/results/eth3d_visibility_sensitivity.md" ] ] } @@ -338,7 +340,7 @@ "refs": [ { "ref": "origin/t3/server", - "sha": "739fb6f747294b7253eb7d8936315392a80d178a" + "sha": "779f0c3c7d2e460b9228444e42104f67d69650e6" } ], "metrics": [ @@ -346,7 +348,7 @@ "id": "S2-1", "text": "pytest green", "ref": "origin/t3/server", - "sha": "739fb6f747294b7253eb7d8936315392a80d178a", + "sha": "779f0c3c7d2e460b9228444e42104f67d69650e6", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -355,13 +357,13 @@ "id": "S2-2", "text": "Every public golden test 01-14 of docs/research/t3-lane-c-review.md present", "ref": "origin/t3/server", - "sha": "739fb6f747294b7253eb7d8936315392a80d178a", + "sha": "779f0c3c7d2e460b9228444e42104f67d69650e6", "status": "met", "evidence": "all 14 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_golden_public.py" ] ] }, @@ -369,17 +371,17 @@ "id": "S2-3", "text": "Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail", "ref": "origin/t3/server", - "sha": "739fb6f747294b7253eb7d8936315392a80d178a", + "sha": "779f0c3c7d2e460b9228444e42104f67d69650e6", "status": "partial", "evidence": "found missing coverage, mirror; missing equality unsure, larger clearance", "links": [ [ "server/tests/test_coderabbit_review.py", - "https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_coderabbit_review.py" + "https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_coderabbit_review.py" ], [ "server/tests/test_properties.py", - "https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_properties.py" + "https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_properties.py" ] ] }, @@ -387,17 +389,17 @@ "id": "S2-4", "text": "Schemas published", "ref": "origin/t3/server", - "sha": "739fb6f747294b7253eb7d8936315392a80d178a", + "sha": "779f0c3c7d2e460b9228444e42104f67d69650e6", "status": "met", - "evidence": "present at 739fb6f7", + "evidence": "present at 779f0c3c", "links": [ [ "server/schemas/scene.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/schemas/scene.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/schemas/scene.schema.json" ], [ "server/schemas/result.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/schemas/result.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/schemas/result.schema.json" ] ] }, @@ -405,21 +407,21 @@ "id": "S2-5", "text": "Schemas validated in tests", "ref": "origin/t3/server", - "sha": "739fb6f747294b7253eb7d8936315392a80d178a", + "sha": "779f0c3c7d2e460b9228444e42104f67d69650e6", "status": "met", "evidence": "all 2 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_golden_public.py" ], [ "server/tests/test_schemas.py", - "https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_schemas.py" + "https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_schemas.py" ], [ "server/tests/test_api.py", - "https://github.com/SamGu-NRX/house-scanning/blob/739fb6f747294b7253eb7d8936315392a80d178a/server/tests/test_api.py" + "https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_api.py" ] ] }, @@ -427,16 +429,16 @@ "id": "S2-6", "text": "A real-derived scene returns a result in under 1 s", "ref": "origin/t3/server", - "sha": "739fb6f747294b7253eb7d8936315392a80d178a", + "sha": "779f0c3c7d2e460b9228444e42104f67d69650e6", "status": "met", - "evidence": "latency_ms = 364.2 (want < 1000) at sha 739fb6f7: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json", + "evidence": "latency_ms = 159.7 (want < 1000) at sha 779f0c3c: ~/house-scanning-data/reports/e2e/20260926-075623-779f0c3c/report.json", "links": [] }, { "id": "S2-7", "text": "ruff clean", "ref": "origin/t3/server", - "sha": "739fb6f747294b7253eb7d8936315392a80d178a", + "sha": "779f0c3c7d2e460b9228444e42104f67d69650e6", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -445,9 +447,9 @@ "id": "S2-8", "text": "Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time", "ref": "origin/t3/server", - "sha": "739fb6f747294b7253eb7d8936315392a80d178a", - "status": "gap", - "evidence": "contract_problem_count = 12 (want 0) at sha 739fb6f7: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json", + "sha": "779f0c3c7d2e460b9228444e42104f67d69650e6", + "status": "met", + "evidence": "contract_problem_count = 0 (want 0) at sha 779f0c3c: ~/house-scanning-data/reports/e2e/20260926-075623-779f0c3c/report.json", "links": [] } ] @@ -458,7 +460,7 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6" + "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a" } ], "metrics": [ @@ -466,9 +468,9 @@ "id": "S3-1", "text": "CI build green", "ref": "origin/t3/ios-mvf", - "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", - "status": "not run", - "evidence": "iOS build: pending at a39d0a50", + "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", + "status": "gap", + "evidence": "iOS build: failure at e8bdfb0e", "links": [ [ "#10", @@ -480,13 +482,13 @@ "id": "S3-2", "text": "Built as Swift 6 with strict concurrency", "ref": "origin/t3/ios-mvf", - "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", + "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", "status": "met", "evidence": "all 2 found", "links": [ [ "ios/project.yml", - "https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/project.yml" + "https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/project.yml" ] ] }, @@ -494,30 +496,30 @@ "id": "S3-3", "text": "Zero build warnings", "ref": "origin/t3/ios-mvf", - "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", - "status": "met", - "evidence": "build.warnings = length 0 (want 0) at sha a39d0a50: ~/house-scanning-data/reports/sim/20260926-071459-t3-ios-mvf-a39d0a50-replay/report.json", + "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", + "status": "stale", + "evidence": "report for sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/sim/20260926-071459-t3-ios-mvf-a39d0a50-replay/report.json", "links": [] }, { "id": "S3-4", "text": "Unit tests for coverage, auto-capture and guidance", "ref": "origin/t3/ios-mvf", - "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", + "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", "status": "partial", "evidence": "found coverage; missing auto-capture, guidance", "links": [ [ "ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift" ] ] }, @@ -525,17 +527,17 @@ "id": "S3-5", "text": "UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop", "ref": "origin/t3/ios-mvf", - "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", + "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", "status": "partial", "evidence": "all 3 found (the tests exist; passing is S3-1)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanUITests/FullFlowUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -543,17 +545,17 @@ "id": "S3-6", "text": "Accessibility audit on every screen", "ref": "origin/t3/ios-mvf", - "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", + "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", "status": "partial", "evidence": "3 matches in 2 files (an audit exists; which screens it covers needs a read)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanUITests/FullFlowUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/a39d0a503412d688dd151f628cdeebc00f8221b6/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -561,16 +563,16 @@ "id": "S3-6b", "text": "Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues", "ref": "origin/t3/ios-mvf", - "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", - "status": "gap", - "evidence": "issue_count = 10 (want 0) at sha a39d0a50: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", + "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", + "status": "stale", + "evidence": "report for sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", "links": [] }, { "id": "S3-7", "text": "Screenshots of each state in the PR", "ref": "origin/t3/ios-mvf", - "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", + "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", "status": "met", "evidence": "18 of 9 matches in the PR body", "links": [ @@ -584,16 +586,16 @@ "id": "S3-8", "text": "Exported scene.json validates against C1", "ref": "origin/t3/ios-mvf", - "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", - "status": "met", - "evidence": "app_export_scene_valid = true (want True) at app_sha a39d0a50: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json", + "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", + "status": "stale", + "evidence": "report for app_sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json", "links": [] }, { "id": "S3-9", "text": "Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design", "ref": "origin/t3/ios-mvf", - "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", + "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -606,11 +608,11 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6" + "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a" }, { "ref": "origin/t3/verification", - "sha": "30e866c6d0c835c025dfa050d339d7b6ff350e69" + "sha": "eb903b7074f330b89802e58dcb9aef64d42e018c" } ], "metrics": [ @@ -618,25 +620,25 @@ "id": "S4-1", "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/ios-mvf", - "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", - "status": "gap", - "evidence": "app_export_passed = false (want True) at app_sha a39d0a50: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json", + "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", + "status": "stale", + "evidence": "report for app_sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json", "links": [] }, { "id": "S4-2", "text": "Screenshot report per app state", "ref": "origin/t3/ios-mvf", - "sha": "a39d0a503412d688dd151f628cdeebc00f8221b6", - "status": "met", - "evidence": "states = length 10 (want at least 9) at sha a39d0a50: ~/house-scanning-data/reports/sim/20260926-071459-t3-ios-mvf-a39d0a50-replay/report.json", + "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", + "status": "stale", + "evidence": "report for sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/sim/20260926-071459-t3-ios-mvf-a39d0a50-replay/report.json", "links": [] }, { "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "30e866c6d0c835c025dfa050d339d7b6ff350e69", + "sha": "eb903b7074f330b89802e58dcb9aef64d42e018c", "status": "met", "evidence": "verification/product covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state.", "links": [] @@ -645,13 +647,13 @@ "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "30e866c6d0c835c025dfa050d339d7b6ff350e69", + "sha": "eb903b7074f330b89802e58dcb9aef64d42e018c", "status": "met", - "evidence": "present at 30e866c6", + "evidence": "present at eb903b70", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/30e866c6d0c835c025dfa050d339d7b6ff350e69/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/eb903b7074f330b89802e58dcb9aef64d42e018c/verification/SCOREBOARD.md" ] ] } diff --git a/verification/tests/test_scoreboard.py b/verification/tests/test_scoreboard.py index fd7d12a3e..5b433f6cd 100644 --- a/verification/tests/test_scoreboard.py +++ b/verification/tests/test_scoreboard.py @@ -456,6 +456,17 @@ def test_report_sha_key_missing_from_the_report_is_stale(tmp_path): ) +def test_stale_cites_the_newest_report_that_records_the_sha(tmp_path): + measured = write_report(tmp_path / "a" / "report.json", {"app_sha": OLD, "passed": True}, 100) + write_report(tmp_path / "b" / "report.json", {"sha": SHA, "passed": True}, 200) + m = report_metric( + tmp_path, key="passed", equals=True, require_current_sha=True, sha_key="app_sha" + ) + assert evaluate(m, SHA, ctx()) == Outcome( + "stale", f"report for app_sha bbbbbbbb, ref at aaaaaaaa: {measured}" + ) + + def test_report_sha_key_defaults_to_sha(): assert metric("report", glob="x", key="k", equals=1).probe["sha_key"] == "sha" From 29b7b7d1cf831e36d8f3783b44962be54bd8d9b3 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 12:28:39 -0500 Subject: [PATCH 36/59] verification: final app run on the hosted API at t3/ios-mvf 194f2eb The app now uploads scene.json alone as JSON, so the end-to-end check sends JSON too (a raw application/zip body only for the compressed-scene input), and make sim-app / make e2e take SERVER_URL for the hosted API. The runner reads the app's longer bundle log line and reports one it cannot read instead of silently skipping the export. Three cases ruled out a pass only because the policy never approved; the demo policy does. A ground request's out_ft now sets how far out it must be seen to be redundant. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/Makefile | 9 +- verification/README.md | 4 +- verification/SCOREBOARD.md | 69 ++-- verification/e2e/cases/README.md | 10 +- .../e2e/cases/d-reach-drift-left.json | 3 - .../e2e/cases/d-reach-drift-right.json | 3 - .../e2e/cases/g06-regulator-footprint.json | 3 - verification/e2e/cases/generate.py | 4 +- verification/hsverify/e2e.py | 26 +- verification/hsverify/resultcheck.py | 7 +- verification/hsverify/simrun.py | 13 +- verification/hsverify/statelog.py | 4 +- verification/scoreboard/scoreboard.json | 306 ++++++++++-------- verification/tests/test_e2e.py | 18 +- verification/tests/test_resultcheck.py | 9 + verification/tests/test_simrun.py | 8 + verification/tests/test_statelog.py | 7 +- 17 files changed, 304 insertions(+), 199 deletions(-) diff --git a/verification/Makefile b/verification/Makefile index 910827cdb..cbe19ce1c 100644 --- a/verification/Makefile +++ b/verification/Makefile @@ -3,6 +3,9 @@ REF ?= origin/t3/ios-mvf SERVER_REF ?= origin/t3/server +# Set SERVER_URL to use a running server (the hosted API) instead of starting one from SERVER_REF. +SERVER_URL ?= +SERVER = $(if $(SERVER_URL),--server-url $(SERVER_URL),--server-ref $(SERVER_REF)) REPLAY ?= $(HOME)/house-scanning-data/replays/advio-20-0040-0075 ARGS ?= @@ -19,17 +22,17 @@ sim: uv run python -m hsverify.simrun --ref $(REF) $(ARGS) # The app at REF on the real ADVIO replay (C3), driven by its autopilot, uploading to the server -# started from SERVER_REF: a screenshot per STATE in ~/house-scanning-data/reports/sim/. Fails +# started from SERVER_REF (or at SERVER_URL): a screenshot per STATE in ~/house-scanning-data/reports/sim/. Fails # unless the run reaches the result state; the scan bundle is copied into the report as scan.zip. sim-app: uv run python -m hsverify.simrun --ref $(REF) --replay $(REPLAY) --autopilot \ - --server-ref $(SERVER_REF) --extra-arg=-autopilotHold --extra-arg=3 \ + $(SERVER) --extra-arg=-autopilotHold --extra-arg=3 \ --idle 60 --timeout 900 --require result --label replay $(ARGS) # Every case in e2e/cases/ plus the ETH3D facade scene through the server's API at SERVER_REF; # each result is validated against server/schemas/result.schema.json at the same ref. e2e: - uv run python -m hsverify.e2e --server-ref $(SERVER_REF) --scene scenes/eth3d-facade/case.json $(ARGS) + uv run python -m hsverify.e2e $(SERVER) --scene scenes/eth3d-facade/case.json $(ARGS) # Self-test of the runner against a fixture app that logs a known STATE sequence. sim-probe: diff --git a/verification/README.md b/verification/README.md index a1184188d..79ed53725 100644 --- a/verification/README.md +++ b/verification/README.md @@ -7,8 +7,8 @@ datasets. Requirements: uv, and Xcode 26 or newer with an iOS Simulator runtime. | Check | Command, from `verification/` | Output | | --- | --- | --- | -| The app at `t3/ios-mvf` on the real ADVIO replay, uploading to a server started from `t3/server`; a screenshot per `STATE` | `make sim-app` (`REF=`, `SERVER_REF=`, `ARGS=` to vary) | `reports/sim//index.html` | -| Every case, the ETH3D facade scene and optional app exports through the server's API | `make e2e` (`ARGS="--app-export "`, `SERVER_REF=`, or `ARGS=--server-url `) | `reports/e2e//report.md` | +| The app at `t3/ios-mvf` on the real ADVIO replay, uploading to a server started from `t3/server`; a screenshot per `STATE` | `make sim-app` (`REF=`, `SERVER_REF=` or `SERVER_URL=` for the hosted API, `ARGS=`) | `reports/sim//index.html` | +| Every case, the ETH3D facade scene and optional app exports through the server's API | `make e2e` (`ARGS="--app-export "`, `SERVER_REF=` or `SERVER_URL=`) | `reports/e2e//report.md` | | Apple's accessibility audit on every screen the app reaches | `uv run python -m hsverify.a11yaudit --ref --replay --autopilot` | `reports/a11y//index.html` | | A replay session is well formed; its poses against ground truth | `uv run python -m hsverify.replaycheck ` | printed | | Every open PR and plan metric with its evidence | `make scoreboard` (`ARGS=--slow` also runs test suites) | `SCOREBOARD.md` | diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index 6f7272980..1b41e6e66 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,24 +1,27 @@ # Scoreboard -Generated 2026-09-26 08:00 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-26 12:28 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | -| [#17](https://github.com/SamGu-NRX/house-scanning/pull/17) | `t3/panel-label` | `c70ebf37` | yes | 3 success | 2026-09-26 07:35 CDT | -| [#16](https://github.com/SamGu-NRX/house-scanning/pull/16) | `t3/meter-closeup` | `944cbe17` | yes | 3 success | 2026-09-26 06:45 CDT | -| [#15](https://github.com/SamGu-NRX/house-scanning/pull/15) | `t3/web-review` | `a7c9b129` | yes | 3 success, 3 failure; failing: Label PR size, Web checks, Vercel | 2026-09-26 06:08 CDT | -| [#14](https://github.com/SamGu-NRX/house-scanning/pull/14) | `t3/docs-cleanup` | `ab03d7fd` | no | 4 success | 2026-09-26 05:17 CDT | -| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `eb903b70` | yes | 3 success | 2026-09-26 07:34 CDT | -| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `24793c32` | yes | 3 success | 2026-09-26 07:30 CDT | -| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `779f0c3c` | no | 3 success | 2026-09-26 07:59 CDT | -| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `e8bdfb0e` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 07:46 CDT | +| [#22](https://github.com/SamGu-NRX/house-scanning/pull/22) | `t3/packet` | `d5439cfe` | no | 3 success | 2026-09-26 12:09 CDT | +| [#21](https://github.com/SamGu-NRX/house-scanning/pull/21) | `t3/ios-map3d` | `66cdcbaa` | yes | 2 success, 1 pending | 2026-09-26 12:02 CDT | +| [#20](https://github.com/SamGu-NRX/house-scanning/pull/20) | `t3/recon` | `c962e092` | yes | 3 success | 2026-09-26 10:59 CDT | +| [#18](https://github.com/SamGu-NRX/house-scanning/pull/18) | `codex/research-findings` | `bd4a7d38` | no | 2 success | 2026-09-26 09:07 CDT | +| [#17](https://github.com/SamGu-NRX/house-scanning/pull/17) | `t3/panel-label` | `cf6e6e19` | no | 4 success | 2026-09-26 08:16 CDT | +| [#16](https://github.com/SamGu-NRX/house-scanning/pull/16) | `t3/meter-closeup` | `b199613b` | no | 4 success | 2026-09-26 08:16 CDT | +| [#15](https://github.com/SamGu-NRX/house-scanning/pull/15) | `t3/web-review` | `1ebc0f40` | yes | 3 success | 2026-09-26 10:32 CDT | +| [#14](https://github.com/SamGu-NRX/house-scanning/pull/14) | `t3/docs-cleanup` | `8afa3313` | no | 2 success | 2026-09-26 11:47 CDT | +| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `52aa87f7` | yes | 3 success | 2026-09-26 08:00 CDT | +| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `24793c32` | no | 4 success | 2026-09-26 08:14 CDT | +| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `68084b5f` | no | 2 success | 2026-09-26 12:08 CDT | +| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `194f2ebf` | no | 3 success | 2026-09-26 11:51 CDT | | [#9](https://github.com/SamGu-NRX/house-scanning/pull/9) | `codex/capture-contract-probe` | `4b6c4309` | yes | 4 success | 2026-09-26 00:08 CDT | -| [#8](https://github.com/SamGu-NRX/house-scanning/pull/8) | `t3/testflight` | `b896d59f` | no | 4 success | 2026-09-26 03:18 CDT | | [#7](https://github.com/SamGu-NRX/house-scanning/pull/7) | `t3/measure-lab` | `45770a3b` | no | 6 success | 2026-09-26 03:19 CDT | | [#6](https://github.com/SamGu-NRX/house-scanning/pull/6) | `t3/research` | `9737e3f0` | no | 4 success | 2026-09-26 05:13 CDT | -| [#4](https://github.com/SamGu-NRX/house-scanning/pull/4) | `t3/scoring-harness` | `77ce830d` | no | 5 success | 2026-09-26 03:17 CDT | +| [#4](https://github.com/SamGu-NRX/house-scanning/pull/4) | `t3/scoring-harness` | `77ce830d` | no | 5 success | 2026-09-26 10:10 CDT | ## S1 Evals on real data @@ -34,52 +37,52 @@ Generated 2026-09-26 08:00 CDT by `make scoreboard` (`uv run python -m hsverify. ## S2 Server and solver -`origin/t3/server` at [`779f0c3c`](https://github.com/SamGu-NRX/house-scanning/tree/779f0c3c7d2e460b9228444e42104f67d69650e6). 5 met, 1 partial, 2 not run. +`origin/t3/server` at [`68084b5f`](https://github.com/SamGu-NRX/house-scanning/tree/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f). 5 met, 1 partial, 2 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S2-1 | pytest green | not run | slow; run with --slow | -| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_golden_public.py) | -| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_properties.py) | -| S2-4 | Schemas published | met | present at 779f0c3c [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/schemas/result.schema.json) | -| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_schemas.py), [server/tests/test_api.py](https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_api.py) | -| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 159.7 (want < 1000) at sha 779f0c3c: ~/house-scanning-data/reports/e2e/20260926-075623-779f0c3c/report.json | +| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_golden_public.py) | +| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_properties.py) | +| S2-4 | Schemas published | met | present at 68084b5f [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/schemas/result.schema.json) | +| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_schemas.py), [server/tests/test_api.py](https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_api.py) | +| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 153.8 (want < 1000) at sha 68084b5f: ~/house-scanning-data/reports/e2e/20260926-122755-68084b5f/report.json | | S2-7 | ruff clean | not run | slow; run with --slow | -| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | met | contract_problem_count = 0 (want 0) at sha 779f0c3c: ~/house-scanning-data/reports/e2e/20260926-075623-779f0c3c/report.json | +| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | met | contract_problem_count = 0 (want 0) at sha 68084b5f: ~/house-scanning-data/reports/e2e/20260926-122755-68084b5f/report.json | ## S3 iOS capture app -`origin/t3/ios-mvf` at [`e8bdfb0e`](https://github.com/SamGu-NRX/house-scanning/tree/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a). 2 met, 3 partial, 2 gap, 3 stale. +`origin/t3/ios-mvf` at [`194f2ebf`](https://github.com/SamGu-NRX/house-scanning/tree/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b). 5 met, 3 partial, 1 gap, 1 stale. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S3-1 | CI build green | gap | iOS build: failure at e8bdfb0e [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/project.yml) | -| S3-3 | Zero build warnings | stale | report for sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/sim/20260926-071459-t3-ios-mvf-a39d0a50-replay/report.json | -| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage; missing auto-capture, guidance [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift) | -| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | +| S3-1 | CI build green | met | iOS build: success at 194f2ebf [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | +| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/project.yml) | +| S3-3 | Zero build warnings | met | build.warnings = length 0 (want 0) at sha 194f2ebf: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json | +| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage; missing auto-capture, guidance [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift) | +| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at 194f2ebf: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | | S3-7 | Screenshots of each state in the PR | met | 18 of 9 matches in the PR body [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-8 | Exported scene.json validates against C1 | stale | report for app_sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json | +| S3-8 | Exported scene.json validates against C1 | met | app_export_scene_valid = true (want True) at app_sha 194f2ebf: ~/house-scanning-data/reports/e2e/20260926-122755-68084b5f/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/ios-mvf` at [`e8bdfb0e`](https://github.com/SamGu-NRX/house-scanning/tree/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a), `origin/t3/verification` at [`eb903b70`](https://github.com/SamGu-NRX/house-scanning/tree/eb903b7074f330b89802e58dcb9aef64d42e018c). 2 met, 2 stale. +`origin/t3/ios-mvf` at [`194f2ebf`](https://github.com/SamGu-NRX/house-scanning/tree/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b), `origin/t3/verification` at [`52aa87f7`](https://github.com/SamGu-NRX/house-scanning/tree/52aa87f79e8dad281e96a6ef51dfe791a48b7779). 4 met. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S4-1 | End to end from a real replay to a placement result passes | stale | report for app_sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json | -| S4-2 | Screenshot report per app state | stale | report for sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/sim/20260926-071459-t3-ios-mvf-a39d0a50-replay/report.json | +| S4-1 | End to end from a real replay to a placement result passes | met | app_export_passed = true (want True) at app_sha 194f2ebf: ~/house-scanning-data/reports/e2e/20260926-122755-68084b5f/report.json | +| S4-2 | Screenshot report per app state | met | states = length 12 (want at least 9) at sha 194f2ebf: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json | | S4-3 | Product description written from the code and checked against screenshots | met | verification/product covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state. | -| S4-4 | Scoreboard of every PR | met | present at eb903b70 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/eb903b7074f330b89802e58dcb9aef64d42e018c/verification/SCOREBOARD.md) | +| S4-4 | Scoreboard of every PR | met | present at 52aa87f7 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/52aa87f79e8dad281e96a6ef51dfe791a48b7779/verification/SCOREBOARD.md) | ## M Maintenance -`origin/t3/docs-cleanup` at [`ab03d7fd`](https://github.com/SamGu-NRX/house-scanning/tree/ab03d7fd735884346f9182e3cb4cbf4aee60994a). 1 met, 1 gap. +`origin/t3/docs-cleanup` at [`8afa3313`](https://github.com/SamGu-NRX/house-scanning/tree/8afa331394deda5e15ad9a802361b7082312d638). 2 gap. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| M-1 | AGENTS.md shorter than on main | met | exit 0: AGENTS.md 39 lines, main 45 | +| M-1 | AGENTS.md shorter than on main | gap | exit 1: AGENTS.md 45 lines, main 45 | | M-2 | AGENTS.md accurate, every path and command real | gap | not yet reviewed | diff --git a/verification/e2e/cases/README.md b/verification/e2e/cases/README.md index c74b5f290..1b1d23d66 100644 --- a/verification/e2e/cases/README.md +++ b/verification/e2e/cases/README.md @@ -17,9 +17,11 @@ uv run python e2e/cases/generate.py ## Why the assertions avoid the decision -`rules.yaml` sets `policy.auto_approve: false`, so any would-be pass or reject becomes -`manual_review`. The cases therefore assert on the sweep, on single checks, on `spot` and on -`missing_evidence`. They assert a decision only where the scene alone forces it: +Whether a would-be pass or reject is decided or sent to a person depends on the policy, which +changes: `rules.yaml` set `policy.auto_approve: false` when the cases were written, and S2's +`cd0da8b` made the public policy `demo`, which decides automatically. The cases therefore assert +on the sweep, on single checks, on `spot` and on `missing_evidence`. They assert a decision only +where the scene alone forces it, under any policy: - An unsure spot or an unobserved area rules out `pass`. - An unexplored end rules out `reject`. So does a missing `coverage`, since its ends then default @@ -255,6 +257,6 @@ errors, corrected once `rules.yaml` was published: - 04: the golden gives w1 outward [0, −1] for a baseline running +x, which contradicts the schema's clockwise rule. Its pad also relies on a ground check whose passing surfaces are not published. The flush rule it tests is exercised by g03 and g12b. -- 11 (d): its expectation is a pass, which `auto_approve: false` can't produce. Its photo-request +- 11 (d): its expectation is a pass, which `auto_approve: false` could not produce when the cases were written. Its photo-request half duplicates g01. - 14: needs a pass. The result schema already requires every field it lists. diff --git a/verification/e2e/cases/d-reach-drift-left.json b/verification/e2e/cases/d-reach-drift-left.json index 674cb4611..9c3a02752 100644 --- a/verification/e2e/cases/d-reach-drift-left.json +++ b/verification/e2e/cases/d-reach-drift-left.json @@ -152,9 +152,6 @@ "outcome": "fail", "why": "25.5 - 5.09 = 20.41 > 20 (b = a + W)" } - ], - "decision_not": [ - "pass" ] } } diff --git a/verification/e2e/cases/d-reach-drift-right.json b/verification/e2e/cases/d-reach-drift-right.json index daafc01fc..9d5826816 100644 --- a/verification/e2e/cases/d-reach-drift-right.json +++ b/verification/e2e/cases/d-reach-drift-right.json @@ -152,9 +152,6 @@ "outcome": "fail", "why": "25.5 - 5.09 = 20.41 > 20" } - ], - "decision_not": [ - "pass" ] } } diff --git a/verification/e2e/cases/g06-regulator-footprint.json b/verification/e2e/cases/g06-regulator-footprint.json index 6c7ca5fa0..c302dc351 100644 --- a/verification/e2e/cases/g06-regulator-footprint.json +++ b/verification/e2e/cases/g06-regulator-footprint.json @@ -152,9 +152,6 @@ "gas": 3.0 }, "expect": { - "decision_not": [ - "pass" - ], "sweep_runs": [ { "wall_id": "w1", diff --git a/verification/e2e/cases/generate.py b/verification/e2e/cases/generate.py index d385284b9..e38bf87a6 100644 --- a/verification/e2e/cases/generate.py +++ b/verification/e2e/cases/generate.py @@ -272,7 +272,6 @@ def straight_scene(x0, x1, z=0.0, meter_x=0.0, objects=(), ground=None, cov=None straight_scene(-3, 16, objects=[gas("w1", (6.5, 7.5), spike)]) | {"facing": [facing("w1", (-3, 16), depth=12.0)]}, { - "decision_not": ["pass"], "sweep_runs": [ {"wall_id": "w1", "start_ft": [3, 9], "outcome": "fail", "failing_match": "gas"} ], @@ -913,7 +912,7 @@ def starts(points): "Straight wall with default (drifting) errors. The route to a start a right of the meter is " "a +/- (0.6 + 0.16(a + W)): pass below a = 12.058, fail above a = 25.016, unsure between.", drift_scene(-1, 29), - {"start_outcomes": starts(reach_points), "decision_not": ["pass"]}, + {"start_outcomes": starts(reach_points)}, {"route_length": MAX_ROUTE}, ) # Mirror: left of the meter the near edge is the battery's right edge b = a + W, route = -b, and @@ -926,7 +925,6 @@ def starts(points): drift_scene(-29, 1), { "start_outcomes": starts([(-s - W, o, why + " (b = a + W)") for s, o, why in reach_points]), - "decision_not": ["pass"], }, {"route_length": MAX_ROUTE}, ) diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index d2ae13f89..3348e455c 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -233,6 +233,7 @@ class Endpoint: path: str content_type: str # "application/json" or "multipart/form-data" file_field: str | None = None + accepts_zip: bool = False # also takes a raw application/zip body (a prebuilt bundle) def discover_endpoint(openapi: dict, override: str | None = None) -> Endpoint: @@ -261,14 +262,16 @@ def discover_endpoint(openapi: dict, override: str | None = None) -> Endpoint: elif len(candidates) > 1: named = [c for c in candidates if re.search(r"scene|place|solve|placement", c[0])] candidates = named if len(named) == 1 else candidates - # One path taking both a JSON body and an upload: use the upload, which is how the app sends - # scene.json with its keyframe JPEGs (C1). + # One path taking both a JSON body and an upload: use JSON, which is how the app sends + # scene.json since t3/ios-mvf 657ab28 (the hosted API refuses bodies over 4.5 MB). if len({c[0] for c in candidates}) == 1 and len(candidates) > 1: - candidates = [c for c in candidates if c[1] == "multipart/form-data"] or candidates[:1] + candidates = [c for c in candidates if c[1] == "application/json"] or candidates[:1] if len(candidates) != 1: listed = ", ".join(f"{p} ({k})" for p, k, _ in candidates) or "none" raise SystemExit(f"Cannot tell which endpoint takes a scene: {listed}. Pass --endpoint.") path, kind, schema = candidates[0] + body = openapi.get("paths", {}).get(path, {}).get("post", {}).get("requestBody", {}) + accepts_zip = "application/zip" in body.get("content", {}) file_field = None if kind == "multipart/form-data": props = resolve_ref(openapi, schema).get("properties", {}) @@ -281,7 +284,7 @@ def discover_endpoint(openapi: dict, override: str | None = None) -> Endpoint: if len(binary) != 1: raise SystemExit(f"{path}: expected one file field in the upload form, got {binary}") file_field = binary[0] - return Endpoint(path, kind, file_field) + return Endpoint(path, kind, file_field, accepts_zip) def resolve_ref(openapi: dict, schema: dict) -> dict: @@ -309,7 +312,9 @@ def post( url: str, endpoint: Endpoint, item: SceneInput, timeout_s: float = 60 ) -> tuple[int | None, bytes, float]: """Send one scene. A timeout or a dropped connection comes back as status None.""" - if endpoint.content_type == "application/json": + if item.bundle is not None and endpoint.accepts_zip: + body, ctype = item.bundle, "application/zip" + elif endpoint.content_type == "application/json": body, ctype = item.raw, "application/json" else: boundary = uuid.uuid4().hex @@ -587,8 +592,15 @@ def judge_hostile( """A refusal (400, 413, 422) or a valid result, either within the time budget and, for a server started here, the memory budget. A crash, a hang, a slow answer or a large allocation is a failure.""" - if item.bundle is not None and endpoint.content_type == "application/json": - return {"status": "skipped", "skip_reason": "the endpoint takes JSON, not a zip"} + if ( + item.bundle is not None + and endpoint.content_type == "application/json" + and not (endpoint.accepts_zip) + ): + return { + "status": "skipped", + "skip_reason": "the endpoint takes neither a zip nor an upload", + } before = limit.reset() if limit else 0.0 status, payload, ms = post(url, endpoint, item, timeout_s=HOSTILE_BUDGET_MS / 1000 + 5) record = {"http_status": status, "latency_ms": round(ms, 1)} diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 076ef6ed8..414a12fd9 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -308,9 +308,10 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: if request["kind"] != "band" or "span_ft" not in request or "band" not in request: continue a, b = sorted(request["span_ft"]) - # A ground request carries no distance out, so it may want deeper ground over a span - # already seen; only ground seen out to GROUND_FAR_FT makes it redundant. - far = GROUND_FAR_FT if request["band"] == "ground" else 0.0 + # A ground request may want deeper ground over a span already seen: it is redundant only + # where ground was seen as far out as it asks (out_ft, S2 55cb4b0), or, when it does not + # say, out to GROUND_FAR_FT. + far = request.get("out_ft", GROUND_FAR_FT) if request["band"] == "ground" else 0.0 if b - a > EPS and covers(observed(scene, request["band"], min_out_ft=far), a, b): problems.append( f"missing_evidence asks for {request['band']} {request['span_ft']}, " diff --git a/verification/hsverify/simrun.py b/verification/hsverify/simrun.py index 128d9f9f7..3e0dc39c6 100644 --- a/verification/hsverify/simrun.py +++ b/verification/hsverify/simrun.py @@ -306,6 +306,14 @@ def missing_required(required: list[str], states: list[ShotRecord]) -> list[str] return [f"required state {name} never appeared" for name in required if name not in seen] +def logged_message(line: str) -> str: + """The message of one ndjson log line, or "" for the banner and anything unreadable.""" + try: + return json.loads(line).get("eventMessage") or "" + except (json.JSONDecodeError, AttributeError): + return "" + + def copy_app_export(out: Path) -> tuple[str | None, list[str]]: """Copy the scan bundle the app last logged into `out` as scan.zip. @@ -313,8 +321,11 @@ def copy_app_export(out: Path) -> tuple[str | None, list[str]]: and no problem; a logged bundle that cannot be copied is a problem. """ log = out / "state.ndjson" - bundles = [b for line in log.open() if (b := parse_bundle_line(line))] if log.exists() else [] + lines = log.read_text().splitlines() if log.exists() else [] + bundles = [b for line in lines if (b := parse_bundle_line(line))] if not bundles: + if any(logged_message(line).startswith("bundle ") for line in lines): + return None, ["the app logged a bundle line this runner cannot read; see state.ndjson"] return None, [] path, keyframes = bundles[-1] try: diff --git a/verification/hsverify/statelog.py b/verification/hsverify/statelog.py index 08dce3ff3..0f2edcaef 100644 --- a/verification/hsverify/statelog.py +++ b/verification/hsverify/statelog.py @@ -19,7 +19,9 @@ ENGINE_CATEGORY = "engine" # The engine logs `bundle with keyframes` once it has written a scan bundle; the path # is on the Mac's disk, inside the Simulator's app container. -_BUNDLE_RE = re.compile(r"^bundle (.+) with (\d+) keyframes$") +# "bundle .zip with N keyframes", optionally followed by ",
    " (app 194f2eb adds +# depth and mesh counts). +_BUNDLE_RE = re.compile(r"^bundle (.+\.zip) with (\d+) keyframes(?:,.*)?$") # A state name is an identifier-like token. Anything after it on the line is detail. _STATE_RE = re.compile(r"STATE=([A-Za-z0-9_.\-]+)") diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index 83c74e149..d69e4ee88 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,15 +1,15 @@ { - "generated_at": "2026-09-26T08:00:08-05:00", + "generated_at": "2026-09-26T12:28:23-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": false, "prs": [ { - "number": 17, - "title": "experiments: panel label reading \u2014 stopped, too few openly licensed photos", - "branch": "t3/panel-label", - "head": "c70ebf3798f588c9b8d63fd8955b377fbe465ca7", - "draft": true, - "updated": "2026-09-26T12:35:06Z", + "number": 22, + "title": "packet: capture packet 1.1 \u2014 spec, validator and samples", + "branch": "t3/packet", + "head": "d5439cfe5b6334f439f0518d2de2992cfe27538c", + "draft": false, + "updated": "2026-09-26T17:09:43Z", "ci": { "counts": { "success": 3, @@ -22,12 +22,30 @@ } }, { - "number": 16, - "title": "experiments: meter close-up reading, number finding and retake checks", - "branch": "t3/meter-closeup", - "head": "944cbe176c684f45ac2e1caccda2bc4b15737e19", + "number": 21, + "title": "ios: live on-device 3D map with occlusion-honest coverage", + "branch": "t3/ios-map3d", + "head": "66cdcbaaca5de3ca4812e7073fd5490b477713a2", "draft": true, - "updated": "2026-09-26T11:45:06Z", + "updated": "2026-09-26T17:02:20Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 1, + "skipped": 0 + }, + "failing": [], + "text": "2 success, 1 pending" + } + }, + { + "number": 20, + "title": "recon: handoff to the server team", + "branch": "t3/recon", + "head": "c962e0929dc29bb890c2c6fb23a359ff5e7f5108", + "draft": true, + "updated": "2026-09-26T15:59:37Z", "ci": { "counts": { "success": 3, @@ -39,53 +57,103 @@ "text": "3 success" } }, + { + "number": 18, + "title": "Research handoff: live survey findings, options and tested prototypes", + "branch": "codex/research-findings", + "head": "bd4a7d387606d8dedb679e2962accb5a5a48a4b2", + "draft": false, + "updated": "2026-09-26T14:07:43Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "2 success" + } + }, + { + "number": 17, + "title": "experiments: panel label reading stopped at the gate; field-day photo list", + "branch": "t3/panel-label", + "head": "cf6e6e1924274523acb9116c354c58b61d8a0826", + "draft": false, + "updated": "2026-09-26T13:16:12Z", + "ci": { + "counts": { + "success": 4, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "4 success" + } + }, + { + "number": 16, + "title": "experiments: meter close-up reading, tap-to-confirm and two retake checks", + "branch": "t3/meter-closeup", + "head": "b199613b439f3d399e34df55df1b9ab4c371b24a", + "draft": false, + "updated": "2026-09-26T13:16:09Z", + "ci": { + "counts": { + "success": 4, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "4 success" + } + }, { "number": 15, "title": "web: placement review page", "branch": "t3/web-review", - "head": "a7c9b129a52d46a87a9542e665dffcff0fc4d167", + "head": "1ebc0f40b4f53b8429f0049feb7aff183ec1768d", "draft": true, - "updated": "2026-09-26T11:08:48Z", + "updated": "2026-09-26T15:32:04Z", "ci": { "counts": { "success": 3, - "failure": 3, + "failure": 0, "pending": 0, "skipped": 0 }, - "failing": [ - "Label PR size", - "Web checks", - "Vercel" - ], - "text": "3 success, 3 failure; failing: Label PR size, Web checks, Vercel" + "failing": [], + "text": "3 success" } }, { "number": 14, - "title": "docs: trim AGENTS.md and fix stale decisions in the team docs", + "title": "docs: rebuild onboarding around the client and server teams", "branch": "t3/docs-cleanup", - "head": "ab03d7fd735884346f9182e3cb4cbf4aee60994a", + "head": "8afa331394deda5e15ad9a802361b7082312d638", "draft": false, - "updated": "2026-09-26T10:17:43Z", + "updated": "2026-09-26T16:47:37Z", "ci": { "counts": { - "success": 4, + "success": 2, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "4 success" + "text": "2 success" } }, { "number": 13, "title": "verification: independent end-to-end checks (S4)", "branch": "t3/verification", - "head": "eb903b7074f330b89802e58dcb9aef64d42e018c", + "head": "52aa87f79e8dad281e96a6ef51dfe791a48b7779", "draft": true, - "updated": "2026-09-26T12:34:24Z", + "updated": "2026-09-26T13:00:35Z", "ci": { "counts": { "success": 3, @@ -99,58 +167,56 @@ }, { "number": 12, - "title": "feat(evals): real-data accuracy evals for ARKit drift and photo-based wall reconstruction", + "title": "feat(evals): real-data evals for AR drift, wall reconstruction, coverage and the field test", "branch": "t3/evals", "head": "24793c328850fba2018ea1f8d951c45ac30ac281", - "draft": true, - "updated": "2026-09-26T12:30:50Z", + "draft": false, + "updated": "2026-09-26T13:14:09Z", "ci": { "counts": { - "success": 3, + "success": 4, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "3 success" + "text": "4 success" } }, { "number": 11, "title": "server: placement solver, rules and API (S2)", "branch": "t3/server", - "head": "779f0c3c7d2e460b9228444e42104f67d69650e6", + "head": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f", "draft": false, - "updated": "2026-09-26T12:59:27Z", + "updated": "2026-09-26T17:08:40Z", "ci": { "counts": { - "success": 3, + "success": 2, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "3 success" + "text": "2 success" } }, { "number": 10, "title": "ios: guided wall scan from the meter to a battery spot", "branch": "t3/ios-mvf", - "head": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", + "head": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", "draft": false, - "updated": "2026-09-26T12:46:16Z", + "updated": "2026-09-26T16:51:33Z", "ci": { "counts": { - "success": 2, - "failure": 1, + "success": 3, + "failure": 0, "pending": 0, "skipped": 0 }, - "failing": [ - "iOS build" - ], - "text": "2 success, 1 failure; failing: iOS build" + "failing": [], + "text": "3 success" } }, { @@ -171,24 +237,6 @@ "text": "4 success" } }, - { - "number": 8, - "title": "ci: add manual TestFlight upload workflow", - "branch": "t3/testflight", - "head": "b896d59fb564321df5fcdea1a5b50b45d27bc7a1", - "draft": false, - "updated": "2026-09-26T08:18:47Z", - "ci": { - "counts": { - "success": 4, - "failure": 0, - "pending": 0, - "skipped": 0 - }, - "failing": [], - "text": "4 success" - } - }, { "number": 7, "title": "experiments: add measure-lab, a no-LiDAR measurement rig", @@ -231,7 +279,7 @@ "branch": "t3/scoring-harness", "head": "77ce830dd08acf0c7982edcec91cd3240e0565ea", "draft": false, - "updated": "2026-09-26T08:17:23Z", + "updated": "2026-09-26T15:10:23Z", "ci": { "counts": { "success": 5, @@ -340,7 +388,7 @@ "refs": [ { "ref": "origin/t3/server", - "sha": "779f0c3c7d2e460b9228444e42104f67d69650e6" + "sha": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f" } ], "metrics": [ @@ -348,7 +396,7 @@ "id": "S2-1", "text": "pytest green", "ref": "origin/t3/server", - "sha": "779f0c3c7d2e460b9228444e42104f67d69650e6", + "sha": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -357,13 +405,13 @@ "id": "S2-2", "text": "Every public golden test 01-14 of docs/research/t3-lane-c-review.md present", "ref": "origin/t3/server", - "sha": "779f0c3c7d2e460b9228444e42104f67d69650e6", + "sha": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f", "status": "met", "evidence": "all 14 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_golden_public.py" ] ] }, @@ -371,17 +419,17 @@ "id": "S2-3", "text": "Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail", "ref": "origin/t3/server", - "sha": "779f0c3c7d2e460b9228444e42104f67d69650e6", + "sha": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f", "status": "partial", "evidence": "found missing coverage, mirror; missing equality unsure, larger clearance", "links": [ [ "server/tests/test_coderabbit_review.py", - "https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_coderabbit_review.py" + "https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_coderabbit_review.py" ], [ "server/tests/test_properties.py", - "https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_properties.py" + "https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_properties.py" ] ] }, @@ -389,17 +437,17 @@ "id": "S2-4", "text": "Schemas published", "ref": "origin/t3/server", - "sha": "779f0c3c7d2e460b9228444e42104f67d69650e6", + "sha": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f", "status": "met", - "evidence": "present at 779f0c3c", + "evidence": "present at 68084b5f", "links": [ [ "server/schemas/scene.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/schemas/scene.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/schemas/scene.schema.json" ], [ "server/schemas/result.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/schemas/result.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/schemas/result.schema.json" ] ] }, @@ -407,21 +455,21 @@ "id": "S2-5", "text": "Schemas validated in tests", "ref": "origin/t3/server", - "sha": "779f0c3c7d2e460b9228444e42104f67d69650e6", + "sha": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f", "status": "met", "evidence": "all 2 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_golden_public.py" ], [ "server/tests/test_schemas.py", - "https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_schemas.py" + "https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_schemas.py" ], [ "server/tests/test_api.py", - "https://github.com/SamGu-NRX/house-scanning/blob/779f0c3c7d2e460b9228444e42104f67d69650e6/server/tests/test_api.py" + "https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_api.py" ] ] }, @@ -429,16 +477,16 @@ "id": "S2-6", "text": "A real-derived scene returns a result in under 1 s", "ref": "origin/t3/server", - "sha": "779f0c3c7d2e460b9228444e42104f67d69650e6", + "sha": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f", "status": "met", - "evidence": "latency_ms = 159.7 (want < 1000) at sha 779f0c3c: ~/house-scanning-data/reports/e2e/20260926-075623-779f0c3c/report.json", + "evidence": "latency_ms = 153.8 (want < 1000) at sha 68084b5f: ~/house-scanning-data/reports/e2e/20260926-122755-68084b5f/report.json", "links": [] }, { "id": "S2-7", "text": "ruff clean", "ref": "origin/t3/server", - "sha": "779f0c3c7d2e460b9228444e42104f67d69650e6", + "sha": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -447,9 +495,9 @@ "id": "S2-8", "text": "Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time", "ref": "origin/t3/server", - "sha": "779f0c3c7d2e460b9228444e42104f67d69650e6", + "sha": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f", "status": "met", - "evidence": "contract_problem_count = 0 (want 0) at sha 779f0c3c: ~/house-scanning-data/reports/e2e/20260926-075623-779f0c3c/report.json", + "evidence": "contract_problem_count = 0 (want 0) at sha 68084b5f: ~/house-scanning-data/reports/e2e/20260926-122755-68084b5f/report.json", "links": [] } ] @@ -460,7 +508,7 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a" + "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b" } ], "metrics": [ @@ -468,9 +516,9 @@ "id": "S3-1", "text": "CI build green", "ref": "origin/t3/ios-mvf", - "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", - "status": "gap", - "evidence": "iOS build: failure at e8bdfb0e", + "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", + "status": "met", + "evidence": "iOS build: success at 194f2ebf", "links": [ [ "#10", @@ -482,13 +530,13 @@ "id": "S3-2", "text": "Built as Swift 6 with strict concurrency", "ref": "origin/t3/ios-mvf", - "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", + "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", "status": "met", "evidence": "all 2 found", "links": [ [ "ios/project.yml", - "https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/project.yml" + "https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/project.yml" ] ] }, @@ -496,30 +544,30 @@ "id": "S3-3", "text": "Zero build warnings", "ref": "origin/t3/ios-mvf", - "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", - "status": "stale", - "evidence": "report for sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/sim/20260926-071459-t3-ios-mvf-a39d0a50-replay/report.json", + "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", + "status": "met", + "evidence": "build.warnings = length 0 (want 0) at sha 194f2ebf: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json", "links": [] }, { "id": "S3-4", "text": "Unit tests for coverage, auto-capture and guidance", "ref": "origin/t3/ios-mvf", - "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", + "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", "status": "partial", "evidence": "found coverage; missing auto-capture, guidance", "links": [ [ "ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" ], [ - "ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift" + "ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift", + "https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift" ], [ - "ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift" + "ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift", + "https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift" ] ] }, @@ -527,17 +575,17 @@ "id": "S3-5", "text": "UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop", "ref": "origin/t3/ios-mvf", - "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", + "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", "status": "partial", "evidence": "all 3 found (the tests exist; passing is S3-1)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanUITests/FullFlowUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -545,17 +593,17 @@ "id": "S3-6", "text": "Accessibility audit on every screen", "ref": "origin/t3/ios-mvf", - "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", + "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", "status": "partial", "evidence": "3 matches in 2 files (an audit exists; which screens it covers needs a read)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanUITests/FullFlowUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/e8bdfb0e1ed1b0333117b49f6ad838c2e233237a/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -563,16 +611,16 @@ "id": "S3-6b", "text": "Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues", "ref": "origin/t3/ios-mvf", - "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", + "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", "status": "stale", - "evidence": "report for sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", + "evidence": "report for sha a39d0a50, ref at 194f2ebf: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", "links": [] }, { "id": "S3-7", "text": "Screenshots of each state in the PR", "ref": "origin/t3/ios-mvf", - "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", + "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", "status": "met", "evidence": "18 of 9 matches in the PR body", "links": [ @@ -586,16 +634,16 @@ "id": "S3-8", "text": "Exported scene.json validates against C1", "ref": "origin/t3/ios-mvf", - "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", - "status": "stale", - "evidence": "report for app_sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json", + "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", + "status": "met", + "evidence": "app_export_scene_valid = true (want True) at app_sha 194f2ebf: ~/house-scanning-data/reports/e2e/20260926-122755-68084b5f/report.json", "links": [] }, { "id": "S3-9", "text": "Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design", "ref": "origin/t3/ios-mvf", - "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", + "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -608,11 +656,11 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a" + "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b" }, { "ref": "origin/t3/verification", - "sha": "eb903b7074f330b89802e58dcb9aef64d42e018c" + "sha": "52aa87f79e8dad281e96a6ef51dfe791a48b7779" } ], "metrics": [ @@ -620,25 +668,25 @@ "id": "S4-1", "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/ios-mvf", - "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", - "status": "stale", - "evidence": "report for app_sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/e2e/20260926-071717-739fb6f7/report.json", + "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", + "status": "met", + "evidence": "app_export_passed = true (want True) at app_sha 194f2ebf: ~/house-scanning-data/reports/e2e/20260926-122755-68084b5f/report.json", "links": [] }, { "id": "S4-2", "text": "Screenshot report per app state", "ref": "origin/t3/ios-mvf", - "sha": "e8bdfb0e1ed1b0333117b49f6ad838c2e233237a", - "status": "stale", - "evidence": "report for sha a39d0a50, ref at e8bdfb0e: ~/house-scanning-data/reports/sim/20260926-071459-t3-ios-mvf-a39d0a50-replay/report.json", + "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", + "status": "met", + "evidence": "states = length 12 (want at least 9) at sha 194f2ebf: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json", "links": [] }, { "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "eb903b7074f330b89802e58dcb9aef64d42e018c", + "sha": "52aa87f79e8dad281e96a6ef51dfe791a48b7779", "status": "met", "evidence": "verification/product covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state.", "links": [] @@ -647,13 +695,13 @@ "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "eb903b7074f330b89802e58dcb9aef64d42e018c", + "sha": "52aa87f79e8dad281e96a6ef51dfe791a48b7779", "status": "met", - "evidence": "present at eb903b70", + "evidence": "present at 52aa87f7", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/eb903b7074f330b89802e58dcb9aef64d42e018c/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/52aa87f79e8dad281e96a6ef51dfe791a48b7779/verification/SCOREBOARD.md" ] ] } @@ -665,7 +713,7 @@ "refs": [ { "ref": "origin/t3/docs-cleanup", - "sha": "ab03d7fd735884346f9182e3cb4cbf4aee60994a" + "sha": "8afa331394deda5e15ad9a802361b7082312d638" } ], "metrics": [ @@ -673,16 +721,16 @@ "id": "M-1", "text": "AGENTS.md shorter than on main", "ref": "origin/t3/docs-cleanup", - "sha": "ab03d7fd735884346f9182e3cb4cbf4aee60994a", - "status": "met", - "evidence": "exit 0: AGENTS.md 39 lines, main 45", + "sha": "8afa331394deda5e15ad9a802361b7082312d638", + "status": "gap", + "evidence": "exit 1: AGENTS.md 45 lines, main 45", "links": [] }, { "id": "M-2", "text": "AGENTS.md accurate, every path and command real", "ref": "origin/t3/docs-cleanup", - "sha": "ab03d7fd735884346f9182e3cb4cbf4aee60994a", + "sha": "8afa331394deda5e15ad9a802361b7082312d638", "status": "gap", "evidence": "not yet reviewed", "links": [] diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index 29c3507d7..eb07c3058 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -6,6 +6,7 @@ import time import zipfile from http.server import BaseHTTPRequestHandler, HTTPServer +from typing import ClassVar import pytest @@ -74,19 +75,20 @@ def test_multipart_endpoint_names_its_file_field(): ) -def test_one_path_with_json_and_upload_uses_the_upload(): +def test_one_path_with_json_and_upload_uses_json_as_the_app_does(): both = { "post": { "requestBody": { "content": { "application/json": {"schema": {}}, + "application/zip": {"schema": {"type": "string", "format": "binary"}}, "multipart/form-data": {"schema": {"$ref": "#/components/schemas/Upload"}}, } } } } assert discover_endpoint(openapi({"/v1/placements": both})) == Endpoint( - "/v1/placements", "multipart/form-data", "bundle" + "/v1/placements", "application/json", None, accepts_zip=True ) @@ -274,8 +276,10 @@ class RefusingServer(BaseHTTPRequestHandler): """Refuses every request with 422 after `delay` seconds.""" delay = 0.0 + content_types: ClassVar[list[str]] = [] def do_POST(self): + RefusingServer.content_types.append(self.headers["Content-Type"]) self.rfile.read(int(self.headers["Content-Length"])) time.sleep(self.delay) self.send_response(422) @@ -333,3 +337,13 @@ def test_a_sim_report_folder_is_an_app_export_with_its_sha(tmp_path): (tmp_path / "report.json").write_text(json.dumps({"sha": "abc123"})) with pytest.raises(SystemExit, match="never reached the upload"): load_app_export(tmp_path) + + +def test_a_prebuilt_zip_goes_as_application_zip_when_the_endpoint_takes_it(refusing_url): + RefusingServer.content_types.clear() + bomb = hostile_inputs()[4] + json_only = Endpoint("/p", "application/json") + assert judge_hostile(refusing_url, json_only, bomb, SCHEMAS)["status"] == "skipped" + zip_too = Endpoint("/p", "application/json", accepts_zip=True) + assert judge_hostile(refusing_url, zip_too, bomb, SCHEMAS)["status"] == "pass" + assert RefusingServer.content_types == ["application/zip"] diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 226d8941b..9f237074b 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -590,3 +590,12 @@ def test_transforms(): "out_ft": 40.0, } assert with_requests_captured(scene, result()) is None + + +def test_a_ground_request_is_redundant_only_as_far_out_as_it_asks(): + r = result() + ask = {"kind": "band", "band": "ground", "span_ft": [-3.0, -1.0], "message": ""} + r["missing_evidence"] = [ask | {"out_ft": 2.5}] # SCENE saw this ground 3 ft out + assert any("lists as observed" in m for m in invariant_problems(SCENE, r, rules=RULES)) + r["missing_evidence"] = [ask | {"out_ft": 3.5}] + assert not any("lists as observed" in m for m in invariant_problems(SCENE, r, rules=RULES)) diff --git a/verification/tests/test_simrun.py b/verification/tests/test_simrun.py index 9b43331f5..09932538d 100644 --- a/verification/tests/test_simrun.py +++ b/verification/tests/test_simrun.py @@ -144,3 +144,11 @@ def test_logged_bundle_that_is_gone_is_a_problem(tmp_path): assert export is None assert len(problems) == 1 assert problems[0].startswith(f"the app logged bundle {gone} with 5 keyframes; copy failed") + + +def test_a_bundle_line_the_runner_cannot_read_is_a_problem(tmp_path): + write_log(tmp_path, log_line("bundle written to a new place, 5 keyframes", category="engine")) + assert copy_app_export(tmp_path) == ( + None, + ["the app logged a bundle line this runner cannot read; see state.ndjson"], + ) diff --git a/verification/tests/test_statelog.py b/verification/tests/test_statelog.py index ee59be7d9..bfc4cc2ad 100644 --- a/verification/tests/test_statelog.py +++ b/verification/tests/test_statelog.py @@ -65,8 +65,11 @@ def test_plain_line(): BUNDLE_PATH = "/Users/x/Library/Developer/CoreSimulator/Devices/D/data/Caches/Scans/S/scan.zip" -def test_reads_the_engine_bundle_line(): - line = entry(f"bundle {BUNDLE_PATH} with 77 keyframes", category="engine") +@pytest.mark.parametrize( + "suffix", ["", ", 0 with depth, no mesh (kept on the phone)", ", 12 with depth, mesh"] +) +def test_reads_the_engine_bundle_line(suffix): + line = entry(f"bundle {BUNDLE_PATH} with 77 keyframes{suffix}", category="engine") assert parse_bundle_line(line) == (BUNDLE_PATH, 77) From 63350b12495e7c69d18e6a4391e053a70f432dbd Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 13:08:49 -0500 Subject: [PATCH 37/59] verification: point the facade scene's rule sources at docs/04 docs/01 is removed by #14; the Base Core height and NEC headroom now cite docs/04's 'What Base does today' and 'Public rule values'. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/scenes/eth3d-facade/build_scene.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/verification/scenes/eth3d-facade/build_scene.py b/verification/scenes/eth3d-facade/build_scene.py index c05bfe24b..5b9fffb4e 100644 --- a/verification/scenes/eth3d-facade/build_scene.py +++ b/verification/scenes/eth3d-facade/build_scene.py @@ -42,8 +42,10 @@ # moves by feet for a few pixels of error. MAX_VIEW_ANGLE_DEG = 75.0 COVERAGE_STEP_FT = 0.25 -BATTERY_HEIGHT_FT = 39.5 / 12 # Base Core height, docs/01 rules table; the wall band's top probe -HEADROOM_FT = 6.5 # NEC 110.26 headroom, docs/01; the overhead band's top probe +# Base Core height, docs/04 (What Base does today); the wall band's top probe +BATTERY_HEIGHT_FT = 39.5 / 12 +# NEC 110.26 headroom, docs/04 (Public rule values); the overhead band's top probe +HEADROOM_FT = 6.5 GROUND_OUT_FT = 6.0 # how far out the ground band claims to have been seen OVERHEAD_OUT_FT = 3.0 # overhead search depth in front of the wall OVERHEAD_MAX_FT = 12.0 # anything higher than this is not reported as an overhead From e0e3ae6e49c996cee123ec8759cc6bea305d5730 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 22:36:41 -0500 Subject: [PATCH 38/59] verification: fix the review's nine findings; move the app reviews to #61 - A run claims the contract only when a scene was answered and checked (contract_ok), and latency only for a real scene that got a result. - Coverage follows the server's "What settles each check": wall height per check, facing and overhead bands unless measured, battery clearance, and the battery's position error by the wall's source (tap, mesh, plane). Requests are redundant only where the view reached their out_ft, on every band. - Bundles are read within entry and size limits, checked before reading. - build_scene.py replaces only its own outputs; case regeneration keeps hand-written cases (generated ones carry generated_by). - make sim-app fails without the app's scan (--require-export). - The memory tests are deterministic and always reap their children. - product/ and ux/ move to docs/app-review/ in #61; the README is shorter. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/Makefile | 2 +- verification/README.md | 95 ++--- verification/e2e/cases/c5-no-coverage.json | 1 + verification/e2e/cases/d-gas-drift.json | 1 + .../e2e/cases/d-reach-drift-left.json | 1 + .../e2e/cases/d-reach-drift-right.json | 1 + .../e2e/cases/g01-unseen-beyond-garage.json | 1 + .../e2e/cases/g03-negative-s-near-edge.json | 1 + .../e2e/cases/g05-inside-corner-gas.json | 1 + .../e2e/cases/g06-regulator-footprint.json | 1 + verification/e2e/cases/g07-headroom-fail.json | 1 + .../e2e/cases/g07-headroom-margin.json | 1 + .../e2e/cases/g07-headroom-partial-span.json | 1 + verification/e2e/cases/g07-headroom-pass.json | 1 + .../e2e/cases/g07-headroom-unobserved.json | 1 + .../e2e/cases/g08a-garage-blocks-route.json | 1 + .../cases/g08b-missing-wall-blocks-route.json | 1 + verification/e2e/cases/g09-reach-14.json | 1 + verification/e2e/cases/g09-reach-15.json | 1 + verification/e2e/cases/g09-reach-16.json | 1 + verification/e2e/cases/g09-reach-20.json | 1 + .../e2e/cases/g09-reach-20p2-pm03.json | 1 + .../e2e/cases/g09-reach-20p4-pm03.json | 1 + verification/e2e/cases/g09-reach-21.json | 1 + verification/e2e/cases/g09-vertical-run.json | 1 + .../e2e/cases/g10-derived-compound-error.json | 1 + verification/e2e/cases/g10-e03-fail.json | 1 + .../e2e/cases/g10-e03-lower-edge.json | 1 + verification/e2e/cases/g10-e03-pass.json | 1 + .../e2e/cases/g10-e03-upper-edge.json | 1 + verification/e2e/cases/g10-e05-fail.json | 1 + .../e2e/cases/g10-e05-lower-edge.json | 1 + verification/e2e/cases/g10-e05-pass.json | 1 + .../e2e/cases/g10-e05-upper-edge.json | 1 + verification/e2e/cases/g10-e15-fail.json | 1 + .../e2e/cases/g10-e15-lower-edge.json | 1 + verification/e2e/cases/g10-e15-pass.json | 1 + .../e2e/cases/g10-e15-upper-edge.json | 1 + .../e2e/cases/g11a-unexplored-corner.json | 1 + .../cases/g11b-both-ends-real-all-fail.json | 1 + .../e2e/cases/g11c-unobserved-ground.json | 1 + .../e2e/cases/g12a-subgrid-start.json | 1 + .../e2e/cases/g12b-exact-fit-last-start.json | 1 + .../g13a-unsure-never-outranks-pass.json | 1 + verification/e2e/cases/g13b-unsure-alone.json | 1 + verification/e2e/cases/generate.py | 18 +- verification/hsverify/e2e.py | 70 +++- verification/hsverify/memory.py | 21 +- verification/hsverify/resultcheck.py | 193 +++++++--- verification/hsverify/simrun.py | 21 +- verification/product/README.md | 78 ---- verification/product/bug-triage.md | 120 ------ .../foundations/coverage-and-guidance.md | 185 --------- verification/product/foundations/flow.md | 145 ------- verification/product/glossary.md | 111 ------ verification/product/screens/find-meter.md | 177 --------- verification/product/screens/gap-request.md | 195 ---------- verification/product/screens/mark-features.md | 220 ----------- .../product/screens/meter-close-up.md | 135 ------- verification/product/screens/onboarding.md | 176 --------- verification/product/screens/result.md | 222 ----------- verification/product/screens/uploading.md | 174 --------- verification/product/screens/wall-walk.md | 362 ------------------ verification/product/verification.md | 47 --- .../scenes/eth3d-facade/build_scene.py | 27 +- verification/scoreboard/metrics.yaml | 12 +- verification/tests/test_build_scene.py | 52 +++ verification/tests/test_e2e.py | 100 ++++- verification/tests/test_generate_cases.py | 41 ++ verification/tests/test_memory.py | 74 ++-- verification/tests/test_resultcheck.py | 164 +++++++- verification/tests/test_simrun.py | 8 + verification/ux/checklist.md | 120 ------ verification/ux/friction.md | 94 ----- verification/ux/review.md | 100 ----- 75 files changed, 755 insertions(+), 2847 deletions(-) delete mode 100644 verification/product/README.md delete mode 100644 verification/product/bug-triage.md delete mode 100644 verification/product/foundations/coverage-and-guidance.md delete mode 100644 verification/product/foundations/flow.md delete mode 100644 verification/product/glossary.md delete mode 100644 verification/product/screens/find-meter.md delete mode 100644 verification/product/screens/gap-request.md delete mode 100644 verification/product/screens/mark-features.md delete mode 100644 verification/product/screens/meter-close-up.md delete mode 100644 verification/product/screens/onboarding.md delete mode 100644 verification/product/screens/result.md delete mode 100644 verification/product/screens/uploading.md delete mode 100644 verification/product/screens/wall-walk.md delete mode 100644 verification/product/verification.md create mode 100644 verification/tests/test_build_scene.py create mode 100644 verification/tests/test_generate_cases.py delete mode 100644 verification/ux/checklist.md delete mode 100644 verification/ux/friction.md delete mode 100644 verification/ux/review.md diff --git a/verification/Makefile b/verification/Makefile index cbe19ce1c..de79d452a 100644 --- a/verification/Makefile +++ b/verification/Makefile @@ -27,7 +27,7 @@ sim: sim-app: uv run python -m hsverify.simrun --ref $(REF) --replay $(REPLAY) --autopilot \ $(SERVER) --extra-arg=-autopilotHold --extra-arg=3 \ - --idle 60 --timeout 900 --require result --label replay $(ARGS) + --idle 60 --timeout 900 --require result --require-export --label replay $(ARGS) # Every case in e2e/cases/ plus the ETH3D facade scene through the server's API at SERVER_REF; # each result is validated against server/schemas/result.schema.json at the same ref. diff --git a/verification/README.md b/verification/README.md index 79ed53725..131855213 100644 --- a/verification/README.md +++ b/verification/README.md @@ -15,73 +15,50 @@ datasets. Requirements: uv, and Xcode 26 or newer with an iOS Simulator runtime. | These tools' own tests | `make test` | pass or fail | | The Simulator runner against a probe app with a known sequence | `make sim-probe` | `reports/sim//` | -Written results: [the UX review](ux/review.md) against [the checklist](ux/checklist.md), [the -friction audit](ux/friction.md) of the homeowner's path, and -[the product description](product/README.md) with its [bug triage](product/bug-triage.md). +The written reviews of the app (product description, UX review, friction audit) are in #61, at +`docs/app-review/`. ## What the checks guarantee -**Simulator runner** (`hsverify/simrun.py`). Builds the ref in a `/tmp` worktree, installs it on -its own Simulator ("HouseScan Verify") and launches it with the C4 arguments. It follows -`STATE=` from subsystem `dev.housescanning.housescan` and screenshots each state once it has -settled for 1.2 s; a state replaced sooner is captured at once and marked *transient*. The report -lists a failed build, a crash, markers logged as ``, and a run with no markers as -problems, and keeps every app log line. `make sim-app` also fails unless the app reaches -`result`, and keeps the scan it uploaded as `scan.zip`. The app is launched only once the log -stream has attached, so early states are not missed. The runner waits while another -`xcodebuild` compiles, because several workers share the Mac. +**Simulator runner** (`hsverify/simrun.py`). Builds the ref in a `/tmp` worktree, runs it on its +own Simulator ("HouseScan Verify") with the C4 arguments, and screenshots each `STATE` once it has +settled for 1.2 s (a state replaced sooner is marked *transient*). A failed build, a crash, +redacted or missing markers, a required state never reached, or, for `make sim-app`, no scan +bundle from the app is a problem, and the run exits 1. The app launches only after the log stream +has attached. The runner waits while another `xcodebuild` compiles, because the Mac is shared. **End-to-end check** (`hsverify/e2e.py`, `hsverify/resultcheck.py`). Starts the server from a ref -and finds its placement endpoint in the OpenAPI document, refusing to guess between candidates. -The server is killed if it passes 3.5 GB, and the report says so. Each scene is validated against -`server/schemas/scene.schema.json` at that ref first, so bad input is not blamed on the server. -Battery size, clearance radii and measurement errors are read from `server/rules.yaml` at the same -ref. Each answer must validate against `server/schemas/result.schema.json` and satisfy, for any -policy: +(killed past 3.5 GB) or uses `SERVER_URL`, and sends each scene the way the app does. Scenes are +checked against the ref's scene schema first, so bad input is not blamed on the server. Rules come +from the ref's `rules.yaml`. Every answer must validate against the result schema and: -- `pass` has a passing spot, all checks passing, no photo request and an approved policy; - `reject` has no spot; `manual_review` gives reasons. -- Each check's outcome follows from its numbers (C5): a minimum T passes only when measured − - error > T and fails only when measured + error < T, a maximum mirrored, a review line also - cleared for a pass; an unsure labelled `margin` lies within its error of a line. -- No battery position passes unless the wall and cable route under it were observed, and every - point within each clearance's radius, widened by the battery's position error, was observed - in the bands a hazard could hide in (gas: wall and ground; openings: wall; AC, drives, pools: - ground). Only points within that reach whatever the wall's shape are required: along the wall - to the radius, and on the ground out to the battery's depth plus the radius in front, tapering - past its ends. So a correct server is never flagged. -- When the result gives an unobserved area as a reason, each check left unsure for that reason - is named by a request of every band it lacks, and no request covers an area already observed. -- Counts add up, `input_sha256` is the hash of the bytes sent, the spot's offset equals its centre - minus the meter. +- have a decision consistent with its spot, checks, requests and policy; +- have each check's outcome follow from its numbers (C5), with an unsure `margin` inside its + error of a line; +- pass no battery position without the coverage each passing check needs, per the server's "What + settles each check": the wall seen high enough over the clearance's reach, the ground out to the + battery's depth plus the reach, and the facing and overhead bands over the battery unless + measured. The reach includes the battery's position error, whose default depends on how the + wall was found (tap, mesh or plane). Only points within reach whatever the wall's shape are + required, so a correct server is not flagged; +- name every check left unsure for lack of coverage in a request for each band it lacks, and ask + for nothing already seen as far as the request needs; +- add up, and hash the bytes sent. -Each scene is also resent changed, and the pairs must stay ordered: the same scene answers the -same; mirrored it gives the same decision and outcome lengths; with no coverage, less coverage, or -ground trimmed just short of the largest clearance nothing new passes; with more error, or tape -measurements re-taken by tap, no outcome moves except to unsure; adding every requested photo, up -to three rounds, leaves no check unsure for lack of coverage. Five hostile inputs (150 objects -with their own errors, coordinates of 1e300, a 5000 ft wall, nesting 5000 deep, and a small zip -holding a 400 MB scene.json) must each be refused with 400, 413 or 422, or answered validly, -within 10 s and, for a server started here, 500 MB of extra memory. `--no-hostile` skips them. +Each scene is also resent changed, and the answers must stay ordered: mirrored, trimmed, with +more error or with requested views added, nothing improves that should not. Five hostile inputs +must be refused or answered within 10 s and 500 MB. A run claims the contract only when at least +one scene was answered and checked, and latency only for a real scene that got a result. Bundles +are read within fixed size limits. -The 43 cases in [`e2e/cases/`](e2e/cases/README.md) were written from the public goldens and C5 -without reading the server's tests, except that the three drift cases take drift at the -battery's far edge as S2's 70ab0b0 states. The hostile inputs were chosen after reading S2's -hardening tests. Each case names the thresholds it assumes; a mismatch is reported as such -rather than as a pass or fail. [`scenes/eth3d-facade`](scenes/eth3d-facade/README.md) -builds a scene from real laser-scanned geometry; real scenes must answer within 1 s. -`--app-export` takes a `make sim-app` report folder and sends the scan the app uploaded, recording -the app's commit. +The 43 generated cases in [`e2e/cases/`](e2e/cases/README.md) come from the public goldens and C5; +regeneration keeps any case written by hand. [`scenes/eth3d-facade`](scenes/eth3d-facade/README.md) +builds a scene from laser-scanned geometry, replacing only its own outputs. -**Accessibility audit** (`hsverify/a11yaudit.py`). Runs the UI test bundle in -`fixtures/a11y-audit`, which launches the installed app by bundle id and calls -`performAccessibilityAudit(for: .all)` on each distinct screen. It also flags buttons labelled -with an SF Symbol name, which Apple's audit accepts. The probe app's middle screen carries an -unlabelled 12 pt button; an audit of the probe must find it. +**Accessibility audit** (`hsverify/a11yaudit.py`) runs Apple's audit on every screen the app +reaches, and flags buttons labelled with an SF Symbol name. -**Replay check** (`hsverify/replaycheck.py`). Checks the fields, JPEG sizes, rigid poses and -spacing gate of a Measure Lab v2 session, then compares ARKit with `ground_truth.json` after -aligning heading and origin. It reports the path-length ratio separately and does not say which -track is right: ADVIO's ground truth has its own scale error. +**Replay check** (`hsverify/replaycheck.py`) checks a Measure Lab v2 session and compares its +poses with ADVIO's ground truth. -Every report records the harness's peak memory and that of the largest child it waited for. +Every report records the harness's peak memory and that of its largest child. diff --git a/verification/e2e/cases/c5-no-coverage.json b/verification/e2e/cases/c5-no-coverage.json index 28058b40a..29e9225b6 100644 --- a/verification/e2e/cases/c5-no-coverage.json +++ b/verification/e2e/cases/c5-no-coverage.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "c5-no-coverage", "source": "scene.schema.json coverage and end descriptions at origin/t3/server 69c6c3c; C5", "description": "A clean straight wall with ground and facing measurements but no coverage field. Nothing is known to be observed, so no start may pass, the result must ask for views, and it cannot reject because both ends default to unexplored.", diff --git a/verification/e2e/cases/d-gas-drift.json b/verification/e2e/cases/d-gas-drift.json index cfc2290ba..97a2809fa 100644 --- a/verification/e2e/cases/d-gas-drift.json +++ b/verification/e2e/cases/d-gas-drift.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "d-gas-drift", "source": "rules.yaml errors.tap_ft, wall_ft, drift_per_ft and clearances.gas_ft at origin/t3/server f2705dd; C5 margin rule", "description": "A tapped gas meter at s = 4 on a wall with default errors. For a start a right of it the gap is (a - 4) +/- (1.24 + 0.16(a + W)): fail below a = 4.609, pass above a = 10.302, unsure between.", diff --git a/verification/e2e/cases/d-reach-drift-left.json b/verification/e2e/cases/d-reach-drift-left.json index 9c3a02752..401e22f08 100644 --- a/verification/e2e/cases/d-reach-drift-left.json +++ b/verification/e2e/cases/d-reach-drift-left.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "d-reach-drift-left", "source": "rules.yaml errors.drift_per_ft and route.* at origin/t3/server f2705dd; C5 margin rule", "description": "Mirror of d-reach-drift-right. Left of the meter the route runs to the battery's right edge b = a + W, so the boundaries sit at a = -12.058 - W and a = -25.016 - W.", diff --git a/verification/e2e/cases/d-reach-drift-right.json b/verification/e2e/cases/d-reach-drift-right.json index 9d5826816..06bd1abb8 100644 --- a/verification/e2e/cases/d-reach-drift-right.json +++ b/verification/e2e/cases/d-reach-drift-right.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "d-reach-drift-right", "source": "rules.yaml errors.drift_per_ft and route.* at origin/t3/server f2705dd; C5 margin rule", "description": "Straight wall with default (drifting) errors. The route to a start a right of the meter is a +/- (0.6 + 0.16(a + W)): pass below a = 12.058, fail above a = 25.016, unsure between.", diff --git a/verification/e2e/cases/g01-unseen-beyond-garage.json b/verification/e2e/cases/g01-unseen-beyond-garage.json index 241755970..d382da71b 100644 --- a/verification/e2e/cases/g01-unseen-beyond-garage.json +++ b/verification/e2e/cases/g01-unseen-beyond-garage.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g01-unseen-beyond-garage", "source": "t3-lane-c-review.md golden 01 at origin/t3/research a7d91f1", "description": "The wall beyond a garage door left of the meter was never observed. No spot there can be reached by a cable, so the result asks for no photos, and every start beyond the garage fails its route.", diff --git a/verification/e2e/cases/g03-negative-s-near-edge.json b/verification/e2e/cases/g03-negative-s-near-edge.json index 1fc150ec0..c7d60170b 100644 --- a/verification/e2e/cases/g03-negative-s-near-edge.json +++ b/verification/e2e/cases/g03-negative-s-near-edge.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g03-negative-s-near-edge", "source": "t3-lane-c-review.md golden 03 at origin/t3/research a7d91f1", "description": "Two straight 3 ft pads, s = [-9, -6] and [7, 10], joined by segments 13/7 ft long that no battery fits on. The left pad's route runs to its near edge (6 ft), so it beats the right pad (7 ft); measuring to the far edge (103/12 ft) would pick the right pad.", diff --git a/verification/e2e/cases/g05-inside-corner-gas.json b/verification/e2e/cases/g05-inside-corner-gas.json index fb8429c76..78e61e185 100644 --- a/verification/e2e/cases/g05-inside-corner-gas.json +++ b/verification/e2e/cases/g05-inside-corner-gas.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g05-inside-corner-gas", "source": "t3-lane-c-review.md golden 05 at origin/t3/research a7d91f1", "description": "A gas meter on the return wall of an inside corner is 65/12 ft away along the unrolled wall but about 2.589 ft away in plan from a battery at s = [6, 6 + 31/12]. Gas clearance is measured in plan, so those starts fail.", diff --git a/verification/e2e/cases/g06-regulator-footprint.json b/verification/e2e/cases/g06-regulator-footprint.json index c302dc351..2d12e8c84 100644 --- a/verification/e2e/cases/g06-regulator-footprint.json +++ b/verification/e2e/cases/g06-regulator-footprint.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g06-regulator-footprint", "source": "t3-lane-c-review.md golden 06 at origin/t3/research a7d91f1", "description": "The gas meter body stands 10 ft from the wall but its regulator reaches [7, 23/6], 2 ft from the front edge of any battery covering x = 7. Clearance is measured to every part of the footprint.", diff --git a/verification/e2e/cases/g07-headroom-fail.json b/verification/e2e/cases/g07-headroom-fail.json index 9e758201d..4cbbcc947 100644 --- a/verification/e2e/cases/g07-headroom-fail.json +++ b/verification/e2e/cases/g07-headroom-fail.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g07-headroom-fail", "source": "t3-lane-c-review.md golden 07 at origin/t3/research a7d91f1", "description": "An overhead over the whole wall has clear height 5.5 +/- 0.5 ft against headroom 6.5 ft, so every start's headroom check is fail.", diff --git a/verification/e2e/cases/g07-headroom-margin.json b/verification/e2e/cases/g07-headroom-margin.json index 6b1fe62d5..385c3d930 100644 --- a/verification/e2e/cases/g07-headroom-margin.json +++ b/verification/e2e/cases/g07-headroom-margin.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g07-headroom-margin", "source": "t3-lane-c-review.md golden 07 at origin/t3/research a7d91f1", "description": "An overhead over the whole wall has clear height 6.8 +/- 0.5 ft against headroom 6.5 ft, so every start's headroom check is unsure.", diff --git a/verification/e2e/cases/g07-headroom-partial-span.json b/verification/e2e/cases/g07-headroom-partial-span.json index 303bedc2e..7da273f41 100644 --- a/verification/e2e/cases/g07-headroom-partial-span.json +++ b/verification/e2e/cases/g07-headroom-partial-span.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g07-headroom-partial-span", "source": "t3-lane-c-review.md golden 07 at origin/t3/research a7d91f1", "description": "A low overhead (5.5 +/- 0.5 ft) covers only s = [8, 8.5]. Every battery whose 31 in stretch overlaps it fails headroom, even when the overhead covers a few inches of it.", diff --git a/verification/e2e/cases/g07-headroom-pass.json b/verification/e2e/cases/g07-headroom-pass.json index 0f80b48d0..5184547de 100644 --- a/verification/e2e/cases/g07-headroom-pass.json +++ b/verification/e2e/cases/g07-headroom-pass.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g07-headroom-pass", "source": "t3-lane-c-review.md golden 07 at origin/t3/research a7d91f1", "description": "An overhead over the whole wall has clear height 7.2 +/- 0.5 ft against headroom 6.5 ft, so every start's headroom check is pass.", diff --git a/verification/e2e/cases/g07-headroom-unobserved.json b/verification/e2e/cases/g07-headroom-unobserved.json index 8aea3a8e8..4f224cdd9 100644 --- a/verification/e2e/cases/g07-headroom-unobserved.json +++ b/verification/e2e/cases/g07-headroom-unobserved.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g07-headroom-unobserved", "source": "t3-lane-c-review.md golden 07 at origin/t3/research a7d91f1", "description": "Nothing overhead was recorded and the overhead band was never observed, so every start's headroom check is unsure for lack of a view, and the result asks for that view.", diff --git a/verification/e2e/cases/g08a-garage-blocks-route.json b/verification/e2e/cases/g08a-garage-blocks-route.json index 7c66472fa..8d44257c0 100644 --- a/verification/e2e/cases/g08a-garage-blocks-route.json +++ b/verification/e2e/cases/g08a-garage-blocks-route.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g08a-garage-blocks-route", "source": "t3-lane-c-review.md golden 08 at origin/t3/research a7d91f1", "description": "A garage door spans s = [1, 5] and the wall ends at s = -0.5, so every start either overlaps the door or lies beyond it, where the cable would have to cross the door.", diff --git a/verification/e2e/cases/g08b-missing-wall-blocks-route.json b/verification/e2e/cases/g08b-missing-wall-blocks-route.json index aa2a95010..6b20e34a5 100644 --- a/verification/e2e/cases/g08b-missing-wall-blocks-route.json +++ b/verification/e2e/cases/g08b-missing-wall-blocks-route.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g08b-missing-wall-blocks-route", "source": "t3-lane-c-review.md golden 08 at origin/t3/research a7d91f1", "description": "The meter's wall w1 is 1.5 ft long, too short for a battery; after a 1 ft stretch with no wall, w2 runs from s = 2 to 12. No cable can cross the gap, so every start on w2 fails its route and no spot exists.", diff --git a/verification/e2e/cases/g09-reach-14.json b/verification/e2e/cases/g09-reach-14.json index 2b8d162b3..c38cb8b27 100644 --- a/verification/e2e/cases/g09-reach-14.json +++ b/verification/e2e/cases/g09-reach-14.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g09-reach-14", "source": "t3-lane-c-review.md golden 09 at origin/t3/research a7d91f1", "description": "Route 14 ft, 1 ft inside the 15 ft confident reach: the route passes.", diff --git a/verification/e2e/cases/g09-reach-15.json b/verification/e2e/cases/g09-reach-15.json index 1e2aba004..2a3beb2ff 100644 --- a/verification/e2e/cases/g09-reach-15.json +++ b/verification/e2e/cases/g09-reach-15.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g09-reach-15", "source": "t3-lane-c-review.md golden 09 at origin/t3/research a7d91f1", "description": "Route exactly at the 15 ft confident reach: not inside it, so unsure.", diff --git a/verification/e2e/cases/g09-reach-16.json b/verification/e2e/cases/g09-reach-16.json index 8302220e3..3d44710e6 100644 --- a/verification/e2e/cases/g09-reach-16.json +++ b/verification/e2e/cases/g09-reach-16.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g09-reach-16", "source": "t3-lane-c-review.md golden 09 at origin/t3/research a7d91f1", "description": "Route 16 ft, between the 15 ft confident reach and the 20 ft maximum: unsure.", diff --git a/verification/e2e/cases/g09-reach-20.json b/verification/e2e/cases/g09-reach-20.json index ddd0eddb2..74e3350cf 100644 --- a/verification/e2e/cases/g09-reach-20.json +++ b/verification/e2e/cases/g09-reach-20.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g09-reach-20", "source": "t3-lane-c-review.md golden 09 at origin/t3/research a7d91f1", "description": "Route exactly at the 20 ft maximum with no error: 20 + 0 is not below 20 and 20 - 0 is not above it, so unsure.", diff --git a/verification/e2e/cases/g09-reach-20p2-pm03.json b/verification/e2e/cases/g09-reach-20p2-pm03.json index 8fb581d32..60625b5f5 100644 --- a/verification/e2e/cases/g09-reach-20p2-pm03.json +++ b/verification/e2e/cases/g09-reach-20p2-pm03.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g09-reach-20p2-pm03", "source": "t3-lane-c-review.md golden 09 at origin/t3/research a7d91f1", "description": "Route 20.2 +/- 0.3 ft (meter tap error): 20.2 - 0.3 = 19.9 is not above 20, so unsure.", diff --git a/verification/e2e/cases/g09-reach-20p4-pm03.json b/verification/e2e/cases/g09-reach-20p4-pm03.json index 635fe8c8e..7ff4ecd72 100644 --- a/verification/e2e/cases/g09-reach-20p4-pm03.json +++ b/verification/e2e/cases/g09-reach-20p4-pm03.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g09-reach-20p4-pm03", "source": "t3-lane-c-review.md golden 09 at origin/t3/research a7d91f1", "description": "Route 20.4 +/- 0.3 ft (meter tap error): 20.4 - 0.3 = 20.1 is above 20, so fail.", diff --git a/verification/e2e/cases/g09-reach-21.json b/verification/e2e/cases/g09-reach-21.json index 266489b6b..2ea2706f7 100644 --- a/verification/e2e/cases/g09-reach-21.json +++ b/verification/e2e/cases/g09-reach-21.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g09-reach-21", "source": "t3-lane-c-review.md golden 09 at origin/t3/research a7d91f1", "description": "Route 21 ft, 1 ft past the 20 ft maximum: the only start on usable ground fails.", diff --git a/verification/e2e/cases/g09-vertical-run.json b/verification/e2e/cases/g09-vertical-run.json index 14ab6aa2d..ea0be5f32 100644 --- a/verification/e2e/cases/g09-vertical-run.json +++ b/verification/e2e/cases/g09-vertical-run.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g09-vertical-run", "source": "t3-lane-c-review.md golden 09 at origin/t3/research a7d91f1", "description": "The pad starts 11 ft along the wall, but the cable must climb over a 3.5 ft box at the 1 ft run height: 11 + 2 x 2.5 = 16 ft routed, past the 15 ft confident reach.", diff --git a/verification/e2e/cases/g10-derived-compound-error.json b/verification/e2e/cases/g10-derived-compound-error.json index 9439c3c6a..d17aeb721 100644 --- a/verification/e2e/cases/g10-derived-compound-error.json +++ b/verification/e2e/cases/g10-derived-compound-error.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g10-derived-compound-error", "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", "description": "Gas gap 3.5 ft between a gas meter at +/- 0.3 and a battery on a wall at +/- 0.3. The gap carries +/- 0.6 (linear sum), so 3.5 - 0.6 = 2.9 is not above 3: unsure by margin. A root-sum-square error of 0.424 would wrongly pass it.", diff --git a/verification/e2e/cases/g10-e03-fail.json b/verification/e2e/cases/g10-e03-fail.json index ced2093ad..9fdef2f3d 100644 --- a/verification/e2e/cases/g10-e03-fail.json +++ b/verification/e2e/cases/g10-e03-fail.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g10-e03-fail", "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", "description": "Every start has gas gap d = 2.69 ft with error 0.3 ft (gas meter +/- 0.3, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", diff --git a/verification/e2e/cases/g10-e03-lower-edge.json b/verification/e2e/cases/g10-e03-lower-edge.json index c27e9d47e..2596a533b 100644 --- a/verification/e2e/cases/g10-e03-lower-edge.json +++ b/verification/e2e/cases/g10-e03-lower-edge.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g10-e03-lower-edge", "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", "description": "Every start has gas gap d = 2.7 ft with error 0.3 ft (gas meter +/- 0.3, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", diff --git a/verification/e2e/cases/g10-e03-pass.json b/verification/e2e/cases/g10-e03-pass.json index 669aa74f3..ecc3a6cd2 100644 --- a/verification/e2e/cases/g10-e03-pass.json +++ b/verification/e2e/cases/g10-e03-pass.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g10-e03-pass", "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", "description": "Every start has gas gap d = 3.31 ft with error 0.3 ft (gas meter +/- 0.3, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", diff --git a/verification/e2e/cases/g10-e03-upper-edge.json b/verification/e2e/cases/g10-e03-upper-edge.json index ef5225b4c..b2014bcc3 100644 --- a/verification/e2e/cases/g10-e03-upper-edge.json +++ b/verification/e2e/cases/g10-e03-upper-edge.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g10-e03-upper-edge", "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", "description": "Every start has gas gap d = 3.3 ft with error 0.3 ft (gas meter +/- 0.3, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", diff --git a/verification/e2e/cases/g10-e05-fail.json b/verification/e2e/cases/g10-e05-fail.json index 948fb024d..b9e3050c5 100644 --- a/verification/e2e/cases/g10-e05-fail.json +++ b/verification/e2e/cases/g10-e05-fail.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g10-e05-fail", "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", "description": "Every start has gas gap d = 2.49 ft with error 0.5 ft (gas meter +/- 0.5, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", diff --git a/verification/e2e/cases/g10-e05-lower-edge.json b/verification/e2e/cases/g10-e05-lower-edge.json index 1f063283c..95736f521 100644 --- a/verification/e2e/cases/g10-e05-lower-edge.json +++ b/verification/e2e/cases/g10-e05-lower-edge.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g10-e05-lower-edge", "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", "description": "Every start has gas gap d = 2.5 ft with error 0.5 ft (gas meter +/- 0.5, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", diff --git a/verification/e2e/cases/g10-e05-pass.json b/verification/e2e/cases/g10-e05-pass.json index 27bf2452d..69f63c6be 100644 --- a/verification/e2e/cases/g10-e05-pass.json +++ b/verification/e2e/cases/g10-e05-pass.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g10-e05-pass", "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", "description": "Every start has gas gap d = 3.51 ft with error 0.5 ft (gas meter +/- 0.5, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", diff --git a/verification/e2e/cases/g10-e05-upper-edge.json b/verification/e2e/cases/g10-e05-upper-edge.json index 9c5f6b391..f59d16188 100644 --- a/verification/e2e/cases/g10-e05-upper-edge.json +++ b/verification/e2e/cases/g10-e05-upper-edge.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g10-e05-upper-edge", "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", "description": "Every start has gas gap d = 3.5 ft with error 0.5 ft (gas meter +/- 0.5, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", diff --git a/verification/e2e/cases/g10-e15-fail.json b/verification/e2e/cases/g10-e15-fail.json index ed1f8bcfd..1f3d62be4 100644 --- a/verification/e2e/cases/g10-e15-fail.json +++ b/verification/e2e/cases/g10-e15-fail.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g10-e15-fail", "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", "description": "Every start has gas gap d = 1.49 ft with error 1.5 ft (gas meter +/- 1.5, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", diff --git a/verification/e2e/cases/g10-e15-lower-edge.json b/verification/e2e/cases/g10-e15-lower-edge.json index e0aa9b7d6..a11fae0c5 100644 --- a/verification/e2e/cases/g10-e15-lower-edge.json +++ b/verification/e2e/cases/g10-e15-lower-edge.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g10-e15-lower-edge", "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", "description": "Every start has gas gap d = 1.5 ft with error 1.5 ft (gas meter +/- 1.5, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", diff --git a/verification/e2e/cases/g10-e15-pass.json b/verification/e2e/cases/g10-e15-pass.json index 794827e40..b5ea1c602 100644 --- a/verification/e2e/cases/g10-e15-pass.json +++ b/verification/e2e/cases/g10-e15-pass.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g10-e15-pass", "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", "description": "Every start has gas gap d = 4.51 ft with error 1.5 ft (gas meter +/- 1.5, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", diff --git a/verification/e2e/cases/g10-e15-upper-edge.json b/verification/e2e/cases/g10-e15-upper-edge.json index 00e86c52b..5b4204ba3 100644 --- a/verification/e2e/cases/g10-e15-upper-edge.json +++ b/verification/e2e/cases/g10-e15-upper-edge.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g10-e15-upper-edge", "source": "t3-lane-c-review.md golden 10 at origin/t3/research a7d91f1", "description": "Every start has gas gap d = 4.5 ft with error 1.5 ft (gas meter +/- 1.5, wall exact), against a 3.0 ft threshold. The gas object's recognition conf of 0.6 must not change the error.", diff --git a/verification/e2e/cases/g11a-unexplored-corner.json b/verification/e2e/cases/g11a-unexplored-corner.json index 660922c32..d837ed903 100644 --- a/verification/e2e/cases/g11a-unexplored-corner.json +++ b/verification/e2e/cases/g11a-unexplored-corner.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g11a-unexplored-corner", "source": "t3-lane-c-review.md golden 11 (a) at origin/t3/research a7d91f1", "description": "Every start on w1 fails gas by 2 ft (gap 1 ft). The walk stopped at the corner at s = 8 without seeing the next wall, so the result cannot reject and must ask for the wall past the corner.", diff --git a/verification/e2e/cases/g11b-both-ends-real-all-fail.json b/verification/e2e/cases/g11b-both-ends-real-all-fail.json index b8648d3d8..60fc39539 100644 --- a/verification/e2e/cases/g11b-both-ends-real-all-fail.json +++ b/verification/e2e/cases/g11b-both-ends-real-all-fail.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g11b-both-ends-real-all-fail", "source": "t3-lane-c-review.md golden 11 (b) at origin/t3/research a7d91f1", "description": "w1 turns an outside corner at s = 8 onto w2, walked to its real end at s = 14. Gas strips 1 ft in front of each wall fail every start. Nothing is unobserved, so there is no spot and no photo request.", diff --git a/verification/e2e/cases/g11c-unobserved-ground.json b/verification/e2e/cases/g11c-unobserved-ground.json index 6163c3319..6ddd3c15f 100644 --- a/verification/e2e/cases/g11c-unobserved-ground.json +++ b/verification/e2e/cases/g11c-unobserved-ground.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g11c-unobserved-ground", "source": "t3-lane-c-review.md golden 11 (c) at origin/t3/research a7d91f1", "description": "Starts below s = 2*sqrt(2) - 0.5 fail gas. Starts beyond it pass gas but stand on ground nobody saw (observed only up to s = 1), so they are unsure, never pass, and the result asks for that ground.", diff --git a/verification/e2e/cases/g12a-subgrid-start.json b/verification/e2e/cases/g12a-subgrid-start.json index 35d6d893e..c05e3a148 100644 --- a/verification/e2e/cases/g12a-subgrid-start.json +++ b/verification/e2e/cases/g12a-subgrid-start.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g12a-subgrid-start", "source": "t3-lane-c-review.md golden 12 at origin/t3/research a7d91f1", "description": "Windows at s = [3, 4] and [10.1 + 31/12, 11.1 + 31/12] with 3 ft opening clearance leave legal starts only in (7, 7.1). A 2 in grid from s = 0 samples 7 and 43/6 and misses them.", diff --git a/verification/e2e/cases/g12b-exact-fit-last-start.json b/verification/e2e/cases/g12b-exact-fit-last-start.json index 646b4c77d..6ad22a994 100644 --- a/verification/e2e/cases/g12b-exact-fit-last-start.json +++ b/verification/e2e/cases/g12b-exact-fit-last-start.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g12b-exact-fit-last-start", "source": "t3-lane-c-review.md golden 12 at origin/t3/research a7d91f1", "description": "Five 1.8 ft segments, then one straight segment exactly 31/12 ft long at s = [6, 6 + 31/12] where the wall ends. Its single start, s = 6, is also the last start and must be enumerated.", diff --git a/verification/e2e/cases/g13a-unsure-never-outranks-pass.json b/verification/e2e/cases/g13a-unsure-never-outranks-pass.json index ee5dd1ba6..90a65f0a7 100644 --- a/verification/e2e/cases/g13a-unsure-never-outranks-pass.json +++ b/verification/e2e/cases/g13a-unsure-never-outranks-pass.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g13a-unsure-never-outranks-pass", "source": "t3-lane-c-review.md golden 13 at origin/t3/research a7d91f1", "description": "Starts right of the meter have gas gap 3.2 +/- 0.3 (unsure) and the shortest routes. Batteries whose right edge is left of s = 0.3 - sqrt(0.65) clear gas and pass. The chosen spot must be a passing one on the left.", diff --git a/verification/e2e/cases/g13b-unsure-alone.json b/verification/e2e/cases/g13b-unsure-alone.json index d4e97a416..a92e83c01 100644 --- a/verification/e2e/cases/g13b-unsure-alone.json +++ b/verification/e2e/cases/g13b-unsure-alone.json @@ -1,4 +1,5 @@ { + "generated_by": "e2e/cases/generate.py", "id": "g13b-unsure-alone", "source": "t3-lane-c-review.md golden 13 at origin/t3/research a7d91f1", "description": "The wall ends at s = -0.5, so every start has gas gap 3.2 +/- 0.3. The best spot is unsure by margin and fully observed: manual review with no photo request.", diff --git a/verification/e2e/cases/generate.py b/verification/e2e/cases/generate.py index e38bf87a6..00dd0a982 100644 --- a/verification/e2e/cases/generate.py +++ b/verification/e2e/cases/generate.py @@ -962,11 +962,21 @@ def starts(points): ) -def main(): - for old in HERE.glob("*.json"): - old.unlink() +# Marks the files this script owns. Regenerating replaces only those, so a regression case +# written by hand in this folder survives. +GENERATED_BY = "e2e/cases/generate.py" + + +def main(folder: Path = HERE) -> None: + ids = {c["id"] for c in CASES} + for old in folder.glob("*.json"): + if json.loads(old.read_text()).get("generated_by") == GENERATED_BY: + old.unlink() + elif old.stem in ids: + raise SystemExit(f"{old.name} was written by hand but a generated case has its id") for c in CASES: - (HERE / f"{c['id']}.json").write_text(json.dumps(c, indent=2) + "\n") + case = {"generated_by": GENERATED_BY, **c} + (folder / f"{c['id']}.json").write_text(json.dumps(case, indent=2) + "\n") print(f"wrote {len(CASES)} cases") diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index 3348e455c..fb45abd54 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -111,16 +111,51 @@ def load_app_export(path: Path) -> SceneInput: return item +# Limits on a scene bundle this harness reads into memory: the server's own scene.json cap, and +# room for a few hundred phone JPEGs, well under the shared Mac's ~4 GB per-process budget. A +# compressed bundle can declare anything, so reads stop at the limit whatever it declares. +MAX_BUNDLE_ENTRIES = 2000 +MAX_SCENE_BYTES = 10 * 2**20 +MAX_IMAGE_BYTES = 32 * 2**20 +MAX_BUNDLE_BYTES = 512 * 2**20 + + +def read_bounded(bundle: zipfile.ZipFile, info: zipfile.ZipInfo, limit: int) -> bytes: + """An entry's bytes, refused when it declares or turns out to hold more than `limit`.""" + if info.file_size > limit: + raise SystemExit(f"{info.filename}: declares {info.file_size} bytes, limit {limit}") + with bundle.open(info) as f: + data = f.read(limit + 1) + if len(data) > limit: + raise SystemExit(f"{info.filename}: holds more than {limit} bytes") + return data + + +def read_bundle(path: Path) -> tuple[bytes, dict[str, bytes]]: + """scene.json and the JPEGs of a C1 zip bundle, within the limits above.""" + with zipfile.ZipFile(path) as bundle: + infos = [i for i in bundle.infolist() if not i.is_dir()] + if len(infos) > MAX_BUNDLE_ENTRIES: + raise SystemExit(f"{path}: {len(infos)} entries, limit {MAX_BUNDLE_ENTRIES}") + declared = sum(i.file_size for i in infos) + if declared > MAX_BUNDLE_BYTES: + raise SystemExit(f"{path}: declares {declared} bytes, limit {MAX_BUNDLE_BYTES}") + scene = next((i for i in infos if i.filename.endswith("scene.json")), None) + if scene is None: + raise SystemExit(f"{path}: no scene.json in the bundle") + raw = read_bounded(bundle, scene, MAX_SCENE_BYTES) + images = { + Path(i.filename).name: read_bounded(bundle, i, MAX_IMAGE_BYTES) + for i in infos + if i.filename.lower().endswith(".jpg") + } + return raw, images + + def load_input(path: Path, real: bool = False) -> SceneInput: """A case file (scene + expect), a bare scene.json, or a C1 zip bundle.""" if path.suffix == ".zip": - with zipfile.ZipFile(path) as bundle: - names = bundle.namelist() - scene_name = next((n for n in names if n.endswith("scene.json")), None) - if scene_name is None: - raise SystemExit(f"{path}: no scene.json in the bundle") - raw = bundle.read(scene_name) - images = {Path(n).name: bundle.read(n) for n in names if n.lower().endswith(".jpg")} + raw, images = read_bundle(path) return SceneInput(path.stem, json.loads(raw), raw, images, real=real, source=str(path)) data = json.loads(path.read_text()) if "scene_path" in data and "expect" in data: @@ -380,7 +415,7 @@ def judge( if item.skip_reason: return record | {"status": "skipped", "problems": [], "skip_reason": item.skip_reason} if item.hostile: - return record | judge_hostile(url, endpoint, item, schemas, limit) + return record | {"hostile": True} | judge_hostile(url, endpoint, item, schemas, limit) input_errors = schema_errors(item.scene, schemas["scene"]) if input_errors: return record | {"status": "bad input", "problems": input_errors[:20]} @@ -635,18 +670,29 @@ def write_report(out: Path, meta: dict, records: list[dict]) -> dict: counts: dict[str, int] = {} for r in records: counts[r["status"]] = counts.get(r["status"], 0) + 1 - real = [r["latency_ms"] for r in records if r.get("real") and "latency_ms" in r] + # A scene counts as checked only when the server answered it and the answer was judged; a + # skipped scene, a refused one or scene input that failed its own schema checks nothing. + checked = [r for r in records if "decision" in r and not r.get("hostile")] + # Latency counts only a real scene the server answered with a result, never a fast refusal. + real = [r["latency_ms"] for r in checked if r.get("real") and r.get("http_status") == 200] exported = [r for r in records if r.get("app_export")] + contract_problems = sum(len(r.get("contract_problems", [])) for r in records) + killed = bool(meta["peak_memory"].get("server_killed")) report = meta | { "counts": counts, "all_passed": all(r["status"] in ("pass", "skipped") for r in records) - and bool(records) - and not meta["peak_memory"].get("server_killed"), + and bool(checked) + and not killed, "latency_ms": max(real) if real else None, "scenes_answered": sum(1 for r in records if "decision" in r), + "scenes_checked": len(checked), + "scenes_skipped": counts.get("skipped", 0), + "scenes_bad_input": counts.get("bad input", 0), # Invariants and properties broken, summed over every answered scene. Case # expectations are counted separately because a case can be wrong itself. - "contract_problem_count": sum(len(r.get("contract_problems", [])) for r in records), + "contract_problem_count": contract_problems, + # The contract holds only on evidence: at least one scene checked, none broken. + "contract_ok": bool(checked) and contract_problems == 0 and not killed, "expectation_problem_count": sum(len(r.get("expectation_problems", [])) for r in records), "real_scene_passed": any(r.get("real") and r["status"] == "pass" for r in records), # Null when the run had no app export, so the scoreboard looks for an older run. diff --git a/verification/hsverify/memory.py b/verification/hsverify/memory.py index 8fcec4ec9..a1841175b 100644 --- a/verification/hsverify/memory.py +++ b/verification/hsverify/memory.py @@ -12,6 +12,7 @@ import subprocess import sys import threading +from collections.abc import Callable MAXRSS_UNIT = 1 if sys.platform == "darwin" else 1024 @@ -27,6 +28,11 @@ def tree_rss_mb(root: int) -> float: listing = subprocess.run( ["ps", "-A", "-o", "pid=,ppid=,rss="], capture_output=True, text=True, check=True ).stdout + return sum_tree_mb(listing, root) + + +def sum_tree_mb(listing: str, root: int) -> float: + """MB resident in `root` and its descendants, from `ps -o pid=,ppid=,rss=` output (KB).""" children: dict[int, list[int]] = {} rss: dict[int, int] = {} for line in listing.splitlines(): @@ -47,8 +53,17 @@ class TreeLimit: A server under test is started in its own process group, so one signal stops `uv` and the Python it started. `peak_mb` and `killed` go into the report.""" - def __init__(self, proc: subprocess.Popen, limit_mb: float, interval_s: float = 0.5): + def __init__( + self, + proc: subprocess.Popen, + limit_mb: float, + interval_s: float = 0.5, + measure: Callable[[int], float] = tree_rss_mb, + ): self.proc, self.limit_mb, self.interval_s = proc, limit_mb, interval_s + # How a process group's memory is read; tests pass a fixed reading, since what the OS + # reports as resident depends on memory compression and cannot be held steady. + self.measure = measure self.peak_mb, self.killed = 0.0, False self._stop = threading.Event() self._thread = threading.Thread(target=self._watch, daemon=True) @@ -63,12 +78,12 @@ def __exit__(self, *exc) -> None: def reset(self) -> float: """Restart the peak from the group's current memory, and return that.""" - self.peak_mb = tree_rss_mb(self.proc.pid) + self.peak_mb = self.measure(self.proc.pid) return self.peak_mb def _watch(self) -> None: while not self._stop.wait(self.interval_s) and self.proc.poll() is None: - used = tree_rss_mb(self.proc.pid) + used = self.measure(self.proc.pid) self.peak_mb = max(self.peak_mb, used) if used > self.limit_mb: self.killed = True diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 414a12fd9..04fe74ef8 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -33,17 +33,31 @@ NUMBER_TOL_FT = 1e-5 # Spot centres and offsets are rounded by the server; a hundredth of a foot is 1/8 inch. OFFSET_TOL_FT = 0.01 -# Clearance check -> the bands an unseen hazard could hide in, and its rules.yaml radius. C1: a -# gas meter hangs on the wall face (wall band) and its regulator stands on the ground (ground -# band); openings are on the wall face; AC units, drives and pools are on the ground. -# ground_surface (radius 0, added in RuleSet.from_yaml) needs the ground under the battery. -CLEARANCE_CHECKS = { - "gas_clearance": (("ground", "wall"), "gas_ft"), - "ac_clearance": (("ground",), "ac_ft"), - "drive_clearance": (("ground",), "drive_ft"), - "pool_clearance": (("ground",), "pool_ft"), - "opening_clearance": (("wall",), "opening_ft"), -} + + +@dataclass(frozen=True) +class Need: + """One piece of coverage a check needs around a battery over [lo, hi] (server/README.md at the + tested ref, "What settles each check"): + + - `ground`: ground over [lo - r, hi + r], out to depth + r in front of the battery; + - `wall`: the wall band over [lo - r, hi + r], seen higher than `height`; + - `facing`, `overhead`: the band over [lo, hi]; where no measurement (`facing`, `overheads`) + covers a stretch, the band's `out_ft` there must reach `height`. + + r is `radius`, plus the battery's position error when `widen`. + """ + + band: str + radius: float = 0.0 + height: float = 0.0 + widen: bool = True + + +# Scene lists whose entries settle facing_gap and headroom where they cover the battery. +MEASUREMENTS = {"facing": "facing", "overhead": "overheads"} +# rules.yaml error keys for a wall's `source`; a wall without one is tapped. +WALL_ERROR = {"tap": "wall", "mesh": "mesh", "plane": "plane"} @dataclass(frozen=True) @@ -52,33 +66,61 @@ class RuleSet: width_ft: float depth_ft: float - # check id -> (bands, radius in feet around the battery footprint that must be observed) - radii: dict[str, tuple[tuple[str, ...], float]] - # default error by object source, for walls and the meter, and drift_per_ft + # check id -> the coverage its pass needs + needs: dict[str, tuple[Need, ...]] + # default error by object source, for walls (tap, mesh, plane) and the meter, and drift_per_ft errors: dict[str, float] + # the height up the wall the cable route must be seen, from the meter to the battery + route_height_ft: float = 0.0 @classmethod def from_yaml(cls, text: str) -> RuleSet: data = yaml.safe_load(text) - def value(*path: str) -> float: + def value(*path: str, default: float | None = None) -> float: node: Any = data for key in path: if not isinstance(node, dict) or key not in node: + if default is not None: + return default raise ValueError(f"rules.yaml has no {'.'.join(path)}") node = node[key] + if node is None and default is not None: + return default return float(node["value"] if isinstance(node, dict) else node) - radii = { - check: (bands, value("clearances", key)) - for check, (bands, key) in CLEARANCE_CHECKS.items() + depth = value("battery", "depth_ft") + headroom = value("headroom", "min_ft") + # Openings need the wall seen to headroom height, or to a lower exempt height the rules set. + opening_height = min( + headroom, value("openings", "exempt_bottom_above_ft", default=headroom) + ) + + def r(key: str) -> float: + return value("clearances", key) + + def reach(key: str) -> Need: + return Need("ground", r(key)) + + needs = { + "ground_surface": (Need("ground", 0.0),), + "wall_backing": (Need("wall", 0.0, value("battery", "height_ft"), widen=False),), + "gas_clearance": (reach("gas_ft"), Need("wall", r("gas_ft"), headroom)), + "battery_clearance": (reach("battery_ft"), Need("wall", r("battery_ft"), headroom)), + "ac_clearance": (reach("ac_ft"),), + "drive_clearance": (reach("drive_ft"),), + "pool_clearance": (reach("pool_ft"),), + "opening_clearance": (Need("wall", r("opening_ft"), opening_height),), + "wall_equipment_above": (Need("wall", r("wall_equipment_ft"), headroom, widen=False),), + "facing_gap": (Need("facing", 0.0, depth + value("facing", "min_ft"), widen=False),), + "headroom": (Need("overhead", 0.0, headroom, widen=False),), } - radii["ground_surface"] = (("ground",), 0.0) errors = { - name: value("errors", f"{name}_ft") for name in ("tap", "vlm", "tape", "wall", "meter") + name: value("errors", f"{name}_ft") + for name in ("tap", "vlm", "tape", "wall", "mesh", "plane", "meter") } errors["drift_per_ft"] = value("errors", "drift_per_ft") - return cls(value("battery", "width_ft"), value("battery", "depth_ft"), radii, errors) + return cls(value("battery", "width_ft"), depth, needs, errors, value("route", "height_ft")) def schema_errors(instance: Any, schema: dict) -> list[str]: @@ -90,13 +132,22 @@ def schema_errors(instance: Any, schema: dict) -> list[str]: # --- Coverage -------------------------------------------------------------------------------- +def reached(entry: dict) -> float: + """How far an observed entry's view reached: its `out_ft`, or for a band other than ground + without one, all the way (C1: the wall to headroom height, facing to whatever faces it, + overhead clear to the sky).""" + default = 0.0 if entry["band"] == "ground" else math.inf + return entry.get("out_ft", default) + + def observed(scene: dict, band: str, min_out_ft: float = 0.0) -> list[tuple[float, float]]: - """Merged observed s-intervals for a band. Ground intervals must reach `min_out_ft` out.""" + """Merged observed s-intervals for a band, counting only entries whose view reached + `min_out_ft` (out from the wall for ground and facing, up for wall and overhead).""" spans = [] for entry in scene.get("coverage", {}).get("observed", []): if entry["band"] != band: continue - if band == "ground" and entry.get("out_ft", 0.0) + EPS < min_out_ft: + if reached(entry) + EPS < min_out_ft: continue spans.append(tuple(sorted(entry["span_ft"]))) spans.sort() @@ -229,55 +280,92 @@ def required_span(lo: float, hi: float, radius: float) -> tuple[float, float]: def battery_error(scene: dict, rules: RuleSet, wall_id: str, lo: float, hi: float) -> float: - """Position error of a battery over [lo, hi] on `wall_id`: the wall's explicit error, or its - default plus drift at the battery's edge further from the meter (S2 70ab0b0).""" + """Position error of a battery over [lo, hi] on `wall_id`: the wall's explicit error, or the + default for its `source` (tap, mesh, plane) plus drift at the battery's edge further from the + meter (S2 70ab0b0).""" wall = next((w for w in scene["walls"] if w["id"] == wall_id), None) if wall is None: raise ValueError(f"the result names wall {wall_id!r}, which the scene does not have") if "plus_minus_ft" in wall: return wall["plus_minus_ft"] - return rules.errors["wall"] + rules.errors["drift_per_ft"] * max(abs(lo), abs(hi)) + default = rules.errors[WALL_ERROR[wall.get("source", "tap")]] + return default + rules.errors["drift_per_ft"] * max(abs(lo), abs(hi)) def reach_gaps( scene: dict, rules: RuleSet, check: str, wall_id: str, lo: float, hi: float ) -> list[tuple[str, str]]: - """(band, what is missing) for each band `check` needs observed around a battery over - [lo, hi] and does not have. + """(band, what is missing) for each Need of `check` a battery over [lo, hi] does not have. Only points within the check's radius R of the battery whatever the wall's shape are required, so a correct server is never flagged. R is widened by the battery's position - error e (C5: a pass needs the margin to exceed the error, and an unseen hazard just past the - seen area is a margin of zero). + error e where the server widens it (C5: a pass needs the margin to exceed the error, and an + unseen hazard just past the seen area is a margin of zero). - wall band: [lo - R - e, hi + R + e] along the wall, since distance along the wall is - never shorter than straight-line distance; + never shorter than straight-line distance, seen up to the check's height; - ground band: out to depth + R + e in front of the battery (a battery sits flush on one straight segment), and at distance d past either end out to R + e - d. The ground point - there is at most d along the wall plus R + e - d out from the battery's end. + there is at most d along the wall plus R + e - d out from the battery's end; + - facing and overhead bands: over the battery, reaching the check's distance unless + measured. """ - bands, radius = rules.radii[check] - reach = radius + battery_error(scene, rules, wall_id, lo, hi) gaps = [] - for band in bands: - if band == "ground": + for need in rules.needs[check]: + reach = need.radius + (battery_error(scene, rules, wall_id, lo, hi) if need.widen else 0.0) + if need.band == "ground": gap = ground_gap(scene, lo, hi, rules.depth_ft, reach) - else: + elif need.band == "wall": a, b = required_span(lo, hi, reach) - seen = covers(observed(scene, band), a, b) - gap = None if seen else f"{band} [{a:.2f}, {b:.2f}] observed" + seen = covers(observed(scene, "wall", min_out_ft=need.height), a, b) + gap = None if seen else f"wall [{a:.2f}, {b:.2f}] observed higher than {need.height} ft" + else: + gap = measured_band_gap(scene, need.band, wall_id, lo, hi, need.height) if gap: - gaps.append((band, gap)) + gaps.append((need.band, gap)) return gaps +def measured_band_gap( + scene: dict, band: str, wall_id: str, lo: float, hi: float, need: float +) -> str | None: + """The first stretch of [lo, hi] the facing or overhead band leaves unsettled: not observed, + or observed short of `need` where no measurement covers it.""" + entries = [ + (*sorted(e["span_ft"]), reached(e)) + for e in scene.get("coverage", {}).get("observed", []) + if e["band"] == band + ] + measured = [ + tuple(sorted(m["span_ft"])) + for m in scene.get(MEASUREMENTS[band], []) + if m.get("wall_id", wall_id) == wall_id + ] + edges = {x for e in entries for x in e[:2]} | {x for m in measured for x in m} + cuts = sorted({lo, hi} | {x for x in edges if lo < x < hi}) + for p, q in itertools.pairwise(cuts): + if q - p <= EPS: + continue + seen = [out for x, y, out in entries if x <= p + EPS and q - EPS <= y] + where = f"{band} [{p:.2f}, {q:.2f}] observed" + if not seen: + return f"{where}, none seen" + settled = any(x <= p + EPS and q - EPS <= y for x, y in measured) + if not settled and max(seen) + EPS < need: + return f"{where} out to {need:.2f} ft or measured, seen {max(seen):.2f} ft" + return None + + def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: """Missing coverage is never a pass (C5), to the reach each check's rule looks out to (see reach_gaps). A sweep run that passes needs every check's area for every start in it, and the wall and cable route back to the meter. No slack: runs list exactly the starts that were evaluated.""" problems: list[str] = [] - wall = observed(scene, "wall") + route_wall = observed(scene, "wall", min_out_ft=rules.route_height_ft) + # A passing run passes every check the server evaluated; a check it did not evaluate (a + # battery clearance with no battery in the scene, say) needs nothing. + evaluated = {c["id"] for c in result.get("checks", [])} or set(rules.needs) if "coverage" not in scene and result["decision"] == "pass": problems.append("decision pass for a scene with no coverage at all") @@ -286,12 +374,13 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: continue lo, hi = run["start_ft"][0], run["start_ft"][1] + rules.width_ft route_lo, route_hi = min(0.0, lo), max(0.0, hi) - if not covers(wall, route_lo, route_hi): + if not covers(route_wall, route_lo, route_hi): problems.append( f"sweep pass for starts {run['start_ft']} but the wall and cable route " - f"[{route_lo:.2f}, {route_hi:.2f}] were not all observed" + f"[{route_lo:.2f}, {route_hi:.2f}] were not all observed higher than " + f"{rules.route_height_ft} ft" ) - for check in sorted(rules.radii): + for check in sorted(set(rules.needs) & evaluated): for _, gap in reach_gaps(scene, rules, check, run["wall_id"], lo, hi): problems.append(f"sweep pass for starts {run['start_ft']} but {check} needs {gap}") @@ -299,7 +388,7 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: if spot is not None: lo, hi = spot["span_ft"] for check in result.get("checks", []): - if check["outcome"] != "pass" or check["id"] not in rules.radii: + if check["outcome"] != "pass" or check["id"] not in rules.needs: continue for _, gap in reach_gaps(scene, rules, check["id"], spot["wall_id"], lo, hi): problems.append(f"check {check['id']} passes but needs {gap}") @@ -308,10 +397,12 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: if request["kind"] != "band" or "span_ft" not in request or "band" not in request: continue a, b = sorted(request["span_ft"]) - # A ground request may want deeper ground over a span already seen: it is redundant only - # where ground was seen as far out as it asks (out_ft, S2 55cb4b0), or, when it does not - # say, out to GROUND_FAR_FT. - far = request.get("out_ft", GROUND_FAR_FT) if request["band"] == "ground" else 0.0 + # A request may want a farther or higher view over a span already seen: it is redundant + # only where the band was seen as far as it asks (`out_ft`, on every band since S2 + # 55cb4b0). Without one, a ground request is redundant past GROUND_FAR_FT, and any other + # only where the band was seen all the way (no `out_ft`). + default = GROUND_FAR_FT if request["band"] == "ground" else math.inf + far = request.get("out_ft", default) if b - a > EPS and covers(observed(scene, request["band"], min_out_ft=far), a, b): problems.append( f"missing_evidence asks for {request['band']} {request['span_ft']}, " @@ -367,7 +458,7 @@ def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = return problems lo, hi = spot["span_ft"] for c in unobserved_checks(result): - if c not in rules.radii: + if c not in rules.needs: continue for band, gap in reach_gaps(scene, rules, c, spot["wall_id"], lo, hi): if (band, c) not in named: @@ -657,7 +748,7 @@ def with_less_coverage(scene: dict, trim_ft: float = 0.5) -> dict | None: def with_ground_short_of(scene: dict, rules: RuleSet, margin_ft: float = 0.1) -> dict | None: """Ground seen out to just short of the largest clearance radius, where any correct server must stop passing the check that needs it.""" - radius = max(r for bands, r in rules.radii.values() if "ground" in bands) + radius = max(n.radius for ns in rules.needs.values() for n in ns if n.band == "ground") reach = rules.depth_ft + radius - margin_ft grounds = [e for e in scene.get("coverage", {}).get("observed", []) if e["band"] == "ground"] if not any(e.get("out_ft", 0.0) > reach for e in grounds): diff --git a/verification/hsverify/simrun.py b/verification/hsverify/simrun.py index 3e0dc39c6..e7da0ef76 100644 --- a/verification/hsverify/simrun.py +++ b/verification/hsverify/simrun.py @@ -9,7 +9,8 @@ captured immediately and flagged `transient`, because its image may already show the next state. The run ends at `--until`, after `--idle` seconds without a new state, or at `--timeout`. A state named with `--require` that never appeared makes the run fail. If the app -logged a scan bundle, it is copied into the report as scan.zip and named in report.json's +logged a scan bundle, it is copied into the report as scan.zip (required with `--require-export`) +and named in report.json's `app_export`, so the report folder can be replayed as an app export. Reports go outside git (default ~/house-scanning-data/reports/sim/) because a replay can put dataset frames on screen. """ @@ -337,6 +338,17 @@ def copy_app_export(out: Path) -> tuple[str | None, list[str]]: return "scan.zip", [] +def app_export(out: Path, required: bool) -> tuple[str | None, list[str]]: + """copy_app_export, where a run that must keep the app's scan fails without one.""" + name, problems = copy_app_export(out) + if required and name is None and not problems: + problems = [ + "The run was required to keep the app's scan (--require-export), but the app logged " + "no bundle." + ] + return name, problems + + def app_running(udid: str, bundle_id: str) -> bool: out = subprocess.run( ["xcrun", "simctl", "spawn", udid, "launchctl", "list"], capture_output=True, text=True @@ -516,6 +528,11 @@ def main(argv: list[str] | None = None) -> int: metavar="STATE", help="a state the run must reach; one that never appears is a problem (exit 1)", ) + parser.add_argument( + "--require-export", + action="store_true", + help="fail unless the app logs a scan bundle, which is copied into the report as scan.zip", + ) parser.add_argument( "--log-ready-timeout", type=float, @@ -607,7 +624,7 @@ def main(argv: list[str] | None = None) -> int: print("Running...", flush=True) follow(args, udid, bundle_id, out, rep) rep.problems += missing_required(args.require, rep.states) - rep.app_export, export_problems = copy_app_export(out) + rep.app_export, export_problems = app_export(out, args.require_export) rep.problems += export_problems rep.crash_reports = collect_crashes(started, out) if rep.crash_reports: diff --git a/verification/product/README.md b/verification/product/README.md deleted file mode 100644 index 9ace0b1f6..000000000 --- a/verification/product/README.md +++ /dev/null @@ -1,78 +0,0 @@ -# House Scan: what the homeowner experiences - -A description of the iOS capture app from the outside: what the homeowner sees on each screen, -what they can do there, and exactly what happens when they do it. It is written from the app's -code and tests, then checked against the app running in the Simulator. Anything that looks -wrong is collected in [bug-triage.md](bug-triage.md). - -## Scope decisions - -- **Surface.** The HouseScan app, one run of the homeowner flow from onboarding to the result, - default settings. The ARKit camera cannot run in the Simulator, so behavior that needs the live - camera is described from code and marked unverified. -- **Source.** `t3/ios-mvf`. Footers and the last Simulator check are at `a39d0a5`; line - references are at `beede15`. Between the two, the only change a homeowner can reach is that the - strip's marks scale with text size and stay inside it; the upload refusal and meter-number - states added to the contract are not wired to anything yet. -- **How it is run.** `make sim-app REF=` from `verification/`: the real ADVIO replay - `advio-20-0040-0075` (C3), the autopilot, and the server started from `t3/server`. -- **Out of scope.** The `-uiDemo` screens (a design preview, not the product), TestFlight - signing, the server's placement logic (checked by `hsverify/e2e.py`). -- **Not yet built.** Panel close-ups and homeowner questions other than "does this window open". - They are not described. - -## Shape of every document - -The unit of interaction is **one screen of the scan** (one `ScanPhase`). Its five phases, the -subsections of "The interaction, event by event", are: - -1. **Arriving:** what the screen shows first and what the engine starts. -2. **Leaving at once:** the ways off the screen before anything is captured or marked. -3. **First capture:** the first photo kept, tap accepted or answer given. -4. **While capturing:** what updates live and what the homeowner can still do. -5. **Advancing:** what is committed and which screen comes next. - -**Variants** (rows of every Modifiers table, columns "at arrival" and "while capturing"): -live camera or replay; autopilot on or off; a server URL or the bundled sample result; larger -text sizes; Reduce Motion. - -**Interrupts** (rows of every Cancel and interrupt table, in this order): -the homeowner's way out on the screen ("I can't get there", "Can't get a clear shot", Cancel); -Start over; tracking limited (moving fast, too little detail, too dark); tracking lost or -relocalizing, including the 20 s reset; the app backgrounded or a call arriving; camera access -off or the camera session failing; the network lost or the upload failing; the app killed. - -**Cross-cutting concerns** (paragraphs of "Interactions with other systems", in this order): -coverage and evidence; stored photos; upload and offline; accessibility (VoiceOver, Dynamic -Type, Reduce Motion); haptics and motion; verification hooks (`STATE=` logs, replay, autopilot). - -Each document then has Summary, The simple case, The interaction (with one -`stateDiagram-v2`), Modifiers, Cancel and interrupt, Interactions with other systems, Edge cases, -and Open questions and verification, ending with the commit it was verified against. -[screens/meter-close-up.md](screens/meter-close-up.md) is the pilot every other document copies. - -## Writing rules - -- Describe the experience, not the code. Technical detail goes in `> Technical note:` quotes, - only where the mechanism changes what the homeowner would expect. -- Use the words in [glossary.md](glossary.md). Numbers and definitions live in the foundation - documents; other documents link to them. -- Every table cell is filled, "No effect." where that is the answer. -- Behavior that looks like a bug is said plainly under "Open questions" with the file and line. - -## Structure and coverage - -| Document | What it covers | Status | -| --- | --- | --- | -| [foundations/flow.md](foundations/flow.md) | The screens in order, what moves between them, the launch options, Start over, failures | drafted; checked in the Simulator | -| [foundations/coverage-and-guidance.md](foundations/coverage-and-guidance.md) | The coverage strip, when a photo is kept, the one instruction and what overrides it | drafted; checked in the Simulator | -| [screens/onboarding.md](screens/onboarding.md) | The intro pages and the failure screens (camera off, unsupported phone, unreadable replay) | drafted; checked in the Simulator | -| [screens/find-meter.md](screens/find-meter.md) | Finding the meter and tapping it to set the wall | drafted; checked in the Simulator | -| [screens/meter-close-up.md](screens/meter-close-up.md) | The self-timed meter photo (pilot) | drafted; checked in the Simulator | -| [screens/wall-walk.md](screens/wall-walk.md) | Walking the wall, marking its ends, "I can't get there" | drafted; checked in the Simulator | -| [screens/mark-features.md](screens/mark-features.md) | Marking gas meters, doors, windows, AC units, driveways and fences; the window question | drafted; checked in the Simulator | -| [screens/gap-request.md](screens/gap-request.md) | One targeted extra view, from the phone or the server | drafted; checked in the Simulator | -| [screens/uploading.md](screens/uploading.md) | Sending the scan and waiting for the result, offline and failure | drafted; checked in the Simulator | -| [screens/result.md](screens/result.md) | The result, its checks and the 3D and AR views | drafted; checked in the Simulator | -| [verification.md](verification.md) | Checks against the running app and their results | current pass at `a39d0a5` | -| [bug-triage.md](bug-triage.md) | Every suspected defect, deduplicated | 16 entries, 2 resolved | diff --git a/verification/product/bug-triage.md b/verification/product/bug-triage.md deleted file mode 100644 index c50fbdb4e..000000000 --- a/verification/product/bug-triage.md +++ /dev/null @@ -1,120 +0,0 @@ -# Suspected defects - -Collected from every document's open questions and from [verification.md](verification.md), -deduplicated. Status checked at `t3/ios-mvf` `a39d0a5`; line references are at `0876e03` unless an -entry names another commit; paths are under `ios/`. **Seen** means observed in the Simulator; **code** means read from the -source and not run. Every blocker and major entry was re-read in the source by a second reader. - -| ID | Severity | Where the homeowner meets it | Evidence | Decision | -| --- | --- | --- | --- | --- | -| B-03 | Blocker | Camera denied, camera failure, unreadable replay: no message, no way on | seen (replay) | fix | -| B-04 | Blocker | Meter close-up: "Can't get a clear shot" may never appear | code | fix | -| B-09 | Blocker | Feature list: "Add something" does nothing | code | fix | -| B-06 | Major | Walk: an end closer than 20 ft can't be marked (answer half fixed at `525ea40`) | code | fix | -| B-08 | Major | "Can't get there" is never sent to the server | code | fix (with S2) | -| B-10 | Major | Feature list: the scan resets after 20 s of relocalizing (result and upload fixed at `beede15`) | code | fix | -| B-11 | Major | Meter tap can land on another wall's extended plane (ground height fixed at `beede15`) | code | fix | -| B-12 | Major | A server request beyond a still-marked wall end can't finish (past-end case fixed at `beede15`) | code | fix | -| B-02 | Major | Upload and camera failures show raw error text | seen at `21a63e7` | fix | -| B-13 | Minor | 3D and AR tint zones stop one battery width short | code | fix | -| B-14 | Minor | Result rule lines hide whether a limit is a minimum or maximum | seen | fix | -| B-16 | Minor | Wording and VoiceOver: "Tap the ac unit"; "ft" and "in" read as letters | seen in part | fix | -| B-01 | Resolved | Every upload failed with 404 | seen fixed at `0876e03` | none | -| B-05 | Resolved | Camera-screen text missing at `21a63e7` | seen fixed | none | -| B-15 | Resolved | AR view did not say a sample result is a sample | seen fixed at `525ea40` | none | -| B-07 | Minor | Scan photos are never deleted from the phone | code | product call | - -## Blockers - -**B-03. Failures other than an unsupported phone never reach the screen.** Only the failure screen -shows a failure, and the app switches to it only for an unsupported phone -(`HouseScan/Runtime/ScanEngine.swift:91-93`). A denied camera, a failed session or an unreadable -replay set the failure without changing screen (lines 105, 379, 381). The homeowner stays on -"Find your electric meter" with no camera and taps do nothing; "Open Settings" is unreachable. -Seen with `-replay /nonexistent`: the app logged "replay unreadable: cannot read -/nonexistent/session.json" and stayed on onboarding (report -`sim/20260926-044618-5520229-5520229d-preview-bad-replay`). - -**B-04. The close-up's way out can fail to appear.** A failed try counts only when one problem -lasts 4 s with no good frame in between (`HouseScanKit/.../Capture/CloseUpGate.swift`, `evaluate`), -and the photo needs 0.6 s of good frames in a row. Frames that alternate between good and blurry -satisfy neither, so "Can't get a clear shot" never appears. S3's full-flow test cannot catch it: -its autopilot calls skip directly. Suggested: count a failed try per 4 s without a photo. - -**B-09. "Add something" on the feature list does nothing.** The six chips start a mark -(`HouseScan/UI/Screens/MarkFeaturesScreen.swift:64`), but the list never shows marking: the circle, -prompt and Mark button exist only on the walk (`WallWalkScreen.swift:28-45`). The homeowner taps a -chip and nothing visible happens; "Looks complete" then uploads without it. Adding a missed -feature is what this screen is for. - -## Major - -**B-06. Wall ends.** Half fixed at `525ea40`: after "Wall ends here" the walk asks "What's at the -left end?", and only "something blocks it" records a real end; a corner or no answer stays -unexplored. Still open: "Wall ends here" is offered only while the walk asks for the end, which -happens at 6.1 m of coverage (`HouseScanKit/.../Guidance/GuidancePlanner.swift`, `preferredTask`), -so a wall that ends at 10 ft can only be recorded through "Can't get there". Suggested: offer -"Wall ends here" whenever the wall's end is in view. - -**B-08. "Can't get there" is never sent.** The upload carries only covered stretches and each end's -kind (`HouseScan/Runtime/ScanEngine+Export.swift:48-58`, `HouseScanKit/.../Scene/SceneExport.swift`), -so a stretch the homeowner can't reach looks to the server like one nobody looked at. The gap -screen's hint promises "An installer will look at this part instead"; the server may ask for the -same view again. C1 has no field for it, so this needs S2 and S3 together. - -**B-10. The scan resets after a lost world frame.** At `beede15` the reset spares the upload, -result and AR screens; it still fires on the feature list, which shows no coaching. Before: The 20 s relocalization reset runs on every screen while -the camera session runs (`ScanEngine.swift:189, 357-364`), including the feature list, the upload -and the result, where no coaching is shown. After a call, the homeowner can come back to "Find your -electric meter" with every photo, mark and the result gone. The reset also keeps a half-finished -mark and the first close-up (`ScanEngine.swift:386-410`, `KeyframeStore.swift:79`). - -**B-11. The meter tap can guess a depth.** At `beede15` the ground height comes from planes as -they arrive; the infinite-plane fallback remains (`LiveCapture.swift:76`). Before: The raycast falls back to `existingPlaneInfinite` -(`HouseScan/Runtime/LiveCapture.swift:76`): a tap at a meter whose wall is not detected yet can pin -it on another wall's plane extended past its edge. The code's own comment says a tap never guesses a -depth. The ground height also stays a guess (1.4 m below the phone) if no ground plane was seen at -the tap (`ScanEngine+Actions.swift:48-57`). - -**B-12. Requests past a wall end.** At `beede15` marking the end again settles a past-end -request, but only the walk offers "Wall ends here"; the gap request screen shows neither it nor -the end question (`WallWalkScreen.swift:137-173`), so a homeowner can't use the fix. The -band-request case below is unchanged. Before: "Capture it now" on a past-end request first clears that end -(`ScanEngine+Actions.swift:206-209`) and nothing sets it again, so the next upload reports the side -as unexplored even after the homeowner looked past it, and the server can ask again. A server band -request whose stretch lies beyond an end that is still marked can never fill: the coverage map -records no cells past a marked end (`HouseScanKit/.../Coverage/CoverageMap.swift:113-118`), so its -bar stays at 0% and "I can't get there" is the only way out. - -**B-02. Failures show raw error text.** "That didn't go through" is followed by the error's -description, for example `The server answered 404: {"error":{"code":"not_found",...}}` (seen at -`21a63e7`; same code at `5520229`: `UI/Copy/ScanCopy.swift:177`, `ScanEngine.swift:533, 560`). The -camera-failure text does the same (`ScanCopy.swift:240`) once B-03 is fixed. - -## Minor - -- **B-13.** Tint zones in the 3D model and the AR view use the sweep's range of battery *start* - positions as if it were where the battery stands, so each ends one battery width short - (`ScanEngine+Export.swift:124`). -- **B-14.** Seen on the sample result: "Measured 3 ft. The rule is 20 ft, and the measurement can be - off by about 4 in." A maximum reads like a minimum; the server's `comparison` and - `review_threshold_ft` are not used (`UI/Copy/ScanCopy.swift:210-218`). -- **B-16.** The AC unit prompt reads "Tap the ac unit", and VoiceOver hears "Mark ac unit" and - "Remove ac unit" (`ScanCopy.swift:122`, `WallWalkScreen.swift:258`, `MarkFeaturesScreen.swift:121`; - "Add ac unit" seen in the accessibility audit). - Distances are spoken as "ft" and "in" in the 3D description and measurement lines, though - `Distance.spoken` exists (`ResultScene3D.swift:296-321`, `ResultScreen.swift:237`). -- **B-07.** Each scan's photos stay in `Caches/Scans//` after upload, Start over or relaunch; - `discardKeyframes` forgets the list but not the files (`KeyframeStore.swift:79`). -- **B-01 (resolved).** Until `0876e03` the app posted to `/v1/scenes`, which the server does not - serve, and every upload failed with 404. At `0876e03` it posts to `/v1/placements`: seen `POST - /v1/placements 200` and a result (report `sim/20260926-050058-t3-ios-mvf-0876e03c-replay`). -- **B-15 (resolved).** At `525ea40` the AR view on a sample result says "Example spot, not your - result" and "No server checked this scan.", and the upload screen says "Making a sample result". -- **B-05 (resolved).** At `21a63e7` the camera screens' cards rendered without text; at `5520229` the - same capture shows every word. The UI lane added the missing colour assets. - -Smaller notes stay in each document's open questions: refusal wording on the meter tap, "Wall ends -here" aiming at a circle that is not drawn, the kind picker sliding under Reduce Motion, two taps on -"Can't get there" finishing an empty walk, and the result appearing more than 1.2 s after the app -logs it on first load. diff --git a/verification/product/foundations/coverage-and-guidance.md b/verification/product/foundations/coverage-and-guidance.md deleted file mode 100644 index 451a366d7..000000000 --- a/verification/product/foundations/coverage-and-guidance.md +++ /dev/null @@ -1,185 +0,0 @@ -# Coverage and guidance - -## Summary - -While the homeowner walks, the app decides three things without being asked: which camera frames -to keep as photos, which stretches of wall and ground those photos have covered, and the one -thing the homeowner should do next. This document owns the rules and numbers behind all three. -The screens that use them, [the wall walk](../screens/wall-walk.md) and -[a gap request](../screens/gap-request.md), link here instead of restating them. None of it -decides whether a battery fits; covered means evidence exists, not that a spot passes. - -## The simple case - -The homeowner walks slowly along the wall with the phone pointed at it. Every half meter or so -the phone keeps a photo; the counter at the top goes up and the fog over the wall clears where -the camera has looked. The strip at the bottom fills in: gray where nothing has been seen, then -covered once a stretch has been seen from two places, top to bottom. The instruction at the top says where to -go ("Walk slowly to your left") and changes only when that is done or after at least 3 s. - -## The interaction, event by event - -```mermaid -stateDiagram-v2 - [*] --> unseen - unseen --> seen : a kept photo shows one of the cell's three sample lines - seen --> covered : every sample line seen from two places 0.25 m apart - unseen --> skipped : "I can't get there" over this cell - seen --> skipped : "I can't get there" over this cell -``` - -The states are those of one 6 in cell of the [coverage strip](../glossary.md#seeing-the-wall). - -### Arriving - -Coverage starts empty when the meter is tapped: the strip is centered on the meter, both bands -gray. Nothing is kept on [finding the meter](../screens/find-meter.md) or during the -[close-up](../screens/meter-close-up.md), apart from the close-up itself. - -### Leaving at once - -A cell that no kept photo shows stays unseen for the rest of the scan. Nothing decays: a covered -cell stays covered. - -### First capture - -#### When a photo is kept - -A frame is kept only when all of these hold: - -- tracking has been normal for at least 0.5 s; -- the phone moves slower than 1.5 m/s and turns slower than 60° per second; -- the frame is at least half as sharp as the recent frames, not too dark (mean brightness at - least 40 of 255) and not glaring (under a quarter of the pixels clipped); -- at least 0.33 s has passed since the last kept photo (at most about three per second); -- the phone has moved 0.5 m or turned 15° since the last kept photo, or the frame would show a - sample line nobody has seen yet in at least three cells. - -A kept photo counts, and coverage moves, only once its file is written to the phone, so the strip -never claims a view the upload lacks. Walk photos give no haptic; the counter flashes instead. - -### While capturing - -Each cell is checked along three **sample lines** across its band, evenly spaced from edge to edge. -On the wall they sit at the foot, 0.99 m up and 1.98 m up (the top of the band); on the ground -at the foot of the wall, 0.6 m out and 1.2 m out. A photo sees a line when both of its points, a -quarter and three quarters of the way along the cell, are no farther than 6 m, no more than 65° -from straight on and away from the outer 3 % of the image, and the phone is in front of the -wall's line. A cell counts as **seen** once any kept photo sees any of its lines. It becomes -**covered** only when every line has been seen from two places at least 0.25 m apart; different -photos may supply different lines. A cell whose top line no photo has reached stays seen, however -often its lower lines are photographed. Photos are kept only with normal tracking, so every kept -photo counts. (Fixed in `beede15`: before, most of a cell's points from two places were enough, -so a band could read covered without its top ever being in view.) - -The strip shows what has been seen plus 2.5 m of fog ahead on each side, and stops at a marked -wall end. When the phone refines the meter's position, the seen stretches and the ends move with -it, in whole cells. - -#### One instruction at a time - -The instruction is chosen in this order, first match wins: - -1. **The end question** on the walk, right after "Wall ends here": "What's at the left end?" (or - right), with "This tells the installer whether the wall keeps going." Nothing else is chosen - until it is answered ([the wall walk](../screens/wall-walk.md#while-capturing)). -2. **Coaching** from the camera, which replaces everything below it until it clears: "Move your phone - slowly" (starting up), "Slow down" ("Walk a little slower so the photos stay sharp."), "Aim at - a corner or somewhere with more texture" ("A plain wall or the sky gives your phone nothing to - follow."), "It's too dark to see the wall", "Point at the meter like this." ("Your phone lost its place for a moment.", - relocalizing), "Your phone lost its place". During the walk, frames kept back for being blurry, - moving or dark show "Slow down" or "It's too dark to see the wall" only once the problem has - lasted 0.7 s, and the coaching clears after 0.5 s without it. - While tracking is not normal, the walk and the gap request hide everything drawn on the camera - image (the fog, the path, the target ring, the meter dot, the ends and the marks): drawn from - a position the phone does not trust, they would sit in the wrong place. They fade out and back - over 0.2 s. The strip is not drawn on the camera image and stays. -3. **"Take a step back"** when the phone is closer than 1.2 m to the wall; satisfied at 1.4 m. -4. **Aim at a lagging band**: within 1 m either side of the phone, if one band is done over at least - 0.45 m (three cells) where the other is not, "Tilt down to show the ground" or "Tilt up to show - more wall", with a ring on the spot. -5. **Walk to the left**, until coverage runs unbroken 6.1 m (20 ft) from the meter on the left. - The ring and dots lead 1 m past whichever is farther along: the unbroken coverage or the - phone itself, so a homeowner already past a thin patch is not sent back to it (fixed in - `beede15`; the aim instructions ask for such patches); - then **"Is this the left end of the wall?"**, with "Aim where the wall stops or turns a - corner, and tap Wall ends here.", the only moment "Wall ends here" is offered. Only after the - left end is set: the same for the right. -6. With both ends set, the first stretch of at least 0.45 m between them where a band is not done - (ground checked before wall), as an aim instruction. -7. **"That's the whole wall"**, with "Tap Done when you're ready". - -A new instruction replaces the current one only when the current one is satisfied or has been up -for 3 s of screen time. An aim instruction is satisfied at 80 % of the cells within 0.3 m of its spot covered; a -walk instruction when that side's end is set; an end question when it is answered. - -### Advancing - -Coverage is sent with the scan as the covered stretches of each band and the kind of each end. An -end is **real** only when it was marked with "Wall ends here" and answered "Something blocks it". -It is **unexplored** when answered "It turns a corner", when left unanswered, or when set with -"Can't get there": the wall may go on past it. Skipped cells are not sent: to the server a stretch the homeowner said they -can't reach looks the same as one nobody looked at, and it treats both as unknown. - -## Modifiers - -| Modifier | At arrival | While capturing | -| --- | --- | --- | -| Live camera or replay | No effect on the rules; a replay's frames go through the same checks. | Same. | -| Autopilot | No effect on the rules. | The replay runs at three times speed, so the 0.33 s between photos is measured on the recording's clock and bites more often. The 3 s per instruction is screen time, so instructions stay up as long as live (fixed in `beede15`). | -| Server or sample result | No effect. | No effect. | -| Larger text sizes | No effect on coverage. | No effect. | -| Reduce Motion | The fog still fades as cells clear, without drifting. | Same. | - -## Cancel and interrupt - -| Event | Before the first kept photo | After it | -| --- | --- | --- | -| The screen's own way out | "Can't get there" (walk) or "I can't get there" (gap request) marks the cells the instruction asks for as skipped; see the screens. | Same. | -| Start over | Coverage is discarded. | Coverage and photos are discarded. | -| Tracking limited | No photo is kept until tracking has been normal for 0.5 s. The fog is hidden. | No photo is kept, so coverage stops growing until tracking is normal again for 0.5 s. The fog is hidden until tracking is normal. | -| Tracking lost or relocalizing | Guidance waits behind the coaching; the fog is hidden. | After 20 s without relocalizing, coverage and photos are discarded and the scan returns to finding the meter. | -| App backgrounded or a call | No effect on coverage. | Coverage is kept. | -| Camera off or session failed | No photos can be kept. | Coverage so far is kept but cannot grow. | -| Network lost or upload failing | No effect. | No effect. | -| App killed | Coverage is lost. | Coverage is lost. | - -## Interactions with other systems - -**Coverage and evidence.** This document. **Stored photos.** Each kept photo is written as it is -kept; see [the flow](flow.md#interactions-with-other-systems). **Upload and offline.** Coverage -travels in the uploaded scene. **Accessibility.** The strip reads to VoiceOver as "Map of the -wall"; whether it also says how much is covered is checked on [the wall walk](../screens/wall-walk.md). -**Haptics and motion.** No haptic per walk photo; the counter flashes and the fog fades where -cells clear. **Verification -hooks.** Each instruction change logs `GUIDANCE=`. - -## Edge cases - -- Walking faster than 1.5 m/s keeps no photos at all; "Slow down" appears after 0.7 s. -- A stretch closer than 0.45 m to a covered stretch on both sides is never asked for. -- "Can't get there" is not sent to the server (see Advancing), so the server may ask for the same - stretch again on the result. -- A walk that reaches 20 ft on the left asks for the left end whether or not the wall ends there. -- Coverage is on the one wall set by the meter tap; wall past a corner never becomes covered. -- Something standing between the phone and the wall (a bush, a bin) does not stop a cell counting - as seen: coverage records where the camera looked, not what it could see. The server re-checks - from the photos (`HouseScanKit/.../CoverageMap.swift:58-59`). The one hidden case the app does - model is the wall itself: a phone on the far side of the wall's line sees nothing. - -## Open questions and verification - -- **Suspected bug: a blocked end nearer than 20 ft can't be recorded** (triage B-06, half - resolved). The end question makes a marked end's kind truthful, but "Wall ends here" is still - offered only when the walk asks for the end, at 6.1 m of coverage - (`HouseScanKit/.../GuidancePlanner.swift:27`, `:119`). A wall blocked by a fence sooner can only - be recorded as unexplored, through "Can't get there". A wall that goes on straight past 20 ft is - recorded as unexplored only by answering "It turns a corner", which is not what the homeowner - sees. -- The walk always sends the homeowner left first. Starting at the right end of a wall means walking - past everything to the left first. -- "About N ft to go" can never appear: the engine always passes no remaining distance - (`Runtime/ScanEngine.swift`, `step(_:)`). -- All thresholds are the code's stated hypotheses (research note), not measured values. - -Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/foundations/flow.md b/verification/product/foundations/flow.md deleted file mode 100644 index e945590b2..000000000 --- a/verification/product/foundations/flow.md +++ /dev/null @@ -1,145 +0,0 @@ -# The flow - -## Summary - -A scan is one pass through a fixed sequence of screens, from an introduction to a result. The -app decides when to move on: most screens advance by themselves when their job is done, and the -homeowner moves on with a button only where a judgment is theirs (they have finished walking, -their marks are right, they want to see the spot in AR). There is no back button between -screens. "Start over", on the result, is the only way to begin again. This document owns the order -of the screens, what moves the app from one to the next, the launch options, and what happens -when the camera, the replay or the phone cannot run the scan. - -## The simple case - -The homeowner opens the app and pages through a short introduction ("Next", or "Skip" to the last page), then taps "Allow camera". The -camera opens and asks them to find their electric meter; they aim at it and tap. The phone takes a -close-up of the meter by itself, then asks them to walk along the wall, first to the left and -then to the right, marking where the wall ends on each side. They tap "Done with this wall" and see a list of the -things they marked, where they can add a gas meter, door, window, AC unit, driveway or fence. - -When they tap "Looks complete", the phone may ask for one more view of a stretch it did not see well. Then it -sends the scan, shows progress while the server works, and shows the result: whether there is a -spot for the battery, where, how much cable it needs, and each check. From the result they can -see the spot in AR on their wall, and come back. - -## The interaction, event by event - -```mermaid -stateDiagram-v2 - [*] --> onboarding - onboarding --> findMeter : "Allow camera" - onboarding --> unsupported : phone can't track motion - findMeter --> meterCloseUp : meter tapped on a detected wall - meterCloseUp --> wallWalk : close-up taken, or "Can't get a clear shot" - wallWalk --> markFeatures : "Done with this wall" (both ends set) - markFeatures --> gapRequest : "Looks complete", and the phone's check finds a gap - markFeatures --> uploading : "Looks complete", no gap - gapRequest --> uploading : view taken, end marked again, or "I can't get there" - uploading --> result : the server answered - result --> gapRequest : "Capture it now" on a missing view - result --> resultAR : "See it on your wall" - resultAR --> result : "Done" - result --> onboarding : "Start over" - wallWalk --> findMeter : phone lost its place for over 20 s (any capture screen) -``` - -The names are the screen names the app logs (see the [glossary](../glossary.md#verification-words)). - -### Arriving - -The app always opens on `onboarding`. At launch it checks one thing: if the phone cannot run -motion tracking at all, it goes straight to `unsupported` and stays there. Nothing else is checked -up front. Camera permission is asked the first time the camera opens, on `findMeter`. - -### Leaving at once - -There is no way to leave a scan early other than closing the app. The next launch starts a new -scan on `onboarding`; the earlier scan's photos stay in the phone's cache but are never reopened. - -### First capture - -The scan's first durable step is the meter tap on `findMeter`. It fixes the wall for the rest of -the scan: the meter's position, the wall's direction and which side is outside. Every later -distance is measured along this wall from the meter. - -### While capturing - -The camera stays open, without a blink, from `findMeter` through `wallWalk`, `markFeatures` and -`gapRequest`; only the instruction and controls above it change. Photos are kept continuously on -`wallWalk` and `gapRequest` as described in -[coverage and guidance](coverage-and-guidance.md#when-a-photo-is-kept). A counter at the top -shows how many photos have been taken; it goes up once each photo is written to the phone. - -### Advancing - -A screen whose job ends by itself advances after about 1.2 s, so the homeowner sees the success: -the close-up after it is taken, a gap request after the view is covered. The result appears only -after the server has answered; nothing on `uploading` is skipped when the network fails. - -## Modifiers - -| Modifier | At arrival | While capturing | -| --- | --- | --- | -| Live camera or replay | With `-replay ` the camera never opens: frames come from a recorded session, the meter tap uses the wall recorded with it, and `unsupported` is not shown even on a phone that can't track. | A replay plays its recording on `meterCloseUp`, `wallWalk` and `gapRequest`; elsewhere it stops. | -| Autopilot | With `-autopilot` (and a replay) the app presses its own buttons through the whole flow and shows a small badge. It needs a replay; with the live camera it logs that and does nothing. | Screens stay up for `-autopilotHold` seconds (default 1.2). The replay plays at three times speed. | -| Server or sample result | With `-serverURL ` the scan is uploaded there. Without it, or with `-sampleResult`, the app answers with a built-in sample result and the result says it is a sample. | No effect. | -| Larger text sizes | Screens reflow; see each screen. | No effect. | -| Reduce Motion | Screen changes crossfade in 0.15 s instead of moving. | No effect. | - -## Cancel and interrupt - -| Event | Before the meter is tapped | After the meter is tapped | -| --- | --- | --- | -| The screen's own way out | See each screen. | See each screen. | -| Start over | Not offered. The failure screen has it for failures other than an unsupported phone, but those never reach it (Open questions). | Offered only on `result`. It forgets the wall, photos, marks and result and returns to `onboarding`. | -| Tracking limited | Coaching replaces the instruction until it clears ([coverage and guidance](coverage-and-guidance.md#one-instruction-at-a-time)). | Same; no photo counts toward coverage meanwhile. | -| Tracking lost or relocalizing | "Point at the meter like this." (relocalizing) or "Your phone lost its place". | If the phone has not found its place within 20 s on a capture screen (`findMeter` through `gapRequest`), the app forgets the wall, photos and marks and returns to `findMeter`. On `uploading`, `result` and `resultAR` the scan and result stay (fixed in `beede15`). | -| App backgrounded or a call | The screen stays; coaching says "Point at the meter like this." when the camera resumes. | Same; photos and marks are kept. | -| Camera off or session failed | Suspected dead end: see Open questions. | Suspected dead end: see Open questions. | -| Network lost or upload failing | No effect. | Only `uploading` is affected; see [the upload](../screens/uploading.md). | -| App killed | The scan is lost. | The scan is lost, including one "saved on this phone" after a failed upload: its files stay in the cache, but nothing reopens or retries them. | - -## Interactions with other systems - -**Coverage and evidence.** Owned by [coverage and guidance](coverage-and-guidance.md). -**Stored photos.** Kept photos and the close-up are written to a new folder in the app's cache -(`Caches/Scans//`) as they are taken, and zipped from there for upload. The folder is never -deleted by the app and never reopened. **Upload and offline.** Only the -upload needs the network; see [the upload](../screens/uploading.md). **Accessibility.** Every -screen groups its elements in one accessibility container. Its identifier (`screen.`) is -for tests and is not spoken. **Haptics and motion.** A light tap -for a deliberate photo (the close-up, a requested view) but not for walk photos; a success haptic -when the meter is set and when the result appears; a firmer tap for each mark placed and a warning -for a refused one. **Verification hooks.** Each screen change -logs `STATE=` publicly under subsystem `dev.housescanning.housescan`, category `state`; -guidance changes log `GUIDANCE=` under category `guidance`. `-autopilotGate ` makes -the autopilot wait on each screen until a file named after it appears, for UI tests. - -## Edge cases - -- The flow has no back navigation. A homeowner who confirms marks too early cannot return to the - walk; they can only continue or Start over from the result. -- Start over resets the photo counter and every mark; on a replay it shows the recording's first - frame again. -- A second `STATE` line for the same screen is never logged: moving to the screen already shown is - ignored. - -## Open questions and verification - -- **Suspected bug: camera access denied is a dead end.** The failure screen text exists ("House - Scan needs your camera ... Turn on Camera for House Scan in Settings") but only the - `unsupported` screen shows failures, and the app switches to it only when motion tracking is - unsupported (`Runtime/ScanEngine.swift:107`). A denied camera, a failed camera session and an - unreadable replay set the failure (`ScanEngine.swift:119, 489, 491`) without changing screen. The - homeowner would stay on "Find your electric meter" with no camera and no Settings button. - Checkable in the Simulator for the replay case: launch with `-replay /nonexistent`. -- "Your scan is saved on this phone" is true only while the app runs: after a relaunch nothing - reopens or retries it. -- Photos of the home accumulate in `Caches/Scans/` across scans and Start over; the app never - deletes them (`Runtime/KeyframeStore.swift:27`). Worth deciding whether a finished or abandoned - scan should be removed. -- The 1.2 s pause after a finished step and the 20 s relocalization limit are read from code; - neither has been timed on a device. - -Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/glossary.md b/verification/product/glossary.md deleted file mode 100644 index 2a7c73af6..000000000 --- a/verification/product/glossary.md +++ /dev/null @@ -1,111 +0,0 @@ -# Glossary - -The words the documents use, each with one meaning. Screen text is quoted as it appears in the -app; everything else is this description's own term. - -## The house and the wall - -- **Meter.** The homeowner's electric meter. Everything is measured from it. -- **Wall.** The straight outside wall the meter is on, as a line along the ground at its foot. - Set once, by the meter tap; the app does not follow the wall round a corner. -- **Left, right.** As seen by someone standing outside, facing the wall. The app's words: "to - your left", "ft left of your meter". -- **Along the wall.** Distance from the meter along the wall line, negative to the left. The app - shows it in feet and inches; the code works in meters. -- **Wall end.** Where the homeowner said the wall stops. A **real end** is one marked with "Wall - ends here" and answered "Something blocks it" (a fence, gate or neighbor's yard). An - **unexplored end** is one answered "It turns a corner", one left unanswered, or one set with - "Can't get there" during the walk: the wall may continue. -- **End question.** "What's at the left end?" (or right), asked right after "Wall ends here", with - the answers "It turns a corner" and "Something blocks it". - -## Seeing the wall - -- **Kept photo.** A camera frame the app saves as evidence and later uploads. The homeowner never - presses a shutter; [coverage and guidance](foundations/coverage-and-guidance.md#when-a-photo-is-kept) - says when a frame is kept. -- **Close-up.** The one deliberate photo of the meter, taken by itself when the meter is centered, - near, sharp and well exposed. See [the meter close-up](screens/meter-close-up.md). The walk - shows it again while the phone is finding its place, as the picture to aim at. -- **Coverage strip.** The map along the bottom of the walk screen: the wall unrolled flat, meter at - zero, in two rows (bands). -- **Band.** One row of the strip. The **wall band** is the wall face from the ground up to - 1.98 m (6 ft 6 in, the electrical code's headroom height); the **ground band** is the ground from the foot of the wall out to 1.2 m - (3 ft 11 in). -- **Cell.** A 6 in (0.1524 m) stretch of one band. Each cell is in one of four states: - - **unseen**: no kept photo shows it; drawn as fog. - - **seen**: at least one of the cell's three **sample lines** (lines across the band, at its - two edges and its middle) is shown in a kept photo, but not every line from two positions - yet. - - **covered**: every sample line shown from two positions at least 0.25 m apart with normal - tracking. Evidence exists; it is not a pass. - - **skipped**: the homeowner said they can't get there. Drawn differently from both fog and - covered, never evidence, and not sent to the server. -- **Fog.** The overlay on the camera image over wall and ground not yet seen. It clears where - kept photos have looked. - -## Being told what to do - -- **Instruction.** The one line (plus an optional second line) at the top of a camera screen. - There is exactly one at a time. -- **Coaching.** An instruction about the phone rather than the wall ("Slow down", - "Move your phone slowly"). It replaces the guidance instruction until the problem clears. -- **Reticle.** The small circle in the middle of the camera view that a tap on Mark (or "This is - my meter") aims with, when the homeowner does not tap the image itself. -- **Mode badge.** The yellow "REPLAY · AUTOPILOT" label shown when the app runs on a recording or - drives itself. -- **Tap ring.** The ring that blooms where a finger touched the camera image. -- **Target.** A ring on the camera image where the homeowner should aim, or a chevron at the - screen edge pointing to it when it is off screen. -- **Path.** Dots on the ground showing where to stand next, at most 3 m long, 1.5 m out from the - wall. -- **Tracking.** Whether the phone knows where it is. **Normal**, **limited** (moving fast, too - little detail, starting up) or **lost**. **Relocalizing** is the phone finding its place again - after an interruption. - -## Marking - -- **Feature.** Something the homeowner marks that affects where a battery can go: gas meter, door, - window, AC unit, driveway, fence. -- **Mark.** One tap on the camera image (or on the reticle with the Mark button) placing a point of - a feature. A door or window takes two (opposite corners), a driveway or fence two (along one - edge), a gas meter or AC unit one. -- **Refusal.** A tap the app does not accept, with the reason shown ("Nothing to pin there"). -- **Kind picker.** The "What do you see?" panel behind "Mark something" on the walk, with one tile - per feature. -- **Feature list.** The `markFeatures` screen, "Anything else near your meter?": the marked - features, the window question and "Add something". - -## After the walk - -- **Can't get there.** The walk's way out ("Can't get there"); the gap request's is "I can't get - there". Both mark the asked-for stretch as skipped. -- **Gap request.** A screen asking for one more specific view ("Show the ground 3 ft right of your - meter"), from the phone's own check or from the server. -- **Phone request, server request.** A gap request from the phone's own check after the feature - list, or from "Capture it now" on the result. A **past-end request** asks for the ground beyond a - wall end. The **requested stretch** is highlighted in amber. -- **Upload.** Sending the scan: the scene description and the kept photos, zipped. -- **Result.** The server's answer, shown as a headline, the battery spot and its cable, and a list - of checks. -- **Placement line.** The line under the result headline: where the spot is and how much cable. -- **3D model, AR view.** The small 3D model of the wall on the result, and the `resultAR` screen - that draws the spot on the camera image. -- **Sweep.** The server's outcome for every stretch of wall where it tried the battery, drawn as - tinted zones in the 3D model and the AR view. -- **"Still needed".** The result's list of missing views, each with "Capture it now" or "An - installer will check this". -- **Check.** One rule the server evaluated at the spot, shown as **"Looks good"** (pass), **"Not - sure yet"** (unsure) or **"Doesn't work"** (fail). -- **Installer review.** What happens to anything the phone can't settle: a person at Base - decides. "An installer will take a look" is the result headline for a manual review. -- **Sample result.** A built-in result the app shows when no server is configured. The screen - says it is a sample. - -## Verification words - -- **Replay.** Running the app on a recorded session instead of the live camera (`-replay`). -- **Autopilot.** The app pressing its own buttons through the flow on a replay (`-autopilot`). -- **Screen name.** The name the app logs as `STATE=` when a screen appears: - `onboarding`, `findMeter`, `meterCloseUp`, `wallWalk`, `markFeatures`, `gapRequest`, - `uploading`, `result`, `resultAR`, `unsupported`. diff --git a/verification/product/screens/find-meter.md b/verification/product/screens/find-meter.md deleted file mode 100644 index fae5aa084..000000000 --- a/verification/product/screens/find-meter.md +++ /dev/null @@ -1,177 +0,0 @@ -# Finding the meter - -## Summary - -Finding the meter is where the scan fixes its one wall. The homeowner points the phone at their -electric meter and either taps "This is my meter", which pins the meter at the circle in the -middle of the screen, or taps the meter itself on the camera image. The phone accepts the tap -only where it has detected a real vertical surface; that surface becomes the wall for the rest of -the scan. It is the first camera screen (screen name `findMeter`), reached from "Allow camera" on -the introduction, and again after the phone loses its place for over 20 s during capture -([the flow](../foundations/flow.md#the-interaction-event-by-event)). Nothing is photographed here. -An accepted tap goes straight to [the meter close-up](meter-close-up.md). - -## The simple case - -The camera opens. A white ring with a dot at its center sits in the middle of the view, and the -instruction reads "Find your electric meter", with "A gray box with a round glass dial or a small -screen, usually on an outside wall." A blue "This is my meter" button fills the bottom of the -screen. The homeowner stands a few steps from the wall, puts the ring on the meter and taps the -button. A ring blooms from the middle of the screen and fades, the phone gives a success haptic, -and the screen crossfades to the close-up. - -## The interaction, event by event - -```mermaid -stateDiagram-v2 - [*] --> looking - looking --> coached : tracking not normal - coached --> looking : tracking normal again - looking --> refused : tap with no detected wall there - coached --> refused : any tap (refusal hidden behind the coaching) - refused --> refused : another refused tap (only the ring shows) - looking --> [*] : tap on a detected wall, the close-up - refused --> [*] : tap on a detected wall, the close-up -``` - -### Arriving - -The camera fades in behind the screen and stays open through the next screens. The instruction is -"Find your electric meter" and nothing is set: no wall, no photos, and no photo counter on this -screen. On the first arrival the phone starts motion tracking, and iOS asks for camera access if -it has not been granted. While tracking starts up, [coaching](../foundations/coverage-and-guidance.md#one-instruction-at-a-time) -replaces the instruction, usually "Move your phone slowly" ("It's getting its bearings."), with an -amber icon beside it. - -Arriving after a 20 s loss of place, the phone has already thrown away the wall, photos and marks -and restarted tracking with a fresh map, so it has to detect the wall again before a tap can -succeed. - -### Leaving at once - -The only way off this screen is an accepted tap. There is no back button, no skip and no Start -over. Nothing is recorded until the tap is accepted. - -### First capture - -Two gestures do the same thing: - -- **"This is my meter"** pins the point under the ring in the middle of the screen. A ring blooms - at the screen's center. -- **A tap on the camera image** pins the point under the finger. A ring blooms where the finger - touched. - -The ring blooms at once for every tap, accepted or not. The tap is accepted when both hold: - -1. Tracking is normal. -2. The point lies on a vertical surface the phone has detected, or on that surface continued past - the part it has detected so far. A surface the phone would only be guessing at does not count. - -If the first check fails, the tap is refused and nothing changes while the coaching is up. If the -second fails, the instruction changes to "Step a little closer to the wall", with "Then aim at -your meter again." The refusal has no warning icon and no haptic, and the new instruction stays -until a tap is accepted. - -> Technical note: when the tap is accepted, the wall's outside is the side the phone is on, and -> the ground at the wall is the highest detected level surface at least 0.3 m (1 ft) below the -> meter whose detected extent comes within 2 m (6 ft 7 in) of it. With no such surface yet, the -> ground is taken as 1.4 m (4 ft 7 in) below the phone (`Runtime/ScanEngine+Actions.swift:41-42`, -> `:53`). The lookup runs again whenever the phone detects or grows a level surface, for the rest -> of the scan: the first match replaces the guess, and a later one replaces the ground when it -> differs by more than 1 cm. Every mark is re-measured against the new ground -> (`Runtime/ScanEngine.swift:227-246`). While the ground is still a guess, the upload marks the -> meter and each door, window, gas meter and AC unit as uncertain by ±0.3 m (`Runtime/ScanEngine+Export.swift:50-55`, `:72`). - -### While capturing - -Nothing is captured on this screen. Only the coaching updates while it is up, as the phone's -tracking changes: "Slow down", "Aim at a corner or somewhere with more texture", "Point at the -meter like this.", "Your phone lost its place". This screen never shows "Hold steady" or "It's too dark to -see the wall", which belong to the walk. The homeowner can tap as often as they like; each refused -tap shows the ring and nothing else. - -### Advancing - -An accepted tap sets the wall: the meter's position, which way the wall runs and which side is -outside. Coverage starts empty, centered on the meter. The phone pins the meter to the wall so its -position is refined as tracking improves. The success haptic fires and the close-up appears at -once, with no pause on this screen. - -## Modifiers - -| Modifier | At arrival | While capturing | -| --- | --- | --- | -| Live camera or replay | A replay shows the recording's first frame as a still. No coaching appears, because that frame is shown for tapping and is not checked. | Any tap or "This is my meter" is accepted at once, wherever it lands. The wall comes from the recording (its recorded wall, or one assumed from the path the camera took), not from the tap. | -| Autopilot | A badge reads "Replay · Autopilot". The screen stays up for the autopilot's hold ([the flow](../foundations/flow.md#modifiers)), then the autopilot presses "This is my meter" itself. With the live camera it does nothing, and the badge reads "Autopilot". | No effect. | -| Server or sample result | No effect. | No effect. | -| Larger text sizes | The instruction and the button label grow. When they no longer fit, the screen scrolls instead of shrinking the text. The ring in the middle does not grow. | No effect. | -| Reduce Motion | The screen crossfades in over 0.15 s. | The ring from a tap fades in place without growing. A changed instruction crossfades without blurring. | - -## Cancel and interrupt - -| Event | Before any tap | After a refused tap | -| --- | --- | --- | -| The screen's own way out | None. | None. | -| Start over | Not offered. | Not offered. | -| Tracking limited | Coaching replaces the instruction, and taps are refused without any change on screen. | Same. When the coaching clears, "Step a little closer to the wall" shows, even if the refusal was about tracking. | -| Tracking lost or relocalizing | "Point at the meter like this." or "Your phone lost its place". Taps are refused. After 20 s of relocalizing, tracking restarts with a fresh map and the screen stays. | Same. The "Step a little closer to the wall" instruction stays after the restart. | -| App backgrounded or a call | The screen stays. When the camera resumes, "Point at the meter like this." shows until tracking is normal. | Same. | -| Camera off or session failed | Suspected dead end: the screen stays with no camera image, and taps only show the ring ([the flow](../foundations/flow.md#open-questions-and-verification)). | Same. | -| Network lost or upload failing | No effect. | No effect. | -| App killed | The scan is lost. The next launch starts at the introduction. | Same. | - -## Interactions with other systems - -**Coverage and evidence.** Coverage begins at the accepted tap, empty and centered on the meter; -the tap's wall is the only wall coverage is ever measured on. **Stored photos.** None; the camera -keeps no frames on this screen. **Upload and offline.** No effect. The uploaded scene measures -everything from the meter, wall direction and ground set here. **Accessibility.** The ring and -the camera image are hidden from VoiceOver, so the only way to pin the meter is "This is my meter", -whose hint is "Pins your electric meter at the circle in the middle of the screen". The -instruction and its second line are read as one element, marked as updating often. **Haptics and -motion.** A success haptic when a tap is accepted and nothing for a refusal. A 0.55 s ring blooms -at every tap, and the button shrinks slightly while pressed. **Verification hooks.** `STATE=findMeter` -on arrival. A refused tap with no wall logs "meter tap refused: no vertical plane" (category -`engine`); a refusal for tracking logs nothing. No `GUIDANCE=` line is logged on this screen. A -replay logs where its wall came from when it loads. - -## Edge cases - -- The meter is placed where the tap landed, not at the meter's middle. A tap on the meter's edge - or just above it moves the meter's height and position by that much. -- A meter on a wall with little texture (smooth stucco, fresh paint) may never be detected as a - surface. Taps are then refused over and over with the same instruction. -- Tapping past the edge of the part of the wall the phone has detected is accepted, because the - surface counts as continuing (see Open questions). -- A replay started with `-replay` whose homeowner taps "Allow camera" before the recording has - loaded starts the live camera as well (`Runtime/ScanEngine.swift:183`). Read from code only. -- A replay that could not be opened leaves this screen dark, with no way on: every tap is - ignored. - -## Open questions and verification - -- The chest-height ground guess is now replaced as soon as a level surface appears, fixed in - `beede15`. A scan where none ever appears still carries the guess, flagged only by the ±0.3 m, - which the code calls a hypothesis with no measured spread behind it - (`Runtime/ScanEngine+Export.swift:70-72`). -- **Suspected bug: the refusal message is misleading and silent on repeat.** Both refusals show - "Step a little closer to the wall" (`Runtime/ScanEngine+Actions.swift:26`, `:33`), though - neither is about distance: one is tracking, and the other is a surface not yet detected, which - can happen at any range. A second refused tap produces the same text with no haptic - (`UI/ScanRootView.swift:95` warns only for feature marks), so it looks as if the tap did - nothing. -- **Question: the wall can be extended into empty space.** The tap may land on a detected wall's - continuation past what was detected (`Runtime/LiveCapture.swift:76`, `existingPlaneInfinite`). A - tap at a meter on a surface that has not been detected yet can pin the meter on another detected - wall's continuation, at the wrong depth. The code's comment says a tap "never guesses a depth". -- A 20 s loss of place now returns here only from the capture screens (finding the meter, the - close-up, the walk, the list and a gap request); the upload and the result keep the scan - (`Runtime/ScanEngine.swift:469-476`), fixed in `beede15`. The reset still leaves an unfinished - feature mark in place (`Runtime/ScanEngine.swift:496-522` does not clear it). -- **Question: "Point at the meter like this." with nothing to match.** The relocalizing coaching - (`UI/Copy/ScanCopy.swift:65-66`) is written for the walk, which shows the saved close-up under - it. Here no close-up exists yet and no picture is shown, so "like this" points at nothing. -- The live tap, the refusals, the coaching and the camera prompt need the live camera and have not - been seen running. The Simulator shows only the replay path, where every tap is accepted. - -Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/screens/gap-request.md b/verification/product/screens/gap-request.md deleted file mode 100644 index b5872e596..000000000 --- a/verification/product/screens/gap-request.md +++ /dev/null @@ -1,195 +0,0 @@ -# A gap request - -## Summary - -A gap request asks the homeowner for one more specific view of the wall or the ground, such as -"Show the ground about 4 ft right of your meter". It comes from two places. The phone asks once, -after "Looks complete" on [marking features](mark-features.md), when its own check finds a stretch -near the meter that no two kept photos show. The server asks when the homeowner taps "Capture it -now" under "Still needed" on [the result](result.md). Either way the screen (screen name -`gapRequest`) is the walk's camera with one target: the requested stretch glows amber, a bar fills -as it is covered, and at 80 % a green check lands and the scan is sent again. "I can't get there" -leaves at any time before that. There is no other way off the screen. - -## The simple case - -The homeowner taps "Looks complete". The camera stays open and the instruction changes to "Show the -ground about 3 ft left of your meter", with "This might be a spot for the battery, so the ground -there needs a clear look from two places." A stretch of ground on the camera image turns amber, a -ring marks its middle, and dots on the ground lead there. Below the instruction a bar reads "0%", -and the strip at the bottom outlines the same stretch in amber. The homeowner walks over and points -the phone down at it. Photos are kept as on [the wall walk](wall-walk.md), each with a light tap, -and the amber clears cell by cell as the bar climbs. At 80 % a large green check appears, the -instruction reads "Got it, thanks" with "That's the view we needed.", and the phone gives a success -haptic. About 1.2 s later [the upload](uploading.md) begins. - -## The interaction, event by event - -```mermaid -stateDiagram-v2 - [*] --> asking : "Looks complete" and the phone finds a gap, or "Capture it now" - asking --> asking : a photo kept (bar moves) - asking --> complete : 80 % of the stretch covered, with a new photo - asking --> [*] : "I can't get there" (upload at once) - asking --> [*] : phone relocalizing for over 20 s (finding the meter) - complete --> [*] : after 1.2 s, the upload -``` - -### Arriving - -The phone's own check runs when the homeowner taps "Looks complete". It looks between the two -marked [wall ends](../glossary.md#the-house-and-the-wall), no farther than 6.1 m (20 ft) from the -meter, for stretches of at least 0.45 m (three cells) that are unseen or seen but not yet -[covered](../foundations/coverage-and-guidance.md#while-capturing). Ground comes first: the ground -stretch nearest the meter is asked for, and only if the ground is complete, the nearest wall -stretch. If nothing is missing, the app goes straight to the upload and this screen never shows. -The phone asks at most once per walk. - -A server request starts from a "Still needed" card that has "Capture it now". A wall or ground -stretch asks for the span the server named. A request to look past a wall end asks for the ground -from that end to 2 m (6 ft 7 in) beyond it, and the app first forgets that end so the stretch -beyond can count. - -The screen crossfades in over the open camera. What it shows: - -| Part | Phone request | Server request | -| --- | --- | --- | -| Instruction | "Show the ground ..." with "This might be a spot for the battery, so the ground there needs a clear look from two places.", or "Show the wall ..." with "Tilt up so the wall above this spot is in view." | "Show the ground ..." or "Show the wall ...", with the server's message as the second line. | -| Place | "around your meter" when the stretch includes the meter or its middle is within 0.3 m; otherwise "about N ft left of your meter" or "right", whole feet from the middle of the stretch. | Same. | - -Under the instruction, inside its card, is "I can't get there". Below it is the progress bar with a -target icon and the percent already covered, and the strip. On the camera image, the requested -cells that are not yet covered glow amber instead of fog. A ring sits on the middle of the -stretch, 1.2 m up the wall or 0.6 m out on the ground, and dots lead from where the homeowner -stands along the wall to it, 1.5 m out. A phone request starts at "0%"; a server request can start -higher. - -### Leaving at once - -"I can't get there" is the only way off before the view is complete. It marks every cell of the -stretch that is not yet covered as skipped (hatched on the camera and slashed on the strip) and -starts the upload at once, with no pause and no haptic. The upload does not carry the skip: see -Open questions. Nothing else leaves the screen short of the 20 s tracking reset or closing the -app. There is no back button and no Start over. - -### First capture - -The first photo is kept by the same rules as on the walk -([when a photo is kept](../foundations/coverage-and-guidance.md#when-a-photo-is-kept)). It gives a -light tap, the counter at the top rises and its camera icon flashes. Any requested cell it covers -loses its amber, and the bar moves. - -### While capturing - -The bar shows the share of the stretch's cells that are covered, rounded to a whole percent. It -counts only covered cells, so a cell adds nothing until each of its three sample lines has been -seen from two places 0.25 m apart -([coverage and guidance](../foundations/coverage-and-guidance.md#while-capturing)). The instruction does not change while the request is open; only -[coaching](../foundations/coverage-and-guidance.md#one-instruction-at-a-time) replaces it, until -the problem clears. The ring and the dots follow the homeowner as they move. "I can't get there" -stays available, coaching or not. - -### Advancing - -The view is complete when 80 % of the stretch is covered and at least one photo has been kept since -the screen opened. At that moment the bar turns green and shows at least "80%", its icon becomes a -check, a large green check mark appears in the middle of the screen, the instruction reads "Got it, -thanks" with "That's the view we needed.", "I can't get there" disappears and the phone gives a -success haptic. After the pause described in [the flow](../foundations/flow.md#advancing) the upload -starts with every photo so far, including any kept during the pause. The server checks the whole -scan again and a new result replaces the old one. The phone never asks a second time; the server can, -through "Still needed" on the new result. - -## Modifiers - -| Modifier | At arrival | While capturing | -| --- | --- | --- | -| Live camera or replay | A replay plays again from where the autopilot held frames back for this request (two frames before them); with no held-back frames, the whole recording from the start. Frames already kept on the walk are not kept twice. | Whether a replay can complete the view depends on the recording. | -| Autopilot | The autopilot waits only on the phone's request. With no held-back frames it presses "I can't get there" after `-autopilotHold` seconds; otherwise it waits up to 150 s for the upload and then presses it. It never taps "Capture it now". | The replay runs at three times speed; the pause after completion is `-autopilotHold` seconds, at least 1.2. | -| Server or sample result | The sample result always lists two capturable views: the ground from 3 to 5.5 ft right of the meter ("Sample: film the ground in front of the spot.") and past the left end. | The upload returns the same sample, so both cards come back after every request. | -| Larger text sizes | The instruction, its button and the bar's percent grow; the chrome scrolls rather than squeeze when it outgrows the screen. | No effect. | -| Reduce Motion | The screen crossfades in 0.15 s; instruction changes fade instead of blurring. | The green check fades in over 0.15 s instead of scaling up; the fog fades without drifting. | - -## Cancel and interrupt - -| Event | Before the view is complete | During the pause after it | -| --- | --- | --- | -| The screen's own way out | "I can't get there": the rest of the stretch is marked skipped and the upload starts at once. | None; the button is gone and the upload follows. | -| Start over | Not offered. | Not offered. | -| Tracking limited | Coaching replaces the instruction; no photo is kept, so the bar stops. The amber, fog, ring and dots fade off the camera image until tracking is normal. "I can't get there" still works. | Coaching replaces "Got it, thanks"; the upload still starts. | -| Tracking lost or relocalizing | "Your phone lost its place" or "Point at the meter like this.", with no meter photo (see Open questions). The amber, fog, ring and dots fade off the camera image. After 20 s the request is dropped with the wall, photos and marks, and the scan returns to [finding the meter](find-meter.md). | The pause is shorter than 20 s, so the upload starts. | -| App backgrounded or a call | The request and its progress stay; "Point at the meter like this." shows until tracking returns. | The upload starts once the pause has run. | -| Camera off or session failed | Suspected dead end ([the flow](../foundations/flow.md#open-questions-and-verification)), but "I can't get there" still leaves to the upload. | The upload starts. | -| Network lost or upload failing | No effect here; the upload that follows needs it. | Same. | -| App killed | The scan is lost. | The scan is lost. | - -## Interactions with other systems - -**Coverage and evidence.** The stretch is chosen from [coverage](../foundations/coverage-and-guidance.md), -and photos here add coverage anywhere they look, not only on the stretch. Covered stretches go to -the server; skipped cells and the request itself do not. **Stored photos.** Photos kept here are -stored like walk photos and uploaded with them. **Upload and offline.** The screen needs no -network. A server request is followed by a new upload; if it fails, the earlier result cannot be -reopened, only retried with "Try again" ([the upload](uploading.md)). **Accessibility.** The bar -is one element, "Captured so far" (then "View captured") with a value such as "40 percent". The -button's hint is "Skips this view. An installer will look at this part instead." The green check is -hidden from VoiceOver; the instruction is marked as frequently updating. The strip reads "Map of the wall" -with the percent seen on each side, and says nothing about the requested stretch. **Haptics and -motion.** A light tap for every photo kept on this screen (walk photos get none) and a success -haptic when the view is complete. The check springs in over 0.45 s with a slight bounce, the bar -eases over 0.25 s and its icon swaps. **Verification hooks.** `STATE=gapRequest` on arrival, -`GUIDANCE=gap` once; the engine logs "gap N satisfied" or "gap N skipped". The autopilot logs -"skipping the gap: no held-back frames to show it" or "gap not satisfied by the held-back frames; -skipping it". With `-autopilotGate`, the upload after a completed view waits for the test's `gapRequest` file. - -## Edge cases - -- A server stretch that is already 80 % covered shows its percent at once but completes only when - one new photo is kept, from anywhere. -- A server stretch lying beyond a marked wall end can never fill: cells past an end are never - recorded, so the bar stays at "0%" and only "I can't get there" leaves. -- The dots run straight from the homeowner to the stretch, however far; they are not limited to the - walk's 3 m. -- A phone request covers only what is between the marked ends and within 20 ft of the meter, even - on a longer wall. -- After "I can't get there" on a phone request the scan uploads without asking for any other gap, - even if one exists. - -## Open questions and verification - -- **Suspected bug: "I can't get there" is not sent.** The button's hint promises "An installer will - look at this part instead.", and the cells turn skipped, but the upload carries only covered - stretches and the end kinds (`Runtime/ScanEngine+Export.swift:57-63`, - `HouseScanKit/.../SceneExport.swift:283-303`). The server cannot tell a skipped stretch from one - never looked at, and a server request the homeowner skipped can come back on the next result. - This also contradicts the Advancing section of - [coverage and guidance](../foundations/coverage-and-guidance.md#advancing), which says skipped - cells are sent. -- **Suspected bug: a past-end request turns a real end into an unexplored one.** "Capture it now" - forgets the end before the request starts (`Runtime/ScanEngine+Actions.swift:239-244`, - `Runtime/ScanEngine.swift:583-588`). Since `beede15` the engine lets that end be marked again - during the request, and marking it and answering the end question settles the request - (`ScanEngine+Actions.swift:71`, `:75`, `:87`; `Runtime/ScanEngine.swift:625-638`). But this screen - has no "Wall ends here" button and never shows the end question (both exist only on the walk, - `UI/Screens/WallWalkScreen.swift:137-173`), so only code can use it. For the homeowner the next - upload still reports that side as unexplored (`ScanEngine+Export.swift:58-59`), and the server - may ask for the same view again. -- **Suspected bug: a second past-end request asks for the wrong place.** With the end forgotten, - the plan falls back to the meter (`HouseScanKit/.../GapPlanner.swift:127, 130`): the ground from - the meter to 2 m toward that side, "about 3 ft left of your meter", which is usually covered - already. Checkable in the Simulator with the sample result: tap "Capture it now" on the left-end - card twice, completing or skipping in between. -- "Capture it now" is offered only when a request can be built from the item: a band item needs - its span, a past-end item its side (`ScanEngine+Export.swift:152`, `GapPlanner.swift:113, 124`). - Fixed in `beede15`; before, the button could do nothing. -- The phone's wording "This might be a spot for the battery" is stronger than its choice: it asks - for the nearest missing ground, not a place it has judged a candidate. -- The 20 s reset keeps the list of skipped requests (`ScanEngine.swift:496-522`); after - re-walking, a phone request identical to a skipped one would be passed over (`ScanEngine.swift:596`). -- **Question: "Point at the meter like this." without the picture.** The walk shows the saved - close-up under this coaching (`UI/Screens/WallWalkScreen.swift:23-27`); this screen shows the - words alone, even when a close-up was taken. -- Nothing here has run in the Simulator. The completion moment, the amber highlight and the - dots need the replay with `-autopilot`, which reaches this screen only when held-back frames exist. - -Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/screens/mark-features.md b/verification/product/screens/mark-features.md deleted file mode 100644 index ba8af0cf3..000000000 --- a/verification/product/screens/mark-features.md +++ /dev/null @@ -1,220 +0,0 @@ -# Marking features - -## Summary - -Some things near the meter change where a battery can go: a gas meter, a door, a window, an AC -unit, a driveway, a fence. The camera does not find them; the homeowner marks each one by tapping -it on the camera image during [the wall walk](wall-walk.md). After "Done with this wall" the -list screen (screen name `markFeatures`) shows every feature marked, lets the homeowner remove one -and answer "Does this window open?" for each window, and ends with "Looks complete". This -document covers that screen and the marking interaction. Where a battery may go is decided by the -server from these marks; the phone only records them. - -## The simple case - -On the walk the homeowner sees a window and taps "Mark something". A chooser headed "What do you -see?" offers six kinds; they tap "Window". The chooser closes, a circle appears in the middle of -the camera image and the instruction reads "Tap the window's bottom-left corner", with "Put the -circle on it and tap Mark, or tap it on screen." They tap the corner on screen; a ring blooms -where the finger landed and the instruction becomes "Now tap its top-right corner". The second tap -records the window: a dashed outline appears on the wall, a pin at each tap, a window icon on the -coverage strip, and the phone gives a firm tap. When the walk is done they tap "Done with this -wall". The camera dims and a panel appears along the bottom: "Anything else near your -meter?", a row "Window", "About 4 ft right of your meter", then "Does this window open?" with -"It opens" and "It stays shut". They tap "It opens", then "Looks complete". - -## The interaction, event by event - -```mermaid -stateDiagram-v2 - [*] --> walking - walking --> chooser : "Mark something" - chooser --> walking : Close - chooser --> firstTap : a kind picked - firstTap --> firstTap : tap refused - firstTap --> secondTap : tap accepted (door, window, driveway, fence) - firstTap --> walking : tap accepted (gas meter, AC unit), feature recorded - secondTap --> secondTap : tap refused - secondTap --> walking : tap accepted, feature recorded - firstTap --> walking : "Cancel" - secondTap --> walking : "Cancel" - walking --> list : "Done with this wall" - list --> list : remove, answer a window, "Add something" (nothing visible) - list --> [*] : "Looks complete" -``` - -`walking` to `list` is the walk ending; the rest of the walk is in [the wall walk](wall-walk.md). - -### Arriving - -Marking starts from "Mark something" on the walk, shown next to "Wall ends here" or "Done with -this wall" when those are offered, and alone otherwise. It opens the chooser "What do you see?": -"Gas meter", "Door", "Window", "AC unit", "Driveway", "Fence", three to a row, with a close -button. Picking one closes the chooser and starts marking that kind. While marking, a circle sits -in the middle of the camera image, the dotted path and the target ring disappear, the instruction -shows the marking prompt, and the controls become "Cancel" and "Mark". - -The list screen arrives when "Done with this wall" is tapped with both wall ends set (it is not -offered while a mark is in progress). The camera stays on screen, darkened, with a light panel -along the bottom: the title "Anything else near your meter?", the line "Gas meters, doors, windows, AC units, -driveways and fences all change where a battery can go.", then one row per feature in the order -marked, or "Nothing marked yet." if there are none. Each row shows the kind's icon, its name and -where it is ("About 4 ft right of your meter", or "Around your meter" when it spans the meter or -its middle is within 0.3 m of it), and a trash button. A window row adds "Does this window open?" -with two full-width answers stacked one above the other, "It opens" and "It stays shut", neither -selected. Below the list, "Add something" offers the six kinds as -buttons, and "Looks complete" stays pinned at the bottom of the panel. There is no instruction -card, no photo counter and no coverage strip on this screen, and no pins on the camera image. - -### Leaving at once - -While marking, "Cancel" forgets the taps so far and returns to the walk; nothing is recorded. The -chooser's close button returns to the walk without starting. - -On the list, "Looks complete" is the only way on. It needs nothing: an empty list and unanswered -windows are both accepted. There is no way back to the walk ([the flow](../foundations/flow.md#edge-cases)). - -### First capture - -A tap counts when it lands on the open camera image (not on a card or button), or when "Mark" is -tapped, which uses the center of the circle. Each tap is checked in this order; the first failure -is a refusal: - -| Check | Refusal shown | -| --- | --- | -| Tracking not normal | "One moment, your phone is still finding its place." | -| The phone is on the far side of the wall's line | "That spot is behind the wall. Tap something on this side." | -| The tap meets no surface: the wall's plane (gas meter, door, window, AC unit) or the ground (driveway, fence) | "Nothing to pin there. Aim at the wall or the ground and try again." | -| A ground tap more than 8 m out from the wall, or behind its line | "That's too far from the wall to matter. Tap something closer." | - -A refusal replaces the instruction with its words in a warning style, keeps the prompt as the -second line, and gives a warning haptic. The step does not advance. The next accepted tap clears -it. What each kind asks for and records: - -| Kind | Taps and prompts | What the mark records | -| --- | --- | --- | -| Gas meter, AC unit | One: "Tap the gas meter" / "Tap the ac unit", with "Put the circle on it and tap Mark, or tap it on screen." | The tapped point on the wall. The list and strip treat it as 0.3 m wide, centered on the tap; that width is nominal, not measured. | -| Door, window | Two: "Tap the door's bottom-left corner" (or "window's"), with the same second line; then "Now tap its top-right corner". | A rectangle on the wall: from the lower tap's height (never below the ground) to the higher, and between the two taps along the wall. Any two opposite corners give the same rectangle. | -| Driveway | Two: "Tap one end of the driveway's edge", with "Use the edge closest to the wall."; then "Now tap the other end of that edge". | The two ground points: one edge. | -| Fence | Two: "Tap the bottom of the fence at one end", with "Where it meets the ground."; then "Now tap the bottom at the other end". | The two ground points where the fence meets the ground; its distance from the wall is the nearer point's. | - -### While capturing - -Between the taps of a two-tap mark only the ripple confirms the first tap; no pin appears until -the mark is complete. The walk carries on underneath: photos are still kept and coverage still -grows, but coaching, the "Can't get there" reply, the path and the target ring stay hidden until -the mark ends. A finished mark draws on the camera image (a dashed outline for a door or window, a -dashed line for a driveway or fence, a pin at every tap) and as its icon on the coverage strip. - -On the list, the trash button removes its row at once, with no confirmation and no undo; the row -fades out. "It opens" or "It stays shut" fills in when chosen and can be changed; once one is chosen there is no -way back to unanswered. The "Add something" buttons do nothing the homeowner can see (see Open -questions). - -### Advancing - -A mark is recorded on its last accepted tap: the marking ends, the walk's controls return, and -the new feature is added to the end of the list. "Looks complete" runs the phone's own gap check -and goes to a [gap request](gap-request.md) or straight to [the upload](uploading.md) (see -[the flow](../foundations/flow.md#the-interaction-event-by-event)). The features are sent in the -scene description with the scan: - -- a door or window as an opening with its span along the wall, bottom and top; for a window, - whether it opens (yes for "It opens", no for "It stays shut", or left empty when unanswered); a door always left empty; -- a gas meter or AC unit as the tapped point; -- a fence as the stretch of wall it faces and its distance out from the wall, taken at the foot - nearer the wall, so a fence angled toward the wall is not placed farther out than its narrow - end (fixed in `beede15`; it was the average of the two feet); -- a driveway as a strip along its two edge points. - -Heights are sent as height above the ground at the wall. When the phone first measures the ground -or the meter's position is refined, every mark's position along the wall, heights and distance out -are recomputed from its taps ([finding the meter](find-meter.md#first-capture)). - -## Modifiers - -| Modifier | At arrival | While capturing | -| --- | --- | --- | -| Live camera or replay | On the list, the live camera keeps running dimmed but keeps no photos; a replay stops on the frame it was showing. | On a replay a tap is measured against the recorded frame on screen. | -| Autopilot | On the walk the autopilot marks a gas meter and a window on covered wall, answers that the window opens, and on the list waits `-autopilotHold` seconds before pressing "Looks complete". It calls the actions directly, never the chooser or "Mark". A mode badge reads "Replay · Autopilot", in capitals. | If the window is still incomplete after two tries at its corners, it cancels it and logs the refusal. | -| Server or sample result | No effect. | No effect. | -| Larger text sizes | At accessibility sizes the chooser and the "Add something" buttons go to one column (from three and two), "Mark something" spans the width, and the panel scrolls with "Looks complete" still pinned. The window answers are stacked at every size. | No effect. | -| Reduce Motion | The screen crossfades in 0.15 s. Rows still fade and scale slightly as they appear and go. | The tap ripple fades without growing. | - -## Cancel and interrupt - -| Event | While a mark is in progress (on the walk) | On the list | -| --- | --- | --- | -| The screen's own way out | "Cancel": the taps so far are forgotten, nothing is recorded. | None; "Looks complete" is the only way on. | -| Start over | Not offered. | Not offered. | -| Tracking limited | Every tap is refused with "One moment, your phone is still finding its place."; the prompt hides the coaching. | Nothing is shown; the list is unaffected. | -| Tracking lost or relocalizing | Taps are refused as above. After 20 s the scan returns to finding the meter and every feature is forgotten, but the half-done mark is not (see Open questions). | No coaching is shown. After 20 s the scan returns to finding the meter and the list is gone, without warning (see Open questions). | -| App backgrounded or a call | The mark in progress and its taps are kept; taps are refused until tracking is normal again. | The list and answers are kept. | -| Camera off or session failed | Suspected dead end ([the flow](../foundations/flow.md#open-questions-and-verification)). A tap is measured against the last frame the camera delivered. | The list still works and "Looks complete" still leads on. | -| Network lost or upload failing | No effect. | No effect here; see [the upload](uploading.md). | -| App killed | The scan is lost. | The scan is lost. | - -## Interactions with other systems - -**Coverage and evidence.** A mark is a claim by the homeowner, not evidence: it keeps no photo and -covers no cell, and nothing checks that the marked stretch was ever seen. Photos keep being kept -during marking on the walk ([coverage and guidance](../foundations/coverage-and-guidance.md#when-a-photo-is-kept)). -**Stored photos.** None are taken for a mark. **Upload and offline.** Marking and the list need no -network; the features travel in the uploaded scene description. **Accessibility.** The title and -"Add something" are headings. A row reads as one element, its name then its place; the trash -button reads "Remove window"; the kind buttons read "Add window" on the list and "Mark window" in -the chooser; the chosen answer carries the selected trait. "Mark" has the -hint "Marks the point under the circle in the middle of the screen"; "Mark something" has "Pin a -gas meter, door, window, AC unit, driveway or fence". Tapping on the camera image is not -available to VoiceOver; "Mark" is the only way. **Haptics and motion.** A firm tap when a feature -is recorded, a warning when a tap is refused, nothing for a removal or an answer. A ring blooms -and fades at each tap in 0.55 s. **Verification hooks.** `STATE=markFeatures` when the list -appears; marking itself logs nothing. Identifiers: `action.markSomething`, `action.closeTray`, -`feature.` (in the chooser and on the list), `action.markPoint`, `action.cancelMarking`, -`action.deleteFeature`, `window.opens.yes`, `window.opens.no`, `action.confirmFeatures`. - -## Edge cases - -- Tapping the same refused spot again repeats the same refusal: the text does not change and the - warning haptic does not fire again; only the ripple shows the tap was received. -- Two taps at the same height make a door or window with no height; nothing checks the size. -- Wall taps have no distance limit. A tap near the horizon can land far along the wall or high up - it, and is recorded there. Only ground taps are limited, to 8 m out. -- A door or window on the wall around a corner is recorded where the tap meets this wall's plane, - extended: at the wrong place. -- A feature marked beyond a wall end is still recorded and sent. -- Removing every row brings back "Nothing marked yet." -- Features cannot be edited after "Looks complete": not on a gap request, the upload or the result. - -## Open questions and verification - -- **Suspected bug: "Add something" does nothing visible.** The buttons start a mark - (`UI/Screens/MarkFeaturesScreen.swift:64`, accepted on this screen by - `Runtime/ScanEngine+Actions.swift:93`), but the list screen never shows the circle, the prompt, - "Mark" or camera taps; those exist only on the walk (`UI/Screens/WallWalkScreen.swift:28-45, - 121-135`). The homeowner who spots a missed gas meter here presses a button and nothing - happens. "Looks complete" does not end the stray mark either (`ScanEngine+Actions.swift:187-190`). -- **Suspected bug: "Tap the ac unit".** The prompt lowercases the kind's name - (`UI/Copy/ScanCopy.swift:122`), so "AC unit" becomes "ac unit" on screen, and VoiceOver hears - "Remove ac unit", "Add ac unit", "Mark ac unit" (`MarkFeaturesScreen.swift:121, 189`, - `WallWalkScreen.swift:282`). -- **Suspected bug: the list can vanish without a word.** The 20 s relocalization reset still - applies on the list, a capture screen (`Runtime/ScanEngine.swift:207, 458-478`), but the list - shows no coaching. After a call, a - homeowner reading the list with the phone pointed away for 20 s is sent back to finding the - meter with every mark and photo gone, never having seen "Point at the meter like this.". -- **Suspected bug: a half-done mark survives the 20 s reset.** `resetSpatialState` clears the - features but not the mark in progress or its taps (`ScanEngine.swift:496-522`, taps held at - `ScanEngine.swift:73`). After the meter is tapped again the walk opens mid-mark, and a first tap - from the old, discarded map can be joined to a new one. The same holds for a stray mark from - "Add something". -- **Wrong words for the wrong-side refusal.** The check is where the phone is, not where the tap - lands (`ScanEngine+Actions.swift:108`). "Tap something on this side" cannot help; stepping back - out in front of the wall's line does. -- The place in a row reads "ft" to VoiceOver (`MarkFeaturesScreen.swift:107`), though the app has - a spelled-out form for this (`UI/Copy/Distance.swift:25`). -- Unverified: all marking needs the live camera or the autopilot, which bypasses the chooser and - "Mark". MF-01 in [verification](../verification.md) was observed on an earlier commit and has - not been rechecked here. - -Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/screens/meter-close-up.md b/verification/product/screens/meter-close-up.md deleted file mode 100644 index c5a086ddc..000000000 --- a/verification/product/screens/meter-close-up.md +++ /dev/null @@ -1,135 +0,0 @@ -# The meter close-up - -## Summary - -The meter close-up takes one clear photo of the electric meter, so an installer can read it -later. The homeowner does not press a shutter: they hold the meter inside a circle and the phone -takes the photo itself once the meter is centered, near, sharp and well lit. It is the screen -right after the meter tap on [finding the meter](find-meter.md) (screen name `meterCloseUp`), -and it ends on its own. A homeowner who cannot get a usable photo can skip it after the second -failed try. - -## The simple case - -The circle appears in the middle of the camera view and the instruction reads "Hold your meter in -the circle", with "Your phone takes the photo by itself." The homeowner points the phone at the -meter from within about 1.5 m. While everything is right the circle's ring fills; after 0.6 s of -holding still it is full, the camera image flashes white under the words, a check mark appears in the circle and the -photo shrinks into the counter at the top right. About 1.2 s later the walk begins -([the wall walk](wall-walk.md)). - -## The interaction, event by event - -```mermaid -stateDiagram-v2 - [*] --> aiming - aiming --> holding : meter centered, near, sharp, well lit - holding --> aiming : any problem (ring empties) - holding --> taken : held 0.6 s (photo saved) - aiming --> aiming : a problem lasting 4 s (one failed try) - aiming --> skipped : "Can't get a clear shot" (from the second failed try) - taken --> [*] : after 1.2 s, the walk - skipped --> [*] : the walk at once -``` - -### Arriving - -The circle scales in and fades up around the middle of the screen. The ring is empty, the -failed-try count is zero and nothing is saved. The meter's position comes from the tap that -opened this screen; the close-up checks where that point appears in each camera frame. - -### Leaving at once - -There is no way off this screen before the second failed try other than taking the photo. No -back button, no skip. Nothing is recorded until the photo is taken. - -### First capture - -The ring starts filling on the first frame where all of these hold: tracking is normal; the meter -appears within the middle half of the image in both directions; the phone is within 1.5 m of it; -the image is not too dark (mean brightness at least 40 of 255) or glaring (mean above 225, or -more than a fifth of the pixels clipped); and the frame is at least half as sharp as the last 15. -While a check fails, one short fix shows under the instruction, first failing check first: - -| Problem | Shown | -| --- | --- | -| Tracking not normal | "Move slowly" | -| Meter outside the middle of the view, or behind the phone | "Center the meter in the circle" | -| Farther than 1.5 m | "Move closer to the meter" | -| Too dark | "Too dark to read. Try your phone's flashlight." | -| Too bright or glaring | "Too much glare. Tilt the phone a little." | -| Blurry | "Hold still" | - -### While capturing - -The ring fills over 0.6 s while every check passes; any failing frame empties it and the hold -starts again. A problem that lasts 4 s without a break counts as one failed try. From the second -failed try on, a button "Can't get a clear shot" appears below the fix. VoiceOver reads the circle -as "Photo of your meter" with its progress ("40 percent ready"). - -### Advancing - -When the ring is full the photo is saved as the scan's meter close-up. If saving fails, that -counts as a failed try and the fix reads "Hold still". Otherwise the screen flashes, the check -mark appears, the counter goes up with a light haptic tap, and 1.2 s later the walk begins. -"Can't get a clear shot" goes straight to the walk without a photo; the scan then carries no -meter close-up and an installer reads the meter instead. - -## Modifiers - -| Modifier | At arrival | While capturing | -| --- | --- | --- | -| Live camera or replay | A replay plays its recording from the start through this screen; it only succeeds if the recording shows the meter. | No effect. | -| Autopilot | The autopilot waits 4 s for the photo, then presses "Can't get a clear shot" itself, whether or not the button is showing. | No effect. | -| Server or sample result | No effect. | No effect. | -| Larger text sizes | The fix and the button grow; the circle does not. | No effect. | -| Reduce Motion | The circle fades in over 0.15 s instead of scaling. | The flash still shows; the photo does not fly into the counter. | - -## Cancel and interrupt - -| Event | Before the second failed try | From the second failed try | -| --- | --- | --- | -| The screen's own way out | None. | "Can't get a clear shot": the walk starts without a close-up. | -| Start over | Not offered. | Not offered. | -| Tracking limited | "Move slowly"; the ring empties and a 4 s problem counts as a failed try. | Same. | -| Tracking lost or relocalizing | Coaching replaces the instruction; the ring stays empty. After 20 s the scan returns to finding the meter. | Same. | -| App backgrounded or a call | The screen stays; the ring restarts when the camera resumes. | Same. | -| Camera off or session failed | Suspected dead end ([the flow](../foundations/flow.md#open-questions-and-verification)). | Same. | -| Network lost or upload failing | No effect. | No effect. | -| App killed | The scan is lost. | The scan is lost. | - -## Interactions with other systems - -**Coverage and evidence.** The close-up is not a walk photo and covers no cells. **Stored -photos.** Saved as the scan's meter close-up and uploaded with it. The walk shows it again -while the phone is finding its place ([the wall walk](wall-walk.md#while-capturing)). **Upload and offline.** No -effect here. **Accessibility.** The circle is one element with a spoken progress value; the fix -appears as a label; the button carries the hint "Skips the close-up. An installer will read the -meter instead." **Haptics and motion.** A success haptic as the screen opens (the meter is -set) and a light tap when the photo is taken. **Verification hooks.** The log records "close-up skipped after N -failed attempts" when the homeowner skips. - -## Edge cases - -- The circle is fixed in the middle of the screen, but the check uses the middle half of the - camera image. On a tall screen these differ: the meter can sit inside the circle and still be - "not centered", or outside it and pass. -- Two meters side by side: only the tapped one is checked. -- Saving can fail after the ring fills; that is the only failed try that is not 4 s long. - -## Open questions and verification - -- **Suspected bug: the way out can fail to appear.** A failed try counts only when one problem - lasts 4 s without a single good frame (`HouseScanKit/.../CloseUpGate.swift`, `evaluate`), while - the hold needs 0.6 s of good frames in a row. A homeowner whose frames alternate between good - and blurry (a shaky hand, flickering light) never fills the ring and never reaches a failed try, - so "Can't get a clear shot" never appears. That is the dead end the first-try note rules out. - Suggested rule: count attempts by elapsed time on the screen without a photo. -- The autopilot skips after 4 s by calling the skip action directly - (`Runtime/Autopilot.swift`), so S3's full-flow UI test cannot catch the dead end above. -- Observed in the Simulator on the ADVIO replay (no meter in it): the screen showed for 4 s and the - app logged "close-up skipped after 2 failed attempts". Its instruction card rendered without - its text in that screenshot; see [verification](../verification.md). -- The circle-versus-image mismatch above is read from code; it needs a device to confirm. - -Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/screens/onboarding.md b/verification/product/screens/onboarding.md deleted file mode 100644 index 27775d5a8..000000000 --- a/verification/product/screens/onboarding.md +++ /dev/null @@ -1,176 +0,0 @@ -# The onboarding - -## Summary - -The onboarding is the first thing the homeowner sees on every launch and after every Start over -(screen name `onboarding`). Three intro pages say what the scan is, that the phone takes the -photos, and how to stay safe; the last page's "Allow camera" opens [finding the -meter](find-meter.md), where iOS asks for the camera. The same document covers the failure -screen (screen name `unsupported`), which has one message per failure: an unsupported phone, -camera access denied, a failed camera session, an unreadable replay. At this commit only the -unsupported phone ever reaches it; the other three are set but never shown (see Open questions). - -## The simple case - -Page 1 shows a drawn wall with a gray meter, a row of blue dots on the ground and a phone sliding -back and forth under an "About 2 min" tag. Its title reads "Let's find a spot for your battery", -with "Walk along the wall by your electric meter for about 2 minutes. Your phone measures as you -go." The homeowner taps "Next". Page 2, "Your phone takes the photos", shows haze clearing off -the drawn wall, with "Just walk slowly. The haze on the wall clears as your phone sees it." They -tap "Next" again. Page 3, "Stay safe out there", lists three safety rules, and "Next" becomes -"Allow camera" with the note "Your phone will ask to use the camera." They tap it, the camera -screen appears with "Find your electric meter", and iOS shows its camera prompt: "House Scan uses -the camera to measure the outside wall around your electric meter." - -## The interaction, event by event - -```mermaid -stateDiagram-v2 - [*] --> page1 - [*] --> unsupported : phone can't track motion (live camera only) - page1 --> page2 : "Next" or swipe - page2 --> page3 : "Next" or swipe - page1 --> page3 : "Skip" - page2 --> page3 : "Skip" - page3 --> page2 : swipe back - page2 --> page1 : swipe back - page3 --> [*] : "Allow camera" (finding the meter, camera prompt) - unsupported --> unsupported : no way out -``` - -### Arriving - -The app opens on page 1 with the first of three page dots filled. At the same moment it checks -one thing: with the live camera, if the phone cannot run motion tracking, it switches straight to -the failure screen, crossfading over the intro in 0.25 s. With a replay it instead starts loading -the recording in the background. Nothing else is checked here; in particular the camera -permission is not asked. A top bar holds "Skip" on the right and, on a replay or with the -autopilot, a small badge on the left ("Replay", "Autopilot" or "Replay · Autopilot"). - -### Leaving at once - -"Skip" (top right, on pages 1 and 2) jumps to page 3; it does not leave the onboarding. There is -no way past the intro except "Allow camera", and no back button to anything before it. Closing -the app loses nothing, since nothing has been captured. - -On the failure screen for an unsupported phone there is no button at all: the homeowner can only -close the app. - -### First capture - -Nothing is captured here. The one committed action is "Allow camera": it moves to finding the -meter, and only then does the app start the camera. The first time, starting the camera makes iOS -show its permission prompt over "Find your electric meter". The app does not ask for permission -itself and does not wait for the answer before switching screens. - -### While capturing - -The pages can also be swiped in either direction; the button below them follows the current page -("Next" on pages 1 and 2, "Allow camera" on page 3) and the dots follow too, the current one -stretching into a pill. Each illustration animates only while its page is showing: the phone on -page 1 walks one lap every 6 s; the haze on page 2 clears column by column over 4 s, holds clear, -then rolls back in, every 6.5 s. Page 3 has no illustration, only three cards: "Stay on the -ground" ("No ladders, roofs or climbing."), "Keep clear of gas pipes" ("Don't lean on or step -over the gas meter.") and "Don't open the meter" ("Leave covers closed and wires alone."). - -### Advancing - -"Allow camera" always goes to finding the meter. What follows depends on the permission answer: - -| Answer | What the homeowner sees | -| --- | --- | -| Allow, or already allowed | The camera image appears behind "Find your electric meter". | -| Declined, now or earlier | Suspected dead end: the screen stays on "Find your electric meter" with no camera image, no message and no way to Settings. Tapping does nothing. | -| The camera session fails | Suspected dead end: the same screen, frozen. | - -The failure screen shows one of these, with an icon above the title: - -| Failure | Title | Detail | Button | Reached at this commit | -| --- | --- | --- | --- | --- | -| Unsupported phone | "This phone can't measure walls" | "House Scan needs an iPhone that supports motion tracking with the camera. Try another iPhone from the last few years." | None | At launch, live camera only | -| Camera denied | "House Scan needs your camera" | "It uses the camera to measure the wall around your meter. Turn on Camera for House Scan in Settings." | "Open Settings" | No | -| Session failed | "The camera stopped" | The system's error text | "Start over" | No | -| Unreadable replay | "This recording can't be opened" | The loader's error text | "Start over" | No | - -"Open Settings" opens the app's page in iOS Settings. "Start over" would do what it does on the -result ([the flow](../foundations/flow.md#cancel-and-interrupt)) and return to page 1. - -## Modifiers - -| Modifier | At arrival | While capturing | -| --- | --- | --- | -| Live camera or replay | With a replay the motion-tracking check is skipped, so the failure screen never appears at launch, and "Allow camera" never opens the camera or triggers the prompt; the replay's first frame shows behind finding the meter. | No effect. | -| Autopilot | Needs a replay. It waits up to 60 s for the recording to load, holds page 1 for the hold time ([the flow](../foundations/flow.md#modifiers)), then moves on as if "Allow camera" were tapped, without paging. If the recording fails to load it stops and the app stays on page 1. | No effect. | -| Server or sample result | No effect. | No effect. | -| Larger text sizes | Each page's title and body grow and the page scrolls; the illustrations stay 260 pt tall. The failure screen scrolls when its text no longer fits. | The buttons and dots stay fixed below the page. | -| Reduce Motion | The illustrations hold still: the phone stops mid-wall and the haze stops partly cleared. | "Next" slides the page in 0.15 s instead of 0.25 s; "Skip" and swiping are unchanged. | - -## Cancel and interrupt - -| Event | On the intro pages | On the failure screen | -| --- | --- | --- | -| The screen's own way out | "Skip" to page 3; "Allow camera" on it. | Unsupported phone: none. The other three would offer "Open Settings" or "Start over", but are never shown. | -| Start over | Not offered. Arriving here after Start over shows page 1 again. | Not offered for an unsupported phone. | -| Tracking limited | No effect: the camera is not running. | No effect. | -| Tracking lost or relocalizing | No effect. | No effect. | -| App backgrounded or a call | The current page is kept. | No effect. | -| Camera off or session failed | No effect here; the failure lands after "Allow camera", on finding the meter, as a suspected dead end. | Would show "House Scan needs your camera" or "The camera stopped", but never does. | -| Network lost or upload failing | No effect. | No effect. | -| App killed | Nothing is lost; the next launch opens page 1. | The next launch shows the same screen. | - -## Interactions with other systems - -**Coverage and evidence.** None; nothing is seen or covered before the meter tap. **Stored -photos.** None are taken. **Upload and offline.** No effect; nothing here needs the network. -**Accessibility.** Each page title is a heading; the illustrations and the page dots are hidden -from VoiceOver; each safety card is read as one element; "Allow camera" carries the hint "Your -phone will ask to use the camera". On the failure screen the icon is hidden and the title is a -heading. The mode badge is read aloud. **Haptics and motion.** No haptics on either screen. Pages -change with a slide, the button swaps with a fade and the dots settle with a 0.4 s spring. -**Verification hooks.** The log records `STATE=onboarding` at launch and `STATE=unsupported` -when the phone check fails; an unreadable replay logs "replay unreadable: ..." without a screen -change. Tests find the controls as `action.onboardingNext`, `action.onboardingSkip`, -`action.finishOnboarding`, `action.openSettings` and `action.startOver`. - -## Edge cases - -- The app declares motion tracking (`arkit`) as a required device capability - (`ios/Config/Info.plist`), so the App Store and TestFlight will not install it on a phone that - lacks it. The unsupported-phone screen is in practice seen only in the Simulator without a - replay. -- On a live launch the intro is drawn for a moment before the phone check runs, so an unsupported - phone briefly shows page 1 before crossfading away. -- The note "Your phone will ask to use the camera." shows on every pass, but iOS asks only once. - After Start over, or on any later launch, no prompt follows. -- Swiping back from page 3 brings "Next" and "Skip" back; nothing on page 3 is committed until - "Allow camera". - -## Open questions and verification - -- **Suspected bug: camera denied, a failed session and an unreadable replay never reach the - screen** (triaged as [B-03](../bug-triage.md)). The engine switches to the failure screen only - for an unsupported phone (`Runtime/ScanEngine.swift:105-107`). A denied camera and a failed - session set the failure (`ScanEngine.swift:489, 491`) and an unreadable replay does too - (`ScanEngine.swift:119`), with no screen change. The homeowner who declines the prompt sits on - "Find your electric meter" with no camera image; the "Open Settings" and "Start over" buttons on - `UI/Screens/UnsupportedScreen.swift:35-48` are unreachable outside the `-uiDemo` preview. -- **Suspected bug: an unsupported phone has no way out.** `UnsupportedScreen.swift:44` hides - "Start over" for that failure, so the screen has no button. That fits (nothing can fix it), but - it contradicts [the flow](../foundations/flow.md#summary), which says "Start over" is offered on - the unsupported screen. One of the two needs updating. -- **Suspected bug: "Allow camera" during replay loading starts the live camera.** On a replay, - `startSourceIfNeeded` (`ScanEngine.swift:184`) starts the camera whenever the replay has not - loaded yet and no failure is set. A homeowner who taps "Skip" and "Allow camera" before a slow - recording loads gets a live session as well as the replay. Affects replays only; not observed. -- **"The camera stopped" would show raw system error text** as its detail - (`UI/Copy/ScanCopy.swift:260`), the same pattern as B-02, if the screen were ever reached. -- **Reduce Motion is applied to "Next" but not to "Skip"** (`OnboardingScreen.swift:37` versus - `:75`). Even "Next" still slides the page sideways under Reduce Motion, only faster. -- The page view hides its dots from VoiceOver on the grounds that it announces its own position - (`OnboardingScreen.swift:158`), but the index display is set to never show (`:54`). Whether - VoiceOver announces the page number needs checking on a device. -- Read from code, not observed: the camera prompt's timing, ARKit reporting a denied camera as a - session error, and the Reduce Motion stills. The unsupported screen and the - `-replay /nonexistent` case can be checked in the Simulator. - -Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/screens/result.md b/verification/product/screens/result.md deleted file mode 100644 index bf51f53b4..000000000 --- a/verification/product/screens/result.md +++ /dev/null @@ -1,222 +0,0 @@ -# The result - -## Summary - -The result is the server's answer, shown once the [upload](uploading.md) has finished. It covers -two screens. The result screen (screen name `result`) has a small 3D model of the homeowner's own -wall with the battery on it, a headline, where the battery goes and how much cable it needs, and -each check with its outcome. From there the homeowner can open the AR view (screen name -`resultAR`), which draws the battery onto the live camera at the chosen spot. They can also fill -in a missing view with "Capture it now", or begin again with "Start over". Nothing is captured on -either screen, and there is no way back to the walk. - -## The simple case - -After "Checking your wall" the result fades in with a success haptic. At the top, a 3D model of -the wall turns once and comes to rest on the battery, with the cable run from the meter and the -ground in front of each stretch tinted by outcome. Below it the headline reads "There's a spot for -your battery" and, in blue, where it is, for example "4 ft 3 in right of your meter, 3 ft of cable". Next come -the server's one-sentence summary, then "See it on your wall". Under that are a note that the -electrical panel still needs an electrician's review and "What we checked", one row per check, and the "Start over" button. The homeowner taps "See it on your wall", points the -phone at the wall and sees the battery standing there, then taps "Done" to go back. - -## The interaction, event by event - -```mermaid -stateDiagram-v2 - [*] --> result : the server answered (success haptic) - result --> resultAR : "See it on your wall" (only with a spot) - resultAR --> result : "Done" - result --> gapRequest : "Capture it now" - result --> onboarding : "Start over" -``` - -### Arriving - -The result screen opens only after the server has answered ([the flow](../foundations/flow.md)). -A success haptic plays, except when coming back from the AR view. The 3D model fills the top -340 points of the screen; the rest scrolls beneath it, in this order: - -1. The headline, with an icon in the decision's color: - - | Decision | Headline | - | --- | --- | - | pass | "There's a spot for your battery" | - | manual review | "An installer will take a look" | - | reject | "This wall doesn't have a spot" | - -2. The placement line, only when the server chose a spot: the spot's middle measured - along the wall ("4 ft 3 in right of your meter", or "At your meter" within 3 in of it), then - the cable length in whole feet ("3 ft of cable"). The server never sends a spot with a reject. -3. The server's summary sentence. If the summary is empty on a reject, the reason of the first - failing check stands in. -4. "See it on your wall", only when there is a spot. -5. "The placement rules aren't final yet, so an installer reviews every result for now.", when - the server's rules are not approved for automatic decisions. -6. "A closer spot may exist on the left of your meter. The scan didn't reach that side." (or - right). It shows when the server asks for a view past a wall end, or when either wall end is - an unexplored end that the server does not mark as beyond cable reach. -7. "Your electrical panel still needs an electrician's review. This scan only covers where the - battery can go.", on every result: the server's answer covers where the battery goes, not the - panel. -8. "What we checked", one row per check, in the server's order (see - [While capturing](#while-capturing)). -9. "Still needed", one card per missing view the server lists (see [Advancing](#advancing)). -10. "Start over". - -The headline block fades in and rises 12 points after 0.35 s. The 3D model sweeps over 1.6 s from -a high three-quarter view down to its resting angle, centered on the battery (or on the meter -when there is no spot). - -### Leaving at once - -"Start over" is always at the bottom. It forgets the wall, photos, marks and result and returns -to the introduction ([the flow](../foundations/flow.md#cancel-and-interrupt)). It asks for no -confirmation. The screen has no back button to the walk or the marks. - -### First capture - -Nothing is captured here. The first thing most homeowners do is turn the model or open the AR -view. - -**The 3D model.** Dragging turns it, up to 70° either side and between 8° and 45° above the -ground; the first drag stops the opening sweep where it is. Pinching zooms between 0.6 and 1.6 -times. The model shows the wall (to the marked wall ends, or 1.5 m past everything the result -mentions), the meter, every marked feature, the cable as a blue tube, and the battery as a white -box with a blue light bar. The server also sends its sweep: the stretches of wall where it tried -the battery, each with one outcome. The server gives each stretch as the range of places the -battery's left edge can start; the app extends it by the chosen battery's width so the tint -covers the wall the battery would stand on (without a chosen spot it covers the starts alone). -Each stretch is a translucent tint on the ground in front of the wall, as deep as the battery: green where a battery passes, amber where it is unsure, red -where it fails. - -**The AR view.** "See it on your wall" crossfades to the camera. The instruction reads "Your -battery could go here" for an approved pass and "The spot an installer will check" otherwise, -with the placement line under it. With the sample result it reads "Example spot, not your -result", with "No server checked this scan." and the placement line. On the camera image the app draws each -sweep stretch as a tinted ground strip with a dashed outline, the cable as a blue line edged in -white, and the battery as a box with a soft shadow and a blue light bar across its front. The box -rises out of the ground as the view opens. Everything is placed relative to the meter's anchor, -so it stays on the wall as the homeowner moves. "Done" is the only control. The drawing is shown -only while tracking is normal (see [While capturing](#while-capturing)). - -### While capturing - -Each row of "What we checked" shows an icon (check, question mark or cross), the check's name -from the server, its reason, and a measurement line when the server measured something: -"Measured 3 ft 1 in. The rule is 3 ft, and the measurement can be off by about 4 in." The second -sentence drops its error clause when the server gives no error, and disappears when there is no -rule value. An unsure row adds one note: "An installer will check this" when the doubt is a -narrow margin, an unknown attribute, a rule that always needs review or no stated cause, or "One -more photo would settle this" when the area was not seen. The rows have no buttons of their own. - -On the AR view the drawing is redrawn from every camera frame. Whenever tracking is not normal -(limited, lost or relocalizing), the drawing disappears at once and the instruction reads "Point -at your meter", with "The battery comes back once your phone finds its place." When tracking is -normal again the drawing returns, without rising again, and the instruction goes back to the -headline above. -No other coaching appears here: the homeowner is not told why tracking is limited. - -### Advancing - -"Done" returns to the result screen. The screen is built again, so the headline fades in again -and the model replays its sweep and loses any turn or zoom. - -Each "Still needed" card shows the server's message. Under it is "Capture it now" when the phone -can build a request from it (a wall or ground view with its stretch, or a view past a wall end -with its side), and "An installer will check this" otherwise. -"Capture it now" opens a [gap request](gap-request.md) for that stretch, using the server's -message as the second line. For a view past a wall end, that end is forgotten first, because the -wall may go on. When the view is taken, or the homeowner taps "I can't get there", the scan is -uploaded again and a new result replaces this one. The old result cannot be reopened meanwhile. - -## Modifiers - -| Modifier | At arrival | While capturing | -| --- | --- | --- | -| Live camera or replay | A replay shows the "Replay" badge over the model. The AR view on a replay shows one still: the recorded frame that best shows the spot, with the battery drawn on it. | On a replay the AR picture does not move. Live, the drawing follows the phone. | -| Autopilot | The autopilot waits `-autopilotHold` seconds, opens the AR view itself (even when the result has no spot), waits again, taps "Done" and logs "AUTOPILOT done". | No effect. | -| Server or sample result | The sample result shows "Sample result, not from the server" over the model. It is always the same: "An installer will take a look", "4 ft 3 in right of your meter, 3 ft of cable", the rules note, the left-side note, the panel note, four checks and two "Capture it now" cards. The AR view says "Example spot, not your result", with "No server checked this scan." | "Capture it now" works, but the upload returns the same sample, so nothing changes. | -| Larger text sizes | The headline, notes and rows wrap and the page scrolls; the model stays 340 points tall. On the AR view the instruction grows, and at the largest sizes the chrome scrolls. | No effect. | -| Reduce Motion | The headline fades in over 0.2 s without rising. The model starts at its resting angle with no sweep. The AR battery rises over 0.2 s instead of springing up. | Dragging and pinching the model work as usual. | - -## Cancel and interrupt - -| Event | On the result | On the AR view | -| --- | --- | --- | -| The screen's own way out | "Start over"; "Capture it now" leaves for a gap request. | "Done", back to the result. | -| Start over | Offered at the bottom; see [the flow](../foundations/flow.md#cancel-and-interrupt). | Not offered. | -| Tracking limited | Nothing shown; the camera is hidden but keeps tracking. | The drawing disappears and the instruction reads "Point at your meter" with "The battery comes back once your phone finds its place." until tracking is normal. | -| Tracking lost or relocalizing | Nothing shown; the result stays however long the phone takes (fixed in `beede15`; it used to reset to finding the meter after 20 s). | The drawing disappears and "Point at your meter" shows, as above, until the phone finds its place. No reset. | -| App backgrounded or a call | The screen stays. When the app returns the phone relocalizes in the background; nothing is lost. | Same, with the camera resuming behind the drawing. | -| Camera off or session failed | No visible effect. | Suspected dead end, as in [the flow](../foundations/flow.md#open-questions-and-verification): no message, no picture. | -| Network lost or upload failing | No effect; the result is already on the phone. "Capture it now" leads to a new upload, which needs the network ([the upload](uploading.md)). | No effect. | -| App killed | The result is lost; nothing reopens it. | The result is lost. | - -## Interactions with other systems - -**Coverage and evidence.** Nothing here adds coverage; "Capture it now" hands over to a gap -request, which follows [coverage and guidance](../foundations/coverage-and-guidance.md). -**Stored photos.** No photos are kept on either screen. **Upload and offline.** The result needs -no network once shown; only "Capture it now" leads to another upload. **Accessibility.** The -headline block is read as one heading. The model is one element, "3D view of your wall", with a -value such as "Battery 3 ft right of your meter, cable 3 ft" and the hint "Drag to turn the view". -Each check row reads as "Distance from the window: Not sure yet", followed by its reason, -measurement line and note. The outcome words are "Looks good", "Not sure yet" and "Doesn't work". -The AR drawing is hidden from VoiceOver. **Haptics and motion.** A success haptic when the result -appears, not when returning from the AR view. **Verification hooks.** `STATE=result` and -`STATE=resultAR`. Identifiers: `result.headline`, `result.placement`, `result.rulesNotFinal`, -`result.unseenSide`, `result.panelReview`, `result.sampleBadge`, `check.`, `action.showAR`, -`action.captureMissing`, `action.startOver`, `action.closeAR`. - -## Edge cases - -- A manual review can come without a spot. The headline then reads "An installer will take a - look", with no placement line and no "See it on your wall". -- On a reject the checks are those of the nearest spot the server considered, and the model - centers on the meter. -- The left-or-right note says "A closer spot may exist" even on a reject, where no spot was found. -- Opened without a spot (only the autopilot can do this), the AR view draws the sweep and says - "Point at your meter", with no second line while tracking is normal. -- The AR battery is drawn over everything in the picture, including a bush or a person standing - in front of the spot. -- With the sample result on a real phone, the AR view places a battery 4 ft 3 in right of the - real meter, whatever the wall is like, under "Example spot, not your result". -- A "Still needed" item about the wall or the ground that comes without a stretch still shows - "Capture it now", but the button does nothing. - -## Open questions and verification - -- The 20 s relocalization reset no longer applies on the result or the AR view - (`Runtime/ScanEngine.swift:472-476`), fixed in `beede15`. -- The tinted stretches now reach one battery width past the last start - (`Runtime/ScanEngine+Export.swift:132-141`; `start_ft` at `HouseScanKit/.../PlacementResult.swift:450`), - fixed in `beede15`. A result with no spot still tints only the starts, which understates each - stretch; the code says so. -- The AR view now names a sample spot as an example (`UI/Screens/ResultARScreen.swift:49-51`), - fixed in `525ea40`. It still has no badge like the result screen's - (`UI/Screens/ResultScreen.swift:96-105`). -- **Question: an end answered "It turns a corner" may read as unseen.** The left-or-right note - shows when the server reports an end as unexplored and not beyond cable reach - (`Runtime/ScanEngine+Export.swift:157-163`), - and a corner answer is sent as unexplored. A homeowner who walked to the corner would then read - that "the scan didn't reach that side". Depends on what the server echoes back; not checked. -- The measurement line says "The rule is 20 ft" for a limit that is a maximum and "The rule is - 3 ft" for a minimum. The app decodes the server's `comparison` but never shows it, and ignores - `review_threshold_ft` (`UI/Copy/ScanCopy.swift:233-237`). -- "Capture it now" is offered only when the phone can build a request from the item - (`Runtime/ScanEngine+Export.swift:152`), fixed in `beede15`; before, it could do nothing. -- After "Capture it now" the old result is gone from view. If the new upload fails, the homeowner - has only "Try again" on the upload screen and cannot reread the earlier answer. -- VoiceOver reads the model's value and the measurement lines with "ft" and "in", which it reads - as letters (`UI/Result/ResultScene3D.swift:296-321`, `UI/Screens/ResultScreen.swift:240`); - `Distance.spoken` exists for this. -- The autopilot opens the AR view without checking for a spot (`Runtime/Autopilot.swift:93`), a - path no homeowner can take. -- Dragging the model sits inside a scrolling page; whether a vertical drag turns the model or - scrolls the page needs a device. -- Both screens were seen at `0876e03`, `24af434` and `525ea40` with the real server's answer, and with the sample result - ([verification](../verification.md), RES-01 to AR-01). Whether the AR drawing lines up with a - real wall, and hiding it while the phone relocalizes, need the live camera. - -Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/screens/uploading.md b/verification/product/screens/uploading.md deleted file mode 100644 index 3429be51d..000000000 --- a/verification/product/screens/uploading.md +++ /dev/null @@ -1,174 +0,0 @@ -# The upload - -## Summary - -The upload screen (screen name `uploading`) sends the scan and waits for the result. The -homeowner does nothing here unless it fails. It shows a large symbol, a title and three steps -that tick off in order: "Get photos ready", "Send photos", "Check clearances". It appears after -"Looks complete" on [marking features](mark-features.md) when the phone finds no gap, or after a -[gap request](gap-request.md) is taken or skipped, and it ends on its own when the answer arrives -([the result](result.md)). If the upload fails, the steps give way to one button, "Try again". -There is no other way off the screen. - -## The simple case - -The camera closes and the screen reads "Getting your photos ready", with a spinner beside "Get -photos ready". A moment later the first step becomes "Photos ready" with a check mark and the -title changes to "Sending your photos", with "Keep the app open. It takes about a minute." The -second step counts up: "Sending photos, 40%". When the server's answer arrives the title briefly -becomes "Checking your wall" ("Measuring clearances around your meter."), and the result appears -with a success haptic. - -## The interaction, event by event - -```mermaid -stateDiagram-v2 - [*] --> packaging - packaging --> sending : scan zipped - packaging --> failed : zipping failed - sending --> analyzing : the server answered (or the sample reached 100%) - sending --> failed : connection error or a non-2xx answer - analyzing --> failed : the answer is not a readable result - analyzing --> [*] : the result - failed --> packaging : "Try again" -``` - -### Arriving - -The screen fades in over 0.25 s on a plain background; the camera view is gone. The symbol is a -stack of photos in a pale circle and pulses while work is going on. The title is "Getting your -photos ready" with no second line. Of the three steps, the first has a spinner and the other two -an empty circle in a muted color. The app first waits for photos still being written to finish, -for up to 10 s, then zips the scene description (`scene.json`) with the photos. - -### Leaving at once - -None. There is no Cancel, no back and no Start over on this screen. Until an answer or a -failure, the homeowner can only wait or close the app. - -### First capture - -Nothing is captured here. The first durable step is the zip: the scene description, every -[kept photo](../glossary.md#seeing-the-wall) of the scan and the [close-up](meter-close-up.md) -if one was taken. Once it is written the symbol becomes an upward arrow, the first step reads -"Photos ready" with a check mark, the title becomes "Sending your photos" and the second step -reads "Sending photos, 0%". - -### While capturing - -The percentage follows the bytes sent to the server, rounded to a whole number and capped at 99% -until the server answers. The digits roll as they change. When the answer arrives, the symbol -becomes a ruler, the second step reads "Photos sent", the third gets the spinner and the title -becomes "Checking your wall" with "Measuring clearances around your meter." With a real server -this lasts only as long as the app takes to read the answer (see Open questions). - -Any failure replaces the three steps with a "Try again" button, stops the pulse and turns the -symbol to the caution color: - -| What failed | Symbol | Title | Second line | -| --- | --- | --- | --- | -| Any connection error (no network, server unreachable, timeout) | Wi-Fi crossed out | "You're offline" | "Your scan is saved on this phone. Try again when you have signal." | -| The server answered with an error, the answer was unreadable, or zipping failed | Warning triangle | "That didn't go through" | The error's own text, for example "The server answered", the status code and the start of the server's reply | - -"Try again" returns to "Getting your photos ready", zips the scan again and sends all of it. - -### Advancing - -When the answer is a readable result, the app stores it, the title becomes "Done" with a check -mark symbol, and the result screen appears with a success haptic -([the flow](../foundations/flow.md#interactions-with-other-systems)). In normal use the switch -is immediate, so "Done" is not seen. Nothing advances without an answer: a failure stays on -screen until "Try again" succeeds. - -## Modifiers - -| Modifier | At arrival | While capturing | -| --- | --- | --- | -| Live camera or replay | A replay stops playing; the scan is zipped the same way. | On the live camera the phone still watches tracking behind this screen (see Cancel and interrupt); a replay does not. | -| Autopilot | A badge at the top left reads "Replay · Autopilot" (in capitals). With the sample result, the `-autopilotHold` time replaces the 1.2 s in the timings below. | If no result arrives within 240 s and the upload has failed, the autopilot presses "Try again" once and waits 90 s more. With `-autopilotGate`, "Done" stays up until the test's gate file appears, at most 120 s. | -| Server or sample result | With `-serverURL` the zip is posted to that server. Without it, or with `-sampleResult`, the scan is still zipped but not sent, and the screen says so: the title is "Making a sample result", with "No server is connected, so your photos stay on this phone. The result you'll see is an example, not a check of your wall.", and the steps read "Get photos ready", "Load the sample result" and "Show the example". | The title stays the same until the result. "Load the sample result" stays up for about 1 s (four pauses of 0.24 s, no percentage shown), then reads "Sample loaded", and "Show the example" holds for 1.2 s before the [sample result](../glossary.md#after-the-walk). A failure still shows the failure titles. It fails only if the sample file is missing from the app. | -| Larger text sizes | Title, second line and steps grow and wrap; the screen scrolls when they no longer fit. The symbol stays 132 pt. | No effect. | -| Reduce Motion | The screen fades in over 0.15 s; the symbol does not pulse. | Check marks still scale in and the symbol still swaps with an animation. | - -## Cancel and interrupt - -| Event | While zipping or sending | After a failure | -| --- | --- | --- | -| The screen's own way out | None. | "Try again": zips and sends the whole scan again. | -| Start over | Not offered. | Not offered. | -| Tracking limited | No effect; this screen shows no coaching. | No effect. | -| Tracking lost or relocalizing | No effect: the 20 s reset does not apply on this screen, so the scan and its answer are kept (fixed in `beede15`). | No effect; the failure and "Try again" stay. | -| App backgrounded or a call | The app asks for no background time, so iOS may suspend the transfer; a broken connection would then show "You're offline". Unverified. | The failure stays. | -| Camera off or session failed | No effect; the upload does not need the camera. | No effect. | -| Network lost or upload failing | The steps give way to the failure and "Try again" (table above). | "Try again" fails the same way until the network or server is back. | -| App killed | The scan is lost. The zip and photos stay in the cache, never reopened. | Same, although the screen said "Your scan is saved on this phone". | - -## Interactions with other systems - -**Coverage and evidence.** Nothing is captured here. The scene description carries the wall, -the covered stretches of each band, whether each wall end is a real end, the marked features, -and each kept photo's camera position and lens data -([coverage and guidance](../foundations/coverage-and-guidance.md)). The server evaluates the whole -scan again on every upload. **Stored photos.** The zip, `scan.zip`, is written into the scan's -cache folder ([the flow](../foundations/flow.md#interactions-with-other-systems)) and replaced on -each attempt. It holds `scene.json`, every kept photo as `k00001.jpg`, `k00002.jpg` and so on -(landscape JPEGs, unrotated, as the camera produced them) and the close-up as `meter_close.jpg`. -**Upload and offline.** The app posts the raw zip to `{serverURL}/v1/placements` with -`Content-Type: application/zip` and `Accept: application/json`. The request times out after -120 s without data. Any answer outside 200–299 is a failure; so is one that does not read as a -result: a schema version other than 1.0, a missing required field, a wrong type, a pair of the -wrong length, or a value outside a fixed set such as the decision or an outcome -(`HouseScanKit/.../PlacementResult.swift:3-12`). Fields the app does not know are ignored, so the server can add -optional ones without breaking the upload. Only connection errors count as offline. **Accessibility.** The title is a header. The -symbol is hidden from VoiceOver. The three steps are one element, read together. No change of -step or percentage is announced, and neither is a failure. **Haptics and motion.** None on -arrival, failure or "Try again"; a success haptic when the result appears. The symbol pulses while -working and swaps with a replace animation; check marks scale in; the title crossfades in 0.2 s. -**Verification hooks.** `STATE=uploading` on arrival. The log records "bundle with N -keyframes" after zipping, and "packaging failed: …" or "upload failed: …" with the error. The -autopilot logs "upload failed; retrying once". Test identifiers: `screen.uploading`, -`upload.steps`, `action.retryUpload`. - -## Edge cases - -- The screen can appear more than once in a scan: after each gap request, including one started - with "Capture it now" on the result. Each visit starts again at "Getting your photos ready" and - sends every photo, not only the new ones. -- A photo whose write has not finished after the 10 s wait is left out of the zip without notice. -- After a relocalization reset earlier in the scan, only photos kept since the reset are sent; - the older files stay in the folder. -- A wrong server URL, a stopped server or a slow server that goes quiet for 120 s all show "You're - offline" and "Try again when you have signal", even with full signal. -- "Try again" does nothing once pressed until the attempt fails again; a second tap is ignored. - -## Open questions and verification - -- The wrong upload route ([bug triage](../bug-triage.md) B-01) is fixed in `0876e03`: the app - posts to `/v1/placements` (`Runtime/ResultClient.swift:24`), and the server answered 200 in the - Simulator run at this commit ([verification](../verification.md), FLOW-02). -- **Suspected bug: raw error text on screen.** The second line of "That didn't go through" is the - error's developer description (`UI/Copy/ScanCopy.swift:196`, fed by - `Runtime/ScanEngine.swift:672, 699`): HTTP status codes, server JSON, decoder errors. B-02. -- **Suspected bug: "You're offline" for any connection error.** `Runtime/ScanEngine.swift:699` - treats every `URLError` as offline, including an unreachable host, refused connection and - timeout. The copy then tells a homeowner with full signal to wait for signal. -- **Suspected bug: "Checking your wall" is skipped with a real server.** Progress is capped at - 99% while sending (`Runtime/ResultClient.swift:68`) and reaches 100% only after the server has - answered (`:40`); the step changes only then (`Runtime/ScanEngine.swift:681, 685`). The - homeowner watches "Sending photos, 99%" for the whole analysis, then the result replaces - "Checking your wall" almost at once. Only the sample result lingers on the third step, where it - reads "Show the example". -- A relocalization reset can no longer wipe the scan during the upload: the reset applies only - on the capture screens (`Runtime/ScanEngine.swift:469-476`), fixed in `beede15`. -- The app asks for no background execution time (no `beginBackgroundTask`), so "Keep the app - open" is the only guard for the transfer. What iOS does to the transfer when the phone locks - has not been tested. -- "It takes about a minute" has no measurement behind it; no upload has been timed with a real - scan. -- The comment on `writeBundle` (`Runtime/KeyframeStore.swift:89–90`) says the zip stays so a - failed upload can be retried, but "Try again" rebuilds it (`Runtime/ScanEngine+Actions.swift:226–228`). - Harmless, but the kept zip is never reused. -- Pass 1 (`21a63e7`) saw "That didn't go through" with no button or symbol visible - ([verification](../verification.md) UP-01, B-05). Not rechecked at this commit. - -Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/screens/wall-walk.md b/verification/product/screens/wall-walk.md deleted file mode 100644 index 6f23a6a8a..000000000 --- a/verification/product/screens/wall-walk.md +++ /dev/null @@ -1,362 +0,0 @@ -# The wall walk - -## Summary - -The wall walk is where the scan collects its evidence. The homeowner walks along the wall with -the phone pointed at it, first to the left of the meter and then to the right, and the phone keeps -photos by itself. Fog over the camera image clears where the phone has looked, the coverage strip -at the bottom fills in, and one instruction at the top says what to do next. The homeowner says -where the wall stops on each side ("Wall ends here", then whether it turns a corner or something -blocks it; or "Can't get there" when they cannot go further), can mark features on the way ("Mark something"), and leaves with "Done with this wall" -once both ends are set. It follows [the meter close-up](meter-close-up.md) (screen name -`wallWalk`) and leads to [marking features](mark-features.md). If the phone loses its place for -too long, the scan starts again from [finding the meter](find-meter.md). - -## The simple case - -The screen opens over the same camera view. The instruction reads "Walk slowly to your left", with -"Keep the wall and the ground in view." Blue dots on the ground show where to walk, and a ring on -the wall shows where to aim. The homeowner walks. The counter at the top right goes up, the fog -lifts off the wall and ground behind them, and the strip turns from gray to amber to green. - -When enough of the left side is covered, the instruction asks "Is this the left end of the wall?" -and a "Wall ends here" button appears. The homeowner aims the middle of the screen at the corner -and taps it. A white line appears on the wall there, and the instruction asks "What's at the left -end?", with "This tells the installer whether the wall keeps going." and two buttons, "It turns a -corner" and "Something blocks it". They tap "It turns a corner", and the instruction switches to -"Walk slowly to your right". They walk back past the meter and do the same on the right. With both ends set, -"Done with this wall" appears; any stretch between the ends that is still thin gets one more aim -instruction, and then "That's the whole wall", with "Tap Done when you're ready." Tapping "Done with -this wall" opens the list of marks. - -## The interaction, event by event - -```mermaid -stateDiagram-v2 - [*] --> walking - state walking { - [*] --> walkLeft - walkLeft --> askLeftEnd : coverage runs far enough left - walkLeft --> walkRight : "Can't get there" (unexplored left end) - askLeftEnd --> whatAtLeft : "Wall ends here" - whatAtLeft --> walkRight : "It turns a corner" (unexplored) or "Something blocks it" (real end) - walkRight --> askRightEnd : coverage runs far enough right - walkRight --> filling : "Can't get there" (unexplored right end) - askRightEnd --> whatAtRight : "Wall ends here" - whatAtRight --> filling : "It turns a corner" (unexplored) or "Something blocks it" (real end) - filling --> complete : nothing lags between the ends - } - walking --> kindPicker : "Mark something" - kindPicker --> walking : close - kindPicker --> marking : a kind tapped - marking --> walking : last mark placed, or "Cancel" - walking --> [*] : "Done with this wall" (both ends set) - walking --> findMeter : relocalizing for over 20 s -``` - -`filling` and `complete` both show "Done with this wall". "Take a step back" and the tilt -instructions can interrupt any walking state; they are left out of the diagram. - -### Arriving - -The camera stays open from the close-up; only the chrome changes. At the top: a mode badge on a -replay, the photo counter (it already counts the close-up, if one was taken), and the instruction -card. At the bottom: a "Mark something" button and the coverage strip. Over the camera image: -fog on the wall band and ground band around the meter, a dot on the meter, and the wayfinding -cues. The first instruction is chosen from the first camera frame, in the order set out in -[coverage and guidance](../foundations/coverage-and-guidance.md#one-instruction-at-a-time). A -homeowner who has just held the meter close is usually nearer the wall than the walk wants, so -the first line they read is likely "Take a step back", with "Your phone needs to see more of the -wall at once." - -The instructions this screen can show, with their second lines: - -| Instruction | Second line | Cue on the camera | -| --- | --- | --- | -| "Take a step back" | "Your phone needs to see more of the wall at once." | None. | -| "Tilt down to show the ground" | "The strip along the wall, 3 ft 4 in left of your meter." (the distance varies) | Ring on the ground, dots toward it. | -| "Tilt up to show more wall" | "Around 3 ft 4 in left of your meter." (the distance varies) | Ring on the wall, dots toward it. | -| "Walk slowly to your left" (or right) | "Keep the wall and the ground in view." | Ring on the wall 1 m past whichever is farther along that side, the unbroken covered stretch or the phone itself; dots toward it. | -| "Is this the left end of the wall?" (or right) | "Aim where the wall stops or turns a corner, and tap Wall ends here." | Ring at the end of the covered stretch, no dots. | -| "What's at the left end?" (or right) | "This tells the installer whether the wall keeps going." | The ring and dots of the previous instruction stay until the answer. | -| "That's the whole wall" | "Tap Done when you're ready." | None. | - -A distance within 3 in of the meter reads "at your meter". Coaching replaces all of these except -the end question while the phone has a problem (see [Cancel and interrupt](#cancel-and-interrupt)). - -The wayfinding cues work together. The **path** is a line of blue dots on the ground, 1.5 m out -from the wall, from where the homeowner stands toward the next place to stand; the dots shrink -with distance like paint on the ground. The **target** is a white-and-blue ring, sized to the -distance, that pulses slowly while its point is comfortably on screen. Near the edges or off -screen it becomes a blue chevron at the side of the screen, pointing toward the spot, kept clear -of the instruction card and the buttons. - -### Leaving at once - -There is no back button and no way to leave without setting both ends. The fastest way off is -"Can't get there" under "Walk slowly to your left" and again under "Walk slowly to your right": -each sets that side's end where coverage stops, which on arrival is the meter itself. The -instruction then reads "That's the whole wall" and "Done with this wall" moves on with no walk -photos at all (see [Open questions](#open-questions-and-verification)). The only other way off is -the phone losing its place for over 20 s, which throws the walk away. - -### First capture - -The first photo is kept by the rules in -[coverage and guidance](../foundations/coverage-and-guidance.md#when-a-photo-is-kept). When it is -saved, the counter's camera icon flashes green and the number rolls up by one. There is no haptic -tap for walk photos in this build. Wall and ground cells the photo shows any sample line of go -from fog to a thinner haze: their fog fades over 0.7 s while drifting upward, like mist lifting. -On the strip those cells turn from gray to amber. A cell turns green on the strip, and its haze -clears completely, once each of its three sample lines has been seen from two places -([coverage and guidance](../foundations/coverage-and-guidance.md#while-capturing)); a wall cell -whose top line, 1.98 m up, never came into view stays amber. - -### While capturing - -**The strip.** A dark rounded panel above the bottom edge. The top row is the wall band, the lower, -thinner row the ground band. Cells are gray while unseen, amber once seen, green once covered, and -slate with a white slash when skipped. A blue line and a lightning-bolt badge mark the meter, a -white pointer under the strip marks where the homeowner stands, and small ticks along the bottom -mark every foot, longer every fifth. A marked end is a tall white cap; a marked feature shows its -icon above the rows. The strip always spans at least 3.5 m (about 11 ft), and its scale changes as -coverage, the ends or the homeowner move outward. - -**The fog.** A frosted haze painted on the wall face (up to the top of the wall band, 1.98 m) and on the -ground in front of it, over what has been seen plus the fog ahead of it on each side. It is a -thin frosted blur under a light white wash (35 % opacity), both scaled per cell: unseen cells at -75 % strength, so the wall's shapes still show through, seen cells at 35 %, covered cells clear. -These strengths were picked by eye on the demo wall and a synthetic replay; the code says to tune -them on a phone in sun. Skipped cells are not fogged: they get a slate tint with white -diagonal hatching, so they read neither as clear nor as unseen. Set ends are white vertical lines -on the wall with a white dot at the foot. - -**The instruction.** It changes only when the current one is satisfied or has been up long enough -([coverage and guidance](../foundations/coverage-and-guidance.md#one-instruction-at-a-time)). The -old line blurs out and the new one blurs in. - -**"Can't get there".** A small pill inside the instruction card, under the text. It shows only -under "Walk slowly to your …", "Tilt down to show the ground" and "Tilt up to show more wall", -and never while coaching is showing, the kind picker is open or a mark is in progress. - -- Under a tilt instruction it marks the cells not yet covered in a 1 m stretch centered on the - spot as skipped, in that band only. The fog there turns to hatching, the strip shows slate, and - the next instruction is chosen at once. -- Under a walk instruction it sets that side's end as an unexplored end, at the point where - coverage stops running unbroken from the meter. The walk turns to the other side, or to the - thin stretches between the ends. - -**"Wall ends here".** Shown, with "Mark something" beside it, only while the instruction asks "Is -this the left end of the wall?" or the right one. Tapping it takes the point in the middle of the -screen, finds where it meets the wall's line, and sets an end there, unexplored until the -homeowner says otherwise. Which end it sets depends on which side of the meter that point is, not -on which end was asked for. The white end line appears on the camera, the cap appears on the -strip, and the end question follows. Nothing on the screen marks the middle of the screen at this -moment, and if the middle of the screen does not meet the wall line (the phone aimed away from the -wall) the tap does nothing. - -**The end question.** The instruction reads "What's at the left end?" (or right, for the end just -set), with "This tells the installer whether the wall keeps going." The buttons give way to two -full-width answers, one above the other: "It turns a corner" and "Something blocks it". "It turns -a corner" keeps the end unexplored: the wall may go on. "Something blocks it" makes it a real end. -Until one is tapped the walk waits: the instruction does not change, coaching does not replace -it, and "Mark something", "Can't get there" and "Done with this wall" are hidden. The answer gives -no haptic; the next instruction is chosen at once. - -**"Mark something".** Always available while walking. It replaces the buttons with the kind -picker: a panel headed "What do you see?", a close button, and six tiles, each an icon and a -name: "Gas meter", "Door", "Window", "AC unit", "Driveway", "Fence". While it is open, -"Can't get there" hides. The close button returns the buttons. Tapping a tile starts marking that -kind and closes the picker. - -**Marking.** The ring and the path disappear, a white circle with a center dot appears in the -middle of the screen, and the bottom row becomes "Cancel" and "Mark". The instruction becomes the -marking prompt: - -| Kind | First prompt | Second prompt | -| --- | --- | --- | -| Door, window | "Tap the door's bottom-left corner" (or window's), with "Put the circle on it and tap Mark, or tap it on screen." | "Now tap its top-right corner" | -| Driveway | "Tap one end of the driveway's edge", with "Use the edge closest to the wall." | "Now tap the other end of that edge" | -| Fence | "Tap the bottom of the fence at one end", with "Where it meets the ground." | "Now tap the bottom at the other end" | -| Gas meter, AC unit | "Tap the gas meter" (the AC unit prompt reads "Tap the ac unit"), with "Put the circle on it and tap Mark, or tap it on screen." | None: one mark. | - -The homeowner can tap the spot on the camera image or put the circle on it and tap "Mark". Either -way a white ring blooms out from the tapped point (the middle of the screen for "Mark"). Doors, -windows, gas meters and AC units are placed on the wall face; driveways and fences on the ground. -A mark the app accepts moves the prompt on; the last one adds the feature, gives a firm haptic -tap, draws it on the camera (white dots on the marks, a dashed outline for a door or window, a -dashed line for a driveway or fence) and adds its icon to the strip. Marking then ends and the -walk buttons return. - -A refused mark keeps the step. The instruction shows the reason with a red warning icon, the -prompt drops to the second line, and the phone gives a warning haptic: - -| Refusal | When | -| --- | --- | -| "One moment, your phone is still finding its place." | Tracking is not normal. | -| "That spot is behind the wall. Tap something on this side." | The phone itself is on the far side of the wall line. | -| "Nothing to pin there. Aim at the wall or the ground and try again." | The tap's line of sight never meets the wall face (or the ground, for a driveway or fence). | -| "That's too far from the wall to matter. Tap something closer." | A driveway or fence mark lands more than 8 m out from the wall, or behind it. | - -"Cancel" drops the marks placed so far for this feature and returns to walking. While marking, -the marking prompt takes the instruction slot even over coaching, and new photos keep being -taken if the homeowner moves. - -**Finding its place again.** While tracking is not normal, the fog, path, ring, meter dot, ends -and marks fade off the camera image over 0.2 s and fade back once it is normal -([coverage and guidance](../foundations/coverage-and-guidance.md#one-instruction-at-a-time)). -While the phone is relocalizing, the instruction reads "Point at the meter like this.", with "Your -phone lost its place for a moment.", and the meter close-up taken at the start appears in the -middle of the screen: a square photo with rounded corners, a white border and a shadow, fading in -and out over 0.2 s. It cannot be tapped. If the close-up was skipped ("Can't get a clear shot") or its -thumbnail could not be read back, there is no photo and only the instruction shows. - -### Advancing - -"Done with this wall" appears once both ends are set, whatever the coverage between them. Tapping -it ends any marking and moves to [marking features](mark-features.md). What moves with it: the -coverage (covered, skipped and unseen cells), each end with its kind (real only when the -homeowner answered "Something blocks it", otherwise unexplored), the -features marked so far and the walk photos. Coverage between the ends that was never covered -stays unseen; the server treats it as unknown -([coverage and guidance](../foundations/coverage-and-guidance.md#advancing)). There is no way back -to the walk from the next screen. - -## Modifiers - -| Modifier | At arrival | While capturing | -| --- | --- | --- | -| Live camera or replay | A replay starts playing its recording from the first frame (leaving out any frames the autopilot holds back for a gap request); the meter's wall comes from the recording. | When the recording ends the image freezes on its last frame and the instruction stops changing. The 20 s reset never happens on a replay. | -| Autopilot | The walk plays at three times speed and the badge reads "Replay · Autopilot". | After the recording ends, the autopilot marks a gas meter and a window by calling the marking actions at projected points, sets both ends, answering "It turns a corner" to each end question after `-autopilotHold` seconds (when no frame shows the spot it sets a real end directly, with no question), and finishes the walk. It never opens the kind picker or presses "Can't get there". | -| Server or sample result | No effect. | No effect. | -| Larger text sizes | The instruction card and buttons grow; the chrome scrolls when it no longer fits. At accessibility sizes "Mark something" fills the row. The two end answers are stacked and full width at every size. | The kind picker shows one tile per row at accessibility sizes instead of three. The strip does not grow. | -| Reduce Motion | The instruction crossfades instead of blurring; the button rows change with a 0.15 s fade. | The fog fades without drifting, the target ring does not pulse, the counter icon flashes without growing, the tap ring fades without expanding. The kind picker still slides up from the bottom. | - -## Cancel and interrupt - -| Event | Before both ends are set | With both ends set | -| --- | --- | --- | -| The screen's own way out | "Can't get there" under a walk instruction sets an unexplored end; under a tilt instruction it skips that patch. "Cancel" leaves marking. The end question has no way out but an answer. | "Done with this wall". "Can't get there" still skips a patch under a tilt instruction; "Cancel" leaves marking. | -| Start over | Not offered. | Not offered. | -| Tracking limited | Coaching ("Slow down", "Aim at a corner or somewhere with more texture", "Move your phone slowly") replaces the instruction and hides "Can't get there". The fog, path, ring and marks fade off the camera image. No photos are kept. "Wall ends here" still works. Marks are refused with "One moment, your phone is still finding its place." | Same; "Done with this wall" still works. | -| Tracking lost or relocalizing | "Point at the meter like this.", with "Your phone lost its place for a moment." and the saved meter close-up in the middle of the screen (or "Your phone lost its place" when tracking is lost, with no photo). The fog, path, ring and marks fade off the camera image. After 20 s of relocalizing, the app forgets the wall, coverage, ends, an unanswered end question, marks and walk photos, sets the counter to 0 and goes back to "Find your electric meter", with no message saying why. | Same; the ends are lost too. | -| App backgrounded or a call | The camera stops; on return the coaching reads "Point at the meter like this." over the saved close-up until the phone finds its place. Photos, coverage, ends and marks are kept. Relocalizing time counts toward the 20 s. | Same. | -| Camera off or session failed | The screen stays with no new frames; coverage cannot grow. Suspected dead end ([the flow](../foundations/flow.md#open-questions-and-verification)). | Same; "Done with this wall" still moves on. | -| Network lost or upload failing | No effect. | No effect. | -| App killed | The scan is lost. | The scan is lost. | - -## Interactions with other systems - -**Coverage and evidence.** This screen is where coverage grows, by the rules in -[coverage and guidance](../foundations/coverage-and-guidance.md). Skipped cells and unexplored ends -are recorded for installer review, never as evidence. - -**Stored photos.** Each walk photo is written to the scan's folder as it is kept; the counter and -coverage move only after the write succeeds. See [the flow](../foundations/flow.md#interactions-with-other-systems). - -**Upload and offline.** Nothing is sent from this screen. - -**Accessibility.** VoiceOver reads the strip as "Map of the wall" with a value such as "Left of -your meter: 40 percent seen. Right of your meter: 0 percent seen. You are 6 feet 2 inches left of -your meter", ending "Both ends marked" once both are set. The counter reads "12 photos taken". The -instruction card is one element that reads its text and marks itself as updating often; the fog, -path, ring and marks are hidden from VoiceOver. Hints: "Pin a gas meter, door, window, AC unit, -driveway or fence" on "Mark something"; "Marks the end of the wall at the circle in the middle of -the screen" on "Wall ends here"; "Marks the point under the circle in the middle of the screen" on -"Mark"; "Skips this part of the wall. An installer will look at it instead." on "Can't get there"; -"A fence, gate, or your neighbor's yard" on "Something blocks it". -The picker's heading is a header, its close button reads "Close", and each tile reads "Mark" and -the kind in lower case. The saved close-up, while relocalizing, reads "Your photo of the meter -from the start of the scan". - -**Haptics and motion.** A firm tap for each feature added and a warning for each refused mark. -No haptic for walk photos, for setting an end, for answering the end question or for "Can't get -there". Fog lifts over 0.7 s; -button rows change with a spring of 0.4 s; the target ring pulses. - -**Verification hooks.** `STATE=wallWalk` on arrival. Each instruction change logs -`GUIDANCE=`: `walk.left`, `walk.right`, `markEnd.left`, `markEnd.right`, `aimAtGround`, -`aimAtWall`, `stepBack`, `walkComplete`. The engine log (category `engine`) records each end -("end left at s=… (unexplored)", or "(limit)" when the autopilot sets one directly), each answer -("end left is limit" or "end left is unexplored"), "cannot access area during ", and "spatial -reset: relocalization timed out". Accessibility identifiers: `action.markSomething`, -`action.markEnd`, `action.endCorner`, `action.endBlocked`, `action.finishWalk`, `action.cannotAccess`, `action.markPoint`, -`action.cancelMarking`, `action.closeTray`, `feature.`, `wallTape`, `photoCount`, -`instruction`, `relocalize.meterPhoto`. - -## Edge cases - -- Asked for the left end while aiming right of the meter, "Wall ends here" sets the right end and - asks "What's at the right end?". After the answer, "Is this the left end of the wall?" comes back. -- A wall end or a door, window, gas meter or AC unit mark lands wherever the aim meets the wall's - line extended, with no limit: past a corner, above the roof line or below ground. A door's bottom - is raised to ground level. -- "Nothing to pin there" appears for a wall mark only when the phone points away from the wall's - line; aiming at the sky over the wall is accepted. -- The same refusal twice in a row gives the warning haptic only the first time. -- A tilt instruction can come back after "Can't get there": the skip covers 1 m, while the lagging - stretch that raised the instruction can be longer. -- The kind picker stays open if the instruction changes underneath it, including to "Is this the - left end of the wall?"; the end buttons return only when it is closed. -- "Done with this wall" shows while a tilt instruction is still up; the homeowner can leave thin - stretches for the server to call unknown. -- VoiceOver's "percent seen" counts only the wall band, and counts amber (seen once) cells as seen. - A side can read 100 percent while none of it is covered and its ground is unseen. -- Standing at the meter, VoiceOver says "You are 0 inches right of your meter". -- "About N ft to go" can never appear on the walk instruction: the engine always passes no - remaining distance (`Runtime/ScanEngine.swift:784`). - -## Open questions and verification - -- **Suspected bug: a blocked end closer than the reach cannot be recorded.** The end question - makes the kind of a marked end truthful, but "Wall ends here" is still shown only while the - instruction is "Is this the … end of the wall?" (`UI/Screens/WallWalkScreen.swift:101`, `:165`), - and the planner asks that only once coverage runs unbroken to 6.1 m, about 20 ft - (`HouseScanKit/Guidance/GuidancePlanner.swift:27`, `:119`). A homeowner whose wall stops at a - fence 10 ft from the meter is told "Walk slowly to your left" with only "Can't get there" to - answer, which records an unexplored end instead of a real one. A corner at 10 ft comes out right - by accident, since a corner is unexplored anyway. -- **Suspected bug: no "Can't get there" at "Is this the … end of the wall?".** The pill is limited - to walk and tilt instructions (`WallWalkScreen.swift:107-112`), although the engine handles it - for this question (`Runtime/ScanEngine+Actions.swift:209`). A wall that goes on straight past - 20 ft can be recorded as unexplored only by marking an end where there is none and answering "It - turns a corner". This sharpens B-06 in [bug-triage.md](../bug-triage.md). -- **Suspected bug: "Wall ends here" aims at a circle that is not drawn.** The end goes where the - middle of the screen meets the wall (`ScanEngine+Actions.swift:73`), but the center circle is - drawn only while marking (`WallWalkScreen.swift:28-33`). The ring on screen is the planner's - guess at the end (`GuidancePlanner.swift:202-203`), so a homeowner who lines the ring up with the - corner sets the end elsewhere. The hint promises "the circle in the middle of the screen". -- **Suspected bug: "Wall ends here" is unchecked and silent on a miss.** It accepts a tap with - limited tracking, unlike marks (`ScanEngine+Actions.swift:70-80` against `:100`); it does nothing - and says nothing when the aim misses the wall (`:73`). A tap that lands gives no haptic; the end - question is its only feedback. -- **Suspected bug: the 20 s reset keeps a mark in progress.** `resetSpatialState` - (`Runtime/ScanEngine.swift:496-522`) clears the wall, marks and photos but not the marking prompt - or its placed taps. After the meter is found again, the walk opens straight into the old prompt - ("Now tap its top-right corner"), and the finished feature joins a point from the lost world - frame to one from the new frame. -- **Suspected bug: two taps on "Can't get there" finish an empty walk.** On arrival each tap sets an - end at the meter (`ScanEngine+Actions.swift:211-213`); with both ends at the same point the - planner finds nothing to fill (`GuidancePlanner.swift:144`) and says "That's the whole wall". -- **Suspected bug: the AC unit prompt reads "Tap the ac unit".** The prompt lower-cases the kind's - name (`UI/Copy/ScanCopy.swift:122`, `:137`); the picker tile's VoiceOver label has the same - problem ("Mark ac unit", `WallWalkScreen.swift:282`). -- **Possible bug: "That spot is behind the wall" judges the phone, not the tap.** The check is on - where the phone stands (`ScanEngine+Actions.swift:108`), so the message blames the spot when it is - the homeowner who is behind the wall line. -- The kind picker slides in under Reduce Motion (`WallWalkScreen.swift:164`). -- The reset returns to "Find your electric meter" without telling the homeowner that the walk was - lost. The old meter close-up stays in the scan (`Runtime/KeyframeStore.swift:84-87` clears only - walk photos); if the second close-up is skipped, the first one is uploaded. -- The foundations say each kept photo gives a light haptic tap. In this build walk photos give none - (`UI/ScanRootView.swift:91`, `:101-104`); [the flow](../foundations/flow.md) and - [coverage and guidance](../foundations/coverage-and-guidance.md) need updating. -- The walk's reply reads "Can't get there"; the gap request's reads "I can't get there" - (`UI/Screens/GapRequestScreen.swift:26`). The glossary uses the second for both. -- Everything on this screen that needs the live camera is read from code: the fog's look on a real - wall, the ring and chevron placement, the overlays hiding and the saved close-up while the phone - finds its place, the end question after a real "Wall ends here", the 20 s reset, and - interruption recovery. The first - Simulator pass showed this screen's chrome without text (B-05 in [bug-triage.md](../bug-triage.md)). - -Verified against house-scanning commit `a39d0a5` (t3/ios-mvf). diff --git a/verification/product/verification.md b/verification/product/verification.md deleted file mode 100644 index be83efaec..000000000 --- a/verification/product/verification.md +++ /dev/null @@ -1,47 +0,0 @@ -# Checks against the running app - -Each row is one claim from a document, checked in the Simulator. A failed row is not automatically -an app bug: sometimes the document is wrong, and the note says which. Screenshots stay in -`~/house-scanning-data/reports/sim/` because replay frames come from a non-commercial dataset. - -## How a pass is run - -`make sim-app` from `verification/` builds `t3/ios-mvf`, runs it on the ADVIO replay -`advio-20-0040-0075` with the autopilot, and uploads to the server started from `t3/server`. Add -`ARGS="--extra-arg=-sampleResult"` for the built-in result, or `--content-size -accessibility-extra-extra-extra-large --appearance dark` for large text. `python -m -hsverify.a11yaudit` runs Apple's audit on every screen. What these cover: the order of screens, -what each shows and speaks, what the app logged, and what the server answered. What they do not -cover: anything needing the live camera (tracking, the meter tap on a real wall, a close-up that -succeeds), haptics, or timing a person feels. - -## Current pass: `t3/ios-mvf` `a39d0a5`, server `739fb6f` - -Report `sim/20260926-071459-t3-ios-mvf-a39d0a50-replay` (real server; the pass at `525ea40` is -`sim/20260926-052535-t3-ios-mvf-525ea401-replay`), the audit at `a39d0a5` (`a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample`, sample result), plus -the variant runs on the preview -`5520229` (the same UI before S3's last fixes) named in [the UX review](../ux/review.md#evidence). - -| ID | Claim | Result | -| --- | --- | --- | -| FLOW-01 | Screens appear in the order of [the flow](foundations/flow.md#the-interaction-event-by-event) | Pass: all ten, onboarding to result, AR and back. | -| FLOW-02 | The result appears only after the server answered | Pass: `POST /v1/placements 200`, then `result`. | -| FLOW-03 | A failure other than an unsupported phone reaches the failure screen | Fail, as the document predicts: `-replay /nonexistent` logs "replay unreadable" and stays on onboarding ([B-03](bug-triage.md#blockers)). | -| FLOW-04 | An unsupported phone sees "This phone can't measure walls" | Pass (no replay, Simulator). | -| CU-01 | Without a usable photo, the close-up is skipped after the second failed try | Pass on the replay: logged "close-up skipped after 2 failed attempts". | -| CU-02 | One instruction and one named fix under it | Pass: "Hold your meter in the circle", "Center the meter in the circle". | -| WW-01 | One instruction, the reply inside the card, the strip below | Pass at default size; fails at the largest text size (UX-01). | -| WW-02 | Guidance moves from walking to tilting to stepping back | Pass: the audit saw "Walk slowly to your left", "Tilt down to show the ground", "Take a step back". | -| MF-01 | The list shows each mark with its distance, the window question and "Add something" | Pass at default size; hidden behind "Looks complete" at the largest text size (UX-01). | -| GAP-01 | A gap request states what to show and why, with a progress bar and "I can't get there" | Pass ("Show the ground around your meter", "0%"). | -| GAP-02 | New photos close the request and the flow moves on | Pass at `24af434` and `525ea40` ("gap 1 satisfied"; "Got it, thanks"). Not exercised since `beede15`: at `a39d0a5` the request appears, but the autopilot holds back no frames that could fill it and answers "I can't get there" ("gap 1 skipped"). | -| WW-03 | After "Wall ends here" the walk asks what is at that end, and a corner stays unexplored | Pass: logged "end left at s=-3.81 (unexplored)", then "answered the left end: turns a corner"; same on the right. | -| UP-01 | Upload progress is shown in steps apart from coverage | Pass: "Photos ready", "Sending photos, 99%", "Check clearances". | -| RES-01 | The result shows the headline, placement line, the server's summary, the rules-not-final note and the panel notice | Pass: "An installer will take a look" with the server's placement and summary (seen at `0876e03`, `24af434`, `525ea40`, `beede15` and `a39d0a5`). | -| RES-02 | A borderline check shows the measurement, the rule and the error | Pass on the sample result: "Measured 3 ft 1 in. The rule is 3 ft, and the measurement can be off by about 4 in." | -| RES-03 | A maximum reads as a maximum | Fail: "Measured 3 ft. The rule is 20 ft" (B-14). | -| RES-04 | The result is visible when the app logs `STATE=result` | Fail, minor: the upload screen was still showing 1.2 s later on first load, in both passes (UX-07). | -| AR-01 | The AR view draws the spot and cable with one headline and "Done" | Pass: "The spot an installer will check" for a manual review (`24af434`). | - -Superseded: at `21a63e7` the upload went to `/v1/scenes` and failed with 404, and the camera -screens rendered without text. Both are fixed on `t3/ios-mvf`. diff --git a/verification/scenes/eth3d-facade/build_scene.py b/verification/scenes/eth3d-facade/build_scene.py index 5b9fffb4e..f6378894c 100644 --- a/verification/scenes/eth3d-facade/build_scene.py +++ b/verification/scenes/eth3d-facade/build_scene.py @@ -604,6 +604,30 @@ def round3(a): return [round(float(x), 3) for x in a] +# Everything this script writes into --out. It replaces only these, so a mistyped --out (say, +# ~/house-scanning-data) fails instead of losing whatever is there. +OUTPUTS = {"bundle", "scene.json", "scene.zip", "build_report.json", "overlays"} + + +def clear_outputs(out: Path) -> None: + """Remove this script's earlier outputs from `out`; refuse a folder holding anything else.""" + if not out.exists(): + return + if not out.is_dir(): + raise SystemExit(f"--out {out} is a file, not a folder") + foreign = sorted(p.name for p in out.iterdir() if p.name not in OUTPUTS) + if foreign: + raise SystemExit( + f"--out {out} holds files this script did not write ({', '.join(foreign[:5])}); " + "choose an empty or new folder" + ) + for p in out.iterdir(): + if p.is_dir() and not p.is_symlink(): + shutil.rmtree(p) + else: + p.unlink() + + def main(): ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter @@ -821,8 +845,7 @@ def main(): # 8. Write ---------------------------------------------------------------------------------- out = args.out - if out.exists(): - shutil.rmtree(out) + clear_outputs(out) (out / "bundle").mkdir(parents=True) img_dir = args.data / "images/dslr_images_undistorted" for k in kfs: diff --git a/verification/scoreboard/metrics.yaml b/verification/scoreboard/metrics.yaml index 42288a71e..bb65ffac4 100644 --- a/verification/scoreboard/metrics.yaml +++ b/verification/scoreboard/metrics.yaml @@ -132,6 +132,7 @@ metrics: text: A real-derived scene returns a result in under 1 s report: glob: ~/house-scanning-data/reports/e2e/*/report.json + # Null unless a real scene got an HTTP 200 result that was checked. key: latency_ms less_than: 1000 require_current_sha: true @@ -155,8 +156,9 @@ metrics: complete, ordering properties, hostile inputs refused in time report: glob: ~/house-scanning-data/reports/e2e/*/report.json - key: contract_problem_count - equals: 0 + # True only when at least one scene was answered and checked and none broke the contract. + key: contract_ok + equals: true require_current_sha: true # --- S3 -------------------------------------------------------------------------------------- @@ -288,9 +290,9 @@ metrics: manual: status: met evidence: >- - verification/product covers every screen of t3/ios-mvf at a39d0a5, checked in the - Simulator there with the real server's result (verification.md); bug-triage.md has 16 - entries, 3 resolved; ux/review.md judges every state. + docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in + the Simulator there with the real server's result; its bug triage has 16 entries, 3 + resolved; docs/app-review/ux/review.md judges every state. - id: S4-4 workstream: S4 diff --git a/verification/tests/test_build_scene.py b/verification/tests/test_build_scene.py new file mode 100644 index 000000000..a5911a7e9 --- /dev/null +++ b/verification/tests/test_build_scene.py @@ -0,0 +1,52 @@ +"""The ETH3D facade scene builder replaces only its own outputs.""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[1] / "scenes" / "eth3d-facade" / "build_scene.py" +spec = importlib.util.spec_from_file_location("build_scene", SCRIPT) +build_scene = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = build_scene # dataclasses look their module up there +spec.loader.exec_module(build_scene) + + +def test_earlier_outputs_are_replaced(tmp_path): + (tmp_path / "bundle").mkdir() + (tmp_path / "bundle" / "k00001.jpg").write_bytes(b"old") + (tmp_path / "scene.json").write_text("{}") + (tmp_path / "overlays").mkdir() + build_scene.clear_outputs(tmp_path) + assert list(tmp_path.iterdir()) == [] + + +def test_a_folder_with_other_files_is_refused_and_left_alone(tmp_path): + (tmp_path / "scene.json").write_text("{}") + (tmp_path / "datasets").mkdir() + (tmp_path / "notes.txt").write_text("keep me") + with pytest.raises(SystemExit, match=r"did not write \(datasets, notes\.txt\)"): + build_scene.clear_outputs(tmp_path) + assert sorted(p.name for p in tmp_path.iterdir()) == ["datasets", "notes.txt", "scene.json"] + + +def test_a_missing_folder_is_fine_and_a_file_is_refused(tmp_path): + build_scene.clear_outputs(tmp_path / "new") + target = tmp_path / "a-file" + target.write_text("x") + with pytest.raises(SystemExit, match="is a file"): + build_scene.clear_outputs(target) + assert target.read_text() == "x" + + +def test_a_symlinked_output_is_unlinked_not_followed(tmp_path): + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + (elsewhere / "precious").write_text("keep") + out = tmp_path / "out" + out.mkdir() + (out / "bundle").symlink_to(elsewhere, target_is_directory=True) + build_scene.clear_outputs(out) + assert (elsewhere / "precious").read_text() == "keep" + assert not (out / "bundle").exists() diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index eb07c3058..f607bc895 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -23,13 +23,22 @@ load_input, zip_bomb, ) -from hsverify.resultcheck import RuleSet +from hsverify.resultcheck import Need, RuleSet RULES = RuleSet( width_ft=31 / 12, depth_ft=22 / 12, - radii={"ground_surface": (("ground",), 0.0), "gas_clearance": (("ground",), 1.0)}, - errors={"tap": 0.3, "vlm": 1.5, "tape": 0.05, "wall": 0.3, "meter": 0.3, "drift_per_ft": 0.16}, + needs={"ground_surface": (Need("ground", 0.0),), "gas_clearance": (Need("ground", 1.0),)}, + errors={ + "tap": 0.3, + "vlm": 1.5, + "tape": 0.05, + "wall": 0.3, + "mesh": 0.5, + "plane": 0.75, + "meter": 0.3, + "drift_per_ft": 0.16, + }, ) SCENE = { @@ -347,3 +356,88 @@ def test_a_prebuilt_zip_goes_as_application_zip_when_the_endpoint_takes_it(refus zip_too = Endpoint("/p", "application/json", accepts_zip=True) assert judge_hostile(refusing_url, zip_too, bomb, SCHEMAS)["status"] == "pass" assert RefusingServer.content_types == ["application/zip"] + + +# --- What a run may claim -------------------------------------------------------------------- + +META = { + "started_at": "2026-09-26T12:00:00-05:00", + "server_ref": "origin/t3/server", + "server_sha": "a" * 40, + "endpoint": "POST /v1/placements (application/json)", + "load_average_1_5_15": [1.0, 1.0, 1.0], + "peak_memory": {"harness_mb": 50.0}, +} +CHECKED = { + "name": "a", + "status": "pass", + "decision": "manual_review", + "http_status": 200, + "latency_ms": 120.0, + "real": True, + "contract_problems": [], +} + + +@pytest.mark.parametrize( + "records", + [ + [], + [{"name": "s", "status": "skipped", "problems": [], "skip_reason": "no data"}], + [{"name": "b", "status": "bad input", "problems": ["scene schema: x"]}], + [{"name": "h", "status": "pass", "hostile": True, "http_status": 422, "problems": []}], + ], + ids=["no scenes", "skipped only", "bad input only", "hostile only"], +) +def test_a_run_that_checked_no_answer_claims_nothing(tmp_path, records): + report = e2e.write_report(tmp_path, META, records) + assert (report["scenes_checked"], report["contract_ok"], report["all_passed"]) == ( + 0, + False, + False, + ) + + +def test_a_checked_scene_credits_the_contract(tmp_path): + report = e2e.write_report(tmp_path, META, [CHECKED]) + assert (report["scenes_checked"], report["contract_ok"], report["latency_ms"]) == ( + 1, + True, + 120.0, + ) + broken = CHECKED | {"status": "fail", "contract_problems": ["x"], "problems": ["x"]} + assert e2e.write_report(tmp_path, META, [broken])["contract_ok"] is False + + +def test_a_fast_refusal_of_a_real_scene_is_not_latency(tmp_path): + refused = { + "name": "r", + "status": "fail", + "real": True, + "http_status": 422, + "latency_ms": 8.0, + "problems": ["HTTP 422"], + "contract_problems": ["HTTP 422"], + } + report = e2e.write_report(tmp_path, META, [refused]) + assert report["latency_ms"] is None and report["scenes_answered"] == 0 + + +def test_an_oversized_bundle_is_refused_before_it_is_read(tmp_path): + archive = tmp_path / "big.zip" + with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as z, z.open("scene.json", "w") as f: + chunk = b" " * 2**20 + for _ in range(e2e.MAX_SCENE_BYTES // 2**20 + 1): + f.write(chunk) + with pytest.raises(SystemExit, match=r"declares \d+ bytes, limit"): + load_input(archive) + + +def test_a_bundle_with_too_many_entries_is_refused(tmp_path, monkeypatch): + monkeypatch.setattr(e2e, "MAX_BUNDLE_ENTRIES", 3) + archive = tmp_path / "many.zip" + with zipfile.ZipFile(archive, "w") as z: + for i in range(4): + z.writestr(f"k{i}.jpg", b"x") + with pytest.raises(SystemExit, match="4 entries, limit 3"): + load_input(archive) diff --git a/verification/tests/test_generate_cases.py b/verification/tests/test_generate_cases.py new file mode 100644 index 000000000..01c7cdfac --- /dev/null +++ b/verification/tests/test_generate_cases.py @@ -0,0 +1,41 @@ +"""Regenerating the e2e cases replaces only the files the generator owns.""" + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[1] / "e2e" / "cases" / "generate.py" +spec = importlib.util.spec_from_file_location("generate_cases", SCRIPT) +generate = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = generate # dataclasses look their module up there +spec.loader.exec_module(generate) + + +def test_hand_written_cases_survive_and_stale_generated_ones_go(tmp_path): + hand = tmp_path / "manually-added-regression.json" + hand.write_text(json.dumps({"id": "manually-added-regression", "scene": {}})) + stale = tmp_path / "renamed-away.json" + stale.write_text(json.dumps({"generated_by": generate.GENERATED_BY, "id": "renamed-away"})) + generate.main(tmp_path) + assert hand.exists() + assert not stale.exists() + written = {p.stem for p in tmp_path.glob("*.json")} - {hand.stem} + assert written == {c["id"] for c in generate.CASES} + + +def test_a_hand_written_file_with_a_generated_id_is_refused(tmp_path): + taken = generate.CASES[0]["id"] + (tmp_path / f"{taken}.json").write_text(json.dumps({"id": taken})) + with pytest.raises(SystemExit, match="written by hand"): + generate.main(tmp_path) + assert json.loads((tmp_path / f"{taken}.json").read_text()) == {"id": taken} + + +def test_the_committed_cases_are_what_the_generator_writes(tmp_path): + generate.main(tmp_path) + committed = SCRIPT.parent + for f in tmp_path.glob("*.json"): + assert f.read_text() == (committed / f.name).read_text(), f.name diff --git a/verification/tests/test_memory.py b/verification/tests/test_memory.py index c23f5d2fe..c06a61dcd 100644 --- a/verification/tests/test_memory.py +++ b/verification/tests/test_memory.py @@ -1,11 +1,24 @@ +import os +import signal import subprocess import sys import time -from hsverify.memory import TreeLimit, peak_rss_mb, tree_rss_mb +from hsverify.memory import TreeLimit, peak_rss_mb, sum_tree_mb, tree_rss_mb -# A child that writes `mb` megabytes, so they are resident, then waits. -HOLD_MB = "import time; b = b'x' * ({mb} * 2**20); time.sleep(10)" + +def start(code: str) -> subprocess.Popen: + return subprocess.Popen( + [sys.executable, "-c", code], stdout=subprocess.PIPE, text=True, start_new_session=True + ) + + +def stop(child: subprocess.Popen) -> None: + """Kill the child's whole group and reap it, whatever state the test left it in.""" + if child.poll() is None: + os.killpg(child.pid, signal.SIGKILL) + child.wait(timeout=10) + child.stdout.close() def test_peak_memory_is_reported_in_megabytes(): @@ -15,32 +28,37 @@ def test_peak_memory_is_reported_in_megabytes(): assert peak["largest_child_mb"] >= 0 -def test_tree_rss_counts_descendants(): - child = subprocess.Popen( - [ - sys.executable, - "-c", - HOLD_MB.format(mb=60), - ], - start_new_session=True, - ) +def test_the_tree_sums_a_process_and_all_its_descendants_only(): + # pid ppid rss_kb: 10 -> 11 -> 12, and 20 is unrelated. + listing = " 10 1 1024\n 11 10 2048\n 12 11 1024\n 20 1 9999\n" + assert sum_tree_mb(listing, 10) == 4.0 + assert sum_tree_mb(listing, 11) == 3.0 + assert sum_tree_mb(listing, 99) == 0.0 + + +def test_this_process_is_measured(): + # What the OS keeps resident moves with memory pressure on a shared Mac, so only the reading + # itself is checked here; the tree arithmetic is checked above on a fixed listing. + assert tree_rss_mb(os.getpid()) > 1 + + +def test_a_process_group_over_the_limit_is_killed(): + child = start("import time; time.sleep(60)") try: - time.sleep(1.0) - assert tree_rss_mb(child.pid) > 50 + with TreeLimit(child, limit_mb=40, interval_s=0.05, measure=lambda pid: 100.0) as limit: + child.wait(timeout=5) + assert limit.killed and limit.peak_mb == 100.0 + assert child.returncode == -signal.SIGKILL finally: - child.kill() - child.wait() + stop(child) -def test_a_process_group_over_the_limit_is_killed(): - child = subprocess.Popen( - [ - sys.executable, - "-c", - HOLD_MB.format(mb=80), - ], - start_new_session=True, - ) - with TreeLimit(child, limit_mb=40, interval_s=0.1) as limit: - child.wait(timeout=5) - assert limit.killed and limit.peak_mb > 40 +def test_a_process_group_under_the_limit_is_left_running(): + child = start("import time; time.sleep(60)") + try: + with TreeLimit(child, limit_mb=40, interval_s=0.05, measure=lambda pid: 10.0) as limit: + time.sleep(0.3) + assert not limit.killed and limit.peak_mb == 10.0 + assert child.poll() is None + finally: + stop(child) diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 9f237074b..295f32828 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -5,8 +5,10 @@ import pytest from hsverify.resultcheck import ( + Need, RuleSet, assumption_mismatches, + battery_error, comparable, expectation_problems, invariant_problems, @@ -29,12 +31,21 @@ RULES = RuleSet( width_ft=31 / 12, depth_ft=22 / 12, - radii={ - "ground_surface": (("ground",), 0.0), - "gas_clearance": (("ground",), 1.0), - "opening_clearance": (("wall",), 1.0), + needs={ + "ground_surface": (Need("ground", 0.0),), + "gas_clearance": (Need("ground", 1.0),), + "opening_clearance": (Need("wall", 1.0, height=6.5),), + }, + errors={ + "tap": 0.3, + "vlm": 1.5, + "tape": 0.05, + "wall": 0.3, + "mesh": 0.5, + "plane": 0.75, + "meter": 0.3, + "drift_per_ft": 0.16, }, - errors={"tap": 0.3, "vlm": 1.5, "tape": 0.05, "wall": 0.3, "meter": 0.3, "drift_per_ft": 0.16}, ) SCENE = { @@ -335,7 +346,7 @@ def test_a_clearance_is_named_under_each_band_it_lacks(): two_band = RuleSet( RULES.width_ft, RULES.depth_ft, - RULES.radii | {"gas_clearance": (("ground", "wall"), 1.0)}, + RULES.needs | {"gas_clearance": (Need("ground", 1.0), Need("wall", 1.0, height=6.5))}, RULES.errors, ) scene = copy.deepcopy(SCENE) @@ -345,8 +356,8 @@ def test_a_clearance_is_named_under_each_band_it_lacks(): ground_only = {"kind": "band", "band": "ground", "span_ft": [0, 5], "checks": ["gas_clearance"]} r["missing_evidence"] = [ground_only | {"message": ""}] assert missing_evidence_problems(scene, r, two_band) == [ - "check gas_clearance is unsure and needs wall [0.00, 4.58] observed, but no wall request " - "names it" + "check gas_clearance is unsure and needs wall [0.00, 4.58] observed higher than 6.5 ft, " + "but no wall request names it" ] r["missing_evidence"].append(ground_only | {"band": "wall", "message": ""}) assert missing_evidence_problems(scene, r, two_band) == [] @@ -354,17 +365,30 @@ def test_a_clearance_is_named_under_each_band_it_lacks(): def test_rules_come_from_rules_yaml_and_a_missing_one_is_loud(): text = """ -battery: {width_ft: {value: 2.5}, depth_ft: {value: 1.5}} +battery: {width_ft: {value: 2.5}, depth_ft: {value: 1.5}, height_ft: {value: 3.25}} errors: {tap_ft: {value: 0.3}, vlm_ft: {value: 1.5}, tape_ft: {value: 0.05}, - wall_ft: {value: 0.3}, meter_ft: {value: 0.3}, drift_per_ft: {value: 0.16}} -clearances: {gas_ft: {value: 3}, ac_ft: {value: 3}, drive_ft: {value: 5}, - pool_ft: {value: 10}, opening_ft: {value: 3}} + wall_ft: {value: 0.3}, mesh_ft: {value: 0.5}, plane_ft: {value: 0.75}, + meter_ft: {value: 0.3}, drift_per_ft: {value: 0.16}} +clearances: {gas_ft: {value: 3}, ac_ft: {value: 3}, battery_ft: {value: 3}, drive_ft: {value: 5}, + pool_ft: {value: 10}, opening_ft: {value: 3}, wall_equipment_ft: {value: 0}} +openings: {exempt_bottom_above_ft: null} +facing: {min_ft: {value: 3}} +headroom: {min_ft: {value: 6.5}} +route: {height_ft: {value: 1}} """ rules = RuleSet.from_yaml(text) - assert rules.radii["pool_clearance"] == (("ground",), 10.0) - assert rules.radii["gas_clearance"] == (("ground", "wall"), 3.0) - assert rules.radii["opening_clearance"] == (("wall",), 3.0) - assert (rules.width_ft, rules.errors["tape"]) == (2.5, 0.05) + assert rules.needs["pool_clearance"] == (Need("ground", 10.0),) + assert rules.needs["gas_clearance"] == (Need("ground", 3.0), Need("wall", 3.0, 6.5)) + assert rules.needs["battery_clearance"] == (Need("ground", 3.0), Need("wall", 3.0, 6.5)) + assert rules.needs["opening_clearance"] == (Need("wall", 3.0, 6.5),) + assert rules.needs["wall_backing"] == (Need("wall", 0.0, 3.25, widen=False),) + assert rules.needs["facing_gap"] == (Need("facing", 0.0, 4.5, widen=False),) + assert rules.needs["headroom"] == (Need("overhead", 0.0, 6.5, widen=False),) + assert (rules.width_ft, rules.errors["plane"], rules.route_height_ft) == (2.5, 0.75, 1.0) + lower = RuleSet.from_yaml( + text.replace("exempt_bottom_above_ft: null", "exempt_bottom_above_ft: 2") + ) + assert lower.needs["opening_clearance"] == (Need("wall", 3.0, 2.0),) with pytest.raises(ValueError, match=r"rules\.yaml has no clearances\.pool_ft"): RuleSet.from_yaml(text.replace("pool_ft: {value: 10}, ", "")) @@ -599,3 +623,111 @@ def test_a_ground_request_is_redundant_only_as_far_out_as_it_asks(): assert any("lists as observed" in m for m in invariant_problems(SCENE, r, rules=RULES)) r["missing_evidence"] = [ask | {"out_ft": 3.5}] assert not any("lists as observed" in m for m in invariant_problems(SCENE, r, rules=RULES)) + + +# --- Every band's reach, and the wall's source --------------------------------------------------- + +PASS_RUN = {"wall_id": "w1", "start_ft": [1.0, 1.0], "outcome": "pass", "failing": [], "unsure": []} + + +def passing(check_id: str) -> dict: + return {**check(), "id": check_id, "measured_ft": None, "plus_minus_ft": None} + + +def with_wall_seen(out_ft: float | None) -> dict: + scene = copy.deepcopy(SCENE) + wall = scene["coverage"]["observed"][0] + wall.pop("out_ft", None) + if out_ft is not None: + wall["out_ft"] = out_ft + return scene + + +def test_a_wall_seen_too_low_does_not_settle_an_opening(): + r = result(checks=[check(), passing("opening_clearance")]) + msgs = invariant_problems(with_wall_seen(1.0), r, rules=RULES) + needs = "check opening_clearance passes but needs wall [0.00, 4.58] observed higher than 6.5 ft" + assert any(needs in m for m in msgs) + assert invariant_problems(with_wall_seen(7.0), r, rules=RULES) == [] + assert invariant_problems(with_wall_seen(None), r, rules=RULES) == [] # seen to headroom + + +def test_a_request_to_see_higher_than_the_view_reached_is_not_redundant(): + r = result() + r["missing_evidence"] = [ + {"kind": "band", "band": "wall", "span_ft": [-3.0, -1.0], "out_ft": 6.6, "message": ""} + ] + low = invariant_problems(with_wall_seen(1.0), r, rules=RULES) + assert not any("lists as observed" in m for m in low) + full = invariant_problems(with_wall_seen(None), r, rules=RULES) + assert any("lists as observed" in m for m in full) + + +BAND_RULES = RuleSet( + RULES.width_ft, + RULES.depth_ft, + RULES.needs + | { + "facing_gap": (Need("facing", 0.0, 22 / 12 + 3.0, widen=False),), + "headroom": (Need("overhead", 0.0, 6.5, widen=False),), + "battery_clearance": (Need("ground", 100.0),), + }, + RULES.errors, +) + + +def test_facing_and_headroom_passes_need_their_bands(): + r = result(checks=[check(), passing("facing_gap"), passing("headroom")]) + msgs = invariant_problems(SCENE, r, rules=BAND_RULES) + assert any( + "facing_gap passes but needs facing [1.00, 3.58] observed, none seen" in m for m in msgs + ) + assert any( + "headroom passes but needs overhead [1.00, 3.58] observed, none seen" in m for m in msgs + ) + + +def test_a_short_facing_view_needs_a_measurement(): + r = result(checks=[check(), passing("facing_gap")]) + scene = copy.deepcopy(SCENE) + scene["coverage"]["observed"].append({"band": "facing", "span_ft": [0.0, 5.0], "out_ft": 3.0}) + msgs = invariant_problems(scene, r, rules=BAND_RULES) + assert any("out to 4.83 ft or measured, seen 3.00 ft" in m for m in msgs) + scene["facing"] = [{"wall_id": "w1", "span_ft": [0.0, 5.0], "depth_ft": 8.0}] + assert invariant_problems(scene, r, rules=BAND_RULES) == [] + + +def test_an_overhead_seen_clear_all_the_way_up_settles_headroom(): + r = result(checks=[check(), passing("headroom")]) + scene = copy.deepcopy(SCENE) + scene["coverage"]["observed"].append({"band": "overhead", "span_ft": [0.0, 5.0]}) + assert invariant_problems(scene, r, rules=BAND_RULES) == [] + scene["coverage"]["observed"][-1]["out_ft"] = 5.0 # clear only to 5 ft, under 6.5 + assert any("overhead [1.00, 3.58]" in m for m in invariant_problems(scene, r, rules=BAND_RULES)) + + +def test_a_check_the_server_did_not_evaluate_needs_nothing(): + evaluated = result(checks=[check()], sweep=[PASS_RUN]) + assert not any( + "battery_clearance" in m for m in invariant_problems(SCENE, evaluated, rules=BAND_RULES) + ) + with_battery = result(checks=[check(), passing("battery_clearance")], sweep=[PASS_RUN]) + assert any( + "battery_clearance" in m for m in invariant_problems(SCENE, with_battery, rules=BAND_RULES) + ) + + +@pytest.mark.parametrize( + ("wall", "error"), + [ + ({}, 0.3), + ({"source": "tap"}, 0.3), + ({"source": "mesh"}, 0.5), + ({"source": "plane"}, 0.75), + ({"source": "mesh", "plus_minus_ft": 0.2}, 0.2), + ], +) +def test_the_wall_source_sets_the_default_error(wall, error): + scene = {"walls": [{"id": "w1", "baseline": [[0, 0], [9, 0]], **wall}]} + drift = 0.0 if "plus_minus_ft" in wall else 0.16 * 1.0 # far edge 1 ft from the meter + assert battery_error(scene, RULES, "w1", 0.0, 1.0) == pytest.approx(error + drift) diff --git a/verification/tests/test_simrun.py b/verification/tests/test_simrun.py index 09932538d..e52fa07eb 100644 --- a/verification/tests/test_simrun.py +++ b/verification/tests/test_simrun.py @@ -6,6 +6,7 @@ from hsverify.simrun import ( ShotRecord, + app_export, compiling, copy_app_export, missing_required, @@ -152,3 +153,10 @@ def test_a_bundle_line_the_runner_cannot_read_is_a_problem(tmp_path): None, ["the app logged a bundle line this runner cannot read; see state.ndjson"], ) + + +def test_a_run_that_must_keep_the_scan_fails_without_one(tmp_path): + write_log(tmp_path, log_line("STATE=result")) + assert app_export(tmp_path, required=False) == (None, []) + name, problems = app_export(tmp_path, required=True) + assert name is None and "--require-export" in problems[0] diff --git a/verification/ux/checklist.md b/verification/ux/checklist.md deleted file mode 100644 index 71abe7d19..000000000 --- a/verification/ux/checklist.md +++ /dev/null @@ -1,120 +0,0 @@ -# HouseScan capture app: UX review checklist - -A reviewer uses this list to judge each screen state of the iOS capture app from three kinds of evidence. The first is Simulator screenshots. The second is the ordered `STATE=` log from contract C4. The third is the SwiftUI source. The product priority is a first-time homeowner finishing the scan on the first try, with a result that doesn't overstate what the phone saw. - -## How to use it - -1. Collect evidence for each state in the per-state table at the end. For each state, take a screenshot at the default text size and a second one at the largest accessibility text size (AX5, `accessibilityExtraExtraExtraLarge`). Also capture the `STATE=` log for the whole run and the Accessibility Inspector readout of each screen. -2. For each item, record **Pass**, **Fail** or **Not verified**, with the evidence file or `path:line`. Mark anything you could not observe as Not verified. Never infer a Pass. -3. The verdict is **Block** if any blocker fails, **Incomplete** if any blocker is Not verified, and **Approve** otherwise. List major and minor failures as work to do. - -Severity: **blocker** means the homeowner can't finish or gets a wrong result. **Major** means likely confusion or retakes. **Minor** means polish. - -Observed by: `SS` screenshot, `SEQ` STATE log order, `SRC` SwiftUI source, `VO` VoiceOver or Accessibility Inspector, `DT` Dynamic Type run at AX5, `RM` Reduce Motion run, `GRAY` screenshot converted to grayscale. - -The Simulator cannot run ARKit world tracking. Screens that depend on it (walk, fog, tracking loss, relocalization, reveal) can be screenshotted only if the app has a preview or fixture mode that injects those states. Without one, check those items from `SRC` and mark the runtime part Not verified. - -Sources: `HLD` is `docs/05-live-guided-survey-hld.md`. `FTC` is `docs/research/t3-first-try-capture.md` on `origin/t3/research`. Skill names refer to the design and accessibility skills the checklist draws on. - -Numeric limits marked *(reviewer threshold)* are review conventions. No user testing backs them. - -## R. First-try reliability - -| ID | Check | Observed by | Pass when | Sev | Source | -|---|---|---|---|---|---| -| R1 | The walk can't end with a silent gap | SEQ, SS | Every strip cell on the last screen before the upload or finish state is green, marked "can't access", or listed by name as open for installer review | blocker | FTC finish rule; HLD §2 | -| R2 | Missing coverage never shows as done | SRC, SS | The cell-to-style mapping has one explicit case per state (unseen, seen but not enough, enough, failed, skipped) with no `default:` that falls through to green. Any progress count counts only green cells | blocker | HLD §2 coverage display | -| R3 | Tracking loss stops coverage credit | SRC, SS | Code that clears fog or marks cells runs only when `trackingState == .normal`. The strip in the tracking-lost screenshot is unchanged from the screenshot before it | blocker | HLD §9 tracking row; FTC recovery | -| R4 | An interruption resumes the walk | SEQ, SS | Backgrounding mid-walk logs walk, then a relocalizing state, then walk again (or tap meter). It never returns to onboarding, and the captured count and strip survive | blocker | FTC recovery; HLD §9 session resumed | -| R5 | Failed relocalization starts a new frame | SEQ, SRC | After relocalization fails, the app asks to tap the meter again and resets the strip's spatial cells. It doesn't keep drawing old world-anchored overlays | blocker | FTC failed relocalization | -| R6 | Finish waits for server acknowledgment | SEQ, SS | Finish or reveal is reachable only after every upload is acknowledged. Offline, the screen says the scan is saved on the phone and not yet sent, and it doesn't say done | blocker | FTC finish rule 3 | -| R7 | Upload progress is separate from evidence completeness | SS | Upload status is its own element with its own wording (for example "3 photos waiting to send"), placed apart from the coverage strip, and it never changes a strip cell | major | HLD §2, §9 | -| R8 | Offline keeps local guidance running | SS | Offline screenshots still show the instruction and strip. Findings that need the server are labelled pending, not pass or fail | major | HLD §9 connectivity row | -| R9 | Camera denied has a way forward | SEQ, SS | A `cameraAccessDenied` state explains in one sentence why the camera is needed and has an "Open Settings" button that opens the app's Settings page | blocker | HLD §2 device check | -| R10 | AR unavailable is explained | SEQ, SS | On a device without world tracking, the app shows a plain explanation and what to do next. There's no black camera view, spinner loop or crash | blocker | HLD §2 device check | -| R11 | Each auto-capture is acknowledged | SS, SRC | Each saved capture triggers a brief visual mark, such as a flash on the reticle or a counter tick, and the capture count goes up | major | HLD §2 | - -## I. One clear instruction per screen - -| ID | Check | Observed by | Pass when | Sev | Source | -|---|---|---|---|---|---| -| I1 | One primary instruction | SS | The largest text is a single instruction sentence. Any other text is visibly smaller and isn't a second command | major | HLD §2; holistic-ux cognitive load | -| I2 | Standing and aiming cues are distinct | SS, SRC | The where-to-stand cue (a ground marker or "step back" text) and the where-to-aim cue (the reticle or target) differ in shape and position. Neither is a recolored copy of the other | major | HLD §2 | -| I3 | Plain words | SRC | A grep of user-visible strings finds none of: raycast, plane, keyframe, anchor, tracking, LiDAR, epoch, relocaliz, OCR, UNSURE, PASS, FAIL | major | frontend-design writing; apple-design simplicity | -| I4 | One question at a time with two large buttons | SS | A question screen shows exactly one question and two full-width buttons, each at least 44 pt tall. Button labels state the answer ("It opens", "It stays shut"), not Yes and No | major | FTC what the homeowner sees 6 | -| I5 | No shutter where auto-capture applies | SS, SRC | The meter close-up, panel shots and walk have no shutter button. Tapping to set the meter anchor is allowed, because that tap marks a location rather than taking a photo | major | HLD §2; FTC 3 | -| I6 | The instruction stays stable | SEQ | The log never shows an A, B, A change of instruction within 3 s unless a task was completed *(reviewer threshold)* | major | HLD §4 next-view planning | -| I7 | No arrow through unseen ground | SS | Guidance toward an unobserved area is text ("Walk around the corner if you can"), not a world-anchored arrow | major | HLD §4 | -| I8 | Anchoring never guesses a depth | SEQ, SS | Tapping the meter with no detected wall plane shows "step closer" and doesn't advance to the step-back state | blocker | FTC 4 | -| I9 | Panel instructions include the safety wording | SS | The panel screen says to open only the hinged door, not to unscrew the cover and not to touch the breakers, and offers Skip | blocker | FTC opening the panel | -| I10 | Recapture requests give a reason and a target | SS | The request is one line naming what to show and where, for example "Show the ground below this part of the wall" | major | HLD §4 capture request | - -## T. Recovery from tracking loss and gaps - -| ID | Check | Observed by | Pass when | Sev | Source | -|---|---|---|---|---|---| -| T1 | Tracking-loss wording matches the cause | SRC, SS | `excessiveMotion` shows "Slow down". `insufficientFeatures` shows "Aim at a corner or somewhere with more texture". These are separate `case` branches with separate strings | major | FTC recovery | -| T2 | No modal on tracking loss | SRC, SS | No `.alert`, `.sheet` or `.fullScreenCover` is bound to tracking state, and the camera stays visible behind the prompt | major | FTC recovery | -| T3 | World-anchored overlays are hidden while tracking is limited | SRC, SS | Arrows, the fog and the candidate box hide or freeze when tracking isn't `.normal`, and they come back only after it returns | major | HLD §9 tracking row | -| T4 | "I can't access this area" is available | SS | The button is on the walk and gap-request screens and takes one tap. Afterwards the cell shows a distinct skipped mark (not green) and the next task starts | major | HLD §2 | -| T5 | "Can't get a clear shot" appears after two tries | SEQ, SS | The option appears after the second failed close-up attempt, not earlier and never absent. Choosing it moves on and doesn't loop | blocker | FTC close-up gates | -| T6 | Close-up failures name the fix | SS, SRC | Blur, glare and a cut-off label each have their own message saying what to do (hold still, tilt away from the glare, fit the whole label in the box) | major | HLD §9 blur row | -| T7 | Relocalization shows the saved meter view | SS | The screen shows the saved meter image and "Point at the meter like this." | major | FTC recovery | - -## H. Result reveal honesty - -| ID | Check | Observed by | Pass when | Sev | Source | -|---|---|---|---|---|---| -| H1 | UNSURE reads as needing review, with a reason | SS, SRC | Each unsure check shows "An installer will check this" plus a one-line reason. Its style is distinct from pass, and it's never hidden or folded into a pass | blocker | FTC intro; HLD §7 | -| H2 | Borderline numbers show the error | SS | A result near its limit shows the measured value, the rule and the error, as in FTC's example ("3 ft 2 in ... rule is 3 ft ... off by about 4 in") | blocker | FTC borderline reveal | -| H3 | Green coverage doesn't imply a pass | SS | The screen shown when coverage is complete uses no approval words ("approved", "all set", "you qualify"). The verdict appears only on the reveal, in wording separate from coverage | blocker | HLD §2 green row | -| H4 | The reveal is anchored to the meter | SRC, SS | The candidate box and cable line are placed relative to the meter's anchor, and they hide with an explanation if tracking isn't normal at reveal | major | AGENTS.md hard rules; FTC 9 | -| H5 | A failure isn't a dead end | SS | A failed spot names the rule and distance, then shows another candidate or says an installer will look for one | major | holistic-ux peak-end | -| H6 | An unseen side is disclosed | SS | If the other side of the meter wasn't seen, the reveal says so ("A closer spot may exist on the left") | minor | FTC early stop | -| H7 | The panel review needs are stated | SS | The reveal says the panel still needs an electrician's review whenever the server marks it that way | major | FTC borderline reveal | - -## A. Accessibility - -| ID | Check | Observed by | Pass when | Sev | Source | -|---|---|---|---|---|---| -| A1 | Every control has a label | VO, SRC | The inspector reads a name for every button. Each icon-only `Button { Image(systemName:) }` has `.accessibilityLabel`. A primary action without one is a blocker | blocker | better-accessibility names | -| A2 | The AR overlay state is exposed | VO | One element summarises coverage in words ("Wall: 4 of 10 sections done, 2 need another look") and the tracking status | major | better-accessibility; HLD §2 | -| A3 | Instruction changes are announced | VO, SRC | A new instruction posts `AccessibilityNotification.Announcement` or updates a focused element's value | major | better-accessibility live regions | -| A4 | Dynamic Type reaches AX sizes | DT, SRC | At AX5 the full instruction wraps with no "…", both answer buttons stay on screen, and there's no `.lineLimit(1)` or `.dynamicTypeSize(...)` cap below AX5 on instruction text | major | apple-design typography | -| A5 | Text stays readable over the camera | SRC, SS | Instruction text sits on a solid or `.regularMaterial` backing and reaches 4.5:1 contrast against both a white and a black backdrop. Text drawn straight on the camera image fails | major | apple-design materials; holistic-ux | -| A6 | Targets are at least 44 pt | SRC, SS | Each tappable element is at least 44 by 44 pt (132 px in a @3x screenshot), and neighbouring targets don't overlap | major | holistic-ux Fitts; better-accessibility | -| A7 | Reduce Motion is honoured | RM, SRC | The code reads `accessibilityReduceMotion`. With it on, fog clearing, strip fills and the reveal crossfade or cut instead of sliding or scaling, and nothing pulses in a loop | major | apple-design §14; emil-design-eng | -| A8 | Haptics are paired with a visual | SRC | Every `.sensoryFeedback` or feedback-generator call fires with a visible change on the same state update. No signal is haptic-only | major | apple-design multimodal | -| A9 | Coverage isn't colour-only | GRAY, SS | In grayscale, unseen, not enough, enough, failed and skipped cells stay distinguishable by fill pattern or icon, and a text legend or label exists | major | better-accessibility colour; HLD §2 | -| A10 | Reduce Transparency and Increase Contrast are honoured | SRC | With either setting on, materials become opaque | minor | apple-design §14 | -| A11 | VoiceOver order is instruction first | VO | Swiping from the top reads the instruction, then the primary action, then secondary controls | minor | better-accessibility structure | - -## V. Visual and motion quality - -| ID | Check | Observed by | Pass when | Sev | Source | -|---|---|---|---|---|---| -| V1 | The camera dominates capture screens | SS | Chrome covers no more than about a third of the screen during the walk and close-ups *(reviewer threshold)* | minor | frontend-design restraint | -| V2 | At most one prominent button per screen | SS | Only one filled or prominent button is visible. Recovery options such as "I can't access this area" use a secondary style | minor | apple-design simplicity | -| V3 | No stacked translucency | SS | No translucent panel sits on another translucent panel | minor | apple-design materials | -| V4 | No decorative animation during capture | SRC, SS | Capture states have no `.repeatForever`, shimmer, confetti or idle pulse. The motion that remains is state feedback: capture acknowledgment, fog clearing, strip update | major | emil-design-eng purpose | -| V5 | Motion never blocks input | SRC | No `.disabled`, `sleep` or `asyncAfter` gates a button while a transition plays | minor | apple-design interruptibility | -| V6 | UI transitions take 300 ms or less | SRC | Screen-to-screen and overlay animations take 300 ms or less. Only the result reveal may take longer | minor | emil-design-eng duration | -| V7 | Buttons respond to presses | SRC | Buttons use a style with a visible pressed state (a system style, or a scale of 0.95 to 0.98) | minor | emil-design-eng | -| V8 | One name per action | SRC | An action keeps its name through the flow: the button that says "I'm here" isn't called "Continue" on the next screen | minor | frontend-design writing | - -## Per-state table - -State names are the raw values of `ScanPhase` in `ios/HouseScan/Contract/ScanContract.swift` on `t3/ios-mvf` (at `6885b7b`), which the app logs as `STATE=`. Several screens the first-try note describes are conditions inside a phase rather than phases of their own; the second column says where to look for them. - -| State | Also covers | Must show | Key items | -|---|---|---|---| -| `onboarding` | camera permission, denied permission | What the scan does and roughly how long it takes, in one or two sentences, and one start button. When camera access is denied, why it is needed and "Open Settings" | I1, I3, A1, A4, V2, R9 | -| `unsupported` | | A plain explanation and what to do instead | R10, I3 | -| `findMeter` | tap the meter, step back | "Go to your electric meter" and **I'm here**; then "Tap the meter", with "step closer" when there is no wall plane; then a standing cue distinct from the aiming cue | I1, I2, I8, V8, A4, A6 | -| `meterCloseUp` | panel close-ups | A reticle, no shutter, a named fix on failure, "Can't get a clear shot" after two tries. Panel shots add the safety wording and Skip | I5, I9, T5, T6, R11, A8 | -| `wallWalk` | tracking lost, relocalizing | One instruction, the strip with non-colour cell marks, upload status kept separate, "I can't access this area". When tracking is limited: a cause-specific prompt, no modal, coverage and arrows frozen. After an interruption: the saved meter view | R1, R2, R3, R4, R5, R7, I6, I7, T1, T2, T3, T4, T7, A2, A3, A9, V1, V4 | -| `gapRequest` | | One line with the reason and target, and a skip option | I10, T4, R1 | -| `markFeatures` | homeowner questions | The saved frame with one instruction for marking a missed object and one confirm button. Questions: one at a time, two large answer buttons | I1, I4, A1, A4, A6, H1 | -| `uploading` | offline | Upload count and pending state, no approval wording. Offline: saved on the phone and not sent, retry | R6, R7, R8, H3 | -| `result` | | Decision in plain words, numbers with error on tap, unsure checks as installer review, the unseen side disclosed | H1, H2, H3, H5, H6, H7 | -| `resultAR` | | The box on the meter anchor and the cable line, hidden with an explanation when tracking is not normal | H4, A7, T3 | diff --git a/verification/ux/friction.md b/verification/ux/friction.md deleted file mode 100644 index 60cfe30f5..000000000 --- a/verification/ux/friction.md +++ /dev/null @@ -1,94 +0,0 @@ -# Friction on the homeowner's path - -Where a first-time homeowner can stall, redo something or have to decide, and what would most -raise the share of scans that finish on the first try. Written for the owner of `t3/ios-mvf`, at -`a39d0a5`. Evidence: - -- the Simulator run `sim/20260926-071459-t3-ios-mvf-a39d0a50-replay` on the ADVIO replay; -- the server's answer to that run's upload, `e2e/20260926-071717-739fb6f7` (server `739fb6f`); -- the source at `a39d0a5`. `Kit/` below is `ios/HouseScanKit/Sources/HouseScanKit/`; other paths are under `ios/HouseScan/`. - -## The run in numbers - -The replay is 34.7 s of walking along a 39.7 m path, 79 frames, with tracking normal throughout; -the app turned it into 26 ft of wall (s −11 to +15 ft). With the autopilot holding each step 3 s, -launch to result took 44 s: - -| Stage | Seconds | -| --- | --- | -| Onboarding | 4.6 | -| Find the meter | 3.2 | -| Meter close-up | 4.0, skipped after 2 failed tries (the replay has no meter) | -| Wall walk | 24.7 | -| Feature list | 3.2 | -| Gap request | 3.4, answered "I can't get there" | -| Upload | 0.7 | - -Without failures, reaching the result takes 10 taps plus iOS's camera dialog: "Next", "Next", -"Allow camera", "This is my meter", "Wall ends here" and its end answer on each side, "Done with -this wall", "Looks complete". Five of them are decisions: where the meter is, where each end is, -and what is at each end. Seeing the spot in AR adds "See it on your wall" and "Done". - -The first try did not finish. The result said "An installer will take a look … 8 checks depend on -areas the scan did not see". The server asked for four more views (ground, wall, facing, overhead) -and for walking past both ends. - -## Where a homeowner can stall, in flow order - -| Stage | Trigger | What they see | Way forward | Kind | -| --- | --- | --- | --- | --- | -| Find meter | Camera denied, camera failure, unreadable replay (`Runtime/ScanEngine.swift:119, 489, 491` set the failure without changing screen) | "Find your electric meter" with no camera; taps do nothing | None; close the app | Stuck (B-03) | -| Find meter | Tap refused: tracking not normal, or no wall under the tap | "Step a little closer to the wall" for both causes | Tap again | Redo | -| Find meter to gap request | Relocalizing for over 20 s (`Runtime/ScanEngine.swift:466-471`) | Back to "Find your electric meter"; wall, marks and photos gone, no explanation | Start the scan again | Redo | -| Close-up | One problem held for 4 s is a failed try; a photo needs 0.6 s of good frames (`Kit/Capture/CloseUpGate.swift:13, 15`) | One instruction and a named fix, "Can't get a clear shot" from the second failed try | Skip; if frames alternate good and blurry, no try fails and the skip never appears | Redo, can stick (B-04) | -| Walk | A side's end is closer than 6.1 m (20 ft); "Wall ends here" is offered only after that much is covered (`Kit/Guidance/GuidancePlanner.swift:27, 119`) | "Walk slowly to your left" with no way to say the wall ends | "Can't get there", which records the end as unexplored | Decision, recorded wrong (B-06) | -| Walk | After "Wall ends here" | "What's at the left end?": "It turns a corner" / "Something blocks it" | Answer; a corner exports as unexplored (`Runtime/ScanEngine+Actions.swift:85`) | Decision, twice | -| Walk | Ground not seen | "Tilt down to show the ground" | Tilt; on the replay ground was seen under 4 ft of the 26 ft | Redo | -| Walk | Marking a door, window, drive or fence takes two taps; each can be refused | Warning haptic and a reason | Tap again or "Cancel" | Redo | -| Feature list | A chip under "Add something" (`UI/Screens/MarkFeaturesScreen.swift:64`) | Nothing visible; the marking circle exists only on the walk | None; "Looks complete" uploads without the feature | Stuck (B-09) | -| Feature list | A marked window | "Does this window open?" | Answer | Decision | -| Gap request | The phone's own check finds a gap after "Looks complete" | "Show the ground around your meter", a progress bar | Fill 80 %, or "I can't get there", which the upload does not carry (B-08) | Redo | -| Upload | Any failure | "That didn't go through" and raw error text (B-02) | "Try again" resends everything | Redo | -| Result | A check depends on an area not seen: every scan at `a39d0a5` (below) | "An installer will take a look", "Still needed" cards with "Capture it now" | Capture and upload again; ground, facing and overhead requests cannot be settled | Redo, cannot finish | -| Result | An end within reach is unexplored | "Keep walking past the left end of the scan (11 ft 0 in left of the meter)" | Walk past it; the server then asks again for ground there (S2 finding, routed) | Redo | - -The autopilot presses buttons directly. It skips the close-up after 4 s and answers every end "It -turns a corner". It never taps "Add something", "Capture it now" or "Try again", and never -triggers a camera denial or the 20 s reset. Those rows come from the source and were not seen -running. - -## Three changes, ranked - -**1. Report how far out the ground was seen.** Every upload says ground was seen out to exactly -1.2 m (3.9 ft). `Runtime/ScanEngine+Export.swift:62` exports the fixed `groundBandDepth` from -`Kit/Coverage/CoverageMap.swift:36`. The clearance checks need ground out to about 6 ft for gas -and AC, 8 ft for a drive and 13 ft for a pool: the battery's 1.83 ft depth, plus the radius, plus -its position error. So these four checks are unsure on every scan, and no "Show the ground" -request can be met however far the homeowner points. This alone guarantees a re-request. - -The fix: record each stretch's out distance from where the camera actually saw the ground, and -have the walk's tilt instruction ask for the distance the rules need. The drive (5 ft) and pool -(10 ft) radii are placeholders in `server/rules.yaml`, so the rules owner can also shrink what -is asked; that is S2's call. - -**2. Report the space above and in front of the wall, or stop requiring it.** The upload lists -only wall and ground coverage (`Kit/Scene/SceneExport.swift:284-288`), so the facing-gap and -headroom checks are unsure on every scan. The phone's gap planner drops overhead and facing -requests (`Kit/Guidance/GapPlanner.swift:118`), so "Capture it now" cannot settle them either. -The walk already counts the wall face up to the 6.5 ft headroom line -(`Kit/Coverage/CoverageMap.swift:34`), so the frames that saw it may also have seen the space in -front. The fix needs one decision shared with S2: which walk frames count as seeing overhead and -facing. The app then exports those bands. - -**3. Let the walk follow a wall round its corners.** The app exports a single wall -(`Kit/Scene/SceneExport.swift:295`), and "It turns a corner" exports the end as unexplored. So -any corner within the 20 ft cable reach becomes a server request to keep walking past it: two of -the six requests on the replay. Separately, a wall ending closer than 20 ft can only be recorded -through "Can't get there" (row 5 above). - -The fix: offer "Wall ends here" whenever the end is in view (`Kit/Guidance/GuidancePlanner.swift:119`, -`UI/Screens/WallWalkScreen.swift`). After "It turns a corner", continue the walk onto the next -wall, which C1's wall chain already supports. - -The blockers B-03 and B-09 stop a homeowner outright, but they occur on uncommon paths. The three -changes above affect every scan. diff --git a/verification/ux/review.md b/verification/ux/review.md deleted file mode 100644 index bf9eaa6a6..000000000 --- a/verification/ux/review.md +++ /dev/null @@ -1,100 +0,0 @@ -# UX review of every app state - -Each state of the HouseScan app judged against [checklist.md](checklist.md). Screenshots and -audit reports stay in `~/house-scanning-data/reports/` (replay frames are non-commercial data). - -## Evidence - -| Run | What it shows | -| --- | --- | -| `sim/20260926-071459-t3-ios-mvf-a39d0a50-replay` | Every state at `a39d0a5`, the current head, with the real server's result; the gap request appears and is answered "I can't get there" | -| `a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample` | Apple's audit at `a39d0a5` on 21 screens: 10 issues, down from 32 at `525ea40` | -| `sim/20260926-051323-t3-ios-mvf-24af4342-replay` | Every state at `t3/ios-mvf` `24af434` with the real server's result | -| `sim/20260926-052535-t3-ios-mvf-525ea401-replay` | Every state at `525ea40` with the real server's result and the end question answered | -| `a11y/20260926-052756-t3-ios-mvf-525ea401-head-sample` | Apple's audit at `525ea40` on 21 screens, including the end question and the sample wording | -| `a11y/20260926-051535-t3-ios-mvf-24af4342-head-sample` | Apple's audit at `24af434` on 17 screens; the gap loop closes on the replay | -| `sim/20260926-050058-t3-ios-mvf-0876e03c-replay` | Every state at `0876e03` | -| `sim/20260926-043152-5520229-5520229d-preview-sample` | Every state of the flow, default text size, light appearance, sample result | -| `sim/20260926-044328-5520229-5520229d-preview-ax5-dark` | The same at the largest accessibility text size (AX5), dark appearance | -| `sim/20260926-044527-5520229-5520229d-preview-unsupported` | The unsupported-phone screen | -| `a11y/20260926-044005-5520229-5520229d-preview-sample` | Apple's accessibility audit on 17 distinct screens, with every label | -| Source at `0876e03` | Items that need the live camera | - -`5520229` is `t3/ios-mvf` `90c01dd` merged with the UI lane `e5dde35`; `0876e03` is the same UI -merged by S3, plus the fixes listed under "Fixed since the preview". The Simulator cannot run -ARKit, so tracking loss, relocalization and the live meter tap are judged from source. - -## Verdict: Block - -Blockers that fail: R9 (camera access denied leads nowhere, [B-03](../product/bug-triage.md#blockers)), -T5 (the close-up's way out can fail to appear, B-04) and R1 (a stretch the homeowner can't reach -is not recorded, B-08). The feature list's "Add something" doing nothing (B-09) blocks adding a -missed feature. - -## By state - -| State | Result | Findings | -| --- | --- | --- | -| `onboarding` | Pass at default size: one message, one primary button, "About 2 min". At AX5 the pill reads "About 2 mi" and the body runs under the page dots. | UX-01 | -| `unsupported` | Pass: "This phone can't measure walls" with what to do instead. Camera denied never reaches this screen. | B-03 | -| `findMeter` | Pass at default size: one instruction, the reticle, "This is my meter". At AX5 the button is pushed partly off screen. Refusal wording ("Step a little closer to the wall") also covers tracking refusals. | UX-01, B-11 | -| `meterCloseUp` | Pass: one instruction, no shutter, a named fix ("Center the meter in the circle"). The problem pill fails contrast; the way out can fail to appear. | UX-03, B-04 | -| `wallWalk` | One instruction with its reply inside the card, as the checklist asks. The strip's states differ by colour alone, the counter fails contrast, and at AX5 the card covers most of the camera and pushes "Mark something" and the strip off screen. At `525ea40` a marked end is followed by "What's at the left end?", so its kind is truthful, but an end closer than 20 ft still can't be marked. | UX-01, UX-02, UX-03, UX-05, UX-08, B-06 | -| `markFeatures` | Pass at default size: clear list, remove buttons labelled, window question with two full-width answers. At AX5 the list is hidden behind "Looks complete". "Add something" does nothing. | UX-01, B-09 | -| `gapRequest` | Pass: one line with the reason and target, a progress bar, "I can't get there". The requested stretch reuses the amber that means "seen". | UX-05, UX-06, B-08 | -| `uploading` | Pass: progress in its own steps, apart from coverage (R7); with no server it says "Making a sample result" (`525ea40`). Four text elements do not scale. A failure shows raw error text. | UX-04, B-02 | -| `result` | Pass on honesty: "An installer will take a look", the rules-not-final note, the unseen side disclosed, the borderline window check with measurement, rule and error, and "An installer will check this". A maximum reads like a minimum; the first load leaves the upload screen up for over 1.2 s. | B-14, UX-07 | -| `resultAR` | Pass: the spot and cable over the camera with one headline and "Done". The headline says "The spot an installer will check" unless the result is an approved pass, and "Example spot, not your result" for a sample (seen at `525ea40`). | none | - -Checks that passed across the flow: one primary instruction per camera screen (I1); plain words, -no jargon in any on-screen string (I3); no shutter where capture is automatic (I5); every control -has a spoken name (A1, from the audit's labels); no hit area below 44 pt (A6, audit). - -## Findings - -**UX-01. Major. Large text breaks the camera screens and the feature list (A4).** At AX5: -"This is my meter" is pushed partly off screen; the feature list shows its heading and "Looks -complete" but not the marked features or the window question; on the walk and a gap request the -instruction card covers most of the camera, and "Mark something", "I can't get there" and the strip -fall off screen; onboarding's "About 2 min" truncates to "About 2 mi". Seen on the preview; the -only layout change at `0876e03` stacks the window answers, which makes the list taller. The audit flags clipped text -on the close-up and walk cards. Suggested: cap the instruction card's type at a large size with -the rest scrollable, keep the primary button pinned above the home indicator, and let the list -scroll behind a pinned "Looks complete". - -**UX-02. Major. Coverage is shown by colour alone (A9).** Covered cells are green and seen cells -amber; in grayscale both are the same light gray, and the strip has no legend or row labels. A -homeowner with red-green colour blindness cannot tell done from not done. Suggested: a pattern or -height difference per state, and "Wall" and "Ground" row labels. - -**UX-03. Minor. Contrast over the camera (A5).** At `a39d0a5` the audit still fails contrast on -the photo counter during the walk; the close-up's problem pill and the walk's "Can't get there" -reply, which failed at `24af434` and `525ea40`, now pass. Failures on text mid-transition (the -result headline and the sample note as they fade in) are left out. - -**UX-04. Minor. Text that does not scale or clips (A4).** At `a39d0a5` one element's font size -cannot change ("Mark something" on the walk), down from ten at `525ea40`; the strip's marks now -scale. Text may clip at large sizes on the close-up, the mark prompt ("Now tap its top-right -corner") and the end question ("What's at the right end?" and its explanation). - -**UX-05. Minor. Two names for one action (V8).** The walk's reply is "Can't get there"; the gap -request's is "I can't get there". - -**UX-06. Minor. One colour, two meanings.** The requested stretch is outlined in the amber that -marks "seen but not enough" cells on the same strip. - -**UX-07. Minor. The result appears late the first time.** On the first visit the upload screen -("Checking your wall") was still showing 1.2 s after the app logged the result; after returning -from AR the result appeared at once. The 3D model's first load is the likely cause. - -**UX-08. Minor, not seen at `a39d0a5`. Text VoiceOver cannot read (A2).** At `525ea40` the audit -flagged text on the walk that is drawn rather than exposed to accessibility; the audit at -`a39d0a5` does not report it. - -## Fixed since the preview (`0876e03`) - -Seen: the window question answers "It opens" and "It stays shut", stacked (I4), and the result -says the electrical panel still needs an electrician's review (H7). From source: tracking coaching says -"Slow down" and "Aim at a corner or somewhere with more texture" (T1). While relocalizing, the -saved meter photo shows with "Point at the meter like this." (T7). Overlays hide while tracking is -not normal (T3). From 5f73553366ce44ca39162f1834c1874c6e48395b Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 22:39:49 -0500 Subject: [PATCH 39/59] verification: count a past-end request as covering what lies beyond the end A band past an unexplored end cannot be settled by a view along the same line, so the server asks to keep walking instead; the completeness check no longer demands a band request there. The drift case's gas meter gets heights above the cable run: without them the server (6c7ca23) leaves every route past it unsure, which is not what the case tests. At 6c7ca23 all 50 scenes pass. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/SCOREBOARD.md | 93 ++--- verification/e2e/cases/README.md | 5 +- verification/e2e/cases/d-gas-drift.json | 2 + verification/e2e/cases/generate.py | 11 +- verification/hsverify/resultcheck.py | 23 +- verification/scoreboard/scoreboard.json | 480 +++++++++++++++--------- verification/tests/test_resultcheck.py | 29 ++ 7 files changed, 424 insertions(+), 219 deletions(-) diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index 1b41e6e66..fa22e1156 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,88 +1,95 @@ # Scoreboard -Generated 2026-09-26 12:28 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-26 22:39 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | -| [#22](https://github.com/SamGu-NRX/house-scanning/pull/22) | `t3/packet` | `d5439cfe` | no | 3 success | 2026-09-26 12:09 CDT | -| [#21](https://github.com/SamGu-NRX/house-scanning/pull/21) | `t3/ios-map3d` | `66cdcbaa` | yes | 2 success, 1 pending | 2026-09-26 12:02 CDT | -| [#20](https://github.com/SamGu-NRX/house-scanning/pull/20) | `t3/recon` | `c962e092` | yes | 3 success | 2026-09-26 10:59 CDT | -| [#18](https://github.com/SamGu-NRX/house-scanning/pull/18) | `codex/research-findings` | `bd4a7d38` | no | 2 success | 2026-09-26 09:07 CDT | -| [#17](https://github.com/SamGu-NRX/house-scanning/pull/17) | `t3/panel-label` | `cf6e6e19` | no | 4 success | 2026-09-26 08:16 CDT | -| [#16](https://github.com/SamGu-NRX/house-scanning/pull/16) | `t3/meter-closeup` | `b199613b` | no | 4 success | 2026-09-26 08:16 CDT | -| [#15](https://github.com/SamGu-NRX/house-scanning/pull/15) | `t3/web-review` | `1ebc0f40` | yes | 3 success | 2026-09-26 10:32 CDT | -| [#14](https://github.com/SamGu-NRX/house-scanning/pull/14) | `t3/docs-cleanup` | `8afa3313` | no | 2 success | 2026-09-26 11:47 CDT | -| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `52aa87f7` | yes | 3 success | 2026-09-26 08:00 CDT | -| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `24793c32` | no | 4 success | 2026-09-26 08:14 CDT | -| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `68084b5f` | no | 2 success | 2026-09-26 12:08 CDT | -| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `194f2ebf` | no | 3 success | 2026-09-26 11:51 CDT | +| [#86](https://github.com/SamGu-NRX/house-scanning/pull/86) | `t3/experience` | `8bbef71d` | yes | 2 success | 2026-09-26 22:26 CDT | +| [#61](https://github.com/SamGu-NRX/house-scanning/pull/61) | `t3/app-review-docs` | `6059eae6` | no | 2 success | 2026-09-26 21:51 CDT | +| [#60](https://github.com/SamGu-NRX/house-scanning/pull/60) | `t3/device-test-4.1-analysis` | `c566490c` | no | 2 success | 2026-09-26 22:39 CDT | +| [#59](https://github.com/SamGu-NRX/house-scanning/pull/59) | `t3/signals-lab` | `0cde05e3` | yes | 2 success | 2026-09-26 21:38 CDT | +| [#58](https://github.com/SamGu-NRX/house-scanning/pull/58) | `t3/ios-ci-no-preboot` | `3c1298ab` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 22:18 CDT | +| [#56](https://github.com/SamGu-NRX/house-scanning/pull/56) | `t3/walk-guidance-gaps` | `cae4f426` | no | 3 success | 2026-09-26 22:18 CDT | +| [#55](https://github.com/SamGu-NRX/house-scanning/pull/55) | `t3/motion-permission` | `9607a481` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 22:23 CDT | +| [#52](https://github.com/SamGu-NRX/house-scanning/pull/52) | `t3/battery-mark` | `68a66a82` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 20:42 CDT | +| [#49](https://github.com/SamGu-NRX/house-scanning/pull/49) | `t3/gap-skip-continues` | `ce901122` | no | 3 success | 2026-09-26 22:18 CDT | +| [#46](https://github.com/SamGu-NRX/house-scanning/pull/46) | `t3/field-test-run2` | `afd02c82` | no | 2 success | 2026-09-26 17:36 CDT | +| [#23](https://github.com/SamGu-NRX/house-scanning/pull/23) | `t3/device-field-test` | `321ea7e2` | no | 2 success | 2026-09-26 19:39 CDT | +| [#22](https://github.com/SamGu-NRX/house-scanning/pull/22) | `t3/packet` | `b2fefa56` | no | 2 success | 2026-09-26 21:44 CDT | +| [#21](https://github.com/SamGu-NRX/house-scanning/pull/21) | `t3/ios-map3d` | `6f883d8b` | yes | 2 success | 2026-09-26 22:37 CDT | +| [#20](https://github.com/SamGu-NRX/house-scanning/pull/20) | `t3/recon` | `6573a79c` | yes | 3 success | 2026-09-26 21:50 CDT | +| [#18](https://github.com/SamGu-NRX/house-scanning/pull/18) | `codex/research-findings` | `bd4a7d38` | no | 2 success | 2026-09-26 16:21 CDT | +| [#16](https://github.com/SamGu-NRX/house-scanning/pull/16) | `t3/meter-closeup` | `b199613b` | no | 4 success | 2026-09-26 22:26 CDT | +| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `e0e3ae6e` | no | 3 success | 2026-09-26 22:36 CDT | +| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `190ed339` | no | 3 success | 2026-09-26 19:16 CDT | +| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `6c7ca23b` | no | 3 success | 2026-09-26 22:37 CDT | +| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `4b71161d` | no | 2 success, 2 pending | 2026-09-26 22:37 CDT | | [#9](https://github.com/SamGu-NRX/house-scanning/pull/9) | `codex/capture-contract-probe` | `4b6c4309` | yes | 4 success | 2026-09-26 00:08 CDT | -| [#7](https://github.com/SamGu-NRX/house-scanning/pull/7) | `t3/measure-lab` | `45770a3b` | no | 6 success | 2026-09-26 03:19 CDT | -| [#6](https://github.com/SamGu-NRX/house-scanning/pull/6) | `t3/research` | `9737e3f0` | no | 4 success | 2026-09-26 05:13 CDT | -| [#4](https://github.com/SamGu-NRX/house-scanning/pull/4) | `t3/scoring-harness` | `77ce830d` | no | 5 success | 2026-09-26 10:10 CDT | +| [#7](https://github.com/SamGu-NRX/house-scanning/pull/7) | `t3/measure-lab` | `7b3ddcc0` | no | 4 success | 2026-09-26 22:27 CDT | +| [#4](https://github.com/SamGu-NRX/house-scanning/pull/4) | `t3/scoring-harness` | `0f3e7532` | no | 3 success | 2026-09-26 22:25 CDT | ## S1 Evals on real data -`origin/t3/evals` at [`24793c32`](https://github.com/SamGu-NRX/house-scanning/tree/24793c328850fba2018ea1f8d951c45ac30ac281). 2 met, 2 partial, 1 not run. +`origin/t3/evals` at [`190ed339`](https://github.com/SamGu-NRX/house-scanning/tree/190ed339f41924ae0244e0b88c5ebe647ddd9eb4). 2 met, 2 partial, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S1-1 | One command regenerates every number from a clean checkout | partial | make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac. | -| S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/README.md) | +| S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/README.md) | | S1-3 | Metric code tested on hand-computed cases | not run | slow; run with --slow | -| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/README.md), [experiments/evals/field/FIELD_SHEET.md](https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/field/FIELD_SHEET.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/results/advio_drift.md) | -| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/README.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/results/eth3d_visibility_sensitivity.md) | +| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/README.md), [experiments/evals/field/FIELD_SHEET.md](https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/field/FIELD_SHEET.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/results/advio_drift.md) | +| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/README.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/results/eth3d_visibility_sensitivity.md) | ## S2 Server and solver -`origin/t3/server` at [`68084b5f`](https://github.com/SamGu-NRX/house-scanning/tree/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f). 5 met, 1 partial, 2 not run. +`origin/t3/server` at [`6c7ca23b`](https://github.com/SamGu-NRX/house-scanning/tree/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75). 5 met, 1 partial, 2 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S2-1 | pytest green | not run | slow; run with --slow | -| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_golden_public.py) | -| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_properties.py) | -| S2-4 | Schemas published | met | present at 68084b5f [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/schemas/result.schema.json) | -| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_schemas.py), [server/tests/test_api.py](https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_api.py) | -| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 153.8 (want < 1000) at sha 68084b5f: ~/house-scanning-data/reports/e2e/20260926-122755-68084b5f/report.json | +| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_golden_public.py) | +| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_properties.py) | +| S2-4 | Schemas published | met | present at 6c7ca23b [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/schemas/result.schema.json) | +| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_existing_battery.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_existing_battery.py), [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_schemas.py) | +| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 171.6 (want < 1000) at sha 6c7ca23b: ~/house-scanning-data/reports/e2e/20260926-223908-6c7ca23b/report.json | | S2-7 | ruff clean | not run | slow; run with --slow | -| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | met | contract_problem_count = 0 (want 0) at sha 68084b5f: ~/house-scanning-data/reports/e2e/20260926-122755-68084b5f/report.json | +| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | met | contract_ok = true (want True) at sha 6c7ca23b: ~/house-scanning-data/reports/e2e/20260926-223908-6c7ca23b/report.json | ## S3 iOS capture app -`origin/t3/ios-mvf` at [`194f2ebf`](https://github.com/SamGu-NRX/house-scanning/tree/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b). 5 met, 3 partial, 1 gap, 1 stale. +`origin/t3/ios-mvf` at [`4b71161d`](https://github.com/SamGu-NRX/house-scanning/tree/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672). 2 met, 3 partial, 1 gap, 3 stale, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S3-1 | CI build green | met | iOS build: success at 194f2ebf [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/project.yml) | -| S3-3 | Zero build warnings | met | build.warnings = length 0 (want 0) at sha 194f2ebf: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json | -| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage; missing auto-capture, guidance [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift) | -| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at 194f2ebf: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | +| S3-1 | CI build green | not run | iOS build: pending at 4b71161d [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | +| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/project.yml) | +| S3-3 | Zero build warnings | stale | report for sha 194f2ebf, ref at 4b71161d: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json | +| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage, guidance; missing auto-capture [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift) | +| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/SourceRestartUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/SourceRestartUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at 4b71161d: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | | S3-7 | Screenshots of each state in the PR | met | 18 of 9 matches in the PR body [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-8 | Exported scene.json validates against C1 | met | app_export_scene_valid = true (want True) at app_sha 194f2ebf: ~/house-scanning-data/reports/e2e/20260926-122755-68084b5f/report.json | +| S3-8 | Exported scene.json validates against C1 | stale | report for app_sha 194f2ebf, ref at 4b71161d: ~/house-scanning-data/reports/e2e/20260926-223908-6c7ca23b/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/ios-mvf` at [`194f2ebf`](https://github.com/SamGu-NRX/house-scanning/tree/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b), `origin/t3/verification` at [`52aa87f7`](https://github.com/SamGu-NRX/house-scanning/tree/52aa87f79e8dad281e96a6ef51dfe791a48b7779). 4 met. +`origin/t3/ios-mvf` at [`4b71161d`](https://github.com/SamGu-NRX/house-scanning/tree/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672), `origin/t3/verification` at [`e0e3ae6e`](https://github.com/SamGu-NRX/house-scanning/tree/e0e3ae6e49c996cee123ec8759cc6bea305d5730). 2 met, 2 stale. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S4-1 | End to end from a real replay to a placement result passes | met | app_export_passed = true (want True) at app_sha 194f2ebf: ~/house-scanning-data/reports/e2e/20260926-122755-68084b5f/report.json | -| S4-2 | Screenshot report per app state | met | states = length 12 (want at least 9) at sha 194f2ebf: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json | -| S4-3 | Product description written from the code and checked against screenshots | met | verification/product covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state. | -| S4-4 | Scoreboard of every PR | met | present at 52aa87f7 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/52aa87f79e8dad281e96a6ef51dfe791a48b7779/verification/SCOREBOARD.md) | +| S4-1 | End to end from a real replay to a placement result passes | stale | report for app_sha 194f2ebf, ref at 4b71161d: ~/house-scanning-data/reports/e2e/20260926-223908-6c7ca23b/report.json | +| S4-2 | Screenshot report per app state | stale | report for sha 194f2ebf, ref at 4b71161d: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json | +| S4-3 | Product description written from the code and checked against screenshots | met | docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state. | +| S4-4 | Scoreboard of every PR | met | present at e0e3ae6e [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/e0e3ae6e49c996cee123ec8759cc6bea305d5730/verification/SCOREBOARD.md) | ## M Maintenance -`origin/t3/docs-cleanup` at [`8afa3313`](https://github.com/SamGu-NRX/house-scanning/tree/8afa331394deda5e15ad9a802361b7082312d638). 2 gap. +`origin/t3/docs-cleanup` at [`3c0dd56c`](https://github.com/SamGu-NRX/house-scanning/tree/3c0dd56c4e74ec872de7a1904c7d9a9a5b860705). 2 gap. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| M-1 | AGENTS.md shorter than on main | gap | exit 1: AGENTS.md 45 lines, main 45 | +| M-1 | AGENTS.md shorter than on main | gap | exit 1: AGENTS.md 59 lines, main 59 | | M-2 | AGENTS.md accurate, every path and command real | gap | not yet reviewed | diff --git a/verification/e2e/cases/README.md b/verification/e2e/cases/README.md index 1b1d23d66..8a56a82aa 100644 --- a/verification/e2e/cases/README.md +++ b/verification/e2e/cases/README.md @@ -188,8 +188,9 @@ Asserted starts: - **Unsure:** 4.8092, 7.5 and 10.1016. - **Pass:** 10.5016 and 11. Both routes also pass: 11 + 0.6 + 2.17 = 13.77 < 15. -All starts sit right of the working space. The route to them crosses the gas meter, which -detours; a gas meter with no heights added no extra length at 2c9348f. +All starts sit right of the working space. The route to them passes under the gas meter, which +sits 2.5 to 4 ft up, above the 1 ft cable run; without heights the server (from 6c7ca23) cannot +tell the detour and leaves every route past it unsure. ### What rules.yaml and the README leave open diff --git a/verification/e2e/cases/d-gas-drift.json b/verification/e2e/cases/d-gas-drift.json index 97a2809fa..e94419104 100644 --- a/verification/e2e/cases/d-gas-drift.json +++ b/verification/e2e/cases/d-gas-drift.json @@ -36,6 +36,8 @@ 4.0, 4.0 ], + "bottom_ft": 2.5, + "top_ft": 4.0, "source": "tap" } ], diff --git a/verification/e2e/cases/generate.py b/verification/e2e/cases/generate.py index 00dd0a982..57ebdba8d 100644 --- a/verification/e2e/cases/generate.py +++ b/verification/e2e/cases/generate.py @@ -936,7 +936,16 @@ def starts(points): G_E = TAP + DRIFT * G_AT G_FAIL = (3.0 + G_AT - G_E - WALL_E - DRIFT * W) / (1 + DRIFT) G_PASS = (3.0 + G_AT + G_E + WALL_E + DRIFT * W) / (1 - DRIFT) -tap_gas = {"type": "gas_meter", "wall_id": "w1", "span_ft": [G_AT, G_AT], "source": "tap"} +# Heights above the 1 ft cable run, so the route passes under it and only the clearance is tested: +# with no heights the server (6c7ca23) calls the detour unknown and every route past it unsure. +tap_gas = { + "type": "gas_meter", + "wall_id": "w1", + "span_ft": [G_AT, G_AT], + "bottom_ft": 2.5, + "top_ft": 4.0, + "source": "tap", +} case( "d-gas-drift", "rules.yaml errors.tap_ft, wall_ft, drift_per_ft and clearances.gas_ft at origin/t3/server " diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 04fe74ef8..23e56f3a5 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -457,10 +457,11 @@ def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = if rules is None or spot is None: return problems lo, hi = spot["span_ft"] + asked = with_past_ends_asked(scene, requests) for c in unobserved_checks(result): if c not in rules.needs: continue - for band, gap in reach_gaps(scene, rules, c, spot["wall_id"], lo, hi): + for band, gap in reach_gaps(asked, rules, c, spot["wall_id"], lo, hi): if (band, c) not in named: problems.append( f"check {c} is unsure and needs {gap}, but no {band} request names it" @@ -468,6 +469,26 @@ def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = return problems +def with_past_ends_asked(scene: dict, requests: list[dict]) -> dict: + """The scene as if everything past each end the result asks to walk past were observed. + + Past an unexplored end the wall may turn, so a view along the same line settles nothing; the + server asks to keep walking (`past_end`) instead of for a band there, and that request covers + every band beyond the end.""" + past = [r for r in requests if r["kind"] == "past_end" and "span_ft" in r and "side" in r] + if not past: + return scene + m = copy.deepcopy(scene) + observed_ = m.setdefault("coverage", {}).setdefault("observed", []) + for r in past: + end = r["span_ft"][0] + span = [end - 1e6, end] if r["side"] == "left" else [end, end + 1e6] + for band in ("wall", "facing", "overhead"): + observed_.append({"band": band, "span_ft": span}) + observed_.append({"band": "ground", "span_ft": span, "out_ft": 1e6}) + return m + + # --- Case expectations ----------------------------------------------------------------------- diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index d69e4ee88..cea2176f6 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,51 +1,107 @@ { - "generated_at": "2026-09-26T12:28:23-05:00", + "generated_at": "2026-09-26T22:39:49-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": false, "prs": [ { - "number": 22, - "title": "packet: capture packet 1.1 \u2014 spec, validator and samples", - "branch": "t3/packet", - "head": "d5439cfe5b6334f439f0518d2de2992cfe27538c", + "number": 86, + "title": "experiments: live dots, a macOS prototype of the holographic capture view", + "branch": "t3/experience", + "head": "8bbef71d4c3158b3f5eb90a65fb5fab70642b170", + "draft": true, + "updated": "2026-09-27T03:26:12Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "2 success" + } + }, + { + "number": 61, + "title": "docs: app review \u2014 product description, UX review and friction audit", + "branch": "t3/app-review-docs", + "head": "6059eae63b26330142bf94db5486ddbf8b42c397", "draft": false, - "updated": "2026-09-26T17:09:43Z", + "updated": "2026-09-27T02:51:14Z", "ci": { "counts": { - "success": 3, + "success": 2, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "3 success" + "text": "2 success" } }, { - "number": 21, - "title": "ios: live on-device 3D map with occlusion-honest coverage", - "branch": "t3/ios-map3d", - "head": "66cdcbaaca5de3ca4812e7073fd5490b477713a2", - "draft": true, - "updated": "2026-09-26T17:02:20Z", + "number": 60, + "title": "docs: build 4.1 field-test analyses (three runs)", + "branch": "t3/device-test-4.1-analysis", + "head": "c566490c5e3fa0d45803895bb463cb260d11a906", + "draft": false, + "updated": "2026-09-27T03:39:25Z", "ci": { "counts": { "success": 2, "failure": 0, - "pending": 1, + "pending": 0, "skipped": 0 }, "failing": [], - "text": "2 success, 1 pending" + "text": "2 success" } }, { - "number": 20, - "title": "recon: handoff to the server team", - "branch": "t3/recon", - "head": "c962e0929dc29bb890c2c6fb23a359ff5e7f5108", + "number": 59, + "title": "experiments: signals lab round 1, which extra phone signals sharpen clearances", + "branch": "t3/signals-lab", + "head": "0cde05e30a7d07b01d8a458a0eeefbf2695ab2da", "draft": true, - "updated": "2026-09-26T15:59:37Z", + "updated": "2026-09-27T02:38:16Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "2 success" + } + }, + { + "number": 58, + "title": "ci(ios): drop the background Simulator boot and the duplicate t3/ios-mvf push run", + "branch": "t3/ios-ci-no-preboot", + "head": "3c1298ab46869730418c5cafc6b4d60eafe0e6fb", + "draft": false, + "updated": "2026-09-27T03:18:03Z", + "ci": { + "counts": { + "success": 2, + "failure": 1, + "pending": 0, + "skipped": 0 + }, + "failing": [ + "iOS build" + ], + "text": "2 success, 1 failure; failing: iOS build" + } + }, + { + "number": 56, + "title": "Tests for #24, #28 and #29: replay field runs 1 and 2 against the walk guidance", + "branch": "t3/walk-guidance-gaps", + "head": "cae4f426f1f5e5a0f4ba4dde97347d04df724d40", + "draft": false, + "updated": "2026-09-27T03:18:01Z", "ci": { "counts": { "success": 3, @@ -58,12 +114,70 @@ } }, { - "number": 18, - "title": "Research handoff: live survey findings, options and tested prototypes", - "branch": "codex/research-findings", - "head": "bd4a7d387606d8dedb679e2962accb5a5a48a4b2", + "number": 55, + "title": "Ask for Motion & Fitness on the onboarding, where it is explained (#43)", + "branch": "t3/motion-permission", + "head": "9607a481359897e405fc4e2acfe7149c4599dd8d", + "draft": false, + "updated": "2026-09-27T03:23:04Z", + "ci": { + "counts": { + "success": 2, + "failure": 1, + "pending": 0, + "skipped": 0 + }, + "failing": [ + "iOS build" + ], + "text": "2 success, 1 failure; failing: iOS build" + } + }, + { + "number": 52, + "title": "Mark an existing battery or another box on the wall (#27)", + "branch": "t3/battery-mark", + "head": "68a66a827c6fd6a56e18f0ffea25387f0cf23396", "draft": false, - "updated": "2026-09-26T14:07:43Z", + "updated": "2026-09-27T01:42:12Z", + "ci": { + "counts": { + "success": 2, + "failure": 1, + "pending": 0, + "skipped": 0 + }, + "failing": [ + "iOS build" + ], + "text": "2 success, 1 failure; failing: iOS build" + } + }, + { + "number": 49, + "title": "ios: 'I can't get there' moves on to the next requested view (#39)", + "branch": "t3/gap-skip-continues", + "head": "ce90112202265f211509e5f73cdbe178f1894506", + "draft": false, + "updated": "2026-09-27T03:18:00Z", + "ci": { + "counts": { + "success": 3, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "3 success" + } + }, + { + "number": 46, + "title": "experiments: field test run 2 on build 3.1", + "branch": "t3/field-test-run2", + "head": "afd02c820bd025b1c077d8e238d3f3421e5d9f3a", + "draft": false, + "updated": "2026-09-26T22:36:21Z", "ci": { "counts": { "success": 2, @@ -76,48 +190,66 @@ } }, { - "number": 17, - "title": "experiments: panel label reading stopped at the gate; field-day photo list", - "branch": "t3/panel-label", - "head": "cf6e6e1924274523acb9116c354c58b61d8a0826", + "number": 23, + "title": "experiments: device field test runbook and run 1 of the TestFlight build", + "branch": "t3/device-field-test", + "head": "321ea7e2705b80f61eaac3b4c7c3ddbb6503c675", "draft": false, - "updated": "2026-09-26T13:16:12Z", + "updated": "2026-09-27T00:39:14Z", "ci": { "counts": { - "success": 4, + "success": 2, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "4 success" + "text": "2 success" } }, { - "number": 16, - "title": "experiments: meter close-up reading, tap-to-confirm and two retake checks", - "branch": "t3/meter-closeup", - "head": "b199613b439f3d399e34df55df1b9ab4c371b24a", + "number": 22, + "title": "packet: point to the server's capture packet 0.4 and propose 0.5 additions", + "branch": "t3/packet", + "head": "b2fefa568d80f807b4310bd3f64ecd58c8b20f81", "draft": false, - "updated": "2026-09-26T13:16:09Z", + "updated": "2026-09-27T02:44:00Z", "ci": { "counts": { - "success": 4, + "success": 2, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "4 success" + "text": "2 success" } }, { - "number": 15, - "title": "web: placement review page", - "branch": "t3/web-review", - "head": "1ebc0f40b4f53b8429f0049feb7aff183ec1768d", + "number": 21, + "title": "ios: live on-device 3D map with occlusion-honest coverage", + "branch": "t3/ios-map3d", + "head": "6f883d8bad71ac4c15dc4d3197784f4d01ed8cfe", "draft": true, - "updated": "2026-09-26T15:32:04Z", + "updated": "2026-09-27T03:37:36Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "2 success" + } + }, + { + "number": 20, + "title": "recon: handoff to the server team", + "branch": "t3/recon", + "head": "6573a79c0331c71c8516d6bc6d60354a6669ee7f", + "draft": true, + "updated": "2026-09-27T02:50:12Z", "ci": { "counts": { "success": 3, @@ -130,12 +262,12 @@ } }, { - "number": 14, - "title": "docs: rebuild onboarding around the client and server teams", - "branch": "t3/docs-cleanup", - "head": "8afa331394deda5e15ad9a802361b7082312d638", + "number": 18, + "title": "Research handoff: live survey findings, options and tested prototypes", + "branch": "codex/research-findings", + "head": "bd4a7d387606d8dedb679e2962accb5a5a48a4b2", "draft": false, - "updated": "2026-09-26T16:47:37Z", + "updated": "2026-09-26T21:21:58Z", "ci": { "counts": { "success": 2, @@ -147,13 +279,31 @@ "text": "2 success" } }, + { + "number": 16, + "title": "experiments: meter close-up reading, tap-to-confirm and two retake checks", + "branch": "t3/meter-closeup", + "head": "b199613b439f3d399e34df55df1b9ab4c371b24a", + "draft": false, + "updated": "2026-09-27T03:26:22Z", + "ci": { + "counts": { + "success": 4, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "4 success" + } + }, { "number": 13, "title": "verification: independent end-to-end checks (S4)", "branch": "t3/verification", - "head": "52aa87f79e8dad281e96a6ef51dfe791a48b7779", - "draft": true, - "updated": "2026-09-26T13:00:35Z", + "head": "e0e3ae6e49c996cee123ec8759cc6bea305d5730", + "draft": false, + "updated": "2026-09-27T03:36:55Z", "ci": { "counts": { "success": 3, @@ -169,54 +319,54 @@ "number": 12, "title": "feat(evals): real-data evals for AR drift, wall reconstruction, coverage and the field test", "branch": "t3/evals", - "head": "24793c328850fba2018ea1f8d951c45ac30ac281", + "head": "190ed339f41924ae0244e0b88c5ebe647ddd9eb4", "draft": false, - "updated": "2026-09-26T13:14:09Z", + "updated": "2026-09-27T00:16:22Z", "ci": { "counts": { - "success": 4, + "success": 3, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "4 success" + "text": "3 success" } }, { "number": 11, "title": "server: placement solver, rules and API (S2)", "branch": "t3/server", - "head": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f", + "head": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75", "draft": false, - "updated": "2026-09-26T17:08:40Z", + "updated": "2026-09-27T03:37:50Z", "ci": { "counts": { - "success": 2, + "success": 3, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "2 success" + "text": "3 success" } }, { "number": 10, "title": "ios: guided wall scan from the meter to a battery spot", "branch": "t3/ios-mvf", - "head": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", + "head": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", "draft": false, - "updated": "2026-09-26T16:51:33Z", + "updated": "2026-09-27T03:37:11Z", "ci": { "counts": { - "success": 3, + "success": 2, "failure": 0, - "pending": 0, + "pending": 2, "skipped": 0 }, "failing": [], - "text": "3 success" + "text": "2 success, 2 pending" } }, { @@ -241,27 +391,9 @@ "number": 7, "title": "experiments: add measure-lab, a no-LiDAR measurement rig", "branch": "t3/measure-lab", - "head": "45770a3b1bd2ab90967e82a974fdfbf4d7bfbecb", + "head": "7b3ddcc046fcdafee6fe698c58eea5f56d017ddf", "draft": false, - "updated": "2026-09-26T08:19:30Z", - "ci": { - "counts": { - "success": 6, - "failure": 0, - "pending": 0, - "skipped": 0 - }, - "failing": [], - "text": "6 success" - } - }, - { - "number": 6, - "title": "docs: add research notes on capture, lane reviews and scoring", - "branch": "t3/research", - "head": "9737e3f0eefe90f2a12a190bf8750e7fed64413f", - "draft": false, - "updated": "2026-09-26T10:13:56Z", + "updated": "2026-09-27T03:27:31Z", "ci": { "counts": { "success": 4, @@ -277,18 +409,18 @@ "number": 4, "title": "Add a scoring harness that compares capture pipelines with a tape survey", "branch": "t3/scoring-harness", - "head": "77ce830dd08acf0c7982edcec91cd3240e0565ea", + "head": "0f3e75323b4469af88b101b983cf8c6d25ad1c3f", "draft": false, - "updated": "2026-09-26T15:10:23Z", + "updated": "2026-09-27T03:25:44Z", "ci": { "counts": { - "success": 5, + "success": 3, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "5 success" + "text": "3 success" } } ], @@ -300,7 +432,7 @@ "refs": [ { "ref": "origin/t3/evals", - "sha": "24793c328850fba2018ea1f8d951c45ac30ac281" + "sha": "190ed339f41924ae0244e0b88c5ebe647ddd9eb4" } ], "metrics": [ @@ -308,7 +440,7 @@ "id": "S1-1", "text": "One command regenerates every number from a clean checkout", "ref": "origin/t3/evals", - "sha": "24793c328850fba2018ea1f8d951c45ac30ac281", + "sha": "190ed339f41924ae0244e0b88c5ebe647ddd9eb4", "status": "partial", "evidence": "make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac.", "links": [] @@ -317,13 +449,13 @@ "id": "S1-2", "text": "Dataset URLs and sha256 recorded", "ref": "origin/t3/evals", - "sha": "24793c328850fba2018ea1f8d951c45ac30ac281", + "sha": "190ed339f41924ae0244e0b88c5ebe647ddd9eb4", "status": "met", "evidence": "all 2 found", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/README.md" ] ] }, @@ -331,7 +463,7 @@ "id": "S1-3", "text": "Metric code tested on hand-computed cases", "ref": "origin/t3/evals", - "sha": "24793c328850fba2018ea1f8d951c45ac30ac281", + "sha": "190ed339f41924ae0244e0b88c5ebe647ddd9eb4", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -340,21 +472,21 @@ "id": "S1-4", "text": "ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft", "ref": "origin/t3/evals", - "sha": "24793c328850fba2018ea1f8d951c45ac30ac281", + "sha": "190ed339f41924ae0244e0b88c5ebe647ddd9eb4", "status": "met", "evidence": "all 5 found", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/README.md" ], [ "experiments/evals/field/FIELD_SHEET.md", - "https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/field/FIELD_SHEET.md" + "https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/field/FIELD_SHEET.md" ], [ "experiments/evals/results/advio_drift.md", - "https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/results/advio_drift.md" + "https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/results/advio_drift.md" ] ] }, @@ -362,21 +494,21 @@ "id": "S1-5", "text": "ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question", "ref": "origin/t3/evals", - "sha": "24793c328850fba2018ea1f8d951c45ac30ac281", + "sha": "190ed339f41924ae0244e0b88c5ebe647ddd9eb4", "status": "partial", "evidence": "found 1-3 m row, 3-10 m row, scale error; missing verdict", "links": [ [ "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/README.md" + "https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/README.md" ], [ "experiments/evals/results/eth3d_recon.md", - "https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/results/eth3d_recon.md" + "https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/results/eth3d_recon.md" ], [ "experiments/evals/results/eth3d_visibility_sensitivity.md", - "https://github.com/SamGu-NRX/house-scanning/blob/24793c328850fba2018ea1f8d951c45ac30ac281/experiments/evals/results/eth3d_visibility_sensitivity.md" + "https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/results/eth3d_visibility_sensitivity.md" ] ] } @@ -388,7 +520,7 @@ "refs": [ { "ref": "origin/t3/server", - "sha": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f" + "sha": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75" } ], "metrics": [ @@ -396,7 +528,7 @@ "id": "S2-1", "text": "pytest green", "ref": "origin/t3/server", - "sha": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f", + "sha": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -405,13 +537,13 @@ "id": "S2-2", "text": "Every public golden test 01-14 of docs/research/t3-lane-c-review.md present", "ref": "origin/t3/server", - "sha": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f", + "sha": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75", "status": "met", "evidence": "all 14 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_golden_public.py" ] ] }, @@ -419,17 +551,17 @@ "id": "S2-3", "text": "Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail", "ref": "origin/t3/server", - "sha": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f", + "sha": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75", "status": "partial", "evidence": "found missing coverage, mirror; missing equality unsure, larger clearance", "links": [ [ "server/tests/test_coderabbit_review.py", - "https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_coderabbit_review.py" + "https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_coderabbit_review.py" ], [ "server/tests/test_properties.py", - "https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_properties.py" + "https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_properties.py" ] ] }, @@ -437,17 +569,17 @@ "id": "S2-4", "text": "Schemas published", "ref": "origin/t3/server", - "sha": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f", + "sha": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75", "status": "met", - "evidence": "present at 68084b5f", + "evidence": "present at 6c7ca23b", "links": [ [ "server/schemas/scene.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/schemas/scene.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/schemas/scene.schema.json" ], [ "server/schemas/result.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/schemas/result.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/schemas/result.schema.json" ] ] }, @@ -455,21 +587,21 @@ "id": "S2-5", "text": "Schemas validated in tests", "ref": "origin/t3/server", - "sha": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f", + "sha": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75", "status": "met", "evidence": "all 2 found", "links": [ [ - "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_golden_public.py" + "server/tests/test_existing_battery.py", + "https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_existing_battery.py" ], [ - "server/tests/test_schemas.py", - "https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_schemas.py" + "server/tests/test_golden_public.py", + "https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_golden_public.py" ], [ - "server/tests/test_api.py", - "https://github.com/SamGu-NRX/house-scanning/blob/68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f/server/tests/test_api.py" + "server/tests/test_schemas.py", + "https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_schemas.py" ] ] }, @@ -477,16 +609,16 @@ "id": "S2-6", "text": "A real-derived scene returns a result in under 1 s", "ref": "origin/t3/server", - "sha": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f", + "sha": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75", "status": "met", - "evidence": "latency_ms = 153.8 (want < 1000) at sha 68084b5f: ~/house-scanning-data/reports/e2e/20260926-122755-68084b5f/report.json", + "evidence": "latency_ms = 171.6 (want < 1000) at sha 6c7ca23b: ~/house-scanning-data/reports/e2e/20260926-223908-6c7ca23b/report.json", "links": [] }, { "id": "S2-7", "text": "ruff clean", "ref": "origin/t3/server", - "sha": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f", + "sha": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -495,9 +627,9 @@ "id": "S2-8", "text": "Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time", "ref": "origin/t3/server", - "sha": "68084b5f764cc9e6dd9a2cd13840ddfbd67f4a3f", + "sha": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75", "status": "met", - "evidence": "contract_problem_count = 0 (want 0) at sha 68084b5f: ~/house-scanning-data/reports/e2e/20260926-122755-68084b5f/report.json", + "evidence": "contract_ok = true (want True) at sha 6c7ca23b: ~/house-scanning-data/reports/e2e/20260926-223908-6c7ca23b/report.json", "links": [] } ] @@ -508,7 +640,7 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b" + "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672" } ], "metrics": [ @@ -516,9 +648,9 @@ "id": "S3-1", "text": "CI build green", "ref": "origin/t3/ios-mvf", - "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", - "status": "met", - "evidence": "iOS build: success at 194f2ebf", + "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", + "status": "not run", + "evidence": "iOS build: pending at 4b71161d", "links": [ [ "#10", @@ -530,13 +662,13 @@ "id": "S3-2", "text": "Built as Swift 6 with strict concurrency", "ref": "origin/t3/ios-mvf", - "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", + "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", "status": "met", "evidence": "all 2 found", "links": [ [ "ios/project.yml", - "https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/project.yml" + "https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/project.yml" ] ] }, @@ -544,30 +676,30 @@ "id": "S3-3", "text": "Zero build warnings", "ref": "origin/t3/ios-mvf", - "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", - "status": "met", - "evidence": "build.warnings = length 0 (want 0) at sha 194f2ebf: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json", + "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", + "status": "stale", + "evidence": "report for sha 194f2ebf, ref at 4b71161d: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json", "links": [] }, { "id": "S3-4", "text": "Unit tests for coverage, auto-capture and guidance", "ref": "origin/t3/ios-mvf", - "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", + "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", "status": "partial", - "evidence": "found coverage; missing auto-capture, guidance", + "evidence": "found coverage, guidance; missing auto-capture", "links": [ [ "ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" ], [ - "ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift" + "ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift", + "https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift" ], [ - "ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanKit/Tests/HouseScanKitTests/SceneExportTests.swift" + "ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift", + "https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift" ] ] }, @@ -575,17 +707,21 @@ "id": "S3-5", "text": "UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop", "ref": "origin/t3/ios-mvf", - "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", + "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", "status": "partial", "evidence": "all 3 found (the tests exist; passing is S3-1)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/FullFlowUITests.swift" + ], + [ + "ios/HouseScanUITests/SourceRestartUITests.swift", + "https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/SourceRestartUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -593,17 +729,17 @@ "id": "S3-6", "text": "Accessibility audit on every screen", "ref": "origin/t3/ios-mvf", - "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", + "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", "status": "partial", "evidence": "3 matches in 2 files (an audit exists; which screens it covers needs a read)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/FullFlowUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/194f2ebfa8b5336c4f9771a04cbe96f7df49b97b/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -611,16 +747,16 @@ "id": "S3-6b", "text": "Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues", "ref": "origin/t3/ios-mvf", - "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", + "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", "status": "stale", - "evidence": "report for sha a39d0a50, ref at 194f2ebf: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", + "evidence": "report for sha a39d0a50, ref at 4b71161d: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", "links": [] }, { "id": "S3-7", "text": "Screenshots of each state in the PR", "ref": "origin/t3/ios-mvf", - "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", + "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", "status": "met", "evidence": "18 of 9 matches in the PR body", "links": [ @@ -634,16 +770,16 @@ "id": "S3-8", "text": "Exported scene.json validates against C1", "ref": "origin/t3/ios-mvf", - "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", - "status": "met", - "evidence": "app_export_scene_valid = true (want True) at app_sha 194f2ebf: ~/house-scanning-data/reports/e2e/20260926-122755-68084b5f/report.json", + "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", + "status": "stale", + "evidence": "report for app_sha 194f2ebf, ref at 4b71161d: ~/house-scanning-data/reports/e2e/20260926-223908-6c7ca23b/report.json", "links": [] }, { "id": "S3-9", "text": "Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design", "ref": "origin/t3/ios-mvf", - "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", + "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -656,11 +792,11 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b" + "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672" }, { "ref": "origin/t3/verification", - "sha": "52aa87f79e8dad281e96a6ef51dfe791a48b7779" + "sha": "e0e3ae6e49c996cee123ec8759cc6bea305d5730" } ], "metrics": [ @@ -668,40 +804,40 @@ "id": "S4-1", "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/ios-mvf", - "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", - "status": "met", - "evidence": "app_export_passed = true (want True) at app_sha 194f2ebf: ~/house-scanning-data/reports/e2e/20260926-122755-68084b5f/report.json", + "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", + "status": "stale", + "evidence": "report for app_sha 194f2ebf, ref at 4b71161d: ~/house-scanning-data/reports/e2e/20260926-223908-6c7ca23b/report.json", "links": [] }, { "id": "S4-2", "text": "Screenshot report per app state", "ref": "origin/t3/ios-mvf", - "sha": "194f2ebfa8b5336c4f9771a04cbe96f7df49b97b", - "status": "met", - "evidence": "states = length 12 (want at least 9) at sha 194f2ebf: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json", + "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", + "status": "stale", + "evidence": "report for sha 194f2ebf, ref at 4b71161d: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json", "links": [] }, { "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "52aa87f79e8dad281e96a6ef51dfe791a48b7779", + "sha": "e0e3ae6e49c996cee123ec8759cc6bea305d5730", "status": "met", - "evidence": "verification/product covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result (verification.md); bug-triage.md has 16 entries, 3 resolved; ux/review.md judges every state.", + "evidence": "docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state.", "links": [] }, { "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "52aa87f79e8dad281e96a6ef51dfe791a48b7779", + "sha": "e0e3ae6e49c996cee123ec8759cc6bea305d5730", "status": "met", - "evidence": "present at 52aa87f7", + "evidence": "present at e0e3ae6e", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/52aa87f79e8dad281e96a6ef51dfe791a48b7779/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/e0e3ae6e49c996cee123ec8759cc6bea305d5730/verification/SCOREBOARD.md" ] ] } @@ -713,7 +849,7 @@ "refs": [ { "ref": "origin/t3/docs-cleanup", - "sha": "8afa331394deda5e15ad9a802361b7082312d638" + "sha": "3c0dd56c4e74ec872de7a1904c7d9a9a5b860705" } ], "metrics": [ @@ -721,16 +857,16 @@ "id": "M-1", "text": "AGENTS.md shorter than on main", "ref": "origin/t3/docs-cleanup", - "sha": "8afa331394deda5e15ad9a802361b7082312d638", + "sha": "3c0dd56c4e74ec872de7a1904c7d9a9a5b860705", "status": "gap", - "evidence": "exit 1: AGENTS.md 45 lines, main 45", + "evidence": "exit 1: AGENTS.md 59 lines, main 59", "links": [] }, { "id": "M-2", "text": "AGENTS.md accurate, every path and command real", "ref": "origin/t3/docs-cleanup", - "sha": "8afa331394deda5e15ad9a802361b7082312d638", + "sha": "3c0dd56c4e74ec872de7a1904c7d9a9a5b860705", "status": "gap", "evidence": "not yet reviewed", "links": [] diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 295f32828..9d627bb06 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -731,3 +731,32 @@ def test_the_wall_source_sets_the_default_error(wall, error): scene = {"walls": [{"id": "w1", "baseline": [[0, 0], [9, 0]], **wall}]} drift = 0.0 if "plus_minus_ft" in wall else 0.16 * 1.0 # far edge 1 ft from the meter assert battery_error(scene, RULES, "w1", 0.0, 1.0) == pytest.approx(error + drift) + + +def test_a_request_to_walk_past_an_end_covers_every_band_beyond_it(): + two_band = RuleSet( + RULES.width_ft, + RULES.depth_ft, + RULES.needs | {"gas_clearance": (Need("ground", 1.0), Need("wall", 1.0, height=6.5))}, + RULES.errors, + ) + scene = copy.deepcopy(SCENE) + scene["coverage"]["observed"][0]["span_ft"] = [1.0, 10.0] # the wall ends at 1, unexplored + r = result(checks=[check("unsure", None, cause="unobserved")]) + r["reasons"] = [{"code": "unobserved_area", "message": ""}] + r["missing_evidence"] = [ + { + "kind": "band", + "band": "ground", + "span_ft": [0, 5], + "checks": ["gas_clearance"], + "message": "", + }, + ] + assert any( + "no wall request names it" in m for m in missing_evidence_problems(scene, r, two_band) + ) + r["missing_evidence"].append( + {"kind": "past_end", "side": "left", "span_ft": [1.0, 1.0], "message": ""} + ) + assert missing_evidence_problems(scene, r, two_band) == [] From 4fe1e4357e6cb831a153c211865d8c54acd50ac7 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 22:55:18 -0500 Subject: [PATCH 40/59] verification: cut the README to the experiment format; move what each check asserts into the docstrings Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/README.md | 94 ++++++++++------------------ verification/hsverify/e2e.py | 18 ++++-- verification/hsverify/memory.py | 4 +- verification/hsverify/resultcheck.py | 10 ++- 4 files changed, 56 insertions(+), 70 deletions(-) diff --git a/verification/README.md b/verification/README.md index 131855213..2751a7c93 100644 --- a/verification/README.md +++ b/verification/README.md @@ -1,64 +1,34 @@ # Verification (S4) -Independent checks of what the other workstreams' commits do, each tied to a commit SHA. Nothing -here fixes their code; findings go to the owning thread. Reports are written to -`~/house-scanning-data/reports/`, outside git, because replay frames come from non-commercial -datasets. Requirements: uv, and Xcode 26 or newer with an iOS Simulator runtime. - -| Check | Command, from `verification/` | Output | -| --- | --- | --- | -| The app at `t3/ios-mvf` on the real ADVIO replay, uploading to a server started from `t3/server`; a screenshot per `STATE` | `make sim-app` (`REF=`, `SERVER_REF=` or `SERVER_URL=` for the hosted API, `ARGS=`) | `reports/sim//index.html` | -| Every case, the ETH3D facade scene and optional app exports through the server's API | `make e2e` (`ARGS="--app-export "`, `SERVER_REF=` or `SERVER_URL=`) | `reports/e2e//report.md` | -| Apple's accessibility audit on every screen the app reaches | `uv run python -m hsverify.a11yaudit --ref --replay --autopilot` | `reports/a11y//index.html` | -| A replay session is well formed; its poses against ground truth | `uv run python -m hsverify.replaycheck ` | printed | -| Every open PR and plan metric with its evidence | `make scoreboard` (`ARGS=--slow` also runs test suites) | `SCOREBOARD.md` | -| These tools' own tests | `make test` | pass or fail | -| The Simulator runner against a probe app with a known sequence | `make sim-probe` | `reports/sim//` | - -The written reviews of the app (product description, UX review, friction audit) are in #61, at -`docs/app-review/`. - -## What the checks guarantee - -**Simulator runner** (`hsverify/simrun.py`). Builds the ref in a `/tmp` worktree, runs it on its -own Simulator ("HouseScan Verify") with the C4 arguments, and screenshots each `STATE` once it has -settled for 1.2 s (a state replaced sooner is marked *transient*). A failed build, a crash, -redacted or missing markers, a required state never reached, or, for `make sim-app`, no scan -bundle from the app is a problem, and the run exits 1. The app launches only after the log stream -has attached. The runner waits while another `xcodebuild` compiles, because the Mac is shared. - -**End-to-end check** (`hsverify/e2e.py`, `hsverify/resultcheck.py`). Starts the server from a ref -(killed past 3.5 GB) or uses `SERVER_URL`, and sends each scene the way the app does. Scenes are -checked against the ref's scene schema first, so bad input is not blamed on the server. Rules come -from the ref's `rules.yaml`. Every answer must validate against the result schema and: - -- have a decision consistent with its spot, checks, requests and policy; -- have each check's outcome follow from its numbers (C5), with an unsure `margin` inside its - error of a line; -- pass no battery position without the coverage each passing check needs, per the server's "What - settles each check": the wall seen high enough over the clearance's reach, the ground out to the - battery's depth plus the reach, and the facing and overhead bands over the battery unless - measured. The reach includes the battery's position error, whose default depends on how the - wall was found (tap, mesh or plane). Only points within reach whatever the wall's shape are - required, so a correct server is not flagged; -- name every check left unsure for lack of coverage in a request for each band it lacks, and ask - for nothing already seen as far as the request needs; -- add up, and hash the bytes sent. - -Each scene is also resent changed, and the answers must stay ordered: mirrored, trimmed, with -more error or with requested views added, nothing improves that should not. Five hostile inputs -must be refused or answered within 10 s and 500 MB. A run claims the contract only when at least -one scene was answered and checked, and latency only for a real scene that got a result. Bundles -are read within fixed size limits. - -The 43 generated cases in [`e2e/cases/`](e2e/cases/README.md) come from the public goldens and C5; -regeneration keeps any case written by hand. [`scenes/eth3d-facade`](scenes/eth3d-facade/README.md) -builds a scene from laser-scanned geometry, replacing only its own outputs. - -**Accessibility audit** (`hsverify/a11yaudit.py`) runs Apple's audit on every screen the app -reaches, and flags buttons labelled with an SF Symbol name. - -**Replay check** (`hsverify/replaycheck.py`) checks a Measure Lab v2 session and compares its -poses with ADVIO's ground truth. - -Every report records the harness's peak memory and that of its largest child. +**Question.** Do the app and server branches do what the plan says? Pass: the app at +`t3/ios-mvf`'s head reaches a result from the real ADVIO replay and keeps its scan; every server +answer at `t3/server`'s head validates against the result schema and never passes without the +coverage its checks need; hostile inputs are refused within 10 s and 500 MB; each plan metric +has evidence at the current head. + +**Run**, from `verification/`, with uv (and Xcode 26 with an iOS Simulator for `sim-app`): + +``` +make sim-app # REF=, SERVER_REF= or SERVER_URL= +make e2e # ARGS="--app-export " +make scoreboard # writes SCOREBOARD.md +make test +``` + +Reports go to `~/house-scanning-data/reports/`, outside git, because replay frames come from +non-commercial datasets. What each check asserts is in the docstrings of `hsverify/e2e.py`, +`resultcheck.py` and `simrun.py`. + +**Result.** At `t3/server` `6c7ca23`, with the app's upload from `194f2eb`, all 50 scenes pass (45 +answered and checked; slowest real scene 0.17 s). + +**What it changed.** S2 fixed seven defects it found: +- unseen ground passing the pool check; +- a short reach cutoff; +- unbounded input cost; +- incomplete photo requests; +- repeated requests past a wall end; +- ground past a real end ignored; +- a facing gap called too close to call. + +The friction audit (#61) became team decision 8. diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index fb45abd54..da6a20f3d 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -8,12 +8,22 @@ endpoint is read from its OpenAPI document; with more than one candidate the run stops and asks for `--endpoint` instead of guessing. +A server started here is killed past 3.5 GB. Scenes go as the app sends them: JSON when the +endpoint takes it, a zip only for the compressed-scene hostile input. Bundles read from disk +stay within fixed entry and size limits, checked before reading. + Each scene is first validated against the scene schema published at the same ref, so a bad input is reported as an input problem, not a server bug. Each response must validate against -the result schema and pass the invariants in `resultcheck`. Case files add the outcomes their -geometry forces. Every scene is also sent three more ways: again (the result must be -identical apart from timing), mirrored left to right (same decision, same length of passing, -unsure and failing wall), and without coverage (never a pass). +the result schema and pass the invariants in `resultcheck`, with rules read from the ref's +`rules.yaml`. Case files add the outcomes their geometry forces. Every scene is also resent +changed, and the answers must stay ordered: again (identical apart from timing), mirrored (same +decision and outcome lengths), without or with less coverage, with ground trimmed just short +of the largest clearance, with more error or tape re-measured by tap (nothing improves), and +with every requested view added, up to three rounds (no check left unsure for coverage). Five +hostile inputs must be refused or answered within 10 s and 500 MB of server growth. + +A run claims the contract (`contract_ok`) only when at least one scene was answered and +checked and none broke it, and real-scene latency only from an HTTP 200 result. Reports go to ~/house-scanning-data/reports/e2e// (outside git; real scenes can reference dataset images). diff --git a/verification/hsverify/memory.py b/verification/hsverify/memory.py index a1841175b..b277e15c5 100644 --- a/verification/hsverify/memory.py +++ b/verification/hsverify/memory.py @@ -61,8 +61,8 @@ def __init__( measure: Callable[[int], float] = tree_rss_mb, ): self.proc, self.limit_mb, self.interval_s = proc, limit_mb, interval_s - # How a process group's memory is read; tests pass a fixed reading, since what the OS - # reports as resident depends on memory compression and cannot be held steady. + # How a process group's memory is read; tests pass a fixed reading, since how much + # the OS keeps resident varies with memory pressure and cannot be held steady. self.measure = measure self.peak_mb, self.killed = 0.0, False self._stop = threading.Event() diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 23e56f3a5..34f82820b 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -8,8 +8,14 @@ - the margin rule: a check's outcome must follow from its measurement, error and threshold (PASS only when the margin beats the error, and clears the review line when the check has one; FAIL only when past the threshold by more than the error; otherwise UNSURE); -- coverage: no battery position whose wall stretch, cable route or ground was never observed - may pass, and nothing may ask for photos of an area the scene says was observed; +- coverage: no battery position passes without the coverage each passing check needs, as the + server's README ("What settles each check") states it: the wall seen high enough over the + clearance's reach, the ground out to the battery's depth plus the reach, the facing and + overhead bands unless measured, and the cable route. The reach includes the battery's + position error, which depends on how the wall was found. Only points within reach whatever + the wall's shape are required, so a correct server is not flagged; +- requests: every check left unsure for coverage is named for each band it lacks, and nothing + is requested that was already seen as far as the request needs; - bookkeeping: counts add up, the input hash matches what was sent, the spot's offset from the meter matches its centre. From 5df746880bf7d1f874a8790037fff3c854be1aed Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 23:06:39 -0500 Subject: [PATCH 41/59] verification: require views strictly past the needed height; credit walk-past requests only at the real end The server requires the wall seen higher than a check's height and the facing and overhead bands strictly beyond their reach; the coverage check now does too, while a request stays settled by a view at least as far as it asks. A past_end request covers what lies beyond an end only at the chain's actual end (computed from the walls and the meter), on a side not marked limit. At 6c7ca23 all 50 scenes still pass. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/hsverify/resultcheck.py | 68 ++++++++++++++++++++++---- verification/tests/test_resultcheck.py | 60 ++++++++++++++++++----- 2 files changed, 106 insertions(+), 22 deletions(-) diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 34f82820b..7f85fb514 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -146,14 +146,19 @@ def reached(entry: dict) -> float: return entry.get("out_ft", default) -def observed(scene: dict, band: str, min_out_ft: float = 0.0) -> list[tuple[float, float]]: +def observed( + scene: dict, band: str, min_out_ft: float = 0.0, beyond: bool = False +) -> list[tuple[float, float]]: """Merged observed s-intervals for a band, counting only entries whose view reached - `min_out_ft` (out from the wall for ground and facing, up for wall and overhead).""" + `min_out_ft` (out from the wall for ground and facing, up for wall and overhead). With + `beyond`, the view must pass it strictly, as the server requires of a wall seen "higher + than" a height (server/scene.py at 6c7ca23).""" spans = [] for entry in scene.get("coverage", {}).get("observed", []): if entry["band"] != band: continue - if reached(entry) + EPS < min_out_ft: + far = reached(entry) + if (far <= min_out_ft) if beyond else (far + EPS < min_out_ft): continue spans.append(tuple(sorted(entry["span_ft"]))) spans.sort() @@ -323,7 +328,7 @@ def reach_gaps( gap = ground_gap(scene, lo, hi, rules.depth_ft, reach) elif need.band == "wall": a, b = required_span(lo, hi, reach) - seen = covers(observed(scene, "wall", min_out_ft=need.height), a, b) + seen = covers(observed(scene, "wall", min_out_ft=need.height, beyond=True), a, b) gap = None if seen else f"wall [{a:.2f}, {b:.2f}] observed higher than {need.height} ft" else: gap = measured_band_gap(scene, need.band, wall_id, lo, hi, need.height) @@ -357,7 +362,8 @@ def measured_band_gap( if not seen: return f"{where}, none seen" settled = any(x <= p + EPS and q - EPS <= y for x, y in measured) - if not settled and max(seen) + EPS < need: + # Strictly beyond, as the server requires (server/solver.py at 6c7ca23). + if not settled and max(seen) <= need: return f"{where} out to {need:.2f} ft or measured, seen {max(seen):.2f} ft" return None @@ -368,7 +374,7 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: wall and cable route back to the meter. No slack: runs list exactly the starts that were evaluated.""" problems: list[str] = [] - route_wall = observed(scene, "wall", min_out_ft=rules.route_height_ft) + route_wall = observed(scene, "wall", min_out_ft=rules.route_height_ft, beyond=True) # A passing run passes every check the server evaluated; a check it did not evaluate (a # battery clearance with no battery in the scene, say) needs nothing. evaluated = {c["id"] for c in result.get("checks", [])} or set(rules.needs) @@ -475,20 +481,62 @@ def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = return problems +# How far a past_end request may sit from the chain's actual end and still name it. +END_TOL_FT = 0.1 + + +def chain_ends_s(scene: dict) -> tuple[float, float] | None: + """s of the wall chain's left and right ends (the meter's projection is s = 0; distance + runs along every wall and across the gaps between them), or None without a meter wall.""" + walls = scene.get("walls", []) + points, owner = [], [] + for wall in walls: + for pt in wall["baseline"]: + points.append(tuple(pt)) + owner.append(wall["id"]) + if len(points) < 2 or "meter" not in scene: + return None + cum = [0.0] + for p, q in itertools.pairwise(points): + cum.append(cum[-1] + math.dist(p, q)) + mx, _, mz = scene["meter"]["pos"] + best = None + for i, (p, q) in enumerate(itertools.pairwise(points)): + if owner[i] != scene["meter"]["wall_id"] or owner[i + 1] != owner[i]: + continue + seg = math.dist(p, q) + if seg == 0: + continue + t = ((mx - p[0]) * (q[0] - p[0]) + (mz - p[1]) * (q[1] - p[1])) / seg**2 + t = min(1.0, max(0.0, t)) + d = math.dist((mx, mz), (p[0] + t * (q[0] - p[0]), p[1] + t * (q[1] - p[1]))) + if best is None or d < best[0]: + best = (d, cum[i] + t * seg) + if best is None: + return None + return -best[1], cum[-1] - best[1] + + def with_past_ends_asked(scene: dict, requests: list[dict]) -> dict: """The scene as if everything past each end the result asks to walk past were observed. Past an unexplored end the wall may turn, so a view along the same line settles nothing; the server asks to keep walking (`past_end`) instead of for a band there, and that request covers - every band beyond the end.""" + every band beyond the end. Only a request at the chain's actual end, on a side not marked + `limit`, earns that credit.""" + ends = chain_ends_s(scene) + kinds = scene.get("coverage", {}).get("ends", {}) past = [r for r in requests if r["kind"] == "past_end" and "span_ft" in r and "side" in r] - if not past: + if not past or ends is None: return scene m = copy.deepcopy(scene) observed_ = m.setdefault("coverage", {}).setdefault("observed", []) for r in past: - end = r["span_ft"][0] - span = [end - 1e6, end] if r["side"] == "left" else [end, end + 1e6] + side = r["side"] + end = ends[0] if side == "left" else ends[1] + if kinds.get(side, {}).get("kind") == "limit" or abs(r["span_ft"][0] - end) > END_TOL_FT: + continue + span = [end - 1e6, end] if side == "left" else [end, end + 1e6] for band in ("wall", "facing", "overhead"): observed_.append({"band": band, "span_ft": span}) observed_.append({"band": "ground", "span_ft": span, "out_ft": 1e6}) diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 9d627bb06..ffdd68910 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -9,6 +9,7 @@ RuleSet, assumption_mismatches, battery_error, + chain_ends_s, comparable, expectation_problems, invariant_problems, @@ -733,15 +734,17 @@ def test_the_wall_source_sets_the_default_error(wall, error): assert battery_error(scene, RULES, "w1", 0.0, 1.0) == pytest.approx(error + drift) -def test_a_request_to_walk_past_an_end_covers_every_band_beyond_it(): - two_band = RuleSet( +def test_a_request_to_walk_past_the_real_end_covers_what_lies_beyond_it(): + wide = RuleSet( RULES.width_ft, RULES.depth_ft, - RULES.needs | {"gas_clearance": (Need("ground", 1.0), Need("wall", 1.0, height=6.5))}, + RULES.needs | {"gas_clearance": (Need("ground", 1.0), Need("wall", 3.0, height=6.5))}, RULES.errors, ) scene = copy.deepcopy(SCENE) - scene["coverage"]["observed"][0]["span_ft"] = [1.0, 10.0] # the wall ends at 1, unexplored + scene["walls"][0]["baseline"] = [[-1.0, 0.0], [12.0, 0.0]] # the chain's left end is s = -1 + scene["coverage"]["ends"]["left"] = {"kind": "unexplored"} + scene["coverage"]["observed"][0]["span_ft"] = [-1.0, 10.0] r = result(checks=[check("unsure", None, cause="unobserved")]) r["reasons"] = [{"code": "unobserved_area", "message": ""}] r["missing_evidence"] = [ @@ -751,12 +754,45 @@ def test_a_request_to_walk_past_an_end_covers_every_band_beyond_it(): "span_ft": [0, 5], "checks": ["gas_clearance"], "message": "", - }, + } ] - assert any( - "no wall request names it" in m for m in missing_evidence_problems(scene, r, two_band) - ) - r["missing_evidence"].append( - {"kind": "past_end", "side": "left", "span_ft": [1.0, 1.0], "message": ""} - ) - assert missing_evidence_problems(scene, r, two_band) == [] + # The spot [1, 3.58] needs the wall over [-2, 6.58]; [-2, -1] lies past the end. + assert any("no wall request names it" in m for m in missing_evidence_problems(scene, r, wide)) + + def past_end(at: float) -> dict: + return {"kind": "past_end", "side": "left", "span_ft": [at, at], "message": ""} + + asked = r | {"missing_evidence": [*r["missing_evidence"], past_end(-1.0)]} + assert missing_evidence_problems(scene, asked, wide) == [] + elsewhere = r | {"missing_evidence": [*r["missing_evidence"], past_end(1.0)]} + assert missing_evidence_problems(scene, elsewhere, wide) != [] # not the chain's end + scene["coverage"]["ends"]["left"] = {"kind": "limit"} + assert missing_evidence_problems(scene, asked, wide) != [] # nothing to walk past + + +def test_chain_ends_are_measured_from_the_meter(): + scene = { + "meter": {"pos": [2.0, 4.0, 0.3], "wall_id": "b"}, + "walls": [ + {"id": "a", "baseline": [[-5.0, 3.0], [-5.0, 0.0]]}, + {"id": "b", "baseline": [[-5.0, 0.0], [5.0, 0.0]]}, + ], + } + assert chain_ends_s(scene) == pytest.approx((-10.0, 3.0)) + + +def test_a_view_that_only_reaches_the_needed_height_is_not_higher(): + r = result(checks=[check(), passing("opening_clearance")]) + msgs = invariant_problems(with_wall_seen(6.5), r, rules=RULES) + assert any("observed higher than 6.5 ft" in m for m in msgs) + assert invariant_problems(with_wall_seen(6.51), r, rules=RULES) == [] + + +def test_a_facing_view_that_only_reaches_the_needed_depth_is_not_beyond(): + r = result(checks=[check(), passing("facing_gap")]) + scene = copy.deepcopy(SCENE) + need = 22 / 12 + 3.0 + scene["coverage"]["observed"].append({"band": "facing", "span_ft": [0.0, 5.0], "out_ft": need}) + assert any("facing [1.00, 3.58]" in m for m in invariant_problems(scene, r, rules=BAND_RULES)) + scene["coverage"]["observed"][-1]["out_ft"] = need + 0.01 + assert invariant_problems(scene, r, rules=BAND_RULES) == [] From 3e953732277e8a72ece1546758db63d31e6adc81 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 23:18:20 -0500 Subject: [PATCH 42/59] verification: check each sweep start alone; survive malformed answers and shared builds - A passing sweep run is checked start by start, at the rules' sweep.step_ft, each with its own battery's position error. As one battery at its farthest start's error the run asked more ground than any start needs and could flag a correct server. - A 200 whose body is not a JSON object is a failure in the report, not a crash. - replaycheck reports malformed intrinsics, poses and a half-set spacing gate as format errors. - The scoreboard skips a report removed while it lists them, and gives `gh pr list` 60 s. - Each Simulator run installs its own copy of the app, taken under the build lock, so a later build into the shared DerivedData cannot replace it. - The cases README says which files the generator owns. At t3/server 14dd051 all 50 scenes pass. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/e2e/cases/README.md | 5 +- verification/hsverify/a11yaudit.py | 1 + verification/hsverify/e2e.py | 38 +++++++++++---- verification/hsverify/replaycheck.py | 22 ++++++++- verification/hsverify/resultcheck.py | 39 ++++++++++++++-- verification/hsverify/scoreboard.py | 28 ++++++++--- verification/hsverify/simrun.py | 19 +++++++- verification/tests/test_e2e.py | 12 +++++ verification/tests/test_replaycheck.py | 12 +++++ verification/tests/test_resultcheck.py | 64 ++++++++++++++++++++------ verification/tests/test_scoreboard.py | 8 ++++ verification/tests/test_simrun.py | 26 +++++++++++ 12 files changed, 233 insertions(+), 41 deletions(-) diff --git a/verification/e2e/cases/README.md b/verification/e2e/cases/README.md index 8a56a82aa..cdc9b6f2f 100644 --- a/verification/e2e/cases/README.md +++ b/verification/e2e/cases/README.md @@ -8,8 +8,9 @@ at origin/t3/server 2c9348f, which is public configuration. Nobody read the serv or fixtures to write them, so a passing server has been checked against the spec, not against its own tests. -`generate.py` writes every JSON file in this folder. Edit it, not the JSON, then run from -`verification/`: +`generate.py` writes the files marked `"generated_by": "e2e/cases/generate.py"`. Edit it, not +those files, then run from `verification/`. A hand-written case without that marker survives +regeneration, but it must not use a generated case's id: ```sh uv run python e2e/cases/generate.py diff --git a/verification/hsverify/a11yaudit.py b/verification/hsverify/a11yaudit.py index 48abb4681..f02b604bb 100644 --- a/verification/hsverify/a11yaudit.py +++ b/verification/hsverify/a11yaudit.py @@ -229,6 +229,7 @@ def main(argv: list[str] | None = None) -> int: simrun.prepare_display(udid, args.appearance, args.content_size) subprocess.run(["xcrun", "simctl", "uninstall", udid, bundle_id], capture_output=True) simrun.simctl("install", udid, str(app)) + simrun.discard_app(app) subprocess.run( ["xcrun", "simctl", "privacy", udid, "grant", "camera", bundle_id], capture_output=True ) diff --git a/verification/hsverify/e2e.py b/verification/hsverify/e2e.py index da6a20f3d..87e8bde35 100644 --- a/verification/hsverify/e2e.py +++ b/verification/hsverify/e2e.py @@ -13,9 +13,10 @@ stay within fixed entry and size limits, checked before reading. Each scene is first validated against the scene schema published at the same ref, so a bad -input is reported as an input problem, not a server bug. Each response must validate against -the result schema and pass the invariants in `resultcheck`, with rules read from the ref's -`rules.yaml`. Case files add the outcomes their geometry forces. Every scene is also resent +input is reported as an input problem, not a server bug. Each response must be a JSON object +that validates against the result schema (anything else is a failure in the report, not a +crash) and passes the invariants in `resultcheck`, with rules read from the ref's `rules.yaml`. +Case files add the outcomes their geometry forces. Every scene is also resent changed, and the answers must stay ordered: again (identical apart from timing), mirrored (same decision and outcome lengths), without or with less coverage, with ground trimmed just short of the largest clearance, with more error or tape re-measured by tap (nothing improves), and @@ -397,6 +398,16 @@ def variant(item: SceneInput, name: str, scene: dict) -> SceneInput: ) +def parse_result(payload: bytes) -> dict | None: + """The body as a JSON object, or None: a server's malformed answer is a failure to report, + not a reason to stop the run before the report is written.""" + try: + body = json.loads(payload) + except ValueError: + return None + return body if isinstance(body, dict) else None + + def answer(url: str, endpoint: Endpoint, item: SceneInput) -> tuple[dict | None, str | None]: """The server's result for a scene, or why there is none.""" status, payload, _ = post(url, endpoint, item) @@ -404,7 +415,10 @@ def answer(url: str, endpoint: Endpoint, item: SceneInput) -> tuple[dict | None, return None, f"no answer: {payload.decode(errors='replace')[:200]}" if status != 200: return None, f"HTTP {status}: {payload[:300]!r}" - return json.loads(payload), None + result = parse_result(payload) + if result is None: + return None, f"HTTP 200 with a body that is not a JSON object: {payload[:200]!r}" + return result, None def judge( @@ -439,7 +453,10 @@ def judge( else f"no answer within 60 s: {payload[:200]!r}" ] return record | {"status": "fail", "problems": failure, "contract_problems": failure} - result = json.loads(payload) + result = parse_result(payload) + if result is None: + failure = [f"HTTP 200 with a body that is not a JSON object: {payload[:200]!r}"] + return record | {"status": "fail", "problems": failure, "contract_problems": failure} record["result"] = result problems = [f"result schema: {e}" for e in schema_errors(result, schemas["result"])] if problems: @@ -487,7 +504,8 @@ def resend(name: str, scene: dict) -> dict | None: # 1. Same input, same answer. status, payload, _ = post(url, endpoint, item) - if status != 200 or comparable(json.loads(payload)) != comparable(result): + again = parse_result(payload) if status == 200 else None + if again is None or comparable(again) != comparable(result): problems.append("sending the same scene twice gave different results") # 2. Mirrored left to right: same decision, same amount of each outcome along the wall. @@ -661,8 +679,12 @@ def judge_hostile( elif status in (400, 413, 422): failure = None elif status == 200: - errors = schema_errors(json.loads(payload), schemas["result"]) - failure = f"result schema: {errors[0]}" if errors else None + result = parse_result(payload) + if result is None: + failure = f"HTTP 200 with a body that is not a JSON object: {payload[:200]!r}" + else: + errors = schema_errors(result, schemas["result"]) + failure = f"result schema: {errors[0]}" if errors else None else: failure = f"HTTP {status}: {payload[:200]!r}" problems = [failure] if failure else [] diff --git a/verification/hsverify/replaycheck.py b/verification/hsverify/replaycheck.py index 87d1ab05f..eaf415414 100644 --- a/verification/hsverify/replaycheck.py +++ b/verification/hsverify/replaycheck.py @@ -24,9 +24,11 @@ import argparse import json +import math import struct import sys from pathlib import Path +from typing import Any import numpy as np @@ -38,6 +40,15 @@ def pose_matrix(pose: list[float]) -> np.ndarray: return np.asarray(pose, dtype=float).reshape(4, 4).T +def numbers(value: Any, count: int) -> list[float] | None: + """`value` as `count` finite numbers, or None if it is anything else.""" + if not isinstance(value, list) or len(value) != count: + return None + if not all(isinstance(v, (int, float)) and not isinstance(v, bool) for v in value): + return None + return [float(v) for v in value] if all(math.isfinite(v) for v in value) else None + + def rigid_problems(m: np.ndarray, tol: float = 1e-3) -> list[str]: problems = [] r = m[:3, :3] @@ -92,6 +103,9 @@ def check_format(folder: Path, session: dict) -> tuple[list[str], dict]: gates = session.get("gates", {}) spacing_m = gates.get("keyframeSpacingMeters") spacing_deg = gates.get("keyframeSpacingDegrees") + if spacing_m and spacing_deg is None: + errors.append("gates set keyframeSpacingMeters without keyframeSpacingDegrees") + spacing_m = None previous = None previous_motion = None gate_violations = 0 @@ -113,10 +127,14 @@ def check_format(folder: Path, session: dict) -> tuple[list[str], dict]: errors.append(f"{name}: JPEG is {size}, session says {(kf['w'], kf['h'])}") except ValueError as exc: errors.append(f"{name}: {exc}") - fx, fy, cx, cy = kf["intrinsics"] + intrinsics, pose = numbers(kf["intrinsics"], 4), numbers(kf["pose"], 16) + if intrinsics is None or pose is None: + errors.append(f"{name}: intrinsics must be 4 finite numbers and pose 16") + continue + fx, fy, cx, cy = intrinsics if not (fx > 0 and fy > 0 and 0 < cx < kf["w"] and 0 < cy < kf["h"]): errors.append(f"{name}: intrinsics {kf['intrinsics']} outside the image") - m = pose_matrix(kf["pose"]) + m = pose_matrix(pose) errors += [f"{name}: {p}" for p in rigid_problems(m)] if previous is not None and kf["timestamp"] <= previous["timestamp"]: errors.append(f"{name}: timestamp does not increase") diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 7f85fb514..ea4f80c82 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -78,6 +78,9 @@ class RuleSet: errors: dict[str, float] # the height up the wall the cable route must be seen, from the meter to the battery route_height_ft: float = 0.0 + # the distance between the battery starts the server evaluates (sweep.step_ft); a sweep run + # lists evaluated starts from its first one at this step + step_ft: float | None = None @classmethod def from_yaml(cls, text: str) -> RuleSet: @@ -126,7 +129,14 @@ def reach(key: str) -> Need: for name in ("tap", "vlm", "tape", "wall", "mesh", "plane", "meter") } errors["drift_per_ft"] = value("errors", "drift_per_ft") - return cls(value("battery", "width_ft"), depth, needs, errors, value("route", "height_ft")) + return cls( + value("battery", "width_ft"), + depth, + needs, + errors, + value("route", "height_ft"), + value("sweep", "step_ft"), + ) def schema_errors(instance: Any, schema: dict) -> list[str]: @@ -368,10 +378,21 @@ def measured_band_gap( return None +def run_starts(span: list[float], step_ft: float | None) -> list[float]: + """The starts a sweep run stands for: its first, then every step to its last.""" + if step_ft is None or step_ft <= 0: + raise ValueError("checking a sweep run needs the rules' sweep.step_ft") + a, b = span + count = max(0, round((b - a) / step_ft)) + return [a + k * step_ft for k in range(count)] + [b] + + def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: """Missing coverage is never a pass (C5), to the reach each check's rule looks out to (see - reach_gaps). A sweep run that passes needs every check's area for every start in it, and the - wall and cable route back to the meter. No slack: runs list exactly the starts that were + reach_gaps). A sweep run that passes needs, for each start in it, every check's area around + that start's battery with that battery's own position error, and the wall and cable route + back to the meter. Checking the whole run as one battery at its farthest start's error would + ask for more than any single start needs. No slack: runs list exactly the starts that were evaluated.""" problems: list[str] = [] route_wall = observed(scene, "wall", min_out_ft=rules.route_height_ft, beyond=True) @@ -393,8 +414,16 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: f"{rules.route_height_ft} ft" ) for check in sorted(set(rules.needs) & evaluated): - for _, gap in reach_gaps(scene, rules, check, run["wall_id"], lo, hi): - problems.append(f"sweep pass for starts {run['start_ft']} but {check} needs {gap}") + gaps = {} # each gap once, at the first start that needs it + for start in run_starts(run["start_ft"], rules.step_ft): + battery = (start, start + rules.width_ft) + for _, gap in reach_gaps(scene, rules, check, run["wall_id"], *battery): + gaps.setdefault(gap, start) + problems += [ + f"sweep pass for starts {run['start_ft']} but at start {start:.2f} {check} " + f"needs {gap}" + for gap, start in gaps.items() + ] spot = result.get("spot") if spot is not None: diff --git a/verification/hsverify/scoreboard.py b/verification/hsverify/scoreboard.py index 078c87c15..3cca83fe9 100644 --- a/verification/hsverify/scoreboard.py +++ b/verification/hsverify/scoreboard.py @@ -470,11 +470,13 @@ def _value_or_none(data: dict, key: str) -> Any: def probe_report(probe: dict, sha: str | None) -> Outcome: - paths = sorted( - (Path(p) for p in globlib.glob(os.path.expanduser(probe["glob"]))), - key=lambda p: (p.stat().st_mtime, str(p)), - reverse=True, - ) + stamped = [] + for p in map(Path, globlib.glob(os.path.expanduser(probe["glob"]))): + try: # other runs write and remove reports while the board is built + stamped.append((p.stat().st_mtime, str(p), p)) + except OSError: + continue + paths = [p for *_, p in sorted(stamped, reverse=True)] if not paths: return Outcome("no evidence yet", f"no report matches {probe['glob']}") reports = [(p, _read_report(p)) for p in paths] @@ -742,12 +744,17 @@ def render_markdown(board: Board) -> str: # --- I/O ------------------------------------------------------------------------------------- +# gh can wait indefinitely for the network or a login; the board is written without PRs instead. +PR_LIST_TIMEOUT_S = 60 + + def list_prs() -> list[dict]: out = subprocess.run( ["gh", "pr", "list", "--repo", GITHUB_REPO, "--state", "open", "--json", PR_FIELDS], check=True, capture_output=True, text=True, + timeout=PR_LIST_TIMEOUT_S, ).stdout return json.loads(out) @@ -782,9 +789,16 @@ def main(argv: list[str] | None = None) -> int: pr_error = None try: prs = list_prs() - except (OSError, subprocess.CalledProcessError, json.JSONDecodeError) as e: + except ( + OSError, + subprocess.CalledProcessError, + subprocess.TimeoutExpired, + json.JSONDecodeError, + ) as e: prs = None - pr_error = (getattr(e, "stderr", "") or str(e)).strip() + # A timeout's stderr is bytes even with text=True; its own message says what happened. + stderr = getattr(e, "stderr", None) + pr_error = (stderr if isinstance(stderr, str) and stderr.strip() else str(e)).strip() print(f"warning: could not list PRs: {pr_error}", file=sys.stderr) with cached(CACHE_PATH) as cache: ctx = Context( diff --git a/verification/hsverify/simrun.py b/verification/hsverify/simrun.py index e7da0ef76..dde3ee4cb 100644 --- a/verification/hsverify/simrun.py +++ b/verification/hsverify/simrun.py @@ -52,6 +52,7 @@ DEFAULT_REPORTS = Path.home() / "house-scanning-data" / "reports" / "sim" DERIVED_DATA = Path("/tmp/hs-verify-derived-data") BUILD_LOCK = Path("/tmp/hs-verify-xcodebuild.lock") +BUILT_APP_DIR = "built-app" # in the report folder, until the Simulator has installed it @dataclass @@ -205,6 +206,14 @@ def build_app( with log_path.open("w") as log: code = subprocess.run(cmd, stdout=log, stderr=subprocess.STDOUT).returncode seconds = time.monotonic() - start + # DerivedData is shared: once the lock is released another run may rebuild over this + # app before it is installed, so this run installs its own copy. + built = DERIVED_DATA / "Build" / "Products" / "Debug-iphonesimulator" / f"{scheme}.app" + app = None + if code == 0 and built.exists(): + app = out / BUILT_APP_DIR / built.name + shutil.rmtree(app.parent, ignore_errors=True) + shutil.copytree(built, app, symlinks=True) text = log_path.read_text(errors="replace") warnings = sorted(set(re.findall(r"^.*: warning: .*$", text, re.M))) errors = sorted(set(re.findall(r"^.*: error: .*$", text, re.M))) @@ -217,8 +226,13 @@ def build_app( "errors": [shorten(e, tree) for e in errors], "log": log_path.name, } - app = DERIVED_DATA / "Build" / "Products" / "Debug-iphonesimulator" / f"{scheme}.app" - return result, (app if code == 0 and app.exists() else None) + return result, app + + +def discard_app(app: Path) -> None: + """Remove this run's copy of the app once the Simulator has installed its own.""" + if app.parent.name == BUILT_APP_DIR: + shutil.rmtree(app.parent, ignore_errors=True) def bundle_id_of(app: Path) -> str: @@ -615,6 +629,7 @@ def main(argv: list[str] | None = None) -> int: prepare_display(udid, args.appearance, args.content_size) subprocess.run(["xcrun", "simctl", "uninstall", udid, bundle_id], capture_output=True) simctl("install", udid, str(app)) + discard_app(app) # The Simulator has no camera, but granting access keeps the permission prompt from # hiding the replay flow. The prompt itself is reviewed on a device. subprocess.run( diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index f607bc895..6675779f0 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -39,6 +39,7 @@ "meter": 0.3, "drift_per_ft": 0.16, }, + step_ft=1 / 6, # the server's 2 in ) SCENE = { @@ -441,3 +442,14 @@ def test_a_bundle_with_too_many_entries_is_refused(tmp_path, monkeypatch): z.writestr(f"k{i}.jpg", b"x") with pytest.raises(SystemExit, match="4 entries, limit 3"): load_input(archive) + + +def test_a_200_that_is_not_a_json_object_is_a_failure_not_a_crash(monkeypatch): + for body in (b"oops", b"[1, 2]"): + monkeypatch.setattr(e2e, "post", lambda *args, body=body, **kwargs: (200, body, 5.0)) + endpoint = Endpoint("/place", "application/json") + record = judge("http://unused", endpoint, item(), SCHEMAS, RULES) + assert record["status"] == "fail" + assert "not a JSON object" in record["contract_problems"][0] + hostile = judge_hostile("http://unused", endpoint, hostile_inputs()[3], SCHEMAS) + assert hostile["status"] == "fail" and "not a JSON object" in hostile["problems"][0] diff --git a/verification/tests/test_replaycheck.py b/verification/tests/test_replaycheck.py index d4a3c2be3..dd830e2e3 100644 --- a/verification/tests/test_replaycheck.py +++ b/verification/tests/test_replaycheck.py @@ -107,6 +107,18 @@ def test_format_errors_are_specific(tmp_path): assert "1 motion keyframes closer than the declared spacing gate" in joined +def test_malformed_fields_are_format_errors_not_crashes(tmp_path): + s = session_with(tmp_path, [[0, 0, -0.6 * i] for i in range(4)]) + del s["gates"]["keyframeSpacingDegrees"] + s["keyframes"][1]["intrinsics"] = [500, 500, 320] + s["keyframes"][2]["pose"] = s["keyframes"][2]["pose"][:15] + s["keyframes"][3]["intrinsics"] = [500, "500", 320, 240] + errors, _ = check_format(tmp_path, s) + assert "gates set keyframeSpacingMeters without keyframeSpacingDegrees" in errors + for name in ("k2", "k3", "k4"): + assert f"{name}: intrinsics must be 4 finite numbers and pose 16" in errors + + @pytest.mark.parametrize("scale", [1.0, 0.64]) def test_accuracy_separates_scale_from_alignment(tmp_path, scale): truth_pos = [[0.0, 0.0, -float(i)] for i in range(40)] # straight walk along -z diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index ffdd68910..4c0bdcf79 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -1,6 +1,7 @@ import copy import hashlib import json +from dataclasses import replace import pytest @@ -11,6 +12,7 @@ battery_error, chain_ends_s, comparable, + coverage_problems, expectation_problems, invariant_problems, less_coverage_problems, @@ -21,6 +23,7 @@ observed, outcome_at, outcome_lengths, + run_starts, tape_to_tap, with_ground_short_of, with_less_coverage, @@ -47,6 +50,7 @@ "meter": 0.3, "drift_per_ft": 0.16, }, + step_ft=1 / 6, # the server's 2 in ) SCENE = { @@ -344,11 +348,9 @@ def test_the_reach_widens_by_the_battery_position_error(): def test_a_clearance_is_named_under_each_band_it_lacks(): - two_band = RuleSet( - RULES.width_ft, - RULES.depth_ft, - RULES.needs | {"gas_clearance": (Need("ground", 1.0), Need("wall", 1.0, height=6.5))}, - RULES.errors, + two_band = replace( + RULES, + needs=RULES.needs | {"gas_clearance": (Need("ground", 1.0), Need("wall", 1.0, height=6.5))}, ) scene = copy.deepcopy(SCENE) scene["coverage"]["observed"] = [] # nothing seen: both bands missing around the spot @@ -376,6 +378,7 @@ def test_rules_come_from_rules_yaml_and_a_missing_one_is_loud(): facing: {min_ft: {value: 3}} headroom: {min_ft: {value: 6.5}} route: {height_ft: {value: 1}} +sweep: {step_ft: {value: 0.166666666667}} """ rules = RuleSet.from_yaml(text) assert rules.needs["pool_clearance"] == (Need("ground", 10.0),) @@ -664,16 +667,14 @@ def test_a_request_to_see_higher_than_the_view_reached_is_not_redundant(): assert any("lists as observed" in m for m in full) -BAND_RULES = RuleSet( - RULES.width_ft, - RULES.depth_ft, - RULES.needs +BAND_RULES = replace( + RULES, + needs=RULES.needs | { "facing_gap": (Need("facing", 0.0, 22 / 12 + 3.0, widen=False),), "headroom": (Need("overhead", 0.0, 6.5, widen=False),), "battery_clearance": (Need("ground", 100.0),), }, - RULES.errors, ) @@ -735,11 +736,9 @@ def test_the_wall_source_sets_the_default_error(wall, error): def test_a_request_to_walk_past_the_real_end_covers_what_lies_beyond_it(): - wide = RuleSet( - RULES.width_ft, - RULES.depth_ft, - RULES.needs | {"gas_clearance": (Need("ground", 1.0), Need("wall", 3.0, height=6.5))}, - RULES.errors, + wide = replace( + RULES, + needs=RULES.needs | {"gas_clearance": (Need("ground", 1.0), Need("wall", 3.0, height=6.5))}, ) scene = copy.deepcopy(SCENE) scene["walls"][0]["baseline"] = [[-1.0, 0.0], [12.0, 0.0]] # the chain's left end is s = -1 @@ -796,3 +795,38 @@ def test_a_facing_view_that_only_reaches_the_needed_depth_is_not_beyond(): assert any("facing [1.00, 3.58]" in m for m in invariant_problems(scene, r, rules=BAND_RULES)) scene["coverage"]["observed"][-1]["out_ft"] = need + 0.01 assert invariant_problems(scene, r, rules=BAND_RULES) == [] + + +def test_each_start_in_a_sweep_run_needs_only_its_own_reach(): + # A pool radius of 10 ft with the default wall error. Start 1's battery [1, 3.58] is 0.87 ft + # uncertain, so it needs ground out to 1.83 + 10 + 0.87 = 12.71 ft; start 10's is 2.31 ft + # uncertain. As one battery at start 10's error, the run would ask 14.15 ft in front of start 1. + pool = replace(RULES, needs={"pool_clearance": (Need("ground", 10.0),)}) + scene = { + "meter": {"pos": [0.0, 4.0, 0.0], "wall_id": "w1"}, + "walls": [{"id": "w1", "baseline": [[-40.0, 0.0], [40.0, 0.0]]}], + "objects": [], + "coverage": { + "ends": {"left": {"kind": "limit"}, "right": {"kind": "limit"}}, + "observed": [ + {"band": "wall", "span_ft": [-40.0, 40.0]}, + {"band": "ground", "span_ft": [-40.0, 2.0], "out_ft": 13.5}, + {"band": "ground", "span_ft": [2.0, 40.0], "out_ft": 25.0}, + ], + }, + } + run = {"wall_id": "w1", "start_ft": [1.0, 10.0], "outcome": "pass", "failing": [], "unsure": []} + r = result(spot=False, checks=[check() | {"id": "pool_clearance"}], sweep=[run]) + assert coverage_problems(scene, r, pool) == [] + scene["coverage"]["observed"][1]["out_ft"] = 12.5 + assert any( + "at start 1.00 pool_clearance needs ground [1.00, 2.00] observed out to 12.71 ft" in m + for m in coverage_problems(scene, r, pool) + ) + + +def test_a_sweep_run_stands_for_its_starts_at_the_rules_step(): + assert run_starts([1.0, 1.5], 1 / 6) == pytest.approx([1.0, 7 / 6, 8 / 6, 1.5]) + assert run_starts([2.0, 2.0], 1 / 6) == [2.0] + with pytest.raises(ValueError, match=r"sweep\.step_ft"): + run_starts([1.0, 2.0], None) diff --git a/verification/tests/test_scoreboard.py b/verification/tests/test_scoreboard.py index 5b433f6cd..21292cb27 100644 --- a/verification/tests/test_scoreboard.py +++ b/verification/tests/test_scoreboard.py @@ -237,6 +237,14 @@ def test_manual_and_sha_free_report_ignore_missing_ref(tmp_path): assert evaluate(metric("report", **probe), None, ctx()).status == "no evidence yet" +def test_a_report_removed_while_the_board_lists_them_is_skipped(tmp_path, monkeypatch): + (tmp_path / "kept.json").write_text(json.dumps({"k": 1})) + listed = [str(tmp_path / "removed.json"), str(tmp_path / "kept.json")] + monkeypatch.setattr(sb.globlib, "glob", lambda pattern: listed) + probe = {"glob": str(tmp_path / "*.json"), "key": "k", "equals": 1} + assert evaluate(metric("report", **probe), None, ctx()).status == "met" + + # --- file_at_ref and grep_at_ref ------------------------------------------------------------- diff --git a/verification/tests/test_simrun.py b/verification/tests/test_simrun.py index e52fa07eb..515f29828 100644 --- a/verification/tests/test_simrun.py +++ b/verification/tests/test_simrun.py @@ -2,8 +2,10 @@ import io import json import queue +import subprocess from pathlib import Path +from hsverify import simrun from hsverify.simrun import ( ShotRecord, app_export, @@ -160,3 +162,27 @@ def test_a_run_that_must_keep_the_scan_fails_without_one(tmp_path): assert app_export(tmp_path, required=False) == (None, []) name, problems = app_export(tmp_path, required=True) assert name is None and "--require-export" in problems[0] + + +def test_a_run_installs_its_own_copy_of_the_app_it_built(tmp_path, monkeypatch): + derived = tmp_path / "derived" + built = derived / "Build" / "Products" / "Debug-iphonesimulator" / "HouseScan.app" + monkeypatch.setattr(simrun, "DERIVED_DATA", derived) + monkeypatch.setattr(simrun, "BUILD_LOCK", tmp_path / "lock") + monkeypatch.setattr(simrun, "wait_for_other_builds", lambda max_wait_s: 0.0) + + def fake_build(cmd, **kwargs): + built.mkdir(parents=True, exist_ok=True) + (built / "Info.plist").write_text("run A") + return subprocess.CompletedProcess(cmd, 0) + + monkeypatch.setattr(simrun.subprocess, "run", fake_build) + tree, out = tmp_path / "tree", tmp_path / "report" + (tree / "ios" / "HouseScan.xcodeproj").mkdir(parents=True) + out.mkdir() + result, app = simrun.build_app(tree, out, 0, "ios/HouseScan.xcodeproj", "HouseScan") + assert result["ok"] and app is not None + (built / "Info.plist").write_text("run B") # another run rebuilds after the lock is released + assert (app / "Info.plist").read_text() == "run A" + simrun.discard_app(app) + assert not app.parent.exists() and out.exists() From 544db3bef37d8ff2a13e27c2f2d77c22a7e6edc5 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 23:22:39 -0500 Subject: [PATCH 43/59] verification: say that sweep coverage samples starts, not the server's list A passing run is checked at its first start, every sweep.step_ft after it and its last. The server also evaluates boundaries and midpoints that a run reports only as its ends, so these are a sample of its starts, not the starts it evaluated. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/hsverify/resultcheck.py | 17 +++++++++-------- verification/tests/test_resultcheck.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index ea4f80c82..128db6a12 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -78,8 +78,7 @@ class RuleSet: errors: dict[str, float] # the height up the wall the cable route must be seen, from the meter to the battery route_height_ft: float = 0.0 - # the distance between the battery starts the server evaluates (sweep.step_ft); a sweep run - # lists evaluated starts from its first one at this step + # the rules' sweep.step_ft: the spacing at which a passing sweep run's starts are sampled step_ft: float | None = None @classmethod @@ -379,7 +378,10 @@ def measured_band_gap( def run_starts(span: list[float], step_ft: float | None) -> list[float]: - """The starts a sweep run stands for: its first, then every step to its last.""" + """Starts sampled from a sweep run: its first, every step after it, and its last. + + A sample, not the server's own list: the server also evaluates boundaries and midpoints + that a run reports only as its two ends.""" if step_ft is None or step_ft <= 0: raise ValueError("checking a sweep run needs the rules' sweep.step_ft") a, b = span @@ -389,11 +391,10 @@ def run_starts(span: list[float], step_ft: float | None) -> list[float]: def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: """Missing coverage is never a pass (C5), to the reach each check's rule looks out to (see - reach_gaps). A sweep run that passes needs, for each start in it, every check's area around - that start's battery with that battery's own position error, and the wall and cable route - back to the meter. Checking the whole run as one battery at its farthest start's error would - ask for more than any single start needs. No slack: runs list exactly the starts that were - evaluated.""" + reach_gaps). A sweep run that passes needs, at each start sampled from it (run_starts), every + check's area around that start's battery with that battery's own position error, and the + wall and cable route back to the meter over the whole run. Checking the whole run as one + battery at its farthest start's error would ask for more than any single start needs.""" problems: list[str] = [] route_wall = observed(scene, "wall", min_out_ft=rules.route_height_ft, beyond=True) # A passing run passes every check the server evaluated; a check it did not evaluate (a diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 4c0bdcf79..e420c6dcd 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -825,7 +825,7 @@ def test_each_start_in_a_sweep_run_needs_only_its_own_reach(): ) -def test_a_sweep_run_stands_for_its_starts_at_the_rules_step(): +def test_sweep_run_starts_are_sampled_at_the_rules_step_plus_the_ends(): assert run_starts([1.0, 1.5], 1 / 6) == pytest.approx([1.0, 7 / 6, 8 / 6, 1.5]) assert run_starts([2.0, 2.0], 1 / 6) == [2.0] with pytest.raises(ValueError, match=r"sweep\.step_ft"): From c114db26ff215289f7f57bc6a7823412d3d67336 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 23:50:29 -0500 Subject: [PATCH 44/59] verification: facing and headroom need the band past the battery by the wall's error Server 903d86f requires the facing and overhead bands over [s0 - e, s1 + e], with e the wall's own error without drift (`piece.plus_minus`). The coverage check now asks the same. Where two walls meet is left for S2's settled rule (#11) and marked TODO in chain_ends_s. Provisional e2e at 903d86f, not the acceptance SHA: all 50 scenes pass. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/hsverify/resultcheck.py | 44 ++++++++++++++++++-------- verification/tests/test_resultcheck.py | 19 ++++++++--- 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 128db6a12..0e2104736 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -48,16 +48,19 @@ class Need: - `ground`: ground over [lo - r, hi + r], out to depth + r in front of the battery; - `wall`: the wall band over [lo - r, hi + r], seen higher than `height`; - - `facing`, `overhead`: the band over [lo, hi]; where no measurement (`facing`, `overheads`) - covers a stretch, the band's `out_ft` there must reach `height`. + - `facing`, `overhead`: the band over [lo - r, hi + r]; where no measurement (`facing`, + `overheads`) covers a stretch, the band's `out_ft` there must be beyond `height`. - r is `radius`, plus the battery's position error when `widen`. + r is `radius`, plus the battery's position error when `widen`: with the drift for its + distance from the meter when `drift`, else the wall's own error alone (the server widens + facing and headroom by `piece.plus_minus`, server/solver.py at 903d86f). """ band: str radius: float = 0.0 height: float = 0.0 widen: bool = True + drift: bool = True # Scene lists whose entries settle facing_gap and headroom where they cover the battery. @@ -120,8 +123,8 @@ def reach(key: str) -> Need: "pool_clearance": (reach("pool_ft"),), "opening_clearance": (Need("wall", r("opening_ft"), opening_height),), "wall_equipment_above": (Need("wall", r("wall_equipment_ft"), headroom, widen=False),), - "facing_gap": (Need("facing", 0.0, depth + value("facing", "min_ft"), widen=False),), - "headroom": (Need("overhead", 0.0, headroom, widen=False),), + "facing_gap": (Need("facing", 0.0, depth + value("facing", "min_ft"), drift=False),), + "headroom": (Need("overhead", 0.0, headroom, drift=False),), } errors = { name: value("errors", f"{name}_ft") @@ -308,8 +311,12 @@ def battery_error(scene: dict, rules: RuleSet, wall_id: str, lo: float, hi: floa raise ValueError(f"the result names wall {wall_id!r}, which the scene does not have") if "plus_minus_ft" in wall: return wall["plus_minus_ft"] - default = rules.errors[WALL_ERROR[wall.get("source", "tap")]] - return default + rules.errors["drift_per_ft"] * max(abs(lo), abs(hi)) + return wall_error(wall, rules) + rules.errors["drift_per_ft"] * max(abs(lo), abs(hi)) + + +def wall_error(wall: dict, rules: RuleSet) -> float: + """A wall's own position error, without drift: its explicit one or its source's default.""" + return wall.get("plus_minus_ft", rules.errors[WALL_ERROR[wall.get("source", "tap")]]) def reach_gaps( @@ -327,12 +334,19 @@ def reach_gaps( - ground band: out to depth + R + e in front of the battery (a battery sits flush on one straight segment), and at distance d past either end out to R + e - d. The ground point there is at most d along the wall plus R + e - d out from the battery's end; - - facing and overhead bands: over the battery, reaching the check's distance unless - measured. + - facing and overhead bands: over [lo - e, hi + e], every place the battery may sit, beyond + the check's distance unless measured. """ gaps = [] + wall = next((w for w in scene["walls"] if w["id"] == wall_id), None) for need in rules.needs[check]: - reach = need.radius + (battery_error(scene, rules, wall_id, lo, hi) if need.widen else 0.0) + if not need.widen: + error = 0.0 + elif need.drift: + error = battery_error(scene, rules, wall_id, lo, hi) + else: + error = wall_error(wall, rules) if wall is not None else 0.0 + reach = need.radius + error if need.band == "ground": gap = ground_gap(scene, lo, hi, rules.depth_ft, reach) elif need.band == "wall": @@ -340,7 +354,8 @@ def reach_gaps( seen = covers(observed(scene, "wall", min_out_ft=need.height, beyond=True), a, b) gap = None if seen else f"wall [{a:.2f}, {b:.2f}] observed higher than {need.height} ft" else: - gap = measured_band_gap(scene, need.band, wall_id, lo, hi, need.height) + a, b = required_span(lo, hi, reach) + gap = measured_band_gap(scene, need.band, wall_id, a, b, need.height) if gap: gaps.append((need.band, gap)) return gaps @@ -517,7 +532,10 @@ def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = def chain_ends_s(scene: dict) -> tuple[float, float] | None: """s of the wall chain's left and right ends (the meter's projection is s = 0; distance - runs along every wall and across the gaps between them), or None without a meter wall.""" + runs along every wall and across the spaces between them), or None without a meter wall. + + TODO: skip the space between walls that meet, as the server does, once S2 settles when two + walls meet (#11); counting it now puts an end up to sweep.wall_join_ft too far out.""" walls = scene.get("walls", []) points, owner = [], [] for wall in walls: @@ -554,9 +572,9 @@ def with_past_ends_asked(scene: dict, requests: list[dict]) -> dict: server asks to keep walking (`past_end`) instead of for a band there, and that request covers every band beyond the end. Only a request at the chain's actual end, on a side not marked `limit`, earns that credit.""" - ends = chain_ends_s(scene) kinds = scene.get("coverage", {}).get("ends", {}) past = [r for r in requests if r["kind"] == "past_end" and "span_ft" in r and "side" in r] + ends = chain_ends_s(scene) if past else None if not past or ends is None: return scene m = copy.deepcopy(scene) diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index e420c6dcd..7bd6abd81 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -386,8 +386,8 @@ def test_rules_come_from_rules_yaml_and_a_missing_one_is_loud(): assert rules.needs["battery_clearance"] == (Need("ground", 3.0), Need("wall", 3.0, 6.5)) assert rules.needs["opening_clearance"] == (Need("wall", 3.0, 6.5),) assert rules.needs["wall_backing"] == (Need("wall", 0.0, 3.25, widen=False),) - assert rules.needs["facing_gap"] == (Need("facing", 0.0, 4.5, widen=False),) - assert rules.needs["headroom"] == (Need("overhead", 0.0, 6.5, widen=False),) + assert rules.needs["facing_gap"] == (Need("facing", 0.0, 4.5, drift=False),) + assert rules.needs["headroom"] == (Need("overhead", 0.0, 6.5, drift=False),) assert (rules.width_ft, rules.errors["plane"], rules.route_height_ft) == (2.5, 0.75, 1.0) lower = RuleSet.from_yaml( text.replace("exempt_bottom_above_ft: null", "exempt_bottom_above_ft: 2") @@ -671,8 +671,8 @@ def test_a_request_to_see_higher_than_the_view_reached_is_not_redundant(): RULES, needs=RULES.needs | { - "facing_gap": (Need("facing", 0.0, 22 / 12 + 3.0, widen=False),), - "headroom": (Need("overhead", 0.0, 6.5, widen=False),), + "facing_gap": (Need("facing", 0.0, 22 / 12 + 3.0, drift=False),), + "headroom": (Need("overhead", 0.0, 6.5, drift=False),), "battery_clearance": (Need("ground", 100.0),), }, ) @@ -830,3 +830,14 @@ def test_sweep_run_starts_are_sampled_at_the_rules_step_plus_the_ends(): assert run_starts([2.0, 2.0], 1 / 6) == [2.0] with pytest.raises(ValueError, match=r"sweep\.step_ft"): run_starts([1.0, 2.0], None) + + +def test_facing_and_headroom_need_the_band_past_the_battery_by_the_wall_error(): + # The spot [1, 3.58] on a wall with 0.3 ft of error needs facing seen over [0.7, 3.88]. + r = result(checks=[passing("facing_gap")]) + scene = copy.deepcopy(SCENE) + scene["walls"][0]["plus_minus_ft"] = 0.3 + scene["coverage"]["observed"].append({"band": "facing", "span_ft": [1.0, 3.6]}) + assert any("facing [0.70, 1.00]" in m for m in invariant_problems(scene, r, rules=BAND_RULES)) + scene["coverage"]["observed"][-1]["span_ft"] = [0.7, 3.9] + assert invariant_problems(scene, r, rules=BAND_RULES) == [] From 1b62f94d8fbd3d1d0d26bf0d054151d7468b7ec6 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 26 Sep 2026 23:54:46 -0500 Subject: [PATCH 45/59] verification: widen facing and headroom by the drifted error, as the server does At 903d86f the server's checks see the wall's error at the battery's far edge, drift included (solver.py evaluate() replaces piece.plus_minus with error_at), so c114db2's no-drift widening asked too little. The test is the caretaker's repro: default wall error, views over [5.7, 8.88], start 6 needs both bands from 4.33. Provisional e2e at 903d86f: all 50 scenes pass. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/hsverify/resultcheck.py | 27 +++++++------------------- verification/tests/test_resultcheck.py | 25 ++++++++++++++++++++---- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 0e2104736..46a61d595 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -51,16 +51,14 @@ class Need: - `facing`, `overhead`: the band over [lo - r, hi + r]; where no measurement (`facing`, `overheads`) covers a stretch, the band's `out_ft` there must be beyond `height`. - r is `radius`, plus the battery's position error when `widen`: with the drift for its - distance from the meter when `drift`, else the wall's own error alone (the server widens - facing and headroom by `piece.plus_minus`, server/solver.py at 903d86f). + r is `radius`, plus the battery's position error when `widen` (battery_error: the server's + checks see the wall's error at the battery's far edge, server/solver.py at 903d86f). """ band: str radius: float = 0.0 height: float = 0.0 widen: bool = True - drift: bool = True # Scene lists whose entries settle facing_gap and headroom where they cover the battery. @@ -123,8 +121,8 @@ def reach(key: str) -> Need: "pool_clearance": (reach("pool_ft"),), "opening_clearance": (Need("wall", r("opening_ft"), opening_height),), "wall_equipment_above": (Need("wall", r("wall_equipment_ft"), headroom, widen=False),), - "facing_gap": (Need("facing", 0.0, depth + value("facing", "min_ft"), drift=False),), - "headroom": (Need("overhead", 0.0, headroom, drift=False),), + "facing_gap": (Need("facing", 0.0, depth + value("facing", "min_ft")),), + "headroom": (Need("overhead", 0.0, headroom),), } errors = { name: value("errors", f"{name}_ft") @@ -311,12 +309,8 @@ def battery_error(scene: dict, rules: RuleSet, wall_id: str, lo: float, hi: floa raise ValueError(f"the result names wall {wall_id!r}, which the scene does not have") if "plus_minus_ft" in wall: return wall["plus_minus_ft"] - return wall_error(wall, rules) + rules.errors["drift_per_ft"] * max(abs(lo), abs(hi)) - - -def wall_error(wall: dict, rules: RuleSet) -> float: - """A wall's own position error, without drift: its explicit one or its source's default.""" - return wall.get("plus_minus_ft", rules.errors[WALL_ERROR[wall.get("source", "tap")]]) + default = rules.errors[WALL_ERROR[wall.get("source", "tap")]] + return default + rules.errors["drift_per_ft"] * max(abs(lo), abs(hi)) def reach_gaps( @@ -338,15 +332,8 @@ def reach_gaps( the check's distance unless measured. """ gaps = [] - wall = next((w for w in scene["walls"] if w["id"] == wall_id), None) for need in rules.needs[check]: - if not need.widen: - error = 0.0 - elif need.drift: - error = battery_error(scene, rules, wall_id, lo, hi) - else: - error = wall_error(wall, rules) if wall is not None else 0.0 - reach = need.radius + error + reach = need.radius + (battery_error(scene, rules, wall_id, lo, hi) if need.widen else 0.0) if need.band == "ground": gap = ground_gap(scene, lo, hi, rules.depth_ft, reach) elif need.band == "wall": diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 7bd6abd81..c18a511be 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -386,8 +386,8 @@ def test_rules_come_from_rules_yaml_and_a_missing_one_is_loud(): assert rules.needs["battery_clearance"] == (Need("ground", 3.0), Need("wall", 3.0, 6.5)) assert rules.needs["opening_clearance"] == (Need("wall", 3.0, 6.5),) assert rules.needs["wall_backing"] == (Need("wall", 0.0, 3.25, widen=False),) - assert rules.needs["facing_gap"] == (Need("facing", 0.0, 4.5, drift=False),) - assert rules.needs["headroom"] == (Need("overhead", 0.0, 6.5, drift=False),) + assert rules.needs["facing_gap"] == (Need("facing", 0.0, 4.5),) + assert rules.needs["headroom"] == (Need("overhead", 0.0, 6.5),) assert (rules.width_ft, rules.errors["plane"], rules.route_height_ft) == (2.5, 0.75, 1.0) lower = RuleSet.from_yaml( text.replace("exempt_bottom_above_ft: null", "exempt_bottom_above_ft: 2") @@ -671,8 +671,8 @@ def test_a_request_to_see_higher_than_the_view_reached_is_not_redundant(): RULES, needs=RULES.needs | { - "facing_gap": (Need("facing", 0.0, 22 / 12 + 3.0, drift=False),), - "headroom": (Need("overhead", 0.0, 6.5, drift=False),), + "facing_gap": (Need("facing", 0.0, 22 / 12 + 3.0),), + "headroom": (Need("overhead", 0.0, 6.5),), "battery_clearance": (Need("ground", 100.0),), }, ) @@ -841,3 +841,20 @@ def test_facing_and_headroom_need_the_band_past_the_battery_by_the_wall_error(): assert any("facing [0.70, 1.00]" in m for m in invariant_problems(scene, r, rules=BAND_RULES)) scene["coverage"]["observed"][-1]["span_ft"] = [0.7, 3.9] assert invariant_problems(scene, r, rules=BAND_RULES) == [] + + +def test_facing_and_headroom_widen_by_the_default_error_with_drift(): + # The caretaker's repro: no explicit wall error, views over [5.7, 8.88] out 9 ft. At start 6 + # the battery [6, 8.58] is 0.3 + 0.16 x 8.58 = 1.67 ft uncertain, so both bands are needed + # over [4.33, 10.25]; the error without drift (0.3) would ask only [5.7, 8.88]. + r = result(checks=[passing("facing_gap"), passing("headroom")], spot=False) + r["sweep"][0]["start_ft"] = [6.0, 6.0] + scene = copy.deepcopy(SCENE) + del scene["walls"][0]["plus_minus_ft"] + for band in ("facing", "overhead"): + scene["coverage"]["observed"].append( + {"band": band, "span_ft": [5.7, 6 + 31 / 12 + 0.3], "out_ft": 9.0} + ) + problems = coverage_problems(scene, r, BAND_RULES) + assert any("facing [4.33, 5.70] observed, none seen" in m for m in problems) + assert any("overhead [4.33, 5.70] observed, none seen" in m for m in problems) From de53ecd52a2af1350b7502ef6bfd62586e84ef01 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 00:16:26 -0500 Subject: [PATCH 46/59] verification: lay walls out and check declared heights as server 7133283 does - Two walls meet when the space between them is within both walls' own errors (no drift), capped at sweep.wall_join_ft, never below the server's float tolerance; s then skips the space. A wider space is a gap that s counts. chain_ends_s now uses this layout, so a past_end request at the server's end is credited. - wall_backing, on the spot or any sampled start of a passing sweep run, may pass only if every wall declaring height_ft within the battery's error is taller than battery.height_ft. e2e at 7133283, the final candidate: all 50 scenes pass, contract_ok. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/hsverify/resultcheck.py | 135 +++++++++++++++++++------ verification/tests/test_e2e.py | 2 + verification/tests/test_resultcheck.py | 74 +++++++++++++- 3 files changed, 176 insertions(+), 35 deletions(-) diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 46a61d595..b326c1870 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -81,6 +81,10 @@ class RuleSet: route_height_ft: float = 0.0 # the rules' sweep.step_ft: the spacing at which a passing sweep run's starts are sampled step_ft: float | None = None + # the rules' sweep.wall_join_ft: the most two walls' ends may be apart and still meet + wall_join_ft: float | None = None + # the rules' battery.height_ft: a wall that declares height_ft must be taller + battery_height_ft: float | None = None @classmethod def from_yaml(cls, text: str) -> RuleSet: @@ -136,6 +140,8 @@ def reach(key: str) -> Need: errors, value("route", "height_ft"), value("sweep", "step_ft"), + value("sweep", "wall_join_ft"), + value("battery", "height_ft"), ) @@ -287,6 +293,7 @@ def invariant_problems( problems += missing_evidence_problems(scene, result, rules) if rules is not None: problems += coverage_problems(scene, result, rules) + problems += declared_height_problems(scene, result, rules) return problems @@ -309,8 +316,12 @@ def battery_error(scene: dict, rules: RuleSet, wall_id: str, lo: float, hi: floa raise ValueError(f"the result names wall {wall_id!r}, which the scene does not have") if "plus_minus_ft" in wall: return wall["plus_minus_ft"] - default = rules.errors[WALL_ERROR[wall.get("source", "tap")]] - return default + rules.errors["drift_per_ft"] * max(abs(lo), abs(hi)) + return wall_error(wall, rules) + rules.errors["drift_per_ft"] * max(abs(lo), abs(hi)) + + +def wall_error(wall: dict, rules: RuleSet) -> float: + """A wall's own position error, without drift: its explicit one or its source's default.""" + return wall.get("plus_minus_ft", rules.errors[WALL_ERROR[wall.get("source", "tap")]]) def reach_gaps( @@ -501,7 +512,7 @@ def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = if rules is None or spot is None: return problems lo, hi = spot["span_ft"] - asked = with_past_ends_asked(scene, requests) + asked = with_past_ends_asked(scene, requests, rules) for c in unobserved_checks(result): if c not in rules.needs: continue @@ -517,42 +528,100 @@ def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = END_TOL_FT = 0.1 -def chain_ends_s(scene: dict) -> tuple[float, float] | None: - """s of the wall chain's left and right ends (the meter's projection is s = 0; distance - runs along every wall and across the spaces between them), or None without a meter wall. +# The server's float tolerance (server/scene.py EPS at 7133283), for comparisons that must +# agree with it exactly: where walls meet and whether a declared wall is taller. +SERVER_EPS = 1e-9 + - TODO: skip the space between walls that meet, as the server does, once S2 settles when two - walls meet (#11); counting it now puts an end up to sweep.wall_join_ft too far out.""" +def wall_spans_s(scene: dict, rules: RuleSet) -> dict[str, tuple[float, float]] | None: + """Each wall's stretch of s (the meter's projection is s = 0), or None without a meter wall. + + s runs along every wall. The space from one wall's end to the next one's start counts only + when it is a gap: wider than both walls' own errors together (no drift), capped at + sweep.wall_join_ft. Narrower, the walls meet and s skips the space (server/scene.py at + 7133283, `meets`).""" + if rules.wall_join_ft is None: + raise ValueError("laying out the walls needs the rules' sweep.wall_join_ft") walls = scene.get("walls", []) - points, owner = [], [] - for wall in walls: - for pt in wall["baseline"]: - points.append(tuple(pt)) - owner.append(wall["id"]) - if len(points) < 2 or "meter" not in scene: + if not walls or "meter" not in scene: return None - cum = [0.0] - for p, q in itertools.pairwise(points): - cum.append(cum[-1] + math.dist(p, q)) + raw: dict[str, tuple[float, float]] = {} + meter_s = None mx, _, mz = scene["meter"]["pos"] - best = None - for i, (p, q) in enumerate(itertools.pairwise(points)): - if owner[i] != scene["meter"]["wall_id"] or owner[i + 1] != owner[i]: - continue - seg = math.dist(p, q) - if seg == 0: - continue - t = ((mx - p[0]) * (q[0] - p[0]) + (mz - p[1]) * (q[1] - p[1])) / seg**2 - t = min(1.0, max(0.0, t)) - d = math.dist((mx, mz), (p[0] + t * (q[0] - p[0]), p[1] + t * (q[1] - p[1]))) - if best is None or d < best[0]: - best = (d, cum[i] + t * seg) - if best is None: + s, prev = 0.0, None + for wall in walls: + pts = [tuple(pt) for pt in wall["baseline"]] + if prev is not None: + space = math.dist(prev[0], pts[0]) + meets = min(rules.wall_join_ft, max(prev[1] + wall_error(wall, rules), SERVER_EPS)) + s += space if space > meets else 0.0 + start = s + for p, q in itertools.pairwise(pts): + seg = math.dist(p, q) + if wall["id"] == scene["meter"]["wall_id"] and seg > 0: + t = ((mx - p[0]) * (q[0] - p[0]) + (mz - p[1]) * (q[1] - p[1])) / seg**2 + t = min(1.0, max(0.0, t)) + d = math.dist((mx, mz), (p[0] + t * (q[0] - p[0]), p[1] + t * (q[1] - p[1]))) + if meter_s is None or d < meter_s[0]: + meter_s = (d, s + t * seg) + s += seg + raw[wall["id"]] = (start, s) + prev = (pts[-1], wall_error(wall, rules)) + if meter_s is None: + return None + return {wid: (a - meter_s[1], b - meter_s[1]) for wid, (a, b) in raw.items()} + + +def chain_ends_s(scene: dict, rules: RuleSet) -> tuple[float, float] | None: + """s of the wall chain's left and right ends, or None without a meter wall.""" + spans = wall_spans_s(scene, rules) + if spans is None: + return None + return min(a for a, _ in spans.values()), max(b for _, b in spans.values()) + + +def declared_height_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: + """A battery backs onto a wall that declares `height_ft` only if the wall is taller than the + battery. The server fails wall_backing for a lower wall surely behind the battery and leaves + it unsure for one within the battery's error of it (server/solver.py at 7133283, + check_backing), so wall_backing passes only if every declared wall overlapping + [s0 - e, s1 + e] is taller than battery.height_ft.""" + declared = {w["id"]: w["height_ft"] for w in scene.get("walls", []) if "height_ft" in w} + if not declared or rules.battery_height_ft is None: + return [] + spans = wall_spans_s(scene, rules) or {} + height = rules.battery_height_ft + + def short_near(wall_id: str, lo: float, hi: float) -> str | None: + e = battery_error(scene, rules, wall_id, lo, hi) + for wid, h in declared.items(): + a, b = spans.get(wid, (math.inf, -math.inf)) + near = a < hi + e - SERVER_EPS and b > lo - e + SERVER_EPS + if near and h - height <= SERVER_EPS: + return f"wall {wid} ({h} ft) is within {e:.2f} ft and not taller than {height} ft" return None - return -best[1], cum[-1] - best[1] + + problems = [] + spot = result.get("spot") + backing = next((c for c in result.get("checks", []) if c["id"] == "wall_backing"), None) + if spot is not None and backing is not None and backing["outcome"] == "pass": + short = short_near(spot["wall_id"], *spot["span_ft"]) + if short: + problems.append(f"check wall_backing passes but {short}") + for run in result.get("sweep", []): + if run["outcome"] != "pass": + continue + for start in run_starts(run["start_ft"], rules.step_ft): + short = short_near(run["wall_id"], start, start + rules.width_ft) + if short: + problems.append( + f"sweep pass for starts {run['start_ft']} but at {start:.2f} {short}" + ) + break + return problems -def with_past_ends_asked(scene: dict, requests: list[dict]) -> dict: +def with_past_ends_asked(scene: dict, requests: list[dict], rules: RuleSet) -> dict: """The scene as if everything past each end the result asks to walk past were observed. Past an unexplored end the wall may turn, so a view along the same line settles nothing; the @@ -561,7 +630,7 @@ def with_past_ends_asked(scene: dict, requests: list[dict]) -> dict: `limit`, earns that credit.""" kinds = scene.get("coverage", {}).get("ends", {}) past = [r for r in requests if r["kind"] == "past_end" and "span_ft" in r and "side" in r] - ends = chain_ends_s(scene) if past else None + ends = chain_ends_s(scene, rules) if past else None if not past or ends is None: return scene m = copy.deepcopy(scene) diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index 6675779f0..87ca6feb6 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -40,6 +40,8 @@ "drift_per_ft": 0.16, }, step_ft=1 / 6, # the server's 2 in + wall_join_ft=0.6, + battery_height_ft=3.25, ) SCENE = { diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index c18a511be..bcf5b9cb6 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -13,6 +13,7 @@ chain_ends_s, comparable, coverage_problems, + declared_height_problems, expectation_problems, invariant_problems, less_coverage_problems, @@ -51,6 +52,8 @@ "drift_per_ft": 0.16, }, step_ft=1 / 6, # the server's 2 in + wall_join_ft=0.6, + battery_height_ft=3.25, ) SCENE = { @@ -378,7 +381,7 @@ def test_rules_come_from_rules_yaml_and_a_missing_one_is_loud(): facing: {min_ft: {value: 3}} headroom: {min_ft: {value: 6.5}} route: {height_ft: {value: 1}} -sweep: {step_ft: {value: 0.166666666667}} +sweep: {step_ft: {value: 0.166666666667}, wall_join_ft: {value: 0.6}} """ rules = RuleSet.from_yaml(text) assert rules.needs["pool_clearance"] == (Need("ground", 10.0),) @@ -388,6 +391,7 @@ def test_rules_come_from_rules_yaml_and_a_missing_one_is_loud(): assert rules.needs["wall_backing"] == (Need("wall", 0.0, 3.25, widen=False),) assert rules.needs["facing_gap"] == (Need("facing", 0.0, 4.5),) assert rules.needs["headroom"] == (Need("overhead", 0.0, 6.5),) + assert (rules.wall_join_ft, rules.battery_height_ft) == (0.6, 3.25) assert (rules.width_ft, rules.errors["plane"], rules.route_height_ft) == (2.5, 0.75, 1.0) lower = RuleSet.from_yaml( text.replace("exempt_bottom_above_ft: null", "exempt_bottom_above_ft: 2") @@ -777,7 +781,7 @@ def test_chain_ends_are_measured_from_the_meter(): {"id": "b", "baseline": [[-5.0, 0.0], [5.0, 0.0]]}, ], } - assert chain_ends_s(scene) == pytest.approx((-10.0, 3.0)) + assert chain_ends_s(scene, RULES) == pytest.approx((-10.0, 3.0)) def test_a_view_that_only_reaches_the_needed_height_is_not_higher(): @@ -858,3 +862,69 @@ def test_facing_and_headroom_widen_by_the_default_error_with_drift(): problems = coverage_problems(scene, r, BAND_RULES) assert any("facing [4.33, 5.70] observed, none seen" in m for m in problems) assert any("overhead [4.33, 5.70] observed, none seen" in m for m in problems) + + +@pytest.mark.parametrize( + ("errors", "right_end"), + [ + ((None, None), 5.0), + ((0.1, 0.1), 5.4), + ((0.3, 0.2), 5.0), + ((0.3, 0.0), 5.4), + ((0.0, 0.0), 5.4), + ], +) +def test_walls_meet_across_a_space_within_both_errors(errors, right_end): + # 0.4 ft between the walls. Within both walls' own errors (0.3 + 0.3 by default, capped at + # 0.6) they meet and s skips the space; wider, it is a gap that s counts. Exact walls keep + # any gap (server/scene.py at 7133283). + scene = { + "meter": {"pos": [0.0, 4.0, 0.0], "wall_id": "a"}, + "walls": [ + {"id": "a", "baseline": [[-5.0, 0.0], [0.0, 0.0]]}, + {"id": "b", "baseline": [[0.4, 0.0], [5.4, 0.0]]}, + ], + } + for wall, error in zip(scene["walls"], errors, strict=True): + if error is not None: + wall["plus_minus_ft"] = error + assert chain_ends_s(scene, RULES) == pytest.approx((-5.0, right_end)) + + +def test_the_join_uses_the_walls_own_error_not_drift(): + # 0.8 ft apart, 20 ft from the meter: drift would make the errors exceed it, but the join + # uses the default 0.3 + 0.3 capped at 0.6, so it is a gap. + scene = { + "meter": {"pos": [0.0, 4.0, 0.0], "wall_id": "a"}, + "walls": [ + {"id": "a", "baseline": [[0.0, 0.0], [20.0, 0.0]]}, + {"id": "b", "baseline": [[20.8, 0.0], [25.0, 0.0]]}, + ], + } + assert chain_ends_s(scene, RULES) == pytest.approx((0.0, 25.0)) + + +def short_wall_scene(height: float, span: list[list[float]]) -> dict: + scene = copy.deepcopy(SCENE) + scene["walls"] = [ + {"id": "w1", "baseline": [[-12.0, 0.0], span[0]], "plus_minus_ft": 0.3}, + {"id": "low", "baseline": span, "plus_minus_ft": 0.3, "height_ft": height}, + ] + return scene + + +@pytest.mark.parametrize(("height", "flagged"), [(3.0, True), (3.25, True), (3.26, False)]) +def test_backing_passes_only_if_every_declared_wall_within_error_is_taller(height, flagged): + # The spot [1, 3.58] on w1, which ends at 3.7; the low wall starts there, within 0.3 ft. + scene = short_wall_scene(height, [[3.7, 0.0], [12.0, 0.0]]) + r = result(checks=[check(), passing("wall_backing")]) + msgs = declared_height_problems(scene, r, RULES) + assert bool(msgs) == flagged + if flagged: + assert "check wall_backing passes but wall low" in msgs[0] + + +def test_a_short_wall_beyond_the_error_does_not_matter(): + scene = short_wall_scene(2.0, [[4.0, 0.0], [12.0, 0.0]]) + r = result(checks=[check(), passing("wall_backing")], sweep=[]) + assert declared_height_problems(scene, r, RULES) == [] From 1a0433ee0d79fbd594372d30f5b35951833b9444 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 00:21:49 -0500 Subject: [PATCH 47/59] verification: scoreboard and result at server 7133283 e2e at 7133283: all 50 scenes pass, contract_ok. The board records S2-1 as a gap: one server test, test_solving_the_example_takes_under_a_second, failed during the full run at a load average near 140 and passed three times alone. S4-1 and S4-2 are stale because the app replay is from 194f2eb; two other Simulators were booted, so none was started for a new one. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/README.md | 4 +- verification/SCOREBOARD.md | 95 ++--- verification/scoreboard/scoreboard.json | 454 +++++++++++++++--------- 3 files changed, 342 insertions(+), 211 deletions(-) diff --git a/verification/README.md b/verification/README.md index 2751a7c93..f0515acfb 100644 --- a/verification/README.md +++ b/verification/README.md @@ -19,8 +19,8 @@ Reports go to `~/house-scanning-data/reports/`, outside git, because replay fram non-commercial datasets. What each check asserts is in the docstrings of `hsverify/e2e.py`, `resultcheck.py` and `simrun.py`. -**Result.** At `t3/server` `6c7ca23`, with the app's upload from `194f2eb`, all 50 scenes pass (45 -answered and checked; slowest real scene 0.17 s). +**Result.** At `t3/server` `7133283`, with the app's upload from `194f2eb`, all 50 scenes pass (45 +answered and checked; slowest real scene 0.42 s). **What it changed.** S2 fixed seven defects it found: - unseen ground passing the pool check; diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index fa22e1156..588b40968 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,89 +1,96 @@ # Scoreboard -Generated 2026-09-26 22:39 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes not run (add ARGS=--slow). Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-27 00:20 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran for S2-1, S2-7 only. Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | -| [#86](https://github.com/SamGu-NRX/house-scanning/pull/86) | `t3/experience` | `8bbef71d` | yes | 2 success | 2026-09-26 22:26 CDT | -| [#61](https://github.com/SamGu-NRX/house-scanning/pull/61) | `t3/app-review-docs` | `6059eae6` | no | 2 success | 2026-09-26 21:51 CDT | -| [#60](https://github.com/SamGu-NRX/house-scanning/pull/60) | `t3/device-test-4.1-analysis` | `c566490c` | no | 2 success | 2026-09-26 22:39 CDT | +| [#104](https://github.com/SamGu-NRX/house-scanning/pull/104) | `t3/41-gate-coaching` | `c4dec620` | yes | 2 success, 2 skipped | 2026-09-27 00:15 CDT | +| [#102](https://github.com/SamGu-NRX/house-scanning/pull/102) | `t3/41-card-replies` | `a4b8b272` | yes | 2 success, 2 skipped | 2026-09-27 00:13 CDT | +| [#100](https://github.com/SamGu-NRX/house-scanning/pull/100) | `t3/41-ar-canvas-fallback` | `41291e82` | yes | 2 success, 2 skipped | 2026-09-27 00:10 CDT | +| [#99](https://github.com/SamGu-NRX/house-scanning/pull/99) | `t3/41-end-preview` | `b5006fca` | yes | 2 success, 2 skipped | 2026-09-27 00:11 CDT | +| [#91](https://github.com/SamGu-NRX/house-scanning/pull/91) | `t3/experience-result` | `00fb017d` | yes | 2 success, 1 skipped | 2026-09-27 00:06 CDT | +| [#90](https://github.com/SamGu-NRX/house-scanning/pull/90) | `t3/check-experiments` | `9d66b31c` | no | 5 success | 2026-09-26 23:39 CDT | +| [#89](https://github.com/SamGu-NRX/house-scanning/pull/89) | `t3/meter-brand` | `0a3a05e2` | no | 4 success, 1 pending | 2026-09-27 00:15 CDT | +| [#88](https://github.com/SamGu-NRX/house-scanning/pull/88) | `t3/autodetect-lab` | `538f60e1` | yes | 2 success | 2026-09-26 23:00 CDT | +| [#87](https://github.com/SamGu-NRX/house-scanning/pull/87) | `t3/41-zip-deflate` | `5150321a` | yes | 2 success, 1 skipped | 2026-09-27 00:07 CDT | +| [#86](https://github.com/SamGu-NRX/house-scanning/pull/86) | `t3/experience` | `b442f0a0` | yes | 2 success, 1 skipped | 2026-09-27 00:00 CDT | +| [#61](https://github.com/SamGu-NRX/house-scanning/pull/61) | `t3/app-review-docs` | `c0b628b6` | no | 4 success | 2026-09-26 23:23 CDT | +| [#60](https://github.com/SamGu-NRX/house-scanning/pull/60) | `t3/device-test-4.1-analysis` | `c566490c` | no | 2 success | 2026-09-27 00:15 CDT | | [#59](https://github.com/SamGu-NRX/house-scanning/pull/59) | `t3/signals-lab` | `0cde05e3` | yes | 2 success | 2026-09-26 21:38 CDT | | [#58](https://github.com/SamGu-NRX/house-scanning/pull/58) | `t3/ios-ci-no-preboot` | `3c1298ab` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 22:18 CDT | -| [#56](https://github.com/SamGu-NRX/house-scanning/pull/56) | `t3/walk-guidance-gaps` | `cae4f426` | no | 3 success | 2026-09-26 22:18 CDT | -| [#55](https://github.com/SamGu-NRX/house-scanning/pull/55) | `t3/motion-permission` | `9607a481` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 22:23 CDT | -| [#52](https://github.com/SamGu-NRX/house-scanning/pull/52) | `t3/battery-mark` | `68a66a82` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 20:42 CDT | -| [#49](https://github.com/SamGu-NRX/house-scanning/pull/49) | `t3/gap-skip-continues` | `ce901122` | no | 3 success | 2026-09-26 22:18 CDT | +| [#56](https://github.com/SamGu-NRX/house-scanning/pull/56) | `t3/walk-guidance-gaps` | `cae4f426` | no | 3 success | 2026-09-26 23:57 CDT | +| [#55](https://github.com/SamGu-NRX/house-scanning/pull/55) | `t3/motion-permission` | `9607a481` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 23:57 CDT | +| [#52](https://github.com/SamGu-NRX/house-scanning/pull/52) | `t3/battery-mark` | `68a66a82` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 23:57 CDT | +| [#49](https://github.com/SamGu-NRX/house-scanning/pull/49) | `t3/gap-skip-continues` | `ce901122` | no | 3 success | 2026-09-26 23:57 CDT | | [#46](https://github.com/SamGu-NRX/house-scanning/pull/46) | `t3/field-test-run2` | `afd02c82` | no | 2 success | 2026-09-26 17:36 CDT | | [#23](https://github.com/SamGu-NRX/house-scanning/pull/23) | `t3/device-field-test` | `321ea7e2` | no | 2 success | 2026-09-26 19:39 CDT | -| [#22](https://github.com/SamGu-NRX/house-scanning/pull/22) | `t3/packet` | `b2fefa56` | no | 2 success | 2026-09-26 21:44 CDT | -| [#21](https://github.com/SamGu-NRX/house-scanning/pull/21) | `t3/ios-map3d` | `6f883d8b` | yes | 2 success | 2026-09-26 22:37 CDT | -| [#20](https://github.com/SamGu-NRX/house-scanning/pull/20) | `t3/recon` | `6573a79c` | yes | 3 success | 2026-09-26 21:50 CDT | +| [#22](https://github.com/SamGu-NRX/house-scanning/pull/22) | `t3/packet` | `89372a0a` | no | 4 success | 2026-09-26 23:54 CDT | +| [#21](https://github.com/SamGu-NRX/house-scanning/pull/21) | `t3/ios-map3d` | `c77577b7` | yes | 2 success, 2 skipped | 2026-09-26 23:04 CDT | +| [#20](https://github.com/SamGu-NRX/house-scanning/pull/20) | `t3/recon` | `d072729b` | yes | 3 success, 1 skipped | 2026-09-27 00:08 CDT | | [#18](https://github.com/SamGu-NRX/house-scanning/pull/18) | `codex/research-findings` | `bd4a7d38` | no | 2 success | 2026-09-26 16:21 CDT | -| [#16](https://github.com/SamGu-NRX/house-scanning/pull/16) | `t3/meter-closeup` | `b199613b` | no | 4 success | 2026-09-26 22:26 CDT | -| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `e0e3ae6e` | no | 3 success | 2026-09-26 22:36 CDT | -| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `190ed339` | no | 3 success | 2026-09-26 19:16 CDT | -| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `6c7ca23b` | no | 3 success | 2026-09-26 22:37 CDT | -| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `4b71161d` | no | 2 success, 2 pending | 2026-09-26 22:37 CDT | +| [#16](https://github.com/SamGu-NRX/house-scanning/pull/16) | `t3/meter-closeup` | `dc4f38da` | no | 4 success, 1 pending, 1 skipped | 2026-09-27 00:16 CDT | +| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `de53ecd5` | no | 3 success, 2 pending | 2026-09-27 00:16 CDT | +| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `40698465` | no | 5 success | 2026-09-27 00:14 CDT | +| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `7133283e` | no | 5 success | 2026-09-27 00:15 CDT | +| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `b9851e15` | no | 3 success, 3 pending | 2026-09-27 00:11 CDT | | [#9](https://github.com/SamGu-NRX/house-scanning/pull/9) | `codex/capture-contract-probe` | `4b6c4309` | yes | 4 success | 2026-09-26 00:08 CDT | -| [#7](https://github.com/SamGu-NRX/house-scanning/pull/7) | `t3/measure-lab` | `7b3ddcc0` | no | 4 success | 2026-09-26 22:27 CDT | -| [#4](https://github.com/SamGu-NRX/house-scanning/pull/4) | `t3/scoring-harness` | `0f3e7532` | no | 3 success | 2026-09-26 22:25 CDT | ## S1 Evals on real data -`origin/t3/evals` at [`190ed339`](https://github.com/SamGu-NRX/house-scanning/tree/190ed339f41924ae0244e0b88c5ebe647ddd9eb4). 2 met, 2 partial, 1 not run. +`origin/t3/evals` at [`40698465`](https://github.com/SamGu-NRX/house-scanning/tree/406984650759894263c60ac4f9f357e3f1fa0de8). 1 met, 2 partial, 1 gap, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S1-1 | One command regenerates every number from a clean checkout | partial | make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac. | -| S1-2 | Dataset URLs and sha256 recorded | met | all 2 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/README.md) | +| S1-2 | Dataset URLs and sha256 recorded | gap | none found in 1 files; missing ADVIO, ETH3D | | S1-3 | Metric code tested on hand-computed cases | not run | slow; run with --slow | -| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/README.md), [experiments/evals/field/FIELD_SHEET.md](https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/field/FIELD_SHEET.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/results/advio_drift.md) | -| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/README.md](https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/README.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/results/eth3d_visibility_sensitivity.md) | +| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/METHODS.md](https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/METHODS.md), [experiments/evals/field/FIELD_SHEET.md](https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/field/FIELD_SHEET.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/results/advio_drift.md) | +| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/METHODS.md](https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/METHODS.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/results/eth3d_visibility_sensitivity.md) | ## S2 Server and solver -`origin/t3/server` at [`6c7ca23b`](https://github.com/SamGu-NRX/house-scanning/tree/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75). 5 met, 1 partial, 2 not run. +`origin/t3/server` at [`7133283e`](https://github.com/SamGu-NRX/house-scanning/tree/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca). 6 met, 1 partial, 1 gap. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S2-1 | pytest green | not run | slow; run with --slow | -| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_golden_public.py) | -| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_properties.py) | -| S2-4 | Schemas published | met | present at 6c7ca23b [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/schemas/result.schema.json) | -| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_existing_battery.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_existing_battery.py), [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_schemas.py) | -| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 171.6 (want < 1000) at sha 6c7ca23b: ~/house-scanning-data/reports/e2e/20260926-223908-6c7ca23b/report.json | -| S2-7 | ruff clean | not run | slow; run with --slow | -| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | met | contract_ok = true (want True) at sha 6c7ca23b: ~/house-scanning-data/reports/e2e/20260926-223908-6c7ca23b/report.json | +| S2-1 | pytest green | gap | exit 1: =========================== short test summary info ============================ / FAILED tests/test_api.py::test_solving_the_example_takes_under_a_second - ass... / 1 failed, 339 passed, 1 warning in 190.75s (0:03:10) | +| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_golden_public.py) | +| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_properties.py) | +| S2-4 | Schemas published | met | present at 7133283e [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/schemas/result.schema.json) | +| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_existing_battery.py](https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_existing_battery.py), [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_schemas.py) | +| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 424.6 (want < 1000) at sha 7133283e: ~/house-scanning-data/reports/e2e/20260927-001445-7133283e/report.json | +| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 82ms / All checks passed! / 31 files already formatted | +| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | met | contract_ok = true (want True) at sha 7133283e: ~/house-scanning-data/reports/e2e/20260927-001445-7133283e/report.json | ## S3 iOS capture app -`origin/t3/ios-mvf` at [`4b71161d`](https://github.com/SamGu-NRX/house-scanning/tree/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672). 2 met, 3 partial, 1 gap, 3 stale, 1 not run. +`origin/t3/ios-mvf` at [`b9851e15`](https://github.com/SamGu-NRX/house-scanning/tree/b9851e15c407b024b526d104e2f991ae02cdf87e). 2 met, 3 partial, 1 gap, 3 stale, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S3-1 | CI build green | not run | iOS build: pending at 4b71161d [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/project.yml) | -| S3-3 | Zero build warnings | stale | report for sha 194f2ebf, ref at 4b71161d: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json | -| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage, guidance; missing auto-capture [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift) | -| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/SourceRestartUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/SourceRestartUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at 4b71161d: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | +| S3-1 | CI build green | not run | iOS build: pending at b9851e15 [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | +| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/project.yml) | +| S3-3 | Zero build warnings | stale | report for sha 194f2ebf, ref at b9851e15: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json | +| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage, guidance; missing auto-capture [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift) | +| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/SourceRestartUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/SourceRestartUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at b9851e15: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | | S3-7 | Screenshots of each state in the PR | met | 18 of 9 matches in the PR body [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-8 | Exported scene.json validates against C1 | stale | report for app_sha 194f2ebf, ref at 4b71161d: ~/house-scanning-data/reports/e2e/20260926-223908-6c7ca23b/report.json | +| S3-8 | Exported scene.json validates against C1 | stale | report for app_sha 194f2ebf, ref at b9851e15: ~/house-scanning-data/reports/e2e/20260927-001445-7133283e/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/ios-mvf` at [`4b71161d`](https://github.com/SamGu-NRX/house-scanning/tree/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672), `origin/t3/verification` at [`e0e3ae6e`](https://github.com/SamGu-NRX/house-scanning/tree/e0e3ae6e49c996cee123ec8759cc6bea305d5730). 2 met, 2 stale. +`origin/t3/ios-mvf` at [`b9851e15`](https://github.com/SamGu-NRX/house-scanning/tree/b9851e15c407b024b526d104e2f991ae02cdf87e), `origin/t3/verification` at [`de53ecd5`](https://github.com/SamGu-NRX/house-scanning/tree/de53ecd52a2af1350b7502ef6bfd62586e84ef01). 2 met, 2 stale. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S4-1 | End to end from a real replay to a placement result passes | stale | report for app_sha 194f2ebf, ref at 4b71161d: ~/house-scanning-data/reports/e2e/20260926-223908-6c7ca23b/report.json | -| S4-2 | Screenshot report per app state | stale | report for sha 194f2ebf, ref at 4b71161d: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json | +| S4-1 | End to end from a real replay to a placement result passes | stale | report for app_sha 194f2ebf, ref at b9851e15: ~/house-scanning-data/reports/e2e/20260927-001445-7133283e/report.json | +| S4-2 | Screenshot report per app state | stale | report for sha 194f2ebf, ref at b9851e15: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json | | S4-3 | Product description written from the code and checked against screenshots | met | docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state. | -| S4-4 | Scoreboard of every PR | met | present at e0e3ae6e [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/e0e3ae6e49c996cee123ec8759cc6bea305d5730/verification/SCOREBOARD.md) | +| S4-4 | Scoreboard of every PR | met | present at de53ecd5 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/de53ecd52a2af1350b7502ef6bfd62586e84ef01/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index cea2176f6..d2173dea9 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,42 +1,207 @@ { - "generated_at": "2026-09-26T22:39:49-05:00", + "generated_at": "2026-09-27T00:20:31-05:00", "command": "uv run python -m hsverify.scoreboard", - "slow": false, + "slow": [ + "S2-1", + "S2-7" + ], "prs": [ + { + "number": 104, + "title": "ios: gate coaching keeps the task card, holds through dark flicker, and says \"Turn more slowly\" for turns", + "branch": "t3/41-gate-coaching", + "head": "c4dec62010f20dd94881978a1dc7bce6a47d8920", + "draft": true, + "updated": "2026-09-27T05:15:03Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 2 + }, + "failing": [], + "text": "2 success, 2 skipped" + } + }, + { + "number": 102, + "title": "ios: card replies name the task, stay under gate coaching, and lock after a new card", + "branch": "t3/41-card-replies", + "head": "a4b8b2723736b5f2b29c5605569dcdb76c6d8a69", + "draft": true, + "updated": "2026-09-27T05:13:33Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 2 + }, + "failing": [], + "text": "2 success, 2 skipped" + } + }, + { + "number": 100, + "title": "ios: draw the AR result with the Canvas until RealityKit is seen drawing it", + "branch": "t3/41-ar-canvas-fallback", + "head": "41291e8287bd7a2452c6b53b36120670c3210317", + "draft": true, + "updated": "2026-09-27T05:10:40Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 2 + }, + "failing": [], + "text": "2 success, 2 skipped" + } + }, + { + "number": 99, + "title": "ios: \"leaves out N ft you walked\" only on walk tasks, and on the end question", + "branch": "t3/41-end-preview", + "head": "b5006fca782aa280caa767e476fd5c62b496a9e2", + "draft": true, + "updated": "2026-09-27T05:11:39Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 2 + }, + "failing": [], + "text": "2 success, 2 skipped" + } + }, + { + "number": 91, + "title": "ios: result screen leads with the answer, shows the nearest rejected spot and links each unsure check to its view", + "branch": "t3/experience-result", + "head": "00fb017dcf446921c0ba1753795d03aad1d52872", + "draft": true, + "updated": "2026-09-27T05:06:46Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 1 + }, + "failing": [], + "text": "2 success, 1 skipped" + } + }, + { + "number": 90, + "title": "build: include available experiment suites in root checks", + "branch": "t3/check-experiments", + "head": "9d66b31c40c940c5bab7baea0aef5c5a87be3c10", + "draft": false, + "updated": "2026-09-27T04:39:38Z", + "ci": { + "counts": { + "success": 5, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "5 success" + } + }, + { + "number": 89, + "title": "ios: show the meter's brand beside the number candidates", + "branch": "t3/meter-brand", + "head": "0a3a05e200fa42daf002d5fc77687c898935cffa", + "draft": false, + "updated": "2026-09-27T05:15:36Z", + "ci": { + "counts": { + "success": 4, + "failure": 0, + "pending": 1, + "skipped": 0 + }, + "failing": [], + "text": "4 success, 1 pending" + } + }, + { + "number": 88, + "title": "experiments: auto-detect lab for wall objects and the meter brand", + "branch": "t3/autodetect-lab", + "head": "538f60e1e9796b26ab80bec50004bcc2eb04d38e", + "draft": true, + "updated": "2026-09-27T04:00:03Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "2 success" + } + }, + { + "number": 87, + "title": "ios: deflate the text entries in the shared scan zip", + "branch": "t3/41-zip-deflate", + "head": "5150321a22f540e669755270dd76caa23957d8a4", + "draft": true, + "updated": "2026-09-27T05:07:01Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 1 + }, + "failing": [], + "text": "2 success, 1 skipped" + } + }, { "number": 86, "title": "experiments: live dots, a macOS prototype of the holographic capture view", "branch": "t3/experience", - "head": "8bbef71d4c3158b3f5eb90a65fb5fab70642b170", + "head": "b442f0a0436daf36f1f929d578a63791e1186637", "draft": true, - "updated": "2026-09-27T03:26:12Z", + "updated": "2026-09-27T05:00:35Z", "ci": { "counts": { "success": 2, "failure": 0, "pending": 0, - "skipped": 0 + "skipped": 1 }, "failing": [], - "text": "2 success" + "text": "2 success, 1 skipped" } }, { "number": 61, "title": "docs: app review \u2014 product description, UX review and friction audit", "branch": "t3/app-review-docs", - "head": "6059eae63b26330142bf94db5486ddbf8b42c397", + "head": "c0b628b6b9ec2b314ae418db956a27bc579ee642", "draft": false, - "updated": "2026-09-27T02:51:14Z", + "updated": "2026-09-27T04:23:04Z", "ci": { "counts": { - "success": 2, + "success": 4, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "2 success" + "text": "4 success" } }, { @@ -45,7 +210,7 @@ "branch": "t3/device-test-4.1-analysis", "head": "c566490c5e3fa0d45803895bb463cb260d11a906", "draft": false, - "updated": "2026-09-27T03:39:25Z", + "updated": "2026-09-27T05:15:59Z", "ci": { "counts": { "success": 2, @@ -101,7 +266,7 @@ "branch": "t3/walk-guidance-gaps", "head": "cae4f426f1f5e5a0f4ba4dde97347d04df724d40", "draft": false, - "updated": "2026-09-27T03:18:01Z", + "updated": "2026-09-27T04:57:46Z", "ci": { "counts": { "success": 3, @@ -119,7 +284,7 @@ "branch": "t3/motion-permission", "head": "9607a481359897e405fc4e2acfe7149c4599dd8d", "draft": false, - "updated": "2026-09-27T03:23:04Z", + "updated": "2026-09-27T04:57:34Z", "ci": { "counts": { "success": 2, @@ -139,7 +304,7 @@ "branch": "t3/battery-mark", "head": "68a66a827c6fd6a56e18f0ffea25387f0cf23396", "draft": false, - "updated": "2026-09-27T01:42:12Z", + "updated": "2026-09-27T04:57:26Z", "ci": { "counts": { "success": 2, @@ -159,7 +324,7 @@ "branch": "t3/gap-skip-continues", "head": "ce90112202265f211509e5f73cdbe178f1894506", "draft": false, - "updated": "2026-09-27T03:18:00Z", + "updated": "2026-09-27T04:57:20Z", "ci": { "counts": { "success": 3, @@ -211,54 +376,54 @@ "number": 22, "title": "packet: point to the server's capture packet 0.4 and propose 0.5 additions", "branch": "t3/packet", - "head": "b2fefa568d80f807b4310bd3f64ecd58c8b20f81", + "head": "89372a0a4143918f0d0e67c7e2bbae5ac60e8023", "draft": false, - "updated": "2026-09-27T02:44:00Z", + "updated": "2026-09-27T04:54:02Z", "ci": { "counts": { - "success": 2, + "success": 4, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "2 success" + "text": "4 success" } }, { "number": 21, "title": "ios: live on-device 3D map with occlusion-honest coverage", "branch": "t3/ios-map3d", - "head": "6f883d8bad71ac4c15dc4d3197784f4d01ed8cfe", + "head": "c77577b7bbf24d927925654df61b776157ca4101", "draft": true, - "updated": "2026-09-27T03:37:36Z", + "updated": "2026-09-27T04:04:04Z", "ci": { "counts": { "success": 2, "failure": 0, "pending": 0, - "skipped": 0 + "skipped": 2 }, "failing": [], - "text": "2 success" + "text": "2 success, 2 skipped" } }, { "number": 20, "title": "recon: handoff to the server team", "branch": "t3/recon", - "head": "6573a79c0331c71c8516d6bc6d60354a6669ee7f", + "head": "d072729bcda5d3d774c864e2918aa42a50507db6", "draft": true, - "updated": "2026-09-27T02:50:12Z", + "updated": "2026-09-27T05:08:36Z", "ci": { "counts": { "success": 3, "failure": 0, "pending": 0, - "skipped": 0 + "skipped": 1 }, "failing": [], - "text": "3 success" + "text": "3 success, 1 skipped" } }, { @@ -283,90 +448,90 @@ "number": 16, "title": "experiments: meter close-up reading, tap-to-confirm and two retake checks", "branch": "t3/meter-closeup", - "head": "b199613b439f3d399e34df55df1b9ab4c371b24a", + "head": "dc4f38da7c7e785bf7bba6e4eb2607eb9ac05487", "draft": false, - "updated": "2026-09-27T03:26:22Z", + "updated": "2026-09-27T05:16:24Z", "ci": { "counts": { "success": 4, "failure": 0, - "pending": 0, - "skipped": 0 + "pending": 1, + "skipped": 1 }, "failing": [], - "text": "4 success" + "text": "4 success, 1 pending, 1 skipped" } }, { "number": 13, "title": "verification: independent end-to-end checks (S4)", "branch": "t3/verification", - "head": "e0e3ae6e49c996cee123ec8759cc6bea305d5730", + "head": "de53ecd52a2af1350b7502ef6bfd62586e84ef01", "draft": false, - "updated": "2026-09-27T03:36:55Z", + "updated": "2026-09-27T05:16:39Z", "ci": { "counts": { "success": 3, "failure": 0, - "pending": 0, + "pending": 2, "skipped": 0 }, "failing": [], - "text": "3 success" + "text": "3 success, 2 pending" } }, { "number": 12, "title": "feat(evals): real-data evals for AR drift, wall reconstruction, coverage and the field test", "branch": "t3/evals", - "head": "190ed339f41924ae0244e0b88c5ebe647ddd9eb4", + "head": "406984650759894263c60ac4f9f357e3f1fa0de8", "draft": false, - "updated": "2026-09-27T00:16:22Z", + "updated": "2026-09-27T05:14:24Z", "ci": { "counts": { - "success": 3, + "success": 5, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "3 success" + "text": "5 success" } }, { "number": 11, "title": "server: placement solver, rules and API (S2)", "branch": "t3/server", - "head": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75", + "head": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca", "draft": false, - "updated": "2026-09-27T03:37:50Z", + "updated": "2026-09-27T05:15:28Z", "ci": { "counts": { - "success": 3, + "success": 5, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "3 success" + "text": "5 success" } }, { "number": 10, "title": "ios: guided wall scan from the meter to a battery spot", "branch": "t3/ios-mvf", - "head": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", + "head": "b9851e15c407b024b526d104e2f991ae02cdf87e", "draft": false, - "updated": "2026-09-27T03:37:11Z", + "updated": "2026-09-27T05:11:44Z", "ci": { "counts": { - "success": 2, + "success": 3, "failure": 0, - "pending": 2, + "pending": 3, "skipped": 0 }, "failing": [], - "text": "2 success, 2 pending" + "text": "3 success, 3 pending" } }, { @@ -386,42 +551,6 @@ "failing": [], "text": "4 success" } - }, - { - "number": 7, - "title": "experiments: add measure-lab, a no-LiDAR measurement rig", - "branch": "t3/measure-lab", - "head": "7b3ddcc046fcdafee6fe698c58eea5f56d017ddf", - "draft": false, - "updated": "2026-09-27T03:27:31Z", - "ci": { - "counts": { - "success": 4, - "failure": 0, - "pending": 0, - "skipped": 0 - }, - "failing": [], - "text": "4 success" - } - }, - { - "number": 4, - "title": "Add a scoring harness that compares capture pipelines with a tape survey", - "branch": "t3/scoring-harness", - "head": "0f3e75323b4469af88b101b983cf8c6d25ad1c3f", - "draft": false, - "updated": "2026-09-27T03:25:44Z", - "ci": { - "counts": { - "success": 3, - "failure": 0, - "pending": 0, - "skipped": 0 - }, - "failing": [], - "text": "3 success" - } } ], "pr_error": null, @@ -432,7 +561,7 @@ "refs": [ { "ref": "origin/t3/evals", - "sha": "190ed339f41924ae0244e0b88c5ebe647ddd9eb4" + "sha": "406984650759894263c60ac4f9f357e3f1fa0de8" } ], "metrics": [ @@ -440,7 +569,7 @@ "id": "S1-1", "text": "One command regenerates every number from a clean checkout", "ref": "origin/t3/evals", - "sha": "190ed339f41924ae0244e0b88c5ebe647ddd9eb4", + "sha": "406984650759894263c60ac4f9f357e3f1fa0de8", "status": "partial", "evidence": "make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac.", "links": [] @@ -449,21 +578,16 @@ "id": "S1-2", "text": "Dataset URLs and sha256 recorded", "ref": "origin/t3/evals", - "sha": "190ed339f41924ae0244e0b88c5ebe647ddd9eb4", - "status": "met", - "evidence": "all 2 found", - "links": [ - [ - "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/README.md" - ] - ] + "sha": "406984650759894263c60ac4f9f357e3f1fa0de8", + "status": "gap", + "evidence": "none found in 1 files; missing ADVIO, ETH3D", + "links": [] }, { "id": "S1-3", "text": "Metric code tested on hand-computed cases", "ref": "origin/t3/evals", - "sha": "190ed339f41924ae0244e0b88c5ebe647ddd9eb4", + "sha": "406984650759894263c60ac4f9f357e3f1fa0de8", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -472,21 +596,21 @@ "id": "S1-4", "text": "ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft", "ref": "origin/t3/evals", - "sha": "190ed339f41924ae0244e0b88c5ebe647ddd9eb4", + "sha": "406984650759894263c60ac4f9f357e3f1fa0de8", "status": "met", "evidence": "all 5 found", "links": [ [ - "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/README.md" + "experiments/evals/METHODS.md", + "https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/METHODS.md" ], [ "experiments/evals/field/FIELD_SHEET.md", - "https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/field/FIELD_SHEET.md" + "https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/field/FIELD_SHEET.md" ], [ "experiments/evals/results/advio_drift.md", - "https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/results/advio_drift.md" + "https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/results/advio_drift.md" ] ] }, @@ -494,21 +618,21 @@ "id": "S1-5", "text": "ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question", "ref": "origin/t3/evals", - "sha": "190ed339f41924ae0244e0b88c5ebe647ddd9eb4", + "sha": "406984650759894263c60ac4f9f357e3f1fa0de8", "status": "partial", "evidence": "found 1-3 m row, 3-10 m row, scale error; missing verdict", "links": [ [ - "experiments/evals/README.md", - "https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/README.md" + "experiments/evals/METHODS.md", + "https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/METHODS.md" ], [ "experiments/evals/results/eth3d_recon.md", - "https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/results/eth3d_recon.md" + "https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/results/eth3d_recon.md" ], [ "experiments/evals/results/eth3d_visibility_sensitivity.md", - "https://github.com/SamGu-NRX/house-scanning/blob/190ed339f41924ae0244e0b88c5ebe647ddd9eb4/experiments/evals/results/eth3d_visibility_sensitivity.md" + "https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/results/eth3d_visibility_sensitivity.md" ] ] } @@ -520,7 +644,7 @@ "refs": [ { "ref": "origin/t3/server", - "sha": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75" + "sha": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca" } ], "metrics": [ @@ -528,22 +652,22 @@ "id": "S2-1", "text": "pytest green", "ref": "origin/t3/server", - "sha": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75", - "status": "not run", - "evidence": "slow; run with --slow", + "sha": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca", + "status": "gap", + "evidence": "exit 1: =========================== short test summary info ============================ / FAILED tests/test_api.py::test_solving_the_example_takes_under_a_second - ass... / 1 failed, 339 passed, 1 warning in 190.75s (0:03:10)", "links": [] }, { "id": "S2-2", "text": "Every public golden test 01-14 of docs/research/t3-lane-c-review.md present", "ref": "origin/t3/server", - "sha": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75", + "sha": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca", "status": "met", "evidence": "all 14 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_golden_public.py" ] ] }, @@ -551,17 +675,17 @@ "id": "S2-3", "text": "Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail", "ref": "origin/t3/server", - "sha": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75", + "sha": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca", "status": "partial", "evidence": "found missing coverage, mirror; missing equality unsure, larger clearance", "links": [ [ "server/tests/test_coderabbit_review.py", - "https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_coderabbit_review.py" + "https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_coderabbit_review.py" ], [ "server/tests/test_properties.py", - "https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_properties.py" + "https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_properties.py" ] ] }, @@ -569,17 +693,17 @@ "id": "S2-4", "text": "Schemas published", "ref": "origin/t3/server", - "sha": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75", + "sha": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca", "status": "met", - "evidence": "present at 6c7ca23b", + "evidence": "present at 7133283e", "links": [ [ "server/schemas/scene.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/schemas/scene.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/schemas/scene.schema.json" ], [ "server/schemas/result.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/schemas/result.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/schemas/result.schema.json" ] ] }, @@ -587,21 +711,21 @@ "id": "S2-5", "text": "Schemas validated in tests", "ref": "origin/t3/server", - "sha": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75", + "sha": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca", "status": "met", "evidence": "all 2 found", "links": [ [ "server/tests/test_existing_battery.py", - "https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_existing_battery.py" + "https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_existing_battery.py" ], [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_golden_public.py" ], [ "server/tests/test_schemas.py", - "https://github.com/SamGu-NRX/house-scanning/blob/6c7ca23b63a5493e5a4224af0a5d12d025f3ba75/server/tests/test_schemas.py" + "https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_schemas.py" ] ] }, @@ -609,27 +733,27 @@ "id": "S2-6", "text": "A real-derived scene returns a result in under 1 s", "ref": "origin/t3/server", - "sha": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75", + "sha": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca", "status": "met", - "evidence": "latency_ms = 171.6 (want < 1000) at sha 6c7ca23b: ~/house-scanning-data/reports/e2e/20260926-223908-6c7ca23b/report.json", + "evidence": "latency_ms = 424.6 (want < 1000) at sha 7133283e: ~/house-scanning-data/reports/e2e/20260927-001445-7133283e/report.json", "links": [] }, { "id": "S2-7", "text": "ruff clean", "ref": "origin/t3/server", - "sha": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75", - "status": "not run", - "evidence": "slow; run with --slow", + "sha": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca", + "status": "met", + "evidence": "exit 0: Installed 33 packages in 82ms / All checks passed! / 31 files already formatted", "links": [] }, { "id": "S2-8", "text": "Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time", "ref": "origin/t3/server", - "sha": "6c7ca23b63a5493e5a4224af0a5d12d025f3ba75", + "sha": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca", "status": "met", - "evidence": "contract_ok = true (want True) at sha 6c7ca23b: ~/house-scanning-data/reports/e2e/20260926-223908-6c7ca23b/report.json", + "evidence": "contract_ok = true (want True) at sha 7133283e: ~/house-scanning-data/reports/e2e/20260927-001445-7133283e/report.json", "links": [] } ] @@ -640,7 +764,7 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672" + "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e" } ], "metrics": [ @@ -648,9 +772,9 @@ "id": "S3-1", "text": "CI build green", "ref": "origin/t3/ios-mvf", - "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", + "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", "status": "not run", - "evidence": "iOS build: pending at 4b71161d", + "evidence": "iOS build: pending at b9851e15", "links": [ [ "#10", @@ -662,13 +786,13 @@ "id": "S3-2", "text": "Built as Swift 6 with strict concurrency", "ref": "origin/t3/ios-mvf", - "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", + "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", "status": "met", "evidence": "all 2 found", "links": [ [ "ios/project.yml", - "https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/project.yml" + "https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/project.yml" ] ] }, @@ -676,30 +800,30 @@ "id": "S3-3", "text": "Zero build warnings", "ref": "origin/t3/ios-mvf", - "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", + "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", "status": "stale", - "evidence": "report for sha 194f2ebf, ref at 4b71161d: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json", + "evidence": "report for sha 194f2ebf, ref at b9851e15: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json", "links": [] }, { "id": "S3-4", "text": "Unit tests for coverage, auto-capture and guidance", "ref": "origin/t3/ios-mvf", - "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", + "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", "status": "partial", "evidence": "found coverage, guidance; missing auto-capture", "links": [ [ "ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift" ] ] }, @@ -707,21 +831,21 @@ "id": "S3-5", "text": "UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop", "ref": "origin/t3/ios-mvf", - "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", + "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", "status": "partial", "evidence": "all 3 found (the tests exist; passing is S3-1)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/FullFlowUITests.swift" ], [ "ios/HouseScanUITests/SourceRestartUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/SourceRestartUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/SourceRestartUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -729,17 +853,17 @@ "id": "S3-6", "text": "Accessibility audit on every screen", "ref": "origin/t3/ios-mvf", - "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", + "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", "status": "partial", "evidence": "3 matches in 2 files (an audit exists; which screens it covers needs a read)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/FullFlowUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -747,16 +871,16 @@ "id": "S3-6b", "text": "Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues", "ref": "origin/t3/ios-mvf", - "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", + "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", "status": "stale", - "evidence": "report for sha a39d0a50, ref at 4b71161d: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", + "evidence": "report for sha a39d0a50, ref at b9851e15: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", "links": [] }, { "id": "S3-7", "text": "Screenshots of each state in the PR", "ref": "origin/t3/ios-mvf", - "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", + "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", "status": "met", "evidence": "18 of 9 matches in the PR body", "links": [ @@ -770,16 +894,16 @@ "id": "S3-8", "text": "Exported scene.json validates against C1", "ref": "origin/t3/ios-mvf", - "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", + "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", "status": "stale", - "evidence": "report for app_sha 194f2ebf, ref at 4b71161d: ~/house-scanning-data/reports/e2e/20260926-223908-6c7ca23b/report.json", + "evidence": "report for app_sha 194f2ebf, ref at b9851e15: ~/house-scanning-data/reports/e2e/20260927-001445-7133283e/report.json", "links": [] }, { "id": "S3-9", "text": "Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design", "ref": "origin/t3/ios-mvf", - "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", + "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -792,11 +916,11 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672" + "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e" }, { "ref": "origin/t3/verification", - "sha": "e0e3ae6e49c996cee123ec8759cc6bea305d5730" + "sha": "de53ecd52a2af1350b7502ef6bfd62586e84ef01" } ], "metrics": [ @@ -804,25 +928,25 @@ "id": "S4-1", "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/ios-mvf", - "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", + "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", "status": "stale", - "evidence": "report for app_sha 194f2ebf, ref at 4b71161d: ~/house-scanning-data/reports/e2e/20260926-223908-6c7ca23b/report.json", + "evidence": "report for app_sha 194f2ebf, ref at b9851e15: ~/house-scanning-data/reports/e2e/20260927-001445-7133283e/report.json", "links": [] }, { "id": "S4-2", "text": "Screenshot report per app state", "ref": "origin/t3/ios-mvf", - "sha": "4b71161d3bcc0b5d8ffd8fe438899f9fbe0fd672", + "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", "status": "stale", - "evidence": "report for sha 194f2ebf, ref at 4b71161d: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json", + "evidence": "report for sha 194f2ebf, ref at b9851e15: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json", "links": [] }, { "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "e0e3ae6e49c996cee123ec8759cc6bea305d5730", + "sha": "de53ecd52a2af1350b7502ef6bfd62586e84ef01", "status": "met", "evidence": "docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state.", "links": [] @@ -831,13 +955,13 @@ "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "e0e3ae6e49c996cee123ec8759cc6bea305d5730", + "sha": "de53ecd52a2af1350b7502ef6bfd62586e84ef01", "status": "met", - "evidence": "present at e0e3ae6e", + "evidence": "present at de53ecd5", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/e0e3ae6e49c996cee123ec8759cc6bea305d5730/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/de53ecd52a2af1350b7502ef6bfd62586e84ef01/verification/SCOREBOARD.md" ] ] } From 77ae8474ef1812fe8be2f01b5bce62d46b04a087 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 00:56:15 -0500 Subject: [PATCH 48/59] verification: a wall view without out_ft reached headroom height, as server 6200098 reads it S2 made the omitted wall out_ft a category default at 6200098: it settles any height up to headroom.min_ft inclusively (within the server's 1e-9), while an explicit out_ft must still exceed the needed height. The verifier read it as seen all the way up, so it could not catch a server that did the same. Other bands are unchanged. The route rule needs no new check: the margin rule already requires a route pass to clear both lines. e2e pinned to 6200098: all 50 scenes pass, contract_ok. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/hsverify/resultcheck.py | 49 +++++++++++++++++++------- verification/tests/test_e2e.py | 1 + verification/tests/test_resultcheck.py | 38 ++++++++++++++++++-- 3 files changed, 72 insertions(+), 16 deletions(-) diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index b326c1870..a2de1062b 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -85,6 +85,8 @@ class RuleSet: wall_join_ft: float | None = None # the rules' battery.height_ft: a wall that declares height_ft must be taller battery_height_ft: float | None = None + # the rules' headroom.min_ft: how high a wall view without out_ft reached + headroom_ft: float | None = None @classmethod def from_yaml(cls, text: str) -> RuleSet: @@ -142,6 +144,7 @@ def reach(key: str) -> Need: value("sweep", "step_ft"), value("sweep", "wall_join_ft"), value("battery", "height_ft"), + headroom, ) @@ -155,24 +158,43 @@ def schema_errors(instance: Any, schema: dict) -> list[str]: def reached(entry: dict) -> float: - """How far an observed entry's view reached: its `out_ft`, or for a band other than ground - without one, all the way (C1: the wall to headroom height, facing to whatever faces it, - overhead clear to the sky).""" + """How far a facing, overhead or ground entry's view reached: its `out_ft`, or without one, + none for ground and all the way for facing (to whatever faces the wall) and overhead (clear + to the sky). A wall entry without one is handled in `observed`.""" default = 0.0 if entry["band"] == "ground" else math.inf return entry.get("out_ft", default) +# The server's float tolerance (server/scene.py EPS), for comparisons that must agree with it +# exactly: where walls meet, whether a declared wall is taller, and the wall view default. +SERVER_EPS = 1e-9 + + def observed( - scene: dict, band: str, min_out_ft: float = 0.0, beyond: bool = False + scene: dict, + band: str, + min_out_ft: float = 0.0, + beyond: bool = False, + wall_default_ft: float | None = None, ) -> list[tuple[float, float]]: """Merged observed s-intervals for a band, counting only entries whose view reached `min_out_ft` (out from the wall for ground and facing, up for wall and overhead). With `beyond`, the view must pass it strictly, as the server requires of a wall seen "higher - than" a height (server/scene.py at 6c7ca23).""" + than" a height (server/scene.py at 6c7ca23). + + A wall entry without `out_ft` reached headroom height (`wall_default_ft`, the rules' + headroom.min_ft) and settles any height up to it inclusively, `beyond` or not; S2 chose + this category default at 6200098 (server/scene.py `observed_intervals`). None keeps the + old reading, all the way up.""" spans = [] for entry in scene.get("coverage", {}).get("observed", []): if entry["band"] != band: continue + if band == "wall" and "out_ft" not in entry and wall_default_ft is not None: + if min_out_ft > wall_default_ft + SERVER_EPS: + continue + spans.append(tuple(sorted(entry["span_ft"]))) + continue far = reached(entry) if (far <= min_out_ft) if beyond else (far + EPS < min_out_ft): continue @@ -349,7 +371,10 @@ def reach_gaps( gap = ground_gap(scene, lo, hi, rules.depth_ft, reach) elif need.band == "wall": a, b = required_span(lo, hi, reach) - seen = covers(observed(scene, "wall", min_out_ft=need.height, beyond=True), a, b) + wall = observed( + scene, "wall", need.height, beyond=True, wall_default_ft=rules.headroom_ft + ) + seen = covers(wall, a, b) gap = None if seen else f"wall [{a:.2f}, {b:.2f}] observed higher than {need.height} ft" else: a, b = required_span(lo, hi, reach) @@ -409,7 +434,9 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: wall and cable route back to the meter over the whole run. Checking the whole run as one battery at its farthest start's error would ask for more than any single start needs.""" problems: list[str] = [] - route_wall = observed(scene, "wall", min_out_ft=rules.route_height_ft, beyond=True) + route_wall = observed( + scene, "wall", rules.route_height_ft, beyond=True, wall_default_ft=rules.headroom_ft + ) # A passing run passes every check the server evaluated; a check it did not evaluate (a # battery clearance with no battery in the scene, say) needs nothing. evaluated = {c["id"] for c in result.get("checks", [])} or set(rules.needs) @@ -458,7 +485,8 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: # only where the band was seen all the way (no `out_ft`). default = GROUND_FAR_FT if request["band"] == "ground" else math.inf far = request.get("out_ft", default) - if b - a > EPS and covers(observed(scene, request["band"], min_out_ft=far), a, b): + seen = observed(scene, request["band"], far, wall_default_ft=rules.headroom_ft) + if b - a > EPS and covers(seen, a, b): problems.append( f"missing_evidence asks for {request['band']} {request['span_ft']}, " "which the scene lists as observed" @@ -528,11 +556,6 @@ def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = END_TOL_FT = 0.1 -# The server's float tolerance (server/scene.py EPS at 7133283), for comparisons that must -# agree with it exactly: where walls meet and whether a declared wall is taller. -SERVER_EPS = 1e-9 - - def wall_spans_s(scene: dict, rules: RuleSet) -> dict[str, tuple[float, float]] | None: """Each wall's stretch of s (the meter's projection is s = 0), or None without a meter wall. diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index 87ca6feb6..a9bc642e9 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -42,6 +42,7 @@ step_ft=1 / 6, # the server's 2 in wall_join_ft=0.6, battery_height_ft=3.25, + headroom_ft=6.5, ) SCENE = { diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index bcf5b9cb6..4666ee985 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -54,6 +54,7 @@ step_ft=1 / 6, # the server's 2 in wall_join_ft=0.6, battery_height_ft=3.25, + headroom_ft=6.5, ) SCENE = { @@ -305,9 +306,10 @@ def test_no_coverage_never_passes(): def test_photo_request_for_an_observed_area_is_flagged(): + # The wall was seen with no out_ft, so to headroom height: a request up to it is redundant. r = result() r["missing_evidence"] = [ - {"kind": "band", "band": "wall", "span_ft": [-3.0, -1.0], "message": ""} + {"kind": "band", "band": "wall", "span_ft": [-3.0, -1.0], "out_ft": 6.5, "message": ""} ] assert any("lists as observed" in m for m in invariant_problems(SCENE, r, rules=RULES)) @@ -391,7 +393,7 @@ def test_rules_come_from_rules_yaml_and_a_missing_one_is_loud(): assert rules.needs["wall_backing"] == (Need("wall", 0.0, 3.25, widen=False),) assert rules.needs["facing_gap"] == (Need("facing", 0.0, 4.5),) assert rules.needs["headroom"] == (Need("overhead", 0.0, 6.5),) - assert (rules.wall_join_ft, rules.battery_height_ft) == (0.6, 3.25) + assert (rules.wall_join_ft, rules.battery_height_ft, rules.headroom_ft) == (0.6, 3.25, 6.5) assert (rules.width_ft, rules.errors["plane"], rules.route_height_ft) == (2.5, 0.75, 1.0) lower = RuleSet.from_yaml( text.replace("exempt_bottom_above_ft: null", "exempt_bottom_above_ft: 2") @@ -667,7 +669,10 @@ def test_a_request_to_see_higher_than_the_view_reached_is_not_redundant(): ] low = invariant_problems(with_wall_seen(1.0), r, rules=RULES) assert not any("lists as observed" in m for m in low) - full = invariant_problems(with_wall_seen(None), r, rules=RULES) + # A view with no out_ft reached headroom (6.5), not 6.6. + default = invariant_problems(with_wall_seen(None), r, rules=RULES) + assert not any("lists as observed" in m for m in default) + full = invariant_problems(with_wall_seen(7.0), r, rules=RULES) assert any("lists as observed" in m for m in full) @@ -928,3 +933,30 @@ def test_a_short_wall_beyond_the_error_does_not_matter(): scene = short_wall_scene(2.0, [[4.0, 0.0], [12.0, 0.0]]) r = result(checks=[check(), passing("wall_backing")], sweep=[]) assert declared_height_problems(scene, r, RULES) == [] + + +@pytest.mark.parametrize( + ("height", "out_ft", "settled"), + [ + (6.5, None, True), # no out_ft: seen to headroom, inclusively + (6.5 + 1e-10, None, True), # within the server's float tolerance + (6.51, None, False), # more than headroom needs an explicit view + (6.5, 6.5, False), # an explicit view must pass the height + (6.5, 6.51, True), + (1.0, None, True), + ], +) +def test_a_wall_view_without_out_ft_reached_headroom_height(height, out_ft, settled): + scene = copy.deepcopy(SCENE) + entry = {"band": "wall", "span_ft": [-12.0, 10.0]} + if out_ft is not None: + entry["out_ft"] = out_ft + scene["coverage"]["observed"][0] = entry + seen = observed(scene, "wall", height, beyond=True, wall_default_ft=RULES.headroom_ft) + assert (seen == [(-12.0, 10.0)]) == settled + + +def test_a_pass_needing_the_wall_above_headroom_is_flagged_without_out_ft(): + tall = replace(RULES, needs=RULES.needs | {"opening_clearance": (Need("wall", 1.0, 7.0),)}) + r = result(checks=[check(), passing("opening_clearance")]) + assert any("observed higher than 7.0 ft" in m for m in invariant_problems(SCENE, r, rules=tall)) From fcfecbac2ded0b0e6792b770ec78e6b80678d6be Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 01:13:24 -0500 Subject: [PATCH 49/59] verification: claim only demonstrated reach in the ETH3D builder; owe requests for a spot's unseen checks - build_scene.py (review 4114226406): every coverage entry now carries the reach its probes showed as out_ft: wall 3.29 (the battery's height), overhead 6.5, facing the smallest probed depth up to 20 ft, ground 6. Without one, a wall view read as headroom and facing and overhead as all the way. Tested with mocked probes; the scene outside git is not rebuilt. - missing_evidence_problems (review 4114226418): a chosen spot with checks unsure for an unobserved area no longer passes when the result drops the unobserved_area reason and its requests. Without that reason, what the checks lack must lie past an end the result asks to walk past (the server's unexplored_end with a past_end request). No spot and no reason keeps the all-fail exception. - simrun --udid runs on a Simulator handed over instead of creating one. e2e pinned to 9176125 (S2's final head, code identical to 6200098): all 50 scenes pass. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/hsverify/resultcheck.py | 39 ++++++-- verification/hsverify/simrun.py | 15 ++- .../scenes/eth3d-facade/build_scene.py | 99 +++++++++++-------- verification/tests/test_build_scene.py | 42 ++++++++ verification/tests/test_resultcheck.py | 51 +++++++++- 5 files changed, 191 insertions(+), 55 deletions(-) diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index a2de1062b..beeafdf7c 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -522,21 +522,46 @@ def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = """When coverage is what stands in the way (reason unobserved_area), every check left unsure for an unobserved area is named by a request, and a clearance check by a request of each band it lacks around the chosen spot: a check that forgets one band leaves the next capture - still unsure. Otherwise no photo would change the answer, and C2 lets missing_evidence stay - empty.""" + still unsure. + + Without that reason, a chosen spot may still have checks unsure for an unobserved area only + where what they lack lies past an unexplored end the result asks to walk past (reason + unexplored_end, a past_end request, which names no checks). Anything else they lack is owed + the reason and its requests; leaving both out would never ask the homeowner for the view + that settles the check. Without a spot and without the reason, every position fails anyway, + no photo would change the answer, and C2 lets missing_evidence stay empty.""" codes = {r["code"] for r in result.get("reasons", [])} - if "unobserved_area" not in codes: - return [] + spot = result.get("spot") + unseen = unobserved_checks(result) requests = result.get("missing_evidence", []) + if "unobserved_area" not in codes: + if spot is None or not unseen: + return [] + if rules is None: + if requests: + return [] + return [ + f"the chosen spot has checks unsure for an unobserved area ({', '.join(unseen)}) " + "but no reason unobserved_area and no missing_evidence" + ] + lo, hi = spot["span_ft"] + asked = with_past_ends_asked(scene, requests, rules) + return [ + f"check {c} is unsure and needs {gap}, not past an end the result asks to walk " + f"past, but there is no reason unobserved_area and no {band} request" + for c in unseen + if c in rules.needs + for band, gap in reach_gaps(asked, rules, c, spot["wall_id"], lo, hi) + ] if not requests: return ["reason unobserved_area but missing_evidence is empty"] + problems: list[str] = [] named = {(r.get("band"), c) for r in requests for c in r.get("checks", [])} - problems = [ + problems += [ f"check {c} is unsure (unobserved) but no missing_evidence entry names it" - for c in unobserved_checks(result) + for c in unseen if not any(check == c for _, check in named) ] - spot = result.get("spot") if rules is None or spot is None: return problems lo, hi = spot["span_ft"] diff --git a/verification/hsverify/simrun.py b/verification/hsverify/simrun.py index dde3ee4cb..235d96d8e 100644 --- a/verification/hsverify/simrun.py +++ b/verification/hsverify/simrun.py @@ -105,8 +105,16 @@ def latest_ios_runtime() -> str: return ios[-1]["identifier"] -def ensure_device() -> dict: - """The runner's own Simulator, so it never disturbs devices other workers are using.""" +def ensure_device(udid: str | None = None) -> dict: + """The runner's own Simulator, so it never disturbs devices other workers are using, or + the existing device `udid` when one has been handed over (creating a device costs disk).""" + if udid is not None: + for runtime, devices in json.loads(simctl("list", "devices", "-j"))["devices"].items(): + for device in devices: + if device["udid"] == udid: + simctl("bootstatus", udid, "-b") + return {"name": device["name"], "udid": udid, "runtime": runtime} + raise SystemExit(f"No Simulator with UDID {udid}; see `xcrun simctl list devices`.") runtime = latest_ios_runtime() devices = json.loads(simctl("list", "devices", "-j"))["devices"].get(runtime, []) for device in devices: @@ -563,6 +571,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--out", type=Path, help="report folder (default under reports/sim)") parser.add_argument("--label", default="", help="short name added to the report folder") parser.add_argument("--keep-worktree", action="store_true") + parser.add_argument("--udid", help="run on this existing Simulator instead of the runner's own") parser.add_argument("--keep-booted", action="store_true", help="leave the Simulator on") parser.add_argument("--build-wait", type=float, default=1200.0) args = parser.parse_args(argv) @@ -599,7 +608,7 @@ def main(argv: list[str] | None = None) -> int: flush=True, ) - device = ensure_device() + device = ensure_device(args.udid) rep = RunReport( ref=args.ref, sha=sha, diff --git a/verification/scenes/eth3d-facade/build_scene.py b/verification/scenes/eth3d-facade/build_scene.py index f6378894c..a5cb9f841 100644 --- a/verification/scenes/eth3d-facade/build_scene.py +++ b/verification/scenes/eth3d-facade/build_scene.py @@ -17,6 +17,7 @@ import argparse import json +import math import re import shutil import sys @@ -441,38 +442,73 @@ def visible(kf: Keyframe, wall: Wall, X, walls) -> bool: return True +FACING_PROBE_MAX_FT = 20.0 # the facing probe looks no further out than this + + +def band_reach(band: str, facing_depth: float | None) -> float | None: + """How far a band's probes reach, and so the most a view that sees them all demonstrates: + up the wall and up above the ground (wall, overhead), out from the wall (ground, facing). + It becomes the entry's `out_ft`: without one, the server reads a wall view as reaching + headroom and facing and overhead views as reaching all the way, more than was checked.""" + if band == "wall": + return BATTERY_HEIGHT_FT + if band == "ground": + return GROUND_OUT_FT + if band == "overhead": + return HEADROOM_FT + if band == "facing": + return None if facing_depth is None else min(facing_depth, FACING_PROBE_MAX_FT) + raise ValueError(band) + + def band_probes(band: str, wall: Wall, s: float, facing_depth: float | None): q = wall.plan_at(s) o = wall.outward + reach = band_reach(band, facing_depth) def at(out, y): p = q + out * o return np.array([p[0], y, p[1]]) + if reach is None: + return None if band == "wall": - return [at(0.0, 0.0), at(0.0, BATTERY_HEIGHT_FT)] + return [at(0.0, 0.0), at(0.0, reach)] if band == "ground": - return [at(0.0, 0.0), at(GROUND_OUT_FT, 0.0)] + return [at(0.0, 0.0), at(reach, 0.0)] if band == "overhead": - return [at(0.0, HEADROOM_FT), at(OVERHEAD_OUT_FT, HEADROOM_FT), at(OVERHEAD_OUT_FT, 0.0)] - if band == "facing": - if facing_depth is None: - return None - return [at(0.0, 3.0), at(min(facing_depth, 20.0), 3.0)] - raise ValueError(band) + return [at(0.0, reach), at(OVERHEAD_OUT_FT, reach), at(OVERHEAD_OUT_FT, 0.0)] + return [at(0.0, 3.0), at(reach, 3.0)] -def intervals(ss, ok, step): - """Sample centres ss with flags ok -> [start, end] spans of consecutive True samples.""" - out = [] - for s, k in zip(ss, ok, strict=True): - if not k: - continue - if out and abs(out[-1][1] - (s - step / 2)) < 1e-9: - out[-1][1] = s + step / 2 +def band_coverage(band, walls, facing, kfs, seen=visible): + """Observed entries for one band: stretches where some keyframe sees every probe, merged + across corners, each with the smallest reach demonstrated inside it as `out_ft`, rounded + down so it never claims more. `seen(kf, wall, X, walls)` is `visible`; tests replace it.""" + samples = [] # (s0, s1, reach) per sample seen + for w in walls: + for s in np.arange(w.s0 + COVERAGE_STEP_FT / 2, w.s0 + w.length, COVERAGE_STEP_FT): + fd = next((v for (wid, a, b, v) in facing if wid == w.id and a <= s < b), None) + probes = band_probes(band, w, s, fd) + if probes is not None and any(all(seen(k, w, X, walls) for X in probes) for k in kfs): + half = COVERAGE_STEP_FT / 2 + samples.append([s - half, s + half, band_reach(band, fd)]) + merged = [] # consecutive samples, also across a corner, keep the smaller reach + for a, b, reach in samples: + if merged and abs(merged[-1][1] - a) < 1e-6: + merged[-1][1] = b + merged[-1][2] = min(merged[-1][2], reach) else: - out.append([s - step / 2, s + step / 2]) - return out + merged.append([a, b, reach]) + entries = [ + { + "band": band, + "span_ft": [round(a, 2), round(b, 2)], + "out_ft": math.floor(reach * 100) / 100, + } + for a, b, reach in merged + ] + return entries, [[a, b] for a, b, _ in merged] WALL_RELIEF_FT = 1.0 # pilasters, sills and downspouts stand out less than this; see scan_profiles @@ -767,31 +803,8 @@ def main(): observed = [] per_band = {} for band in ("wall", "ground", "overhead", "facing"): - spans = [] - for w in walls: - ss = np.arange(w.s0 + COVERAGE_STEP_FT / 2, w.s0 + w.length, COVERAGE_STEP_FT) - ok = [] - for s in ss: - fd = next((v for (wid, a, b, v) in facing if wid == w.id and a <= s < b), None) - probes = band_probes(band, w, s, fd) - ok.append( - probes is not None - and any(all(visible(k, w, X, walls) for X in probes) for k in kfs) - ) - spans += [(w.id, a, b) for a, b in intervals(ss, ok, COVERAGE_STEP_FT)] - # join spans that touch across a corner - merged = [] - for _, a, b in spans: - if merged and abs(merged[-1][1] - a) < 1e-6: - merged[-1][1] = b - else: - merged.append([a, b]) - per_band[band] = merged - for a, b in merged: - entry = {"band": band, "span_ft": [round(a, 2), round(b, 2)]} - if band == "ground": - entry["out_ft"] = GROUND_OUT_FT - observed.append(entry) + entries, per_band[band] = band_coverage(band, walls, facing, kfs) + observed += entries # 7. Assemble ---------------------------------------------------------------------------- scene = { diff --git a/verification/tests/test_build_scene.py b/verification/tests/test_build_scene.py index a5911a7e9..d2983bfb7 100644 --- a/verification/tests/test_build_scene.py +++ b/verification/tests/test_build_scene.py @@ -4,6 +4,7 @@ import sys from pathlib import Path +import numpy as np import pytest SCRIPT = Path(__file__).resolve().parents[1] / "scenes" / "eth3d-facade" / "build_scene.py" @@ -50,3 +51,44 @@ def test_a_symlinked_output_is_unlinked_not_followed(tmp_path): build_scene.clear_outputs(out) assert (elsewhere / "precious").read_text() == "keep" assert not (out / "bundle").exists() + + +def wall(wid: str, s0: float, length: float): + return build_scene.Wall(wid, np.array([s0, 0.0]), np.array([s0 + length, 0.0]), s0=s0) + + +def seen_everywhere(kf, w, X, walls): + return True + + +@pytest.mark.parametrize( + ("band", "out_ft"), + [ + ("wall", 3.29), # up to the battery's height, not the headroom an omitted view claims + ("ground", build_scene.GROUND_OUT_FT), + ("overhead", build_scene.HEADROOM_FT), # not clear all the way up + ], +) +def test_every_band_claims_only_the_reach_its_probes_showed(band, out_ft): + entries, spans = build_scene.band_coverage( + band, [wall("w", 0.0, 2.0)], [], ["kf"], seen_everywhere + ) + assert entries == [{"band": band, "span_ft": [0.0, 2.0], "out_ft": out_ft}] + assert spans == [[0.0, 2.0]] + + +def test_facing_claims_the_smallest_depth_probed_and_nothing_past_20_ft(): + walls = [wall("a", 0.0, 1.0), wall("b", 1.0, 1.0)] # meet at a corner: one merged entry + facing = [("a", 0.0, 1.0, 35.0), ("b", 1.0, 1.5, 12.345), ("b", 1.5, 2.0, 30.0)] + entries, _ = build_scene.band_coverage("facing", walls, facing, ["kf"], seen_everywhere) + assert entries == [{"band": "facing", "span_ft": [0.0, 2.0], "out_ft": 12.34}] + entries, _ = build_scene.band_coverage("facing", walls[:1], facing, ["kf"], seen_everywhere) + assert entries[0]["out_ft"] == 20.0 + + +def test_an_unseen_probe_leaves_the_stretch_out(): + def only_low(kf, w, X, walls): + return X[1] <= 1.0 # the top of the wall probe is never seen + + entries, _ = build_scene.band_coverage("wall", [wall("w", 0.0, 2.0)], [], ["kf"], only_low) + assert entries == [] diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 4666ee985..cf538a27f 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -317,8 +317,6 @@ def test_photo_request_for_an_observed_area_is_flagged(): def test_every_unobserved_check_needs_a_request_naming_it(): unseen = check("unsure", None, cause="unobserved") r = result(checks=[unseen]) - # No unobserved_area reason: every position fails anyway, so no photo is owed. - assert missing_evidence_problems(SCENE, r) == [] r["reasons"] = [{"code": "unobserved_area", "message": ""}] assert missing_evidence_problems(SCENE, r) == [ "reason unobserved_area but missing_evidence is empty" @@ -960,3 +958,52 @@ def test_a_pass_needing_the_wall_above_headroom_is_flagged_without_out_ft(): tall = replace(RULES, needs=RULES.needs | {"opening_clearance": (Need("wall", 1.0, 7.0),)}) r = result(checks=[check(), passing("opening_clearance")]) assert any("observed higher than 7.0 ft" in m for m in invariant_problems(SCENE, r, rules=tall)) + + +def test_without_a_spot_or_the_reason_no_photo_is_owed(): + # Every position fails anyway: no spot, no unobserved_area reason, nothing to capture. + unseen = check("unsure", None, cause="unobserved") + r = result(decision="reject", spot=False, checks=[unseen]) + assert missing_evidence_problems(SCENE, r, RULES) == [] + + +def test_a_spot_with_an_unobserved_check_needs_the_reason_and_the_request(): + # The server marks the spot's check unsure for lack of coverage but drops both the reason + # and the request: the homeowner would never be asked for the view that settles it. + scene = copy.deepcopy(SCENE) + scene["coverage"]["observed"] = [scene["coverage"]["observed"][0]] # no ground seen + r = result(checks=[check("unsure", None, cause="unobserved")]) + msgs = missing_evidence_problems(scene, r, RULES) + assert msgs and all("no reason unobserved_area and no ground request" in m for m in msgs) + assert any("no reason unobserved_area" in m for m in invariant_problems(scene, r, rules=RULES)) + r["reasons"].append({"code": "unobserved_area", "message": ""}) + r["missing_evidence"] = [ + { + "kind": "band", + "band": "ground", + "span_ft": [-1, 5], + "out_ft": 3.0, + "checks": ["gas_clearance"], + "message": "", + } + ] + assert missing_evidence_problems(scene, r, RULES) == [] + + +def test_what_lies_past_an_unexplored_end_needs_only_the_walk_past_it(): + # As the server answers once every band view is in: reason unexplored_end, a past_end + # request naming no checks, and checks unsure only for what lies beyond the end. + wide = replace( + RULES, needs=RULES.needs | {"gas_clearance": (Need("ground", 1.0), Need("wall", 3.0, 6.5))} + ) + scene = copy.deepcopy(SCENE) + scene["walls"][0]["baseline"] = [[-1.0, 0.0], [12.0, 0.0]] # the chain's left end is s = -1 + scene["coverage"]["ends"]["left"] = {"kind": "unexplored"} + scene["coverage"]["observed"][0]["span_ft"] = [-1.0, 10.0] + r = result(checks=[check("unsure", None, cause="unobserved")]) + r["reasons"] = [{"code": "unexplored_end", "message": ""}] + assert any("wall [-2.00, 6.58]" in m for m in missing_evidence_problems(scene, r, wide)) + r["missing_evidence"] = [ + {"kind": "past_end", "side": "left", "span_ft": [-1.0, -1.0], "message": ""} + ] + assert missing_evidence_problems(scene, r, wide) == [] From 36ac02f9d1fb3d654ff12dde652de65a134d1cfd Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 01:23:18 -0500 Subject: [PATCH 50/59] verification: scoreboard and result at server 9176125 S2-1 to S2-8 met at 9176125: e2e 50 of 50, server pytest 346 passed, ruff clean. An earlier probe run under a load average near 140 failed one server test, which then passed alone three times and in a full rerun. S4-1 and S4-2 stay stale (app replay from 194f2eb, follow-up #121). Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/README.md | 4 +- verification/SCOREBOARD.md | 92 ++--- verification/scoreboard/scoreboard.json | 496 ++++++++++++------------ 3 files changed, 292 insertions(+), 300 deletions(-) diff --git a/verification/README.md b/verification/README.md index f0515acfb..f4b9e4928 100644 --- a/verification/README.md +++ b/verification/README.md @@ -19,8 +19,8 @@ Reports go to `~/house-scanning-data/reports/`, outside git, because replay fram non-commercial datasets. What each check asserts is in the docstrings of `hsverify/e2e.py`, `resultcheck.py` and `simrun.py`. -**Result.** At `t3/server` `7133283`, with the app's upload from `194f2eb`, all 50 scenes pass (45 -answered and checked; slowest real scene 0.42 s). +**Result.** At `t3/server` `9176125`, with the app's upload from `194f2eb`, all 50 scenes pass (45 +answered and checked; slowest real scene 0.24 s). **What it changed.** S2 fixed seven defects it found: - unseen ground passing the pool check; diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index 588b40968..46c05b11e 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,23 +1,31 @@ # Scoreboard -Generated 2026-09-27 00:20 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran for S2-1, S2-7 only. Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-27 01:23 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran for S2-1, S2-7 only. Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | -| [#104](https://github.com/SamGu-NRX/house-scanning/pull/104) | `t3/41-gate-coaching` | `c4dec620` | yes | 2 success, 2 skipped | 2026-09-27 00:15 CDT | -| [#102](https://github.com/SamGu-NRX/house-scanning/pull/102) | `t3/41-card-replies` | `a4b8b272` | yes | 2 success, 2 skipped | 2026-09-27 00:13 CDT | -| [#100](https://github.com/SamGu-NRX/house-scanning/pull/100) | `t3/41-ar-canvas-fallback` | `41291e82` | yes | 2 success, 2 skipped | 2026-09-27 00:10 CDT | -| [#99](https://github.com/SamGu-NRX/house-scanning/pull/99) | `t3/41-end-preview` | `b5006fca` | yes | 2 success, 2 skipped | 2026-09-27 00:11 CDT | -| [#91](https://github.com/SamGu-NRX/house-scanning/pull/91) | `t3/experience-result` | `00fb017d` | yes | 2 success, 1 skipped | 2026-09-27 00:06 CDT | +| [#124](https://github.com/SamGu-NRX/house-scanning/pull/124) | `t3/integration-viewer` | `1fa19b33` | no | 4 success | 2026-09-27 01:17 CDT | +| [#123](https://github.com/SamGu-NRX/house-scanning/pull/123) | `t3/hunter-integration` | `60023d28` | yes | 2 success, 2 skipped | 2026-09-27 01:21 CDT | +| [#122](https://github.com/SamGu-NRX/house-scanning/pull/122) | `t3/41-server-requests-text` | `a22c2d45` | yes | 3 success, 1 skipped | 2026-09-27 00:37 CDT | +| [#120](https://github.com/SamGu-NRX/house-scanning/pull/120) | `t3/41-walk-task-hold` | `6ea58d50` | yes | 2 success, 1 failure, 1 skipped; failing: iOS build | 2026-09-27 01:03 CDT | +| [#119](https://github.com/SamGu-NRX/house-scanning/pull/119) | `t3/41-anchored-marks` | `f141a621` | yes | 2 success, 2 skipped | 2026-09-27 00:34 CDT | +| [#117](https://github.com/SamGu-NRX/house-scanning/pull/117) | `t3/41-target-ring` | `97048675` | yes | 2 success, 2 skipped | 2026-09-27 00:36 CDT | +| [#109](https://github.com/SamGu-NRX/house-scanning/pull/109) | `t3/41-gap-loop-result` | `5c15da87` | yes | 2 success, 2 skipped | 2026-09-27 01:00 CDT | +| [#108](https://github.com/SamGu-NRX/house-scanning/pull/108) | `t3/41-ground-and-meter-tap` | `9c2a5676` | yes | 2 success, 2 skipped | 2026-09-27 00:25 CDT | +| [#104](https://github.com/SamGu-NRX/house-scanning/pull/104) | `t3/41-gate-coaching` | `9b0acc58` | yes | 2 success, 2 skipped | 2026-09-27 01:00 CDT | +| [#102](https://github.com/SamGu-NRX/house-scanning/pull/102) | `t3/41-card-replies` | `a4b8b272` | yes | 2 success, 2 skipped | 2026-09-27 01:18 CDT | +| [#100](https://github.com/SamGu-NRX/house-scanning/pull/100) | `t3/41-ar-canvas-fallback` | `41291e82` | yes | 2 success, 2 skipped | 2026-09-27 00:57 CDT | +| [#99](https://github.com/SamGu-NRX/house-scanning/pull/99) | `t3/41-end-preview` | `b5006fca` | yes | 2 success, 2 skipped | 2026-09-27 00:56 CDT | +| [#91](https://github.com/SamGu-NRX/house-scanning/pull/91) | `t3/experience-result` | `9583b129` | no | 4 success, 1 pending | 2026-09-27 01:21 CDT | | [#90](https://github.com/SamGu-NRX/house-scanning/pull/90) | `t3/check-experiments` | `9d66b31c` | no | 5 success | 2026-09-26 23:39 CDT | -| [#89](https://github.com/SamGu-NRX/house-scanning/pull/89) | `t3/meter-brand` | `0a3a05e2` | no | 4 success, 1 pending | 2026-09-27 00:15 CDT | +| [#89](https://github.com/SamGu-NRX/house-scanning/pull/89) | `t3/meter-brand` | `0a3a05e2` | no | 5 success | 2026-09-27 01:18 CDT | | [#88](https://github.com/SamGu-NRX/house-scanning/pull/88) | `t3/autodetect-lab` | `538f60e1` | yes | 2 success | 2026-09-26 23:00 CDT | -| [#87](https://github.com/SamGu-NRX/house-scanning/pull/87) | `t3/41-zip-deflate` | `5150321a` | yes | 2 success, 1 skipped | 2026-09-27 00:07 CDT | +| [#87](https://github.com/SamGu-NRX/house-scanning/pull/87) | `t3/41-zip-deflate` | `5150321a` | yes | 2 success, 1 skipped | 2026-09-27 00:31 CDT | | [#86](https://github.com/SamGu-NRX/house-scanning/pull/86) | `t3/experience` | `b442f0a0` | yes | 2 success, 1 skipped | 2026-09-27 00:00 CDT | -| [#61](https://github.com/SamGu-NRX/house-scanning/pull/61) | `t3/app-review-docs` | `c0b628b6` | no | 4 success | 2026-09-26 23:23 CDT | -| [#60](https://github.com/SamGu-NRX/house-scanning/pull/60) | `t3/device-test-4.1-analysis` | `c566490c` | no | 2 success | 2026-09-27 00:15 CDT | +| [#61](https://github.com/SamGu-NRX/house-scanning/pull/61) | `t3/app-review-docs` | `c0b628b6` | no | 4 success | 2026-09-27 01:10 CDT | +| [#60](https://github.com/SamGu-NRX/house-scanning/pull/60) | `t3/device-test-4.1-analysis` | `c566490c` | no | 2 success | 2026-09-27 00:34 CDT | | [#59](https://github.com/SamGu-NRX/house-scanning/pull/59) | `t3/signals-lab` | `0cde05e3` | yes | 2 success | 2026-09-26 21:38 CDT | | [#58](https://github.com/SamGu-NRX/house-scanning/pull/58) | `t3/ios-ci-no-preboot` | `3c1298ab` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 22:18 CDT | | [#56](https://github.com/SamGu-NRX/house-scanning/pull/56) | `t3/walk-guidance-gaps` | `cae4f426` | no | 3 success | 2026-09-26 23:57 CDT | @@ -26,71 +34,63 @@ Generated 2026-09-27 00:20 CDT by `make scoreboard` (`uv run python -m hsverify. | [#49](https://github.com/SamGu-NRX/house-scanning/pull/49) | `t3/gap-skip-continues` | `ce901122` | no | 3 success | 2026-09-26 23:57 CDT | | [#46](https://github.com/SamGu-NRX/house-scanning/pull/46) | `t3/field-test-run2` | `afd02c82` | no | 2 success | 2026-09-26 17:36 CDT | | [#23](https://github.com/SamGu-NRX/house-scanning/pull/23) | `t3/device-field-test` | `321ea7e2` | no | 2 success | 2026-09-26 19:39 CDT | -| [#22](https://github.com/SamGu-NRX/house-scanning/pull/22) | `t3/packet` | `89372a0a` | no | 4 success | 2026-09-26 23:54 CDT | -| [#21](https://github.com/SamGu-NRX/house-scanning/pull/21) | `t3/ios-map3d` | `c77577b7` | yes | 2 success, 2 skipped | 2026-09-26 23:04 CDT | -| [#20](https://github.com/SamGu-NRX/house-scanning/pull/20) | `t3/recon` | `d072729b` | yes | 3 success, 1 skipped | 2026-09-27 00:08 CDT | -| [#18](https://github.com/SamGu-NRX/house-scanning/pull/18) | `codex/research-findings` | `bd4a7d38` | no | 2 success | 2026-09-26 16:21 CDT | -| [#16](https://github.com/SamGu-NRX/house-scanning/pull/16) | `t3/meter-closeup` | `dc4f38da` | no | 4 success, 1 pending, 1 skipped | 2026-09-27 00:16 CDT | -| [#13](https://github.com/SamGu-NRX/house-scanning/pull/13) | `t3/verification` | `de53ecd5` | no | 3 success, 2 pending | 2026-09-27 00:16 CDT | -| [#12](https://github.com/SamGu-NRX/house-scanning/pull/12) | `t3/evals` | `40698465` | no | 5 success | 2026-09-27 00:14 CDT | -| [#11](https://github.com/SamGu-NRX/house-scanning/pull/11) | `t3/server` | `7133283e` | no | 5 success | 2026-09-27 00:15 CDT | -| [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | `t3/ios-mvf` | `b9851e15` | no | 3 success, 3 pending | 2026-09-27 00:11 CDT | -| [#9](https://github.com/SamGu-NRX/house-scanning/pull/9) | `codex/capture-contract-probe` | `4b6c4309` | yes | 4 success | 2026-09-26 00:08 CDT | +| [#22](https://github.com/SamGu-NRX/house-scanning/pull/22) | `t3/packet` | `89372a0a` | no | 4 success | 2026-09-27 01:05 CDT | +| [#21](https://github.com/SamGu-NRX/house-scanning/pull/21) | `t3/ios-map3d` | `c77577b7` | yes | 2 success, 2 skipped | 2026-09-27 00:28 CDT | ## S1 Evals on real data -`origin/t3/evals` at [`40698465`](https://github.com/SamGu-NRX/house-scanning/tree/406984650759894263c60ac4f9f357e3f1fa0de8). 1 met, 2 partial, 1 gap, 1 not run. +`origin/t3/evals` at [`a7f68ba1`](https://github.com/SamGu-NRX/house-scanning/tree/a7f68ba15f7ffc589e28270d3fe84968e9d3becb). 1 met, 2 partial, 1 gap, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S1-1 | One command regenerates every number from a clean checkout | partial | make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac. | | S1-2 | Dataset URLs and sha256 recorded | gap | none found in 1 files; missing ADVIO, ETH3D | | S1-3 | Metric code tested on hand-computed cases | not run | slow; run with --slow | -| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/METHODS.md](https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/METHODS.md), [experiments/evals/field/FIELD_SHEET.md](https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/field/FIELD_SHEET.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/results/advio_drift.md) | -| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/METHODS.md](https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/METHODS.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/results/eth3d_visibility_sensitivity.md) | +| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/METHODS.md](https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/METHODS.md), [experiments/evals/field/FIELD_SHEET.md](https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/field/FIELD_SHEET.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/results/advio_drift.md) | +| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/METHODS.md](https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/METHODS.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/results/eth3d_visibility_sensitivity.md) | ## S2 Server and solver -`origin/t3/server` at [`7133283e`](https://github.com/SamGu-NRX/house-scanning/tree/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca). 6 met, 1 partial, 1 gap. +`origin/t3/server` at [`9176125d`](https://github.com/SamGu-NRX/house-scanning/tree/9176125df0c9890c247818d55edc82ea46053cd8). 7 met, 1 partial. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S2-1 | pytest green | gap | exit 1: =========================== short test summary info ============================ / FAILED tests/test_api.py::test_solving_the_example_takes_under_a_second - ass... / 1 failed, 339 passed, 1 warning in 190.75s (0:03:10) | -| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_golden_public.py) | -| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_properties.py) | -| S2-4 | Schemas published | met | present at 7133283e [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/schemas/result.schema.json) | -| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_existing_battery.py](https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_existing_battery.py), [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_schemas.py) | -| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 424.6 (want < 1000) at sha 7133283e: ~/house-scanning-data/reports/e2e/20260927-001445-7133283e/report.json | -| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 82ms / All checks passed! / 31 files already formatted | -| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | met | contract_ok = true (want True) at sha 7133283e: ~/house-scanning-data/reports/e2e/20260927-001445-7133283e/report.json | +| S2-1 | pytest green | met | exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 346 passed, 1 warning in 40.16s | +| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_golden_public.py) | +| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_properties.py) | +| S2-4 | Schemas published | met | present at 9176125d [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/schemas/result.schema.json) | +| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_existing_battery.py](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_existing_battery.py), [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_schemas.py) | +| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 238.4 (want < 1000) at sha 9176125d: ~/house-scanning-data/reports/e2e/20260927-011249-9176125d/report.json | +| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 42ms / All checks passed! / 31 files already formatted | +| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | met | contract_ok = true (want True) at sha 9176125d: ~/house-scanning-data/reports/e2e/20260927-011249-9176125d/report.json | ## S3 iOS capture app -`origin/t3/ios-mvf` at [`b9851e15`](https://github.com/SamGu-NRX/house-scanning/tree/b9851e15c407b024b526d104e2f991ae02cdf87e). 2 met, 3 partial, 1 gap, 3 stale, 1 not run. +`origin/t3/ios-mvf` at [`f14947eb`](https://github.com/SamGu-NRX/house-scanning/tree/f14947eb9bff5b707660c8d873b72e60ba056631). 1 met, 3 partial, 1 gap, 3 stale, 2 no evidence yet. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S3-1 | CI build green | not run | iOS build: pending at b9851e15 [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/project.yml) | -| S3-3 | Zero build warnings | stale | report for sha 194f2ebf, ref at b9851e15: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json | -| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage, guidance; missing auto-capture [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift) | -| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/SourceRestartUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/SourceRestartUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at b9851e15: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | -| S3-7 | Screenshots of each state in the PR | met | 18 of 9 matches in the PR body [#10](https://github.com/SamGu-NRX/house-scanning/pull/10) | -| S3-8 | Exported scene.json validates against C1 | stale | report for app_sha 194f2ebf, ref at b9851e15: ~/house-scanning-data/reports/e2e/20260927-001445-7133283e/report.json | +| S3-1 | CI build green | no evidence yet | no open PR for t3/ios-mvf | +| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/project.yml) | +| S3-3 | Zero build warnings | stale | report for sha 194f2ebf, ref at f14947eb: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json | +| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage, guidance; missing auto-capture [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift) | +| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/SourceRestartUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/SourceRestartUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/AccessibilityAudit.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/AccessibilityAudit.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at f14947eb: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | +| S3-7 | Screenshots of each state in the PR | no evidence yet | no open PR for t3/ios-mvf | +| S3-8 | Exported scene.json validates against C1 | stale | report for app_sha 194f2ebf, ref at f14947eb: ~/house-scanning-data/reports/e2e/20260927-011249-9176125d/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/ios-mvf` at [`b9851e15`](https://github.com/SamGu-NRX/house-scanning/tree/b9851e15c407b024b526d104e2f991ae02cdf87e), `origin/t3/verification` at [`de53ecd5`](https://github.com/SamGu-NRX/house-scanning/tree/de53ecd52a2af1350b7502ef6bfd62586e84ef01). 2 met, 2 stale. +`origin/t3/ios-mvf` at [`f14947eb`](https://github.com/SamGu-NRX/house-scanning/tree/f14947eb9bff5b707660c8d873b72e60ba056631), `origin/t3/verification` at [`fcfecbac`](https://github.com/SamGu-NRX/house-scanning/tree/fcfecbac2ded0b0e6792b770ec78e6b80678d6be). 2 met, 2 stale. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S4-1 | End to end from a real replay to a placement result passes | stale | report for app_sha 194f2ebf, ref at b9851e15: ~/house-scanning-data/reports/e2e/20260927-001445-7133283e/report.json | -| S4-2 | Screenshot report per app state | stale | report for sha 194f2ebf, ref at b9851e15: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json | +| S4-1 | End to end from a real replay to a placement result passes | stale | report for app_sha 194f2ebf, ref at f14947eb: ~/house-scanning-data/reports/e2e/20260927-011249-9176125d/report.json | +| S4-2 | Screenshot report per app state | stale | report for sha 194f2ebf, ref at f14947eb: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json | | S4-3 | Product description written from the code and checked against screenshots | met | docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state. | -| S4-4 | Scoreboard of every PR | met | present at de53ecd5 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/de53ecd52a2af1350b7502ef6bfd62586e84ef01/verification/SCOREBOARD.md) | +| S4-4 | Scoreboard of every PR | met | present at fcfecbac [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/fcfecbac2ded0b0e6792b770ec78e6b80678d6be/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index d2173dea9..08b478d5f 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,18 +1,164 @@ { - "generated_at": "2026-09-27T00:20:31-05:00", + "generated_at": "2026-09-27T01:23:01-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": [ "S2-1", "S2-7" ], "prs": [ + { + "number": 124, + "title": "tools: viewer that films a capture reaching the server, as a labeled replay or read-only live view", + "branch": "t3/integration-viewer", + "head": "1fa19b3349803ab57da5393b47f05ea0fc5b2add", + "draft": false, + "updated": "2026-09-27T06:17:49Z", + "ci": { + "counts": { + "success": 4, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "4 success" + } + }, + { + "number": 123, + "title": "ios: House Scan Integration build uploads each scan to the capture API while scanning", + "branch": "t3/hunter-integration", + "head": "60023d2817870ded5128b2674879a642f7b0177f", + "draft": true, + "updated": "2026-09-27T06:21:47Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 2 + }, + "failing": [], + "text": "2 success, 2 skipped" + } + }, + { + "number": 122, + "title": "server: stop requests at the wall's ends, keep placeholder distances out of them, name things without ids", + "branch": "t3/41-server-requests-text", + "head": "a22c2d45351c202cff91defe9638197b5e87eb52", + "draft": true, + "updated": "2026-09-27T05:37:19Z", + "ci": { + "counts": { + "success": 3, + "failure": 0, + "pending": 0, + "skipped": 1 + }, + "failing": [], + "text": "3 success, 1 skipped" + } + }, + { + "number": 120, + "title": "ios: hold a walk aim until it is met or stalls, and ask for the meter ground only near the meter", + "branch": "t3/41-walk-task-hold", + "head": "6ea58d50562b0c3008b0edea57b0226c3de3aa92", + "draft": true, + "updated": "2026-09-27T06:03:09Z", + "ci": { + "counts": { + "success": 2, + "failure": 1, + "pending": 0, + "skipped": 1 + }, + "failing": [ + "iOS build" + ], + "text": "2 success, 1 failure, 1 skipped; failing: iOS build" + } + }, + { + "number": 119, + "title": "ios: log meter-anchor corrections so marks can be checked against drift", + "branch": "t3/41-anchored-marks", + "head": "f141a6212f3bba8c26695bf3cb5cb40ef0f1cf9d", + "draft": true, + "updated": "2026-09-27T05:34:51Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 2 + }, + "failing": [], + "text": "2 success, 2 skipped" + } + }, + { + "number": 117, + "title": "ios: aim ring fills as its stretch is captured, with a legend and an unambiguous arrow", + "branch": "t3/41-target-ring", + "head": "970486757fcd13391653b93563a669710f475db9", + "draft": true, + "updated": "2026-09-27T05:36:33Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 2 + }, + "failing": [], + "text": "2 success, 2 skipped" + } + }, + { + "number": 109, + "title": "ios: gap-loop follow-ups, upload ticks, the nearer unseen end and request stretches (#39, #31, #83, #75)", + "branch": "t3/41-gap-loop-result", + "head": "5c15da8705ef09bef33a71d1d1dca27deddb7480", + "draft": true, + "updated": "2026-09-27T06:00:21Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 2 + }, + "failing": [], + "text": "2 success, 2 skipped" + } + }, + { + "number": 108, + "title": "ios: ground needs a plausible meter height and never rises; estimated meter taps are checked and re-fitted to the detected wall", + "branch": "t3/41-ground-and-meter-tap", + "head": "9c2a5676fff6b31b1eb1e4bde63a5473ae61b5f3", + "draft": true, + "updated": "2026-09-27T05:25:24Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 2 + }, + "failing": [], + "text": "2 success, 2 skipped" + } + }, { "number": 104, "title": "ios: gate coaching keeps the task card, holds through dark flicker, and says \"Turn more slowly\" for turns", "branch": "t3/41-gate-coaching", - "head": "c4dec62010f20dd94881978a1dc7bce6a47d8920", + "head": "9b0acc5851e7fa69a1da560724c919f1d63bcc5e", "draft": true, - "updated": "2026-09-27T05:15:03Z", + "updated": "2026-09-27T06:00:23Z", "ci": { "counts": { "success": 2, @@ -30,7 +176,7 @@ "branch": "t3/41-card-replies", "head": "a4b8b2723736b5f2b29c5605569dcdb76c6d8a69", "draft": true, - "updated": "2026-09-27T05:13:33Z", + "updated": "2026-09-27T06:18:35Z", "ci": { "counts": { "success": 2, @@ -48,7 +194,7 @@ "branch": "t3/41-ar-canvas-fallback", "head": "41291e8287bd7a2452c6b53b36120670c3210317", "draft": true, - "updated": "2026-09-27T05:10:40Z", + "updated": "2026-09-27T05:57:54Z", "ci": { "counts": { "success": 2, @@ -66,7 +212,7 @@ "branch": "t3/41-end-preview", "head": "b5006fca782aa280caa767e476fd5c62b496a9e2", "draft": true, - "updated": "2026-09-27T05:11:39Z", + "updated": "2026-09-27T05:56:48Z", "ci": { "counts": { "success": 2, @@ -82,18 +228,18 @@ "number": 91, "title": "ios: result screen leads with the answer, shows the nearest rejected spot and links each unsure check to its view", "branch": "t3/experience-result", - "head": "00fb017dcf446921c0ba1753795d03aad1d52872", - "draft": true, - "updated": "2026-09-27T05:06:46Z", + "head": "9583b1294882f40c963f5d21af9b053a6d3d367d", + "draft": false, + "updated": "2026-09-27T06:21:00Z", "ci": { "counts": { - "success": 2, + "success": 4, "failure": 0, - "pending": 0, - "skipped": 1 + "pending": 1, + "skipped": 0 }, "failing": [], - "text": "2 success, 1 skipped" + "text": "4 success, 1 pending" } }, { @@ -120,16 +266,16 @@ "branch": "t3/meter-brand", "head": "0a3a05e200fa42daf002d5fc77687c898935cffa", "draft": false, - "updated": "2026-09-27T05:15:36Z", + "updated": "2026-09-27T06:18:33Z", "ci": { "counts": { - "success": 4, + "success": 5, "failure": 0, - "pending": 1, + "pending": 0, "skipped": 0 }, "failing": [], - "text": "4 success, 1 pending" + "text": "5 success" } }, { @@ -156,7 +302,7 @@ "branch": "t3/41-zip-deflate", "head": "5150321a22f540e669755270dd76caa23957d8a4", "draft": true, - "updated": "2026-09-27T05:07:01Z", + "updated": "2026-09-27T05:31:34Z", "ci": { "counts": { "success": 2, @@ -192,7 +338,7 @@ "branch": "t3/app-review-docs", "head": "c0b628b6b9ec2b314ae418db956a27bc579ee642", "draft": false, - "updated": "2026-09-27T04:23:04Z", + "updated": "2026-09-27T06:10:04Z", "ci": { "counts": { "success": 4, @@ -210,7 +356,7 @@ "branch": "t3/device-test-4.1-analysis", "head": "c566490c5e3fa0d45803895bb463cb260d11a906", "draft": false, - "updated": "2026-09-27T05:15:59Z", + "updated": "2026-09-27T05:34:34Z", "ci": { "counts": { "success": 2, @@ -378,7 +524,7 @@ "branch": "t3/packet", "head": "89372a0a4143918f0d0e67c7e2bbae5ac60e8023", "draft": false, - "updated": "2026-09-27T04:54:02Z", + "updated": "2026-09-27T06:05:00Z", "ci": { "counts": { "success": 4, @@ -392,11 +538,11 @@ }, { "number": 21, - "title": "ios: live on-device 3D map with occlusion-honest coverage", + "title": "ios: live 3D map as the coverage model where depth exists", "branch": "t3/ios-map3d", "head": "c77577b7bbf24d927925654df61b776157ca4101", "draft": true, - "updated": "2026-09-27T04:04:04Z", + "updated": "2026-09-27T05:28:18Z", "ci": { "counts": { "success": 2, @@ -407,150 +553,6 @@ "failing": [], "text": "2 success, 2 skipped" } - }, - { - "number": 20, - "title": "recon: handoff to the server team", - "branch": "t3/recon", - "head": "d072729bcda5d3d774c864e2918aa42a50507db6", - "draft": true, - "updated": "2026-09-27T05:08:36Z", - "ci": { - "counts": { - "success": 3, - "failure": 0, - "pending": 0, - "skipped": 1 - }, - "failing": [], - "text": "3 success, 1 skipped" - } - }, - { - "number": 18, - "title": "Research handoff: live survey findings, options and tested prototypes", - "branch": "codex/research-findings", - "head": "bd4a7d387606d8dedb679e2962accb5a5a48a4b2", - "draft": false, - "updated": "2026-09-26T21:21:58Z", - "ci": { - "counts": { - "success": 2, - "failure": 0, - "pending": 0, - "skipped": 0 - }, - "failing": [], - "text": "2 success" - } - }, - { - "number": 16, - "title": "experiments: meter close-up reading, tap-to-confirm and two retake checks", - "branch": "t3/meter-closeup", - "head": "dc4f38da7c7e785bf7bba6e4eb2607eb9ac05487", - "draft": false, - "updated": "2026-09-27T05:16:24Z", - "ci": { - "counts": { - "success": 4, - "failure": 0, - "pending": 1, - "skipped": 1 - }, - "failing": [], - "text": "4 success, 1 pending, 1 skipped" - } - }, - { - "number": 13, - "title": "verification: independent end-to-end checks (S4)", - "branch": "t3/verification", - "head": "de53ecd52a2af1350b7502ef6bfd62586e84ef01", - "draft": false, - "updated": "2026-09-27T05:16:39Z", - "ci": { - "counts": { - "success": 3, - "failure": 0, - "pending": 2, - "skipped": 0 - }, - "failing": [], - "text": "3 success, 2 pending" - } - }, - { - "number": 12, - "title": "feat(evals): real-data evals for AR drift, wall reconstruction, coverage and the field test", - "branch": "t3/evals", - "head": "406984650759894263c60ac4f9f357e3f1fa0de8", - "draft": false, - "updated": "2026-09-27T05:14:24Z", - "ci": { - "counts": { - "success": 5, - "failure": 0, - "pending": 0, - "skipped": 0 - }, - "failing": [], - "text": "5 success" - } - }, - { - "number": 11, - "title": "server: placement solver, rules and API (S2)", - "branch": "t3/server", - "head": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca", - "draft": false, - "updated": "2026-09-27T05:15:28Z", - "ci": { - "counts": { - "success": 5, - "failure": 0, - "pending": 0, - "skipped": 0 - }, - "failing": [], - "text": "5 success" - } - }, - { - "number": 10, - "title": "ios: guided wall scan from the meter to a battery spot", - "branch": "t3/ios-mvf", - "head": "b9851e15c407b024b526d104e2f991ae02cdf87e", - "draft": false, - "updated": "2026-09-27T05:11:44Z", - "ci": { - "counts": { - "success": 3, - "failure": 0, - "pending": 3, - "skipped": 0 - }, - "failing": [], - "text": "3 success, 3 pending" - } - }, - { - "number": 9, - "title": "experiments: validate camera metadata for reconstruction", - "branch": "codex/capture-contract-probe", - "head": "4b6c430956bb35edff546f140e3152ed75c56246", - "draft": true, - "updated": "2026-09-26T05:08:13Z", - "ci": { - "counts": { - "success": 4, - "failure": 0, - "pending": 0, - "skipped": 0 - }, - "failing": [], - "text": "4 success" - } } ], "pr_error": null, @@ -561,7 +563,7 @@ "refs": [ { "ref": "origin/t3/evals", - "sha": "406984650759894263c60ac4f9f357e3f1fa0de8" + "sha": "a7f68ba15f7ffc589e28270d3fe84968e9d3becb" } ], "metrics": [ @@ -569,7 +571,7 @@ "id": "S1-1", "text": "One command regenerates every number from a clean checkout", "ref": "origin/t3/evals", - "sha": "406984650759894263c60ac4f9f357e3f1fa0de8", + "sha": "a7f68ba15f7ffc589e28270d3fe84968e9d3becb", "status": "partial", "evidence": "make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac.", "links": [] @@ -578,7 +580,7 @@ "id": "S1-2", "text": "Dataset URLs and sha256 recorded", "ref": "origin/t3/evals", - "sha": "406984650759894263c60ac4f9f357e3f1fa0de8", + "sha": "a7f68ba15f7ffc589e28270d3fe84968e9d3becb", "status": "gap", "evidence": "none found in 1 files; missing ADVIO, ETH3D", "links": [] @@ -587,7 +589,7 @@ "id": "S1-3", "text": "Metric code tested on hand-computed cases", "ref": "origin/t3/evals", - "sha": "406984650759894263c60ac4f9f357e3f1fa0de8", + "sha": "a7f68ba15f7ffc589e28270d3fe84968e9d3becb", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -596,21 +598,21 @@ "id": "S1-4", "text": "ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft", "ref": "origin/t3/evals", - "sha": "406984650759894263c60ac4f9f357e3f1fa0de8", + "sha": "a7f68ba15f7ffc589e28270d3fe84968e9d3becb", "status": "met", "evidence": "all 5 found", "links": [ [ "experiments/evals/METHODS.md", - "https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/METHODS.md" + "https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/METHODS.md" ], [ "experiments/evals/field/FIELD_SHEET.md", - "https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/field/FIELD_SHEET.md" + "https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/field/FIELD_SHEET.md" ], [ "experiments/evals/results/advio_drift.md", - "https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/results/advio_drift.md" + "https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/results/advio_drift.md" ] ] }, @@ -618,21 +620,21 @@ "id": "S1-5", "text": "ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question", "ref": "origin/t3/evals", - "sha": "406984650759894263c60ac4f9f357e3f1fa0de8", + "sha": "a7f68ba15f7ffc589e28270d3fe84968e9d3becb", "status": "partial", "evidence": "found 1-3 m row, 3-10 m row, scale error; missing verdict", "links": [ [ "experiments/evals/METHODS.md", - "https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/METHODS.md" + "https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/METHODS.md" ], [ "experiments/evals/results/eth3d_recon.md", - "https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/results/eth3d_recon.md" + "https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/results/eth3d_recon.md" ], [ "experiments/evals/results/eth3d_visibility_sensitivity.md", - "https://github.com/SamGu-NRX/house-scanning/blob/406984650759894263c60ac4f9f357e3f1fa0de8/experiments/evals/results/eth3d_visibility_sensitivity.md" + "https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/results/eth3d_visibility_sensitivity.md" ] ] } @@ -644,7 +646,7 @@ "refs": [ { "ref": "origin/t3/server", - "sha": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca" + "sha": "9176125df0c9890c247818d55edc82ea46053cd8" } ], "metrics": [ @@ -652,22 +654,22 @@ "id": "S2-1", "text": "pytest green", "ref": "origin/t3/server", - "sha": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca", - "status": "gap", - "evidence": "exit 1: =========================== short test summary info ============================ / FAILED tests/test_api.py::test_solving_the_example_takes_under_a_second - ass... / 1 failed, 339 passed, 1 warning in 190.75s (0:03:10)", + "sha": "9176125df0c9890c247818d55edc82ea46053cd8", + "status": "met", + "evidence": "exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 346 passed, 1 warning in 40.16s", "links": [] }, { "id": "S2-2", "text": "Every public golden test 01-14 of docs/research/t3-lane-c-review.md present", "ref": "origin/t3/server", - "sha": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca", + "sha": "9176125df0c9890c247818d55edc82ea46053cd8", "status": "met", "evidence": "all 14 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_golden_public.py" ] ] }, @@ -675,17 +677,17 @@ "id": "S2-3", "text": "Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail", "ref": "origin/t3/server", - "sha": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca", + "sha": "9176125df0c9890c247818d55edc82ea46053cd8", "status": "partial", "evidence": "found missing coverage, mirror; missing equality unsure, larger clearance", "links": [ [ "server/tests/test_coderabbit_review.py", - "https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_coderabbit_review.py" + "https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_coderabbit_review.py" ], [ "server/tests/test_properties.py", - "https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_properties.py" + "https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_properties.py" ] ] }, @@ -693,17 +695,17 @@ "id": "S2-4", "text": "Schemas published", "ref": "origin/t3/server", - "sha": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca", + "sha": "9176125df0c9890c247818d55edc82ea46053cd8", "status": "met", - "evidence": "present at 7133283e", + "evidence": "present at 9176125d", "links": [ [ "server/schemas/scene.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/schemas/scene.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/schemas/scene.schema.json" ], [ "server/schemas/result.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/schemas/result.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/schemas/result.schema.json" ] ] }, @@ -711,21 +713,21 @@ "id": "S2-5", "text": "Schemas validated in tests", "ref": "origin/t3/server", - "sha": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca", + "sha": "9176125df0c9890c247818d55edc82ea46053cd8", "status": "met", "evidence": "all 2 found", "links": [ [ "server/tests/test_existing_battery.py", - "https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_existing_battery.py" + "https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_existing_battery.py" ], [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_golden_public.py" ], [ "server/tests/test_schemas.py", - "https://github.com/SamGu-NRX/house-scanning/blob/7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca/server/tests/test_schemas.py" + "https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_schemas.py" ] ] }, @@ -733,27 +735,27 @@ "id": "S2-6", "text": "A real-derived scene returns a result in under 1 s", "ref": "origin/t3/server", - "sha": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca", + "sha": "9176125df0c9890c247818d55edc82ea46053cd8", "status": "met", - "evidence": "latency_ms = 424.6 (want < 1000) at sha 7133283e: ~/house-scanning-data/reports/e2e/20260927-001445-7133283e/report.json", + "evidence": "latency_ms = 238.4 (want < 1000) at sha 9176125d: ~/house-scanning-data/reports/e2e/20260927-011249-9176125d/report.json", "links": [] }, { "id": "S2-7", "text": "ruff clean", "ref": "origin/t3/server", - "sha": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca", + "sha": "9176125df0c9890c247818d55edc82ea46053cd8", "status": "met", - "evidence": "exit 0: Installed 33 packages in 82ms / All checks passed! / 31 files already formatted", + "evidence": "exit 0: Installed 33 packages in 42ms / All checks passed! / 31 files already formatted", "links": [] }, { "id": "S2-8", "text": "Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time", "ref": "origin/t3/server", - "sha": "7133283ef6fb672f4b2a4c8e5e2648e75b13c2ca", + "sha": "9176125df0c9890c247818d55edc82ea46053cd8", "status": "met", - "evidence": "contract_ok = true (want True) at sha 7133283e: ~/house-scanning-data/reports/e2e/20260927-001445-7133283e/report.json", + "evidence": "contract_ok = true (want True) at sha 9176125d: ~/house-scanning-data/reports/e2e/20260927-011249-9176125d/report.json", "links": [] } ] @@ -764,7 +766,7 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e" + "sha": "f14947eb9bff5b707660c8d873b72e60ba056631" } ], "metrics": [ @@ -772,27 +774,22 @@ "id": "S3-1", "text": "CI build green", "ref": "origin/t3/ios-mvf", - "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", - "status": "not run", - "evidence": "iOS build: pending at b9851e15", - "links": [ - [ - "#10", - "https://github.com/SamGu-NRX/house-scanning/pull/10" - ] - ] + "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", + "status": "no evidence yet", + "evidence": "no open PR for t3/ios-mvf", + "links": [] }, { "id": "S3-2", "text": "Built as Swift 6 with strict concurrency", "ref": "origin/t3/ios-mvf", - "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", + "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", "status": "met", "evidence": "all 2 found", "links": [ [ "ios/project.yml", - "https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/project.yml" + "https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/project.yml" ] ] }, @@ -800,30 +797,30 @@ "id": "S3-3", "text": "Zero build warnings", "ref": "origin/t3/ios-mvf", - "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", + "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", "status": "stale", - "evidence": "report for sha 194f2ebf, ref at b9851e15: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json", + "evidence": "report for sha 194f2ebf, ref at f14947eb: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json", "links": [] }, { "id": "S3-4", "text": "Unit tests for coverage, auto-capture and guidance", "ref": "origin/t3/ios-mvf", - "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", + "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", "status": "partial", "evidence": "found coverage, guidance; missing auto-capture", "links": [ [ "ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift" ] ] }, @@ -831,21 +828,21 @@ "id": "S3-5", "text": "UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop", "ref": "origin/t3/ios-mvf", - "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", + "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", "status": "partial", "evidence": "all 3 found (the tests exist; passing is S3-1)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/FullFlowUITests.swift" ], [ "ios/HouseScanUITests/SourceRestartUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/SourceRestartUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/SourceRestartUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -853,17 +850,17 @@ "id": "S3-6", "text": "Accessibility audit on every screen", "ref": "origin/t3/ios-mvf", - "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", + "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", "status": "partial", "evidence": "3 matches in 2 files (an audit exists; which screens it covers needs a read)", "links": [ [ - "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/FullFlowUITests.swift" + "ios/HouseScanUITests/AccessibilityAudit.swift", + "https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/AccessibilityAudit.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/b9851e15c407b024b526d104e2f991ae02cdf87e/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -871,39 +868,34 @@ "id": "S3-6b", "text": "Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues", "ref": "origin/t3/ios-mvf", - "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", + "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", "status": "stale", - "evidence": "report for sha a39d0a50, ref at b9851e15: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", + "evidence": "report for sha a39d0a50, ref at f14947eb: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", "links": [] }, { "id": "S3-7", "text": "Screenshots of each state in the PR", "ref": "origin/t3/ios-mvf", - "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", - "status": "met", - "evidence": "18 of 9 matches in the PR body", - "links": [ - [ - "#10", - "https://github.com/SamGu-NRX/house-scanning/pull/10" - ] - ] + "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", + "status": "no evidence yet", + "evidence": "no open PR for t3/ios-mvf", + "links": [] }, { "id": "S3-8", "text": "Exported scene.json validates against C1", "ref": "origin/t3/ios-mvf", - "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", + "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", "status": "stale", - "evidence": "report for app_sha 194f2ebf, ref at b9851e15: ~/house-scanning-data/reports/e2e/20260927-001445-7133283e/report.json", + "evidence": "report for app_sha 194f2ebf, ref at f14947eb: ~/house-scanning-data/reports/e2e/20260927-011249-9176125d/report.json", "links": [] }, { "id": "S3-9", "text": "Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design", "ref": "origin/t3/ios-mvf", - "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", + "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -916,11 +908,11 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e" + "sha": "f14947eb9bff5b707660c8d873b72e60ba056631" }, { "ref": "origin/t3/verification", - "sha": "de53ecd52a2af1350b7502ef6bfd62586e84ef01" + "sha": "fcfecbac2ded0b0e6792b770ec78e6b80678d6be" } ], "metrics": [ @@ -928,25 +920,25 @@ "id": "S4-1", "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/ios-mvf", - "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", + "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", "status": "stale", - "evidence": "report for app_sha 194f2ebf, ref at b9851e15: ~/house-scanning-data/reports/e2e/20260927-001445-7133283e/report.json", + "evidence": "report for app_sha 194f2ebf, ref at f14947eb: ~/house-scanning-data/reports/e2e/20260927-011249-9176125d/report.json", "links": [] }, { "id": "S4-2", "text": "Screenshot report per app state", "ref": "origin/t3/ios-mvf", - "sha": "b9851e15c407b024b526d104e2f991ae02cdf87e", + "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", "status": "stale", - "evidence": "report for sha 194f2ebf, ref at b9851e15: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json", + "evidence": "report for sha 194f2ebf, ref at f14947eb: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json", "links": [] }, { "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "de53ecd52a2af1350b7502ef6bfd62586e84ef01", + "sha": "fcfecbac2ded0b0e6792b770ec78e6b80678d6be", "status": "met", "evidence": "docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state.", "links": [] @@ -955,13 +947,13 @@ "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "de53ecd52a2af1350b7502ef6bfd62586e84ef01", + "sha": "fcfecbac2ded0b0e6792b770ec78e6b80678d6be", "status": "met", - "evidence": "present at de53ecd5", + "evidence": "present at fcfecbac", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/de53ecd52a2af1350b7502ef6bfd62586e84ef01/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/fcfecbac2ded0b0e6792b770ec78e6b80678d6be/verification/SCOREBOARD.md" ] ] } From 64d70d1f34396e822adcb5d502fd3d8c55b0dca1 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 01:50:14 -0500 Subject: [PATCH 51/59] verification: app rows at t3/ios-mvf f14947e; read its bundle line; tolerance for requests - statelog reads the bundle line app f14947e logs ("bundle : packet 1.1 with N photos"). - A view reaches a request's out_ft only within the server's 1e-9: the server sets out_ft 1e-6 past a rule's line, so the old 1e-6 slack called a wall seen to exactly 6.5 ft enough for a request past headroom, and flagged the server's correct request on the app's new scan. - Scoreboard: S4-1 and S4-2 met at f14947e. make sim-app ran on A4EE0B20 with the synthetic replay against the hosted server (12 states, one server gap round); the scan passes e2e pinned to 9176125. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/README.md | 4 +- verification/SCOREBOARD.md | 46 +++++++-------- verification/hsverify/resultcheck.py | 5 +- verification/hsverify/statelog.py | 13 +++-- verification/scoreboard/scoreboard.json | 76 ++++++++++++------------- verification/tests/test_resultcheck.py | 13 +++++ verification/tests/test_statelog.py | 10 ++++ 7 files changed, 99 insertions(+), 68 deletions(-) diff --git a/verification/README.md b/verification/README.md index f4b9e4928..0d6a28dca 100644 --- a/verification/README.md +++ b/verification/README.md @@ -19,8 +19,8 @@ Reports go to `~/house-scanning-data/reports/`, outside git, because replay fram non-commercial datasets. What each check asserts is in the docstrings of `hsverify/e2e.py`, `resultcheck.py` and `simrun.py`. -**Result.** At `t3/server` `9176125`, with the app's upload from `194f2eb`, all 50 scenes pass (45 -answered and checked; slowest real scene 0.24 s). +**Result.** At `t3/server` `9176125`, with the app's upload from `f14947e` (synthetic replay, hosted +server), all 50 scenes pass (45 answered and checked; slowest real scene 0.24 s). **What it changed.** S2 fixed seven defects it found: - unseen ground passing the pool check; diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index 46c05b11e..a84a2ca76 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,31 +1,31 @@ # Scoreboard -Generated 2026-09-27 01:23 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran for S2-1, S2-7 only. Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-27 01:50 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran for S2-1, S2-7 only. Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | -| [#124](https://github.com/SamGu-NRX/house-scanning/pull/124) | `t3/integration-viewer` | `1fa19b33` | no | 4 success | 2026-09-27 01:17 CDT | -| [#123](https://github.com/SamGu-NRX/house-scanning/pull/123) | `t3/hunter-integration` | `60023d28` | yes | 2 success, 2 skipped | 2026-09-27 01:21 CDT | -| [#122](https://github.com/SamGu-NRX/house-scanning/pull/122) | `t3/41-server-requests-text` | `a22c2d45` | yes | 3 success, 1 skipped | 2026-09-27 00:37 CDT | -| [#120](https://github.com/SamGu-NRX/house-scanning/pull/120) | `t3/41-walk-task-hold` | `6ea58d50` | yes | 2 success, 1 failure, 1 skipped; failing: iOS build | 2026-09-27 01:03 CDT | +| [#124](https://github.com/SamGu-NRX/house-scanning/pull/124) | `t3/integration-viewer` | `ed87212d` | no | 3 success, 1 pending | 2026-09-27 01:48 CDT | +| [#123](https://github.com/SamGu-NRX/house-scanning/pull/123) | `t3/hunter-integration` | `294ddecb` | yes | 2 success, 2 skipped | 2026-09-27 01:43 CDT | +| [#122](https://github.com/SamGu-NRX/house-scanning/pull/122) | `t3/41-server-requests-text` | `a22c2d45` | yes | 3 success, 1 skipped | 2026-09-27 01:46 CDT | +| [#120](https://github.com/SamGu-NRX/house-scanning/pull/120) | `t3/41-walk-task-hold` | `6ea58d50` | yes | 2 success, 1 failure, 1 skipped; failing: iOS build | 2026-09-27 01:46 CDT | | [#119](https://github.com/SamGu-NRX/house-scanning/pull/119) | `t3/41-anchored-marks` | `f141a621` | yes | 2 success, 2 skipped | 2026-09-27 00:34 CDT | | [#117](https://github.com/SamGu-NRX/house-scanning/pull/117) | `t3/41-target-ring` | `97048675` | yes | 2 success, 2 skipped | 2026-09-27 00:36 CDT | | [#109](https://github.com/SamGu-NRX/house-scanning/pull/109) | `t3/41-gap-loop-result` | `5c15da87` | yes | 2 success, 2 skipped | 2026-09-27 01:00 CDT | | [#108](https://github.com/SamGu-NRX/house-scanning/pull/108) | `t3/41-ground-and-meter-tap` | `9c2a5676` | yes | 2 success, 2 skipped | 2026-09-27 00:25 CDT | -| [#104](https://github.com/SamGu-NRX/house-scanning/pull/104) | `t3/41-gate-coaching` | `9b0acc58` | yes | 2 success, 2 skipped | 2026-09-27 01:00 CDT | -| [#102](https://github.com/SamGu-NRX/house-scanning/pull/102) | `t3/41-card-replies` | `a4b8b272` | yes | 2 success, 2 skipped | 2026-09-27 01:18 CDT | -| [#100](https://github.com/SamGu-NRX/house-scanning/pull/100) | `t3/41-ar-canvas-fallback` | `41291e82` | yes | 2 success, 2 skipped | 2026-09-27 00:57 CDT | -| [#99](https://github.com/SamGu-NRX/house-scanning/pull/99) | `t3/41-end-preview` | `b5006fca` | yes | 2 success, 2 skipped | 2026-09-27 00:56 CDT | -| [#91](https://github.com/SamGu-NRX/house-scanning/pull/91) | `t3/experience-result` | `9583b129` | no | 4 success, 1 pending | 2026-09-27 01:21 CDT | +| [#104](https://github.com/SamGu-NRX/house-scanning/pull/104) | `t3/41-gate-coaching` | `9b0acc58` | yes | 2 success, 2 skipped | 2026-09-27 01:47 CDT | +| [#102](https://github.com/SamGu-NRX/house-scanning/pull/102) | `t3/41-card-replies` | `a4b8b272` | yes | 2 success, 2 skipped | 2026-09-27 01:46 CDT | +| [#100](https://github.com/SamGu-NRX/house-scanning/pull/100) | `t3/41-ar-canvas-fallback` | `41291e82` | yes | 2 success, 2 skipped | 2026-09-27 01:47 CDT | +| [#99](https://github.com/SamGu-NRX/house-scanning/pull/99) | `t3/41-end-preview` | `b5006fca` | yes | 2 success, 2 skipped | 2026-09-27 01:44 CDT | +| [#91](https://github.com/SamGu-NRX/house-scanning/pull/91) | `t3/experience-result` | `9583b129` | no | 5 success | 2026-09-27 01:46 CDT | | [#90](https://github.com/SamGu-NRX/house-scanning/pull/90) | `t3/check-experiments` | `9d66b31c` | no | 5 success | 2026-09-26 23:39 CDT | | [#89](https://github.com/SamGu-NRX/house-scanning/pull/89) | `t3/meter-brand` | `0a3a05e2` | no | 5 success | 2026-09-27 01:18 CDT | | [#88](https://github.com/SamGu-NRX/house-scanning/pull/88) | `t3/autodetect-lab` | `538f60e1` | yes | 2 success | 2026-09-26 23:00 CDT | -| [#87](https://github.com/SamGu-NRX/house-scanning/pull/87) | `t3/41-zip-deflate` | `5150321a` | yes | 2 success, 1 skipped | 2026-09-27 00:31 CDT | +| [#87](https://github.com/SamGu-NRX/house-scanning/pull/87) | `t3/41-zip-deflate` | `5150321a` | yes | 2 success, 1 skipped | 2026-09-27 01:43 CDT | | [#86](https://github.com/SamGu-NRX/house-scanning/pull/86) | `t3/experience` | `b442f0a0` | yes | 2 success, 1 skipped | 2026-09-27 00:00 CDT | | [#61](https://github.com/SamGu-NRX/house-scanning/pull/61) | `t3/app-review-docs` | `c0b628b6` | no | 4 success | 2026-09-27 01:10 CDT | -| [#60](https://github.com/SamGu-NRX/house-scanning/pull/60) | `t3/device-test-4.1-analysis` | `c566490c` | no | 2 success | 2026-09-27 00:34 CDT | +| [#60](https://github.com/SamGu-NRX/house-scanning/pull/60) | `t3/device-test-4.1-analysis` | `c566490c` | no | 2 success | 2026-09-27 01:48 CDT | | [#59](https://github.com/SamGu-NRX/house-scanning/pull/59) | `t3/signals-lab` | `0cde05e3` | yes | 2 success | 2026-09-26 21:38 CDT | | [#58](https://github.com/SamGu-NRX/house-scanning/pull/58) | `t3/ios-ci-no-preboot` | `3c1298ab` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 22:18 CDT | | [#56](https://github.com/SamGu-NRX/house-scanning/pull/56) | `t3/walk-guidance-gaps` | `cae4f426` | no | 3 success | 2026-09-26 23:57 CDT | @@ -55,42 +55,42 @@ Generated 2026-09-27 01:23 CDT by `make scoreboard` (`uv run python -m hsverify. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S2-1 | pytest green | met | exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 346 passed, 1 warning in 40.16s | +| S2-1 | pytest green | met | exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 346 passed, 1 warning in 65.76s (0:01:05) | | S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_golden_public.py) | | S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_properties.py) | | S2-4 | Schemas published | met | present at 9176125d [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/schemas/result.schema.json) | | S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_existing_battery.py](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_existing_battery.py), [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_schemas.py) | -| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 238.4 (want < 1000) at sha 9176125d: ~/house-scanning-data/reports/e2e/20260927-011249-9176125d/report.json | -| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 42ms / All checks passed! / 31 files already formatted | -| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | met | contract_ok = true (want True) at sha 9176125d: ~/house-scanning-data/reports/e2e/20260927-011249-9176125d/report.json | +| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 239.4 (want < 1000) at sha 9176125d: ~/house-scanning-data/reports/e2e/20260927-014805-9176125d/report.json | +| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 48ms / All checks passed! / 31 files already formatted | +| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | met | contract_ok = true (want True) at sha 9176125d: ~/house-scanning-data/reports/e2e/20260927-014805-9176125d/report.json | ## S3 iOS capture app -`origin/t3/ios-mvf` at [`f14947eb`](https://github.com/SamGu-NRX/house-scanning/tree/f14947eb9bff5b707660c8d873b72e60ba056631). 1 met, 3 partial, 1 gap, 3 stale, 2 no evidence yet. +`origin/t3/ios-mvf` at [`f14947eb`](https://github.com/SamGu-NRX/house-scanning/tree/f14947eb9bff5b707660c8d873b72e60ba056631). 3 met, 3 partial, 1 gap, 1 stale, 2 no evidence yet. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S3-1 | CI build green | no evidence yet | no open PR for t3/ios-mvf | | S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/project.yml) | -| S3-3 | Zero build warnings | stale | report for sha 194f2ebf, ref at f14947eb: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json | +| S3-3 | Zero build warnings | met | build.warnings = length 0 (want 0) at sha f14947eb: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | | S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage, guidance; missing auto-capture [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift) | | S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/SourceRestartUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/SourceRestartUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/ScreenStatesUITests.swift) | | S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/AccessibilityAudit.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/AccessibilityAudit.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/ScreenStatesUITests.swift) | | S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at f14947eb: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | | S3-7 | Screenshots of each state in the PR | no evidence yet | no open PR for t3/ios-mvf | -| S3-8 | Exported scene.json validates against C1 | stale | report for app_sha 194f2ebf, ref at f14947eb: ~/house-scanning-data/reports/e2e/20260927-011249-9176125d/report.json | +| S3-8 | Exported scene.json validates against C1 | met | app_export_scene_valid = true (want True) at app_sha f14947eb: ~/house-scanning-data/reports/e2e/20260927-014805-9176125d/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/ios-mvf` at [`f14947eb`](https://github.com/SamGu-NRX/house-scanning/tree/f14947eb9bff5b707660c8d873b72e60ba056631), `origin/t3/verification` at [`fcfecbac`](https://github.com/SamGu-NRX/house-scanning/tree/fcfecbac2ded0b0e6792b770ec78e6b80678d6be). 2 met, 2 stale. +`origin/t3/ios-mvf` at [`f14947eb`](https://github.com/SamGu-NRX/house-scanning/tree/f14947eb9bff5b707660c8d873b72e60ba056631), `origin/t3/verification` at [`36ac02f9`](https://github.com/SamGu-NRX/house-scanning/tree/36ac02f9d1fb3d654ff12dde652de65a134d1cfd). 4 met. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S4-1 | End to end from a real replay to a placement result passes | stale | report for app_sha 194f2ebf, ref at f14947eb: ~/house-scanning-data/reports/e2e/20260927-011249-9176125d/report.json | -| S4-2 | Screenshot report per app state | stale | report for sha 194f2ebf, ref at f14947eb: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json | +| S4-1 | End to end from a real replay to a placement result passes | met | app_export_passed = true (want True) at app_sha f14947eb: ~/house-scanning-data/reports/e2e/20260927-014805-9176125d/report.json | +| S4-2 | Screenshot report per app state | met | states = length 12 (want at least 9) at sha f14947eb: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | | S4-3 | Product description written from the code and checked against screenshots | met | docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state. | -| S4-4 | Scoreboard of every PR | met | present at fcfecbac [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/fcfecbac2ded0b0e6792b770ec78e6b80678d6be/verification/SCOREBOARD.md) | +| S4-4 | Scoreboard of every PR | met | present at 36ac02f9 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/36ac02f9d1fb3d654ff12dde652de65a134d1cfd/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index beeafdf7c..ed8f1a6c9 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -196,7 +196,10 @@ def observed( spans.append(tuple(sorted(entry["span_ft"]))) continue far = reached(entry) - if (far <= min_out_ft) if beyond else (far + EPS < min_out_ft): + # Without `beyond` the view must reach min_out_ft itself: a request's out_ft is set just + # past a rule's line (server _above, 1e-6 over it), so any looser slack than the + # server's own would count a view at the line as reaching past it. + if (far <= min_out_ft) if beyond else (far + SERVER_EPS < min_out_ft): continue spans.append(tuple(sorted(entry["span_ft"]))) spans.sort() diff --git a/verification/hsverify/statelog.py b/verification/hsverify/statelog.py index 0f2edcaef..b13f7ccd9 100644 --- a/verification/hsverify/statelog.py +++ b/verification/hsverify/statelog.py @@ -20,8 +20,10 @@ # The engine logs `bundle with keyframes` once it has written a scan bundle; the path # is on the Mac's disk, inside the Simulator's app container. # "bundle .zip with N keyframes", optionally followed by ",
    " (app 194f2eb adds -# depth and mesh counts). -_BUNDLE_RE = re.compile(r"^bundle (.+\.zip) with (\d+) keyframes(?:,.*)?$") +# depth and mesh counts); from app f14947e, "bundle .zip: packet 1.1 with N photos ...". +_BUNDLE_RE = re.compile( + r"^bundle (.+\.zip)(?: with (\d+) keyframes(?:,.*)?|: .*?\bwith (\d+) photos\b.*)$" +) # A state name is an identifier-like token. Anything after it on the line is detail. _STATE_RE = re.compile(r"STATE=([A-Za-z0-9_.\-]+)") @@ -56,10 +58,13 @@ def _app_entry(line: str, category: str) -> dict | None: def parse_bundle_line(line: str) -> tuple[str, int] | None: - """(zip path, keyframe count) from the engine's bundle line, or None for any other line.""" + """(zip path, keyframe or photo count) from the engine's bundle line, or None for any other + line.""" entry = _app_entry(line, ENGINE_CATEGORY) match = _BUNDLE_RE.match((entry or {}).get("eventMessage") or "") - return (match.group(1), int(match.group(2))) if match else None + if not match: + return None + return match.group(1), int(match.group(2) or match.group(3)) def parse_ndjson_line(line: str) -> StateEvent | None: diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index 08b478d5f..37cd0aed9 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-09-27T01:23:01-05:00", + "generated_at": "2026-09-27T01:50:01-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": [ "S2-1", @@ -10,27 +10,27 @@ "number": 124, "title": "tools: viewer that films a capture reaching the server, as a labeled replay or read-only live view", "branch": "t3/integration-viewer", - "head": "1fa19b3349803ab57da5393b47f05ea0fc5b2add", + "head": "ed87212dc02a87844b5240d3facd5d05616c11f9", "draft": false, - "updated": "2026-09-27T06:17:49Z", + "updated": "2026-09-27T06:48:01Z", "ci": { "counts": { - "success": 4, + "success": 3, "failure": 0, - "pending": 0, + "pending": 1, "skipped": 0 }, "failing": [], - "text": "4 success" + "text": "3 success, 1 pending" } }, { "number": 123, - "title": "ios: House Scan Integration build uploads each scan to the capture API while scanning", + "title": "ios: House Scan Integration records each scan as a capture packet and, with consent, uploads it while scanning", "branch": "t3/hunter-integration", - "head": "60023d2817870ded5128b2674879a642f7b0177f", + "head": "294ddecb7a1d3fe303cbf21b73bfb3f3f5392daa", "draft": true, - "updated": "2026-09-27T06:21:47Z", + "updated": "2026-09-27T06:43:08Z", "ci": { "counts": { "success": 2, @@ -48,7 +48,7 @@ "branch": "t3/41-server-requests-text", "head": "a22c2d45351c202cff91defe9638197b5e87eb52", "draft": true, - "updated": "2026-09-27T05:37:19Z", + "updated": "2026-09-27T06:46:29Z", "ci": { "counts": { "success": 3, @@ -66,7 +66,7 @@ "branch": "t3/41-walk-task-hold", "head": "6ea58d50562b0c3008b0edea57b0226c3de3aa92", "draft": true, - "updated": "2026-09-27T06:03:09Z", + "updated": "2026-09-27T06:46:35Z", "ci": { "counts": { "success": 2, @@ -158,7 +158,7 @@ "branch": "t3/41-gate-coaching", "head": "9b0acc5851e7fa69a1da560724c919f1d63bcc5e", "draft": true, - "updated": "2026-09-27T06:00:23Z", + "updated": "2026-09-27T06:47:54Z", "ci": { "counts": { "success": 2, @@ -176,7 +176,7 @@ "branch": "t3/41-card-replies", "head": "a4b8b2723736b5f2b29c5605569dcdb76c6d8a69", "draft": true, - "updated": "2026-09-27T06:18:35Z", + "updated": "2026-09-27T06:46:10Z", "ci": { "counts": { "success": 2, @@ -194,7 +194,7 @@ "branch": "t3/41-ar-canvas-fallback", "head": "41291e8287bd7a2452c6b53b36120670c3210317", "draft": true, - "updated": "2026-09-27T05:57:54Z", + "updated": "2026-09-27T06:47:25Z", "ci": { "counts": { "success": 2, @@ -212,7 +212,7 @@ "branch": "t3/41-end-preview", "head": "b5006fca782aa280caa767e476fd5c62b496a9e2", "draft": true, - "updated": "2026-09-27T05:56:48Z", + "updated": "2026-09-27T06:44:16Z", "ci": { "counts": { "success": 2, @@ -230,16 +230,16 @@ "branch": "t3/experience-result", "head": "9583b1294882f40c963f5d21af9b053a6d3d367d", "draft": false, - "updated": "2026-09-27T06:21:00Z", + "updated": "2026-09-27T06:46:57Z", "ci": { "counts": { - "success": 4, + "success": 5, "failure": 0, - "pending": 1, + "pending": 0, "skipped": 0 }, "failing": [], - "text": "4 success, 1 pending" + "text": "5 success" } }, { @@ -302,7 +302,7 @@ "branch": "t3/41-zip-deflate", "head": "5150321a22f540e669755270dd76caa23957d8a4", "draft": true, - "updated": "2026-09-27T05:31:34Z", + "updated": "2026-09-27T06:43:53Z", "ci": { "counts": { "success": 2, @@ -356,7 +356,7 @@ "branch": "t3/device-test-4.1-analysis", "head": "c566490c5e3fa0d45803895bb463cb260d11a906", "draft": false, - "updated": "2026-09-27T05:34:34Z", + "updated": "2026-09-27T06:48:45Z", "ci": { "counts": { "success": 2, @@ -656,7 +656,7 @@ "ref": "origin/t3/server", "sha": "9176125df0c9890c247818d55edc82ea46053cd8", "status": "met", - "evidence": "exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 346 passed, 1 warning in 40.16s", + "evidence": "exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 346 passed, 1 warning in 65.76s (0:01:05)", "links": [] }, { @@ -737,7 +737,7 @@ "ref": "origin/t3/server", "sha": "9176125df0c9890c247818d55edc82ea46053cd8", "status": "met", - "evidence": "latency_ms = 238.4 (want < 1000) at sha 9176125d: ~/house-scanning-data/reports/e2e/20260927-011249-9176125d/report.json", + "evidence": "latency_ms = 239.4 (want < 1000) at sha 9176125d: ~/house-scanning-data/reports/e2e/20260927-014805-9176125d/report.json", "links": [] }, { @@ -746,7 +746,7 @@ "ref": "origin/t3/server", "sha": "9176125df0c9890c247818d55edc82ea46053cd8", "status": "met", - "evidence": "exit 0: Installed 33 packages in 42ms / All checks passed! / 31 files already formatted", + "evidence": "exit 0: Installed 33 packages in 48ms / All checks passed! / 31 files already formatted", "links": [] }, { @@ -755,7 +755,7 @@ "ref": "origin/t3/server", "sha": "9176125df0c9890c247818d55edc82ea46053cd8", "status": "met", - "evidence": "contract_ok = true (want True) at sha 9176125d: ~/house-scanning-data/reports/e2e/20260927-011249-9176125d/report.json", + "evidence": "contract_ok = true (want True) at sha 9176125d: ~/house-scanning-data/reports/e2e/20260927-014805-9176125d/report.json", "links": [] } ] @@ -798,8 +798,8 @@ "text": "Zero build warnings", "ref": "origin/t3/ios-mvf", "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", - "status": "stale", - "evidence": "report for sha 194f2ebf, ref at f14947eb: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json", + "status": "met", + "evidence": "build.warnings = length 0 (want 0) at sha f14947eb: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", "links": [] }, { @@ -887,8 +887,8 @@ "text": "Exported scene.json validates against C1", "ref": "origin/t3/ios-mvf", "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", - "status": "stale", - "evidence": "report for app_sha 194f2ebf, ref at f14947eb: ~/house-scanning-data/reports/e2e/20260927-011249-9176125d/report.json", + "status": "met", + "evidence": "app_export_scene_valid = true (want True) at app_sha f14947eb: ~/house-scanning-data/reports/e2e/20260927-014805-9176125d/report.json", "links": [] }, { @@ -912,7 +912,7 @@ }, { "ref": "origin/t3/verification", - "sha": "fcfecbac2ded0b0e6792b770ec78e6b80678d6be" + "sha": "36ac02f9d1fb3d654ff12dde652de65a134d1cfd" } ], "metrics": [ @@ -921,8 +921,8 @@ "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/ios-mvf", "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", - "status": "stale", - "evidence": "report for app_sha 194f2ebf, ref at f14947eb: ~/house-scanning-data/reports/e2e/20260927-011249-9176125d/report.json", + "status": "met", + "evidence": "app_export_passed = true (want True) at app_sha f14947eb: ~/house-scanning-data/reports/e2e/20260927-014805-9176125d/report.json", "links": [] }, { @@ -930,15 +930,15 @@ "text": "Screenshot report per app state", "ref": "origin/t3/ios-mvf", "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", - "status": "stale", - "evidence": "report for sha 194f2ebf, ref at f14947eb: ~/house-scanning-data/reports/sim/20260926-121648-t3-ios-mvf-194f2ebf-replay/report.json", + "status": "met", + "evidence": "states = length 12 (want at least 9) at sha f14947eb: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", "links": [] }, { "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "fcfecbac2ded0b0e6792b770ec78e6b80678d6be", + "sha": "36ac02f9d1fb3d654ff12dde652de65a134d1cfd", "status": "met", "evidence": "docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state.", "links": [] @@ -947,13 +947,13 @@ "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "fcfecbac2ded0b0e6792b770ec78e6b80678d6be", + "sha": "36ac02f9d1fb3d654ff12dde652de65a134d1cfd", "status": "met", - "evidence": "present at fcfecbac", + "evidence": "present at 36ac02f9", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/fcfecbac2ded0b0e6792b770ec78e6b80678d6be/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/36ac02f9d1fb3d654ff12dde652de65a134d1cfd/verification/SCOREBOARD.md" ] ] } diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index cf538a27f..6dce1c401 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -1007,3 +1007,16 @@ def test_what_lies_past_an_unexplored_end_needs_only_the_walk_past_it(): {"kind": "past_end", "side": "left", "span_ft": [-1.0, -1.0], "message": ""} ] assert missing_evidence_problems(scene, r, wide) == [] + + +def test_a_view_at_headroom_does_not_settle_a_request_just_past_it(): + # App f14947e reports the wall seen to exactly 6.5 ft; the server then asks for a view past + # headroom, out_ft 6.500001. That request is not redundant. + r = result() + r["missing_evidence"] = [ + {"kind": "band", "band": "wall", "span_ft": [-3.0, -1.0], "out_ft": 6.500001, "message": ""} + ] + msgs = invariant_problems(with_wall_seen(6.5), r, rules=RULES) + assert not any("lists as observed" in m for m in msgs) + msgs = invariant_problems(with_wall_seen(6.500001), r, rules=RULES) + assert any("lists as observed" in m for m in msgs) diff --git a/verification/tests/test_statelog.py b/verification/tests/test_statelog.py index bfc4cc2ad..6796ba25e 100644 --- a/verification/tests/test_statelog.py +++ b/verification/tests/test_statelog.py @@ -84,3 +84,13 @@ def test_reads_the_engine_bundle_line(suffix): ) def test_bundle_line_ignores_everything_else(line): assert parse_bundle_line(line) is None + + +def test_reads_the_packet_bundle_line_from_app_f14947e(): + line = entry( + f"bundle {BUNDLE_PATH}: packet 1.1 with 33 photos (0 with depth), 0 depth frames, " + "41 trajectory samples", + category="engine", + ) + assert parse_bundle_line(line) == (BUNDLE_PATH, 33) + assert parse_bundle_line(entry(f"bundle {BUNDLE_PATH}: packet 1.1", category="engine")) is None From 33811e42605028100cfa40500b84e747c83293f8 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 02:12:48 -0500 Subject: [PATCH 52/59] verification: close three false certifications; pin to server 4a9fa44 Each failing first at 64d70d1: - with_requests_captured claimed ground 40 ft out whatever a request asked (4114352249). A captured view now reaches exactly the request's out_ft. - run_starts([0, 0.4], 1) returned only 0.4 (4114352251). A run shorter than a step keeps its first start. - A passing spot inside a failing sweep run passed the invariants (4114352261). The spot's outcome must match the sweep at its start. For 4a9fa44, wall_backing needs the wall over the battery widened by its error. The exact capture exposed a checker false positive in an inside corner (g05): ground in front of one wall seen in the next wall's band. Ground now counts where it lies in any wall's observed band in plan, sampled every 0.05 ft. S4-1 is capped at partial: met on the synthetic replay at f14947e; the real ADVIO replay is blocked by its data. e2e pinned to 4a9fa44: 49 of 50. The ETH3D scene never settles: each round the server asks for ground 0.005 ft further, and drive and pool stay unsure. contract_ok is false. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/README.md | 5 +- verification/SCOREBOARD.md | 52 ++++----- verification/hsverify/resultcheck.py | 147 +++++++++++++++++++++--- verification/scoreboard/metrics.yaml | 5 + verification/scoreboard/scoreboard.json | 120 +++++++++---------- verification/tests/test_resultcheck.py | 89 +++++++++++++- 6 files changed, 314 insertions(+), 104 deletions(-) diff --git a/verification/README.md b/verification/README.md index 0d6a28dca..b602ec1af 100644 --- a/verification/README.md +++ b/verification/README.md @@ -19,8 +19,9 @@ Reports go to `~/house-scanning-data/reports/`, outside git, because replay fram non-commercial datasets. What each check asserts is in the docstrings of `hsverify/e2e.py`, `resultcheck.py` and `simrun.py`. -**Result.** At `t3/server` `9176125`, with the app's upload from `f14947e` (synthetic replay, hosted -server), all 50 scenes pass (45 answered and checked; slowest real scene 0.24 s). +**Result.** At `t3/server` `4a9fa44`, with the app's upload from `f14947e` (synthetic replay, hosted +server), 49 of 50 scenes pass (45 answered and checked; slowest real scene 0.56 s). The ETH3D scene +fails: capturing exactly the ground the server asks for never settles it. **What it changed.** S2 fixed seven defects it found: - unseen ground passing the pool check; diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index a84a2ca76..358be434a 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,21 +1,21 @@ # Scoreboard -Generated 2026-09-27 01:50 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran for S2-1, S2-7 only. Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-27 02:12 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran for S2-1, S2-7 only. Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | -| [#124](https://github.com/SamGu-NRX/house-scanning/pull/124) | `t3/integration-viewer` | `ed87212d` | no | 3 success, 1 pending | 2026-09-27 01:48 CDT | -| [#123](https://github.com/SamGu-NRX/house-scanning/pull/123) | `t3/hunter-integration` | `294ddecb` | yes | 2 success, 2 skipped | 2026-09-27 01:43 CDT | -| [#122](https://github.com/SamGu-NRX/house-scanning/pull/122) | `t3/41-server-requests-text` | `a22c2d45` | yes | 3 success, 1 skipped | 2026-09-27 01:46 CDT | -| [#120](https://github.com/SamGu-NRX/house-scanning/pull/120) | `t3/41-walk-task-hold` | `6ea58d50` | yes | 2 success, 1 failure, 1 skipped; failing: iOS build | 2026-09-27 01:46 CDT | -| [#119](https://github.com/SamGu-NRX/house-scanning/pull/119) | `t3/41-anchored-marks` | `f141a621` | yes | 2 success, 2 skipped | 2026-09-27 00:34 CDT | -| [#117](https://github.com/SamGu-NRX/house-scanning/pull/117) | `t3/41-target-ring` | `97048675` | yes | 2 success, 2 skipped | 2026-09-27 00:36 CDT | -| [#109](https://github.com/SamGu-NRX/house-scanning/pull/109) | `t3/41-gap-loop-result` | `5c15da87` | yes | 2 success, 2 skipped | 2026-09-27 01:00 CDT | -| [#108](https://github.com/SamGu-NRX/house-scanning/pull/108) | `t3/41-ground-and-meter-tap` | `9c2a5676` | yes | 2 success, 2 skipped | 2026-09-27 00:25 CDT | -| [#104](https://github.com/SamGu-NRX/house-scanning/pull/104) | `t3/41-gate-coaching` | `9b0acc58` | yes | 2 success, 2 skipped | 2026-09-27 01:47 CDT | -| [#102](https://github.com/SamGu-NRX/house-scanning/pull/102) | `t3/41-card-replies` | `a4b8b272` | yes | 2 success, 2 skipped | 2026-09-27 01:46 CDT | +| [#124](https://github.com/SamGu-NRX/house-scanning/pull/124) | `t3/integration-viewer` | `980cee89` | no | 3 success, 1 pending | 2026-09-27 02:10 CDT | +| [#123](https://github.com/SamGu-NRX/house-scanning/pull/123) | `t3/hunter-integration` | `9a2368c8` | yes | 2 success, 2 skipped | 2026-09-27 02:08 CDT | +| [#122](https://github.com/SamGu-NRX/house-scanning/pull/122) | `t3/41-server-requests-text` | `c8d3c9d5` | yes | 3 success, 1 skipped | 2026-09-27 01:55 CDT | +| [#120](https://github.com/SamGu-NRX/house-scanning/pull/120) | `t3/41-walk-task-hold` | `8487a02f` | yes | 2 success, 1 pending, 1 skipped | 2026-09-27 02:07 CDT | +| [#119](https://github.com/SamGu-NRX/house-scanning/pull/119) | `t3/41-anchored-marks` | `f141a621` | yes | 2 success, 2 skipped | 2026-09-27 01:55 CDT | +| [#117](https://github.com/SamGu-NRX/house-scanning/pull/117) | `t3/41-target-ring` | `6d82e62a` | yes | 2 success, 2 skipped | 2026-09-27 02:04 CDT | +| [#109](https://github.com/SamGu-NRX/house-scanning/pull/109) | `t3/41-gap-loop-result` | `a44780a2` | yes | 2 success, 2 skipped | 2026-09-27 01:59 CDT | +| [#108](https://github.com/SamGu-NRX/house-scanning/pull/108) | `t3/41-ground-and-meter-tap` | `9c2a5676` | yes | 2 success, 2 skipped | 2026-09-27 01:49 CDT | +| [#104](https://github.com/SamGu-NRX/house-scanning/pull/104) | `t3/41-gate-coaching` | `633c26d7` | yes | 2 success, 2 skipped | 2026-09-27 02:03 CDT | +| [#102](https://github.com/SamGu-NRX/house-scanning/pull/102) | `t3/41-card-replies` | `98d96caa` | yes | 2 success, 2 skipped | 2026-09-27 01:59 CDT | | [#100](https://github.com/SamGu-NRX/house-scanning/pull/100) | `t3/41-ar-canvas-fallback` | `41291e82` | yes | 2 success, 2 skipped | 2026-09-27 01:47 CDT | | [#99](https://github.com/SamGu-NRX/house-scanning/pull/99) | `t3/41-end-preview` | `b5006fca` | yes | 2 success, 2 skipped | 2026-09-27 01:44 CDT | | [#91](https://github.com/SamGu-NRX/house-scanning/pull/91) | `t3/experience-result` | `9583b129` | no | 5 success | 2026-09-27 01:46 CDT | @@ -24,8 +24,8 @@ Generated 2026-09-27 01:50 CDT by `make scoreboard` (`uv run python -m hsverify. | [#88](https://github.com/SamGu-NRX/house-scanning/pull/88) | `t3/autodetect-lab` | `538f60e1` | yes | 2 success | 2026-09-26 23:00 CDT | | [#87](https://github.com/SamGu-NRX/house-scanning/pull/87) | `t3/41-zip-deflate` | `5150321a` | yes | 2 success, 1 skipped | 2026-09-27 01:43 CDT | | [#86](https://github.com/SamGu-NRX/house-scanning/pull/86) | `t3/experience` | `b442f0a0` | yes | 2 success, 1 skipped | 2026-09-27 00:00 CDT | -| [#61](https://github.com/SamGu-NRX/house-scanning/pull/61) | `t3/app-review-docs` | `c0b628b6` | no | 4 success | 2026-09-27 01:10 CDT | -| [#60](https://github.com/SamGu-NRX/house-scanning/pull/60) | `t3/device-test-4.1-analysis` | `c566490c` | no | 2 success | 2026-09-27 01:48 CDT | +| [#61](https://github.com/SamGu-NRX/house-scanning/pull/61) | `t3/app-review-docs` | `c0b628b6` | no | 4 success | 2026-09-27 01:56 CDT | +| [#60](https://github.com/SamGu-NRX/house-scanning/pull/60) | `t3/device-test-4.1-analysis` | `c566490c` | no | 2 success | 2026-09-27 02:01 CDT | | [#59](https://github.com/SamGu-NRX/house-scanning/pull/59) | `t3/signals-lab` | `0cde05e3` | yes | 2 success | 2026-09-26 21:38 CDT | | [#58](https://github.com/SamGu-NRX/house-scanning/pull/58) | `t3/ios-ci-no-preboot` | `3c1298ab` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 22:18 CDT | | [#56](https://github.com/SamGu-NRX/house-scanning/pull/56) | `t3/walk-guidance-gaps` | `cae4f426` | no | 3 success | 2026-09-26 23:57 CDT | @@ -51,18 +51,18 @@ Generated 2026-09-27 01:50 CDT by `make scoreboard` (`uv run python -m hsverify. ## S2 Server and solver -`origin/t3/server` at [`9176125d`](https://github.com/SamGu-NRX/house-scanning/tree/9176125df0c9890c247818d55edc82ea46053cd8). 7 met, 1 partial. +`origin/t3/server` at [`4a9fa44f`](https://github.com/SamGu-NRX/house-scanning/tree/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf). 6 met, 1 partial, 1 gap. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S2-1 | pytest green | met | exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 346 passed, 1 warning in 65.76s (0:01:05) | -| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_golden_public.py) | -| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, mirror; missing equality unsure, larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_properties.py) | -| S2-4 | Schemas published | met | present at 9176125d [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/schemas/result.schema.json) | -| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_existing_battery.py](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_existing_battery.py), [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_schemas.py) | -| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 239.4 (want < 1000) at sha 9176125d: ~/house-scanning-data/reports/e2e/20260927-014805-9176125d/report.json | -| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 48ms / All checks passed! / 31 files already formatted | -| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | met | contract_ok = true (want True) at sha 9176125d: ~/house-scanning-data/reports/e2e/20260927-014805-9176125d/report.json | +| S2-1 | pytest green | met | exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 352 passed, 1 warning in 89.34s (0:01:29) | +| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_golden_public.py) | +| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, equality unsure, mirror; missing larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_properties.py), [server/tests/test_final_review.py](https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_final_review.py) | +| S2-4 | Schemas published | met | present at 4a9fa44f [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/schemas/result.schema.json) | +| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_existing_battery.py](https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_existing_battery.py), [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_schemas.py) | +| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 561.4 (want < 1000) at sha 4a9fa44f: ~/house-scanning-data/reports/e2e/20260927-020953-4a9fa44f/report.json | +| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 33ms / All checks passed! / 32 files already formatted | +| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | gap | contract_ok = false (want True) at sha 4a9fa44f: ~/house-scanning-data/reports/e2e/20260927-020953-4a9fa44f/report.json | ## S3 iOS capture app @@ -78,19 +78,19 @@ Generated 2026-09-27 01:50 CDT by `make scoreboard` (`uv run python -m hsverify. | S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/AccessibilityAudit.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/AccessibilityAudit.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/ScreenStatesUITests.swift) | | S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at f14947eb: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | | S3-7 | Screenshots of each state in the PR | no evidence yet | no open PR for t3/ios-mvf | -| S3-8 | Exported scene.json validates against C1 | met | app_export_scene_valid = true (want True) at app_sha f14947eb: ~/house-scanning-data/reports/e2e/20260927-014805-9176125d/report.json | +| S3-8 | Exported scene.json validates against C1 | met | app_export_scene_valid = true (want True) at app_sha f14947eb: ~/house-scanning-data/reports/e2e/20260927-020953-4a9fa44f/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/ios-mvf` at [`f14947eb`](https://github.com/SamGu-NRX/house-scanning/tree/f14947eb9bff5b707660c8d873b72e60ba056631), `origin/t3/verification` at [`36ac02f9`](https://github.com/SamGu-NRX/house-scanning/tree/36ac02f9d1fb3d654ff12dde652de65a134d1cfd). 4 met. +`origin/t3/ios-mvf` at [`f14947eb`](https://github.com/SamGu-NRX/house-scanning/tree/f14947eb9bff5b707660c8d873b72e60ba056631), `origin/t3/verification` at [`64d70d1f`](https://github.com/SamGu-NRX/house-scanning/tree/64d70d1f34396e822adcb5d502fd3d8c55b0dca1). 3 met, 1 partial. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S4-1 | End to end from a real replay to a placement result passes | met | app_export_passed = true (want True) at app_sha f14947eb: ~/house-scanning-data/reports/e2e/20260927-014805-9176125d/report.json | +| S4-1 | End to end from a real replay to a placement result passes | partial | app_export_passed = true (want True) at app_sha f14947eb: ~/house-scanning-data/reports/e2e/20260927-020953-4a9fa44f/report.json (met on the synthetic replay against the hosted server at f14947e (12 states, one server gap request); the real ADVIO replay is blocked by its data (the walk never covers the ground by the meter), last real run 194f2eb) | | S4-2 | Screenshot report per app state | met | states = length 12 (want at least 9) at sha f14947eb: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | | S4-3 | Product description written from the code and checked against screenshots | met | docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state. | -| S4-4 | Scoreboard of every PR | met | present at 36ac02f9 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/36ac02f9d1fb3d654ff12dde652de65a134d1cfd/verification/SCOREBOARD.md) | +| S4-4 | Scoreboard of every PR | met | present at 64d70d1f [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/64d70d1f34396e822adcb5d502fd3d8c55b0dca1/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index ed8f1a6c9..a085926ea 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -119,7 +119,7 @@ def reach(key: str) -> Need: needs = { "ground_surface": (Need("ground", 0.0),), - "wall_backing": (Need("wall", 0.0, value("battery", "height_ft"), widen=False),), + "wall_backing": (Need("wall", 0.0, value("battery", "height_ft")),), "gas_clearance": (reach("gas_ft"), Need("wall", r("gas_ft"), headroom)), "battery_clearance": (reach("battery_ft"), Need("wall", r("battery_ft"), headroom)), "ac_clearance": (reach("ac_ft"),), @@ -315,6 +315,7 @@ def invariant_problems( if max(abs(want[0] - got[0]), abs(want[1] - got[1])) > OFFSET_TOL_FT: problems.append(f"spot.meter_offset_ft {got} != centre - meter {want}") + problems += spot_sweep_problems(result) problems += missing_evidence_problems(scene, result, rules) if rules is not None: problems += coverage_problems(scene, result, rules) @@ -322,6 +323,24 @@ def invariant_problems( return problems +def spot_sweep_problems(result: dict) -> list[str]: + """The chosen spot is one of the evaluated starts, so the sweep must give its start the + spot's own outcome, and a passing spot must lie in a passing run. A result could otherwise + report a pass at a start its own sweep says fails, with stats that agree with the sweep.""" + spot = result.get("spot") + if spot is None: + return [] + start = spot["span_ft"][0] + got = outcome_at(result, spot["wall_id"], start) + if got == spot["outcome"]: + return [] + verb = {"pass": "passes", "fail": "fails"}.get(spot["outcome"], "is unsure") + where = f"start {start:.2f} on {spot['wall_id']}" + if got is None: + return [f"spot {verb} but no sweep run has {where}"] + return [f"spot {verb} but the sweep has {where} as {got}"] + + def required_span(lo: float, hi: float, radius: float) -> tuple[float, float]: """Along-wall stretch within `radius` of a battery covering [lo, hi]. @@ -371,7 +390,8 @@ def reach_gaps( for need in rules.needs[check]: reach = need.radius + (battery_error(scene, rules, wall_id, lo, hi) if need.widen else 0.0) if need.band == "ground": - gap = ground_gap(scene, lo, hi, rules.depth_ft, reach) + plan = GroundPlan(scene, rules) if rules.wall_join_ft is not None else None + gap = ground_gap(scene, lo, hi, rules.depth_ft, reach, plan) elif need.band == "wall": a, b = required_span(lo, hi, reach) wall = observed( @@ -426,7 +446,8 @@ def run_starts(span: list[float], step_ft: float | None) -> list[float]: if step_ft is None or step_ft <= 0: raise ValueError("checking a sweep run needs the rules' sweep.step_ft") a, b = span - count = max(0, round((b - a) / step_ft)) + # Every step strictly before the last start, the first included however short the run. + count = max(0, math.ceil((b - a) / step_ft - EPS)) return [a + k * step_ft for k in range(count)] + [b] @@ -497,9 +518,17 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: return problems -def ground_gap(scene: dict, lo: float, hi: float, depth: float, radius: float) -> str | None: +def ground_gap( + scene: dict, + lo: float, + hi: float, + depth: float, + radius: float, + plan: GroundPlan | None = None, +) -> str | None: """The first stretch of ground seen less far out than a battery over [lo, hi] needs (see - coverage_problems), or None.""" + coverage_problems), or None. With `plan`, ground not seen in the battery's own wall's band + still counts where it lies in another wall's observed band in plan (GroundPlan).""" entries = [ (*sorted(e["span_ft"]), e.get("out_ft", 0.0)) for e in scene.get("coverage", {}).get("observed", []) @@ -514,6 +543,8 @@ def ground_gap(scene: dict, lo: float, hi: float, depth: float, radius: float) - need = depth + radius if inside else radius - min(abs(lo - q), abs(p - hi)) seen = max((out for x, y, out in entries if x <= p + EPS and q - EPS <= y), default=None) if seen is None or seen + EPS < need: + if plan is not None and plan.covers((lo + hi) / 2, p, q, seen or 0.0, need): + continue where = f"ground [{p:.2f}, {q:.2f}] observed" if seen is None: return f"{where}, none seen" @@ -584,8 +615,12 @@ def missing_evidence_problems(scene: dict, result: dict, rules: RuleSet | None = END_TOL_FT = 0.1 -def wall_spans_s(scene: dict, rules: RuleSet) -> dict[str, tuple[float, float]] | None: - """Each wall's stretch of s (the meter's projection is s = 0), or None without a meter wall. +Point = tuple[float, float] + + +def segments_s(scene: dict, rules: RuleSet) -> list[tuple[str, float, float, Point, Point]] | None: + """Each straight baseline segment as (wall id, s at its start, s at its end, start point, + end point), with s measured from the meter's projection, or None without a meter wall. s runs along every wall. The space from one wall's end to the next one's start counts only when it is a gap: wider than both walls' own errors together (no drift), capped at @@ -596,17 +631,16 @@ def wall_spans_s(scene: dict, rules: RuleSet) -> dict[str, tuple[float, float]] walls = scene.get("walls", []) if not walls or "meter" not in scene: return None - raw: dict[str, tuple[float, float]] = {} + raw: list[tuple[str, float, float, Point, Point]] = [] meter_s = None mx, _, mz = scene["meter"]["pos"] s, prev = 0.0, None for wall in walls: - pts = [tuple(pt) for pt in wall["baseline"]] + pts = [(float(pt[0]), float(pt[1])) for pt in wall["baseline"]] if prev is not None: space = math.dist(prev[0], pts[0]) meets = min(rules.wall_join_ft, max(prev[1] + wall_error(wall, rules), SERVER_EPS)) s += space if space > meets else 0.0 - start = s for p, q in itertools.pairwise(pts): seg = math.dist(p, q) if wall["id"] == scene["meter"]["wall_id"] and seg > 0: @@ -615,12 +649,85 @@ def wall_spans_s(scene: dict, rules: RuleSet) -> dict[str, tuple[float, float]] d = math.dist((mx, mz), (p[0] + t * (q[0] - p[0]), p[1] + t * (q[1] - p[1]))) if meter_s is None or d < meter_s[0]: meter_s = (d, s + t * seg) + raw.append((wall["id"], s, s + seg, p, q)) s += seg - raw[wall["id"]] = (start, s) prev = (pts[-1], wall_error(wall, rules)) if meter_s is None: return None - return {wid: (a - meter_s[1], b - meter_s[1]) for wid, (a, b) in raw.items()} + return [(wid, a - meter_s[1], b - meter_s[1], p, q) for wid, a, b, p, q in raw] + + +def wall_spans_s(scene: dict, rules: RuleSet) -> dict[str, tuple[float, float]] | None: + """Each wall's stretch of s (see segments_s), or None without a meter wall.""" + segments = segments_s(scene, rules) + if segments is None: + return None + spans: dict[str, tuple[float, float]] = {} + for wid, a, b, _, _ in segments: + lo, hi = spans.get(wid, (a, b)) + spans[wid] = (min(lo, a), max(hi, b)) + return spans + + +class GroundPlan: + """The observed ground in plan: each entry's stretch of every segment it runs along, out to + its `out_ft` on the segment's outward side (the baseline turned 90 degrees clockwise seen + from above, scene.schema.json). A point in front of one wall can lie in a neighbouring + wall's band, as in an inside corner, where the second wall faces back over the first.""" + + STEP_FT = 0.05 # sampling step for coverage tests; a sliver thinner than this can be missed + + def __init__(self, scene: dict, rules: RuleSet) -> None: + self.segments = segments_s(scene, rules) or [] + self.strips = [] # (origin, along, outward, from s, to s, out) + for e in scene.get("coverage", {}).get("observed", []): + if e["band"] != "ground": + continue + a, b = sorted(e["span_ft"]) + for _, s0, s1, p, q in self.segments: + if min(b, s1) - max(a, s0) > EPS and s1 - s0 > EPS: + along, outward = self._frame(p, q) + strip = (p, along, outward, max(a, s0) - s0, min(b, s1) - s0) + self.strips.append((*strip, e.get("out_ft", 0.0))) + + @staticmethod + def _frame(p: Point, q: Point) -> tuple[Point, Point]: + length = math.dist(p, q) + along = ((q[0] - p[0]) / length, (q[1] - p[1]) / length) + return along, (-along[1], along[0]) + + def point(self, s_mid: float, s: float, out: float) -> Point | None: + """The plan point at s along the line of the segment under s_mid, `out` in front.""" + seg = next((g for g in self.segments if g[1] - EPS <= s_mid <= g[2] + EPS), None) + if seg is None: + return None + _, s0, _, p, q = seg + along, outward = self._frame(p, q) + t = s - s0 + return (p[0] + t * along[0] + out * outward[0], p[1] + t * along[1] + out * outward[1]) + + def seen(self, x: Point) -> bool: + for origin, along, outward, t0, t1, out in self.strips: + dx, dz = x[0] - origin[0], x[1] - origin[1] + t = dx * along[0] + dz * along[1] + n = dx * outward[0] + dz * outward[1] + if t0 - EPS <= t <= t1 + EPS and -EPS <= n <= out + SERVER_EPS: + return True + return False + + def covers(self, s_mid: float, p: float, q: float, out_lo: float, out_hi: float) -> bool: + """Every sampled point from s = p to q, out_lo to out_hi in front of the battery's + segment, lies in some observed ground band.""" + if not self.strips: + return False + ns = max(1, math.ceil((q - p) / self.STEP_FT)) + no = max(1, math.ceil((out_hi - out_lo) / self.STEP_FT)) + for i in range(ns + 1): + for j in range(no + 1): + x = self.point(s_mid, p + (q - p) * i / ns, out_lo + (out_hi - out_lo) * j / no) + if x is None or not self.seen(x): + return False + return True def chain_ends_s(scene: dict, rules: RuleSet) -> tuple[float, float] | None: @@ -996,11 +1103,19 @@ def with_ground_short_of(scene: dict, rules: RuleSet, margin_ft: float = 0.1) -> GROUND_FAR_FT = 40.0 -def with_requests_captured(scene: dict, result: dict, out_ft: float = GROUND_FAR_FT) -> dict | None: - """The scene as if the homeowner had shown every band the result asked for.""" +def with_requests_captured(scene: dict, result: dict) -> dict | None: + """The scene as if the homeowner had shown every band the result asked for, exactly as far + as each request asks (its `out_ft`, which the result schema says to report back as the + observed entry's), never further. A ground request without one is read as far as the + redundancy check reads it, GROUND_FAR_FT.""" + + def reach(request: dict) -> dict: + if "out_ft" in request: + return {"out_ft": request["out_ft"]} + return {"out_ft": GROUND_FAR_FT} if request["band"] == "ground" else {} + added = [ - {"band": r["band"], "span_ft": sorted(r["span_ft"])} - | ({"out_ft": out_ft} if r["band"] == "ground" else {}) + {"band": r["band"], "span_ft": sorted(r["span_ft"])} | reach(r) for r in result.get("missing_evidence", []) if r["kind"] == "band" and "band" in r and "span_ft" in r ] diff --git a/verification/scoreboard/metrics.yaml b/verification/scoreboard/metrics.yaml index bb65ffac4..6a2c4673c 100644 --- a/verification/scoreboard/metrics.yaml +++ b/verification/scoreboard/metrics.yaml @@ -271,6 +271,11 @@ metrics: equals: true require_current_sha: true sha_key: app_sha + max_status: partial + cap_reason: >- + met on the synthetic replay against the hosted server at f14947e (12 states, one server + gap request); the real ADVIO replay is blocked by its data (the walk never covers the + ground by the meter), last real run 194f2eb - id: S4-2 workstream: S4 diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index 37cd0aed9..3a4291479 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-09-27T01:50:01-05:00", + "generated_at": "2026-09-27T02:12:37-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": [ "S2-1", @@ -10,9 +10,9 @@ "number": 124, "title": "tools: viewer that films a capture reaching the server, as a labeled replay or read-only live view", "branch": "t3/integration-viewer", - "head": "ed87212dc02a87844b5240d3facd5d05616c11f9", + "head": "980cee892d79e27b33ac1a4a7f5696da9755abdb", "draft": false, - "updated": "2026-09-27T06:48:01Z", + "updated": "2026-09-27T07:10:18Z", "ci": { "counts": { "success": 3, @@ -28,9 +28,9 @@ "number": 123, "title": "ios: House Scan Integration records each scan as a capture packet and, with consent, uploads it while scanning", "branch": "t3/hunter-integration", - "head": "294ddecb7a1d3fe303cbf21b73bfb3f3f5392daa", + "head": "9a2368c86a172e8840e3fb56299d4dfb58163416", "draft": true, - "updated": "2026-09-27T06:43:08Z", + "updated": "2026-09-27T07:08:17Z", "ci": { "counts": { "success": 2, @@ -46,9 +46,9 @@ "number": 122, "title": "server: stop requests at the wall's ends, keep placeholder distances out of them, name things without ids", "branch": "t3/41-server-requests-text", - "head": "a22c2d45351c202cff91defe9638197b5e87eb52", + "head": "c8d3c9d5d4575e8e52ef86052fdb9c48987fbf94", "draft": true, - "updated": "2026-09-27T06:46:29Z", + "updated": "2026-09-27T06:55:58Z", "ci": { "counts": { "success": 3, @@ -64,20 +64,18 @@ "number": 120, "title": "ios: hold a walk aim until it is met or stalls, and ask for the meter ground only near the meter", "branch": "t3/41-walk-task-hold", - "head": "6ea58d50562b0c3008b0edea57b0226c3de3aa92", + "head": "8487a02f811b12fd5e8eda3bd9f79e5b84089f1e", "draft": true, - "updated": "2026-09-27T06:46:35Z", + "updated": "2026-09-27T07:07:24Z", "ci": { "counts": { "success": 2, - "failure": 1, - "pending": 0, + "failure": 0, + "pending": 1, "skipped": 1 }, - "failing": [ - "iOS build" - ], - "text": "2 success, 1 failure, 1 skipped; failing: iOS build" + "failing": [], + "text": "2 success, 1 pending, 1 skipped" } }, { @@ -86,7 +84,7 @@ "branch": "t3/41-anchored-marks", "head": "f141a6212f3bba8c26695bf3cb5cb40ef0f1cf9d", "draft": true, - "updated": "2026-09-27T05:34:51Z", + "updated": "2026-09-27T06:55:12Z", "ci": { "counts": { "success": 2, @@ -102,9 +100,9 @@ "number": 117, "title": "ios: aim ring fills as its stretch is captured, with a legend and an unambiguous arrow", "branch": "t3/41-target-ring", - "head": "970486757fcd13391653b93563a669710f475db9", + "head": "6d82e62a9c1d08a2247694d36e0b3be7ae05673c", "draft": true, - "updated": "2026-09-27T05:36:33Z", + "updated": "2026-09-27T07:04:02Z", "ci": { "counts": { "success": 2, @@ -120,9 +118,9 @@ "number": 109, "title": "ios: gap-loop follow-ups, upload ticks, the nearer unseen end and request stretches (#39, #31, #83, #75)", "branch": "t3/41-gap-loop-result", - "head": "5c15da8705ef09bef33a71d1d1dca27deddb7480", + "head": "a44780a21d021b65515c61f93b6b84c1a3de97be", "draft": true, - "updated": "2026-09-27T06:00:21Z", + "updated": "2026-09-27T06:59:30Z", "ci": { "counts": { "success": 2, @@ -140,7 +138,7 @@ "branch": "t3/41-ground-and-meter-tap", "head": "9c2a5676fff6b31b1eb1e4bde63a5473ae61b5f3", "draft": true, - "updated": "2026-09-27T05:25:24Z", + "updated": "2026-09-27T06:49:26Z", "ci": { "counts": { "success": 2, @@ -156,9 +154,9 @@ "number": 104, "title": "ios: gate coaching keeps the task card, holds through dark flicker, and says \"Turn more slowly\" for turns", "branch": "t3/41-gate-coaching", - "head": "9b0acc5851e7fa69a1da560724c919f1d63bcc5e", + "head": "633c26d751fc24ecf8968756a29139fa6b969126", "draft": true, - "updated": "2026-09-27T06:47:54Z", + "updated": "2026-09-27T07:03:52Z", "ci": { "counts": { "success": 2, @@ -174,9 +172,9 @@ "number": 102, "title": "ios: card replies name the task, stay under gate coaching, and lock after a new card", "branch": "t3/41-card-replies", - "head": "a4b8b2723736b5f2b29c5605569dcdb76c6d8a69", + "head": "98d96caa41b24dc3140ca5a5fbaec4a0c610205f", "draft": true, - "updated": "2026-09-27T06:46:10Z", + "updated": "2026-09-27T06:59:29Z", "ci": { "counts": { "success": 2, @@ -338,7 +336,7 @@ "branch": "t3/app-review-docs", "head": "c0b628b6b9ec2b314ae418db956a27bc579ee642", "draft": false, - "updated": "2026-09-27T06:10:04Z", + "updated": "2026-09-27T06:56:32Z", "ci": { "counts": { "success": 4, @@ -356,7 +354,7 @@ "branch": "t3/device-test-4.1-analysis", "head": "c566490c5e3fa0d45803895bb463cb260d11a906", "draft": false, - "updated": "2026-09-27T06:48:45Z", + "updated": "2026-09-27T07:01:07Z", "ci": { "counts": { "success": 2, @@ -646,7 +644,7 @@ "refs": [ { "ref": "origin/t3/server", - "sha": "9176125df0c9890c247818d55edc82ea46053cd8" + "sha": "4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf" } ], "metrics": [ @@ -654,22 +652,22 @@ "id": "S2-1", "text": "pytest green", "ref": "origin/t3/server", - "sha": "9176125df0c9890c247818d55edc82ea46053cd8", + "sha": "4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf", "status": "met", - "evidence": "exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 346 passed, 1 warning in 65.76s (0:01:05)", + "evidence": "exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 352 passed, 1 warning in 89.34s (0:01:29)", "links": [] }, { "id": "S2-2", "text": "Every public golden test 01-14 of docs/research/t3-lane-c-review.md present", "ref": "origin/t3/server", - "sha": "9176125df0c9890c247818d55edc82ea46053cd8", + "sha": "4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf", "status": "met", "evidence": "all 14 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_golden_public.py" ] ] }, @@ -677,17 +675,21 @@ "id": "S2-3", "text": "Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail", "ref": "origin/t3/server", - "sha": "9176125df0c9890c247818d55edc82ea46053cd8", + "sha": "4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf", "status": "partial", - "evidence": "found missing coverage, mirror; missing equality unsure, larger clearance", + "evidence": "found missing coverage, equality unsure, mirror; missing larger clearance", "links": [ [ "server/tests/test_coderabbit_review.py", - "https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_coderabbit_review.py" + "https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_coderabbit_review.py" ], [ "server/tests/test_properties.py", - "https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_properties.py" + "https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_properties.py" + ], + [ + "server/tests/test_final_review.py", + "https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_final_review.py" ] ] }, @@ -695,17 +697,17 @@ "id": "S2-4", "text": "Schemas published", "ref": "origin/t3/server", - "sha": "9176125df0c9890c247818d55edc82ea46053cd8", + "sha": "4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf", "status": "met", - "evidence": "present at 9176125d", + "evidence": "present at 4a9fa44f", "links": [ [ "server/schemas/scene.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/schemas/scene.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/schemas/scene.schema.json" ], [ "server/schemas/result.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/schemas/result.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/schemas/result.schema.json" ] ] }, @@ -713,21 +715,21 @@ "id": "S2-5", "text": "Schemas validated in tests", "ref": "origin/t3/server", - "sha": "9176125df0c9890c247818d55edc82ea46053cd8", + "sha": "4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf", "status": "met", "evidence": "all 2 found", "links": [ [ "server/tests/test_existing_battery.py", - "https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_existing_battery.py" + "https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_existing_battery.py" ], [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_golden_public.py" ], [ "server/tests/test_schemas.py", - "https://github.com/SamGu-NRX/house-scanning/blob/9176125df0c9890c247818d55edc82ea46053cd8/server/tests/test_schemas.py" + "https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_schemas.py" ] ] }, @@ -735,27 +737,27 @@ "id": "S2-6", "text": "A real-derived scene returns a result in under 1 s", "ref": "origin/t3/server", - "sha": "9176125df0c9890c247818d55edc82ea46053cd8", + "sha": "4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf", "status": "met", - "evidence": "latency_ms = 239.4 (want < 1000) at sha 9176125d: ~/house-scanning-data/reports/e2e/20260927-014805-9176125d/report.json", + "evidence": "latency_ms = 561.4 (want < 1000) at sha 4a9fa44f: ~/house-scanning-data/reports/e2e/20260927-020953-4a9fa44f/report.json", "links": [] }, { "id": "S2-7", "text": "ruff clean", "ref": "origin/t3/server", - "sha": "9176125df0c9890c247818d55edc82ea46053cd8", + "sha": "4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf", "status": "met", - "evidence": "exit 0: Installed 33 packages in 48ms / All checks passed! / 31 files already formatted", + "evidence": "exit 0: Installed 33 packages in 33ms / All checks passed! / 32 files already formatted", "links": [] }, { "id": "S2-8", "text": "Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time", "ref": "origin/t3/server", - "sha": "9176125df0c9890c247818d55edc82ea46053cd8", - "status": "met", - "evidence": "contract_ok = true (want True) at sha 9176125d: ~/house-scanning-data/reports/e2e/20260927-014805-9176125d/report.json", + "sha": "4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf", + "status": "gap", + "evidence": "contract_ok = false (want True) at sha 4a9fa44f: ~/house-scanning-data/reports/e2e/20260927-020953-4a9fa44f/report.json", "links": [] } ] @@ -888,7 +890,7 @@ "ref": "origin/t3/ios-mvf", "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", "status": "met", - "evidence": "app_export_scene_valid = true (want True) at app_sha f14947eb: ~/house-scanning-data/reports/e2e/20260927-014805-9176125d/report.json", + "evidence": "app_export_scene_valid = true (want True) at app_sha f14947eb: ~/house-scanning-data/reports/e2e/20260927-020953-4a9fa44f/report.json", "links": [] }, { @@ -912,7 +914,7 @@ }, { "ref": "origin/t3/verification", - "sha": "36ac02f9d1fb3d654ff12dde652de65a134d1cfd" + "sha": "64d70d1f34396e822adcb5d502fd3d8c55b0dca1" } ], "metrics": [ @@ -921,8 +923,8 @@ "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/ios-mvf", "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", - "status": "met", - "evidence": "app_export_passed = true (want True) at app_sha f14947eb: ~/house-scanning-data/reports/e2e/20260927-014805-9176125d/report.json", + "status": "partial", + "evidence": "app_export_passed = true (want True) at app_sha f14947eb: ~/house-scanning-data/reports/e2e/20260927-020953-4a9fa44f/report.json (met on the synthetic replay against the hosted server at f14947e (12 states, one server gap request); the real ADVIO replay is blocked by its data (the walk never covers the ground by the meter), last real run 194f2eb)", "links": [] }, { @@ -938,7 +940,7 @@ "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "36ac02f9d1fb3d654ff12dde652de65a134d1cfd", + "sha": "64d70d1f34396e822adcb5d502fd3d8c55b0dca1", "status": "met", "evidence": "docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state.", "links": [] @@ -947,13 +949,13 @@ "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "36ac02f9d1fb3d654ff12dde652de65a134d1cfd", + "sha": "64d70d1f34396e822adcb5d502fd3d8c55b0dca1", "status": "met", - "evidence": "present at 36ac02f9", + "evidence": "present at 64d70d1f", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/36ac02f9d1fb3d654ff12dde652de65a134d1cfd/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/64d70d1f34396e822adcb5d502fd3d8c55b0dca1/verification/SCOREBOARD.md" ] ] } diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 6dce1c401..f320a5c17 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -388,7 +388,7 @@ def test_rules_come_from_rules_yaml_and_a_missing_one_is_loud(): assert rules.needs["gas_clearance"] == (Need("ground", 3.0), Need("wall", 3.0, 6.5)) assert rules.needs["battery_clearance"] == (Need("ground", 3.0), Need("wall", 3.0, 6.5)) assert rules.needs["opening_clearance"] == (Need("wall", 3.0, 6.5),) - assert rules.needs["wall_backing"] == (Need("wall", 0.0, 3.25, widen=False),) + assert rules.needs["wall_backing"] == (Need("wall", 0.0, 3.25),) assert rules.needs["facing_gap"] == (Need("facing", 0.0, 4.5),) assert rules.needs["headroom"] == (Need("overhead", 0.0, 6.5),) assert (rules.wall_join_ft, rules.battery_height_ft, rules.headroom_ft) == (0.6, 3.25, 6.5) @@ -1020,3 +1020,90 @@ def test_a_view_at_headroom_does_not_settle_a_request_just_past_it(): assert not any("lists as observed" in m for m in msgs) msgs = invariant_problems(with_wall_seen(6.500001), r, rules=RULES) assert any("lists as observed" in m for m in msgs) + + +def test_a_captured_request_claims_only_the_reach_it_asked_for(): + # Review 4114352249: a ground request out 0.5 ft was captured as seen 40 ft out. + r = result() + r["missing_evidence"] = [ + {"kind": "band", "band": "ground", "span_ft": [0, 5], "out_ft": 0.5, "message": ""}, + {"kind": "band", "band": "wall", "span_ft": [2, 3], "out_ft": 6.500001, "message": ""}, + {"kind": "band", "band": "facing", "span_ft": [2, 3], "message": ""}, + ] + added = with_requests_captured(SCENE, r)["coverage"]["observed"][-3:] + assert added == [ + {"band": "ground", "span_ft": [0, 5], "out_ft": 0.5}, + {"band": "wall", "span_ft": [2, 3], "out_ft": 6.500001}, + {"band": "facing", "span_ft": [2, 3]}, + ] + + +def test_a_sweep_run_shorter_than_a_step_keeps_its_first_start(): + # Review 4114352251: run_starts([0, 0.4], 1) returned only 0.4. + assert run_starts([0.0, 0.4], 1.0) == [0.0, 0.4] + + +def test_a_passing_spot_must_pass_in_the_sweep_too(): + # Review 4114352261: the spot passes but its start is in a failing run, stats consistent. + run = { + "wall_id": "w1", + "start_ft": [1.0, 1.0], + "outcome": "fail", + "failing": ["gas_clearance"], + "unsure": [], + } + r = result(sweep=[run]) + r["stats"] |= {"pass": 0, "fail": 1} + assert any( + "spot passes but the sweep has start 1.00 on w1 as fail" in m + for m in invariant_problems(SCENE, r, rules=RULES) + ) + + +def test_wall_backing_needs_the_wall_seen_past_the_battery_by_its_error(): + # Server 4a9fa44: the wall behind every place the battery may sit, [s0 - e, s1 + e]. + backing = replace(RULES, needs={"wall_backing": (Need("wall", 0.0, 3.25),)}) + scene = copy.deepcopy(SCENE) + scene["walls"][0]["plus_minus_ft"] = 0.3 + scene["coverage"]["observed"][0] = {"band": "wall", "span_ft": [1.0, 3.6], "out_ft": 4.0} + r = result(checks=[passing("wall_backing")]) + assert any("wall [0.70, 3.88]" in m for m in coverage_problems(scene, r, backing)) + scene["coverage"]["observed"][0]["span_ft"] = [0.0, 3.9] + assert coverage_problems(scene, r, backing) == [] + + +def inside_corner(w2_out: float) -> dict: + # g05: w1 runs +x with its outside at +z; w2 turns up +z from its end, facing back over w1. + return { + "meter": {"pos": [-6.0, 4.0, 0.0], "wall_id": "w1"}, + "walls": [ + {"id": "w1", "baseline": [[-7.0, 0.0], [4.0, 0.0]], "plus_minus_ft": 0.0}, + {"id": "w2", "baseline": [[4.0, 0.0], [4.0, 12.0]], "plus_minus_ft": 0.0}, + ], + "objects": [], + "coverage": { + "ends": {"left": {"kind": "limit"}, "right": {"kind": "limit"}}, + "observed": [ + {"band": "wall", "span_ft": [-1.0, 22.0]}, + {"band": "ground", "span_ft": [-7.0, 10.0], "out_ft": 10.5}, + {"band": "ground", "span_ft": [10.0, 22.0], "out_ft": w2_out}, + ], + }, + } + + +def test_ground_in_front_of_one_wall_may_be_seen_from_the_next_in_an_inside_corner(): + # The battery at s 4.04 (x -1.96 to 0.62) needs a 10 ft pool radius, ground out to 11.83 in + # front; w1's band reaches 10.5. From x = 4, w2's band reaching 10.66 or more covers it. + pool = replace(RULES, needs={"pool_clearance": (Need("ground", 10.0),)}) + run = { + "wall_id": "w1", + "start_ft": [4.04, 4.04], + "outcome": "pass", + "failing": [], + "unsure": [], + } + r = result(spot=False, checks=[check() | {"id": "pool_clearance"}], sweep=[run]) + assert coverage_problems(inside_corner(10.7), r, pool) == [] + short = coverage_problems(inside_corner(4.0), r, pool) # w2's band stops at x = 0 + assert any("pool_clearance needs ground [4.04, 6.62] observed out to 11.83" in m for m in short) From 7761447b7c54ed950c843222418375cdab8daf80 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 03:12:26 -0500 Subject: [PATCH 53/59] verification: sample ground cells at their centres as well as their edges At 33811e4 GroundPlan.covers tested only each sample cell's edges. A ground gap narrower than a cell between two observed bands has both edges on a band and its middle on none, so the 0.019 ft gap 5a4b4f4 passed over (ground seen [-40, 7.5] and [7.519, 40]) counted as seen. It now tests each cell's centre too; edges stay, so a gap showing only at a boundary is still caught. Samples remain finite and cannot prove every point was seen. The regression test uses that witness and fails at 33811e4. Both 5a4b4f4 witnesses from the caretaker are now flagged: the 0.019 ft gap (10 problems) and the opposite-facade house (25). Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/hsverify/resultcheck.py | 25 ++++++++++++++++++------- verification/tests/test_resultcheck.py | 21 +++++++++++++++++++++ 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index a085926ea..9d3fe603d 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -675,7 +675,9 @@ class GroundPlan: from above, scene.schema.json). A point in front of one wall can lie in a neighbouring wall's band, as in an inside corner, where the second wall faces back over the first.""" - STEP_FT = 0.05 # sampling step for coverage tests; a sliver thinner than this can be missed + # Sampling step for coverage tests. Samples are finite, so they cannot prove that every + # point of a region was seen: a thin gap shaped to avoid them can still be missed. + STEP_FT = 0.05 def __init__(self, scene: dict, rules: RuleSet) -> None: self.segments = segments_s(scene, rules) or [] @@ -717,18 +719,27 @@ def seen(self, x: Point) -> bool: def covers(self, s_mid: float, p: float, q: float, out_lo: float, out_hi: float) -> bool: """Every sampled point from s = p to q, out_lo to out_hi in front of the battery's - segment, lies in some observed ground band.""" + segment, lies in some observed ground band. + + Samples are each cell's edges and its centre, in s and out. Edges alone missed a gap + narrower than a cell between two bands (0.019 ft at 5a4b4f4): both its edges lie on a + band, its middle on none.""" if not self.strips: return False - ns = max(1, math.ceil((q - p) / self.STEP_FT)) - no = max(1, math.ceil((out_hi - out_lo) / self.STEP_FT)) - for i in range(ns + 1): - for j in range(no + 1): - x = self.point(s_mid, p + (q - p) * i / ns, out_lo + (out_hi - out_lo) * j / no) + for s in self._samples(p, q): + for out in self._samples(out_lo, out_hi): + x = self.point(s_mid, s, out) if x is None or not self.seen(x): return False return True + @classmethod + def _samples(cls, a: float, b: float) -> list[float]: + n = max(1, math.ceil((b - a) / cls.STEP_FT)) + edges = [a + (b - a) * i / n for i in range(n + 1)] + centres = [a + (b - a) * (i + 0.5) / n for i in range(n)] + return sorted(edges + centres) + def chain_ends_s(scene: dict, rules: RuleSet) -> tuple[float, float] | None: """s of the wall chain's left and right ends, or None without a meter wall.""" diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index f320a5c17..d138519fd 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -6,6 +6,7 @@ import pytest from hsverify.resultcheck import ( + GroundPlan, Need, RuleSet, assumption_mismatches, @@ -1107,3 +1108,23 @@ def test_ground_in_front_of_one_wall_may_be_seen_from_the_next_in_an_inside_corn assert coverage_problems(inside_corner(10.7), r, pool) == [] short = coverage_problems(inside_corner(4.0), r, pool) # w2's band stops at x = 0 assert any("pool_clearance needs ground [4.04, 6.62] observed out to 11.83" in m for m in short) + + +def test_a_ground_gap_narrower_than_the_sampling_step_is_not_counted_seen(): + # 5a4b4f4 passed a spot over ground seen [-40, 7.5] and [7.519, 40], out 30 ft: both edges + # of the 0.019 ft gap lie on a band, but its middle, s = 7.5095, was never seen. + rules = replace(RULES, needs={"gas_clearance": (Need("ground", 1.0),)}) + scene = copy.deepcopy(SCENE) + scene["walls"][0]["baseline"] = [[-40.0, 0.0], [40.0, 0.0]] + scene["coverage"]["observed"] = [ + {"band": "wall", "span_ft": [-40.0, 40.0]}, + {"band": "ground", "span_ft": [-40.0, 7.5], "out_ft": 30.0}, + {"band": "ground", "span_ft": [7.519, 40.0], "out_ft": 30.0}, + ] + r = result(checks=[check()], sweep=[]) + r["spot"]["span_ft"] = [6.0, 6.0 + 31 / 12] + msgs = coverage_problems(scene, r, rules) + assert any("ground [7.50, 7.52] observed, none seen" in m for m in msgs) + assert GroundPlan(scene, rules).seen((7.5095, 1.0)) is False + scene["coverage"]["observed"][2]["span_ft"] = [7.5, 40.0] + assert coverage_problems(scene, r, rules) == [] From 371c5123c97124ddfd1f78aa809fc3170abd1139 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 03:28:02 -0500 Subject: [PATCH 54/59] verification: pin to server 86b2b4d; scoreboard and result e2e at 86b2b4d with the app's f14947e scan: 50 of 50, contract_ok. The caretaker's three 5a4b4f4 witnesses (opposite-facade house, 0.019 and 0.011 ft ground gaps) are no longer passes there, and the checker flags them at 5a4b4f4. S2-1 to S2-8 met except S2-3 partial. S4-1 and S4-2 are stale: t3/ios-mvf moved to e426883 after the f14947e run. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/README.md | 5 +- verification/SCOREBOARD.md | 86 ++--- verification/scoreboard/scoreboard.json | 420 ++++++++++++------------ 3 files changed, 258 insertions(+), 253 deletions(-) diff --git a/verification/README.md b/verification/README.md index b602ec1af..c4857b26f 100644 --- a/verification/README.md +++ b/verification/README.md @@ -19,9 +19,8 @@ Reports go to `~/house-scanning-data/reports/`, outside git, because replay fram non-commercial datasets. What each check asserts is in the docstrings of `hsverify/e2e.py`, `resultcheck.py` and `simrun.py`. -**Result.** At `t3/server` `4a9fa44`, with the app's upload from `f14947e` (synthetic replay, hosted -server), 49 of 50 scenes pass (45 answered and checked; slowest real scene 0.56 s). The ETH3D scene -fails: capturing exactly the ground the server asks for never settles it. +**Result.** At `t3/server` `86b2b4d`, with the app's upload from `f14947e` (synthetic replay, hosted +server), all 50 scenes pass (45 answered and checked; slowest real scene 0.52 s). **What it changed.** S2 fixed seven defects it found: - unseen ground passing the pool check; diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index 358be434a..68ffed5a3 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,31 +1,34 @@ # Scoreboard -Generated 2026-09-27 02:12 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran for S2-1, S2-7 only. Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-27 03:27 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran for S2-1, S2-7 only. Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | -| [#124](https://github.com/SamGu-NRX/house-scanning/pull/124) | `t3/integration-viewer` | `980cee89` | no | 3 success, 1 pending | 2026-09-27 02:10 CDT | -| [#123](https://github.com/SamGu-NRX/house-scanning/pull/123) | `t3/hunter-integration` | `9a2368c8` | yes | 2 success, 2 skipped | 2026-09-27 02:08 CDT | -| [#122](https://github.com/SamGu-NRX/house-scanning/pull/122) | `t3/41-server-requests-text` | `c8d3c9d5` | yes | 3 success, 1 skipped | 2026-09-27 01:55 CDT | -| [#120](https://github.com/SamGu-NRX/house-scanning/pull/120) | `t3/41-walk-task-hold` | `8487a02f` | yes | 2 success, 1 pending, 1 skipped | 2026-09-27 02:07 CDT | -| [#119](https://github.com/SamGu-NRX/house-scanning/pull/119) | `t3/41-anchored-marks` | `f141a621` | yes | 2 success, 2 skipped | 2026-09-27 01:55 CDT | -| [#117](https://github.com/SamGu-NRX/house-scanning/pull/117) | `t3/41-target-ring` | `6d82e62a` | yes | 2 success, 2 skipped | 2026-09-27 02:04 CDT | -| [#109](https://github.com/SamGu-NRX/house-scanning/pull/109) | `t3/41-gap-loop-result` | `a44780a2` | yes | 2 success, 2 skipped | 2026-09-27 01:59 CDT | -| [#108](https://github.com/SamGu-NRX/house-scanning/pull/108) | `t3/41-ground-and-meter-tap` | `9c2a5676` | yes | 2 success, 2 skipped | 2026-09-27 01:49 CDT | -| [#104](https://github.com/SamGu-NRX/house-scanning/pull/104) | `t3/41-gate-coaching` | `633c26d7` | yes | 2 success, 2 skipped | 2026-09-27 02:03 CDT | -| [#102](https://github.com/SamGu-NRX/house-scanning/pull/102) | `t3/41-card-replies` | `98d96caa` | yes | 2 success, 2 skipped | 2026-09-27 01:59 CDT | -| [#100](https://github.com/SamGu-NRX/house-scanning/pull/100) | `t3/41-ar-canvas-fallback` | `41291e82` | yes | 2 success, 2 skipped | 2026-09-27 01:47 CDT | -| [#99](https://github.com/SamGu-NRX/house-scanning/pull/99) | `t3/41-end-preview` | `b5006fca` | yes | 2 success, 2 skipped | 2026-09-27 01:44 CDT | +| [#137](https://github.com/SamGu-NRX/house-scanning/pull/137) | `t3/41-review-followups-2` | `5c83ad49` | yes | 2 success, 2 skipped | 2026-09-27 03:12 CDT | +| [#136](https://github.com/SamGu-NRX/house-scanning/pull/136) | `t3/41-ends-corner` | `32fc0754` | yes | 2 success, 2 skipped | 2026-09-27 03:22 CDT | +| [#135](https://github.com/SamGu-NRX/house-scanning/pull/135) | `t3/41-walk-exit` | `057cfd42` | yes | 2 success, 2 skipped | 2026-09-27 03:11 CDT | +| [#134](https://github.com/SamGu-NRX/house-scanning/pull/134) | `shrey/readme` | `9ccb79e2` | no | 4 success | 2026-09-27 03:25 CDT | +| [#124](https://github.com/SamGu-NRX/house-scanning/pull/124) | `t3/integration-viewer` | `d966a09f` | no | 4 success | 2026-09-27 03:17 CDT | +| [#123](https://github.com/SamGu-NRX/house-scanning/pull/123) | `t3/hunter-integration` | `f9d0afa0` | yes | 2 success, 2 skipped | 2026-09-27 02:30 CDT | +| [#122](https://github.com/SamGu-NRX/house-scanning/pull/122) | `t3/41-server-requests-text` | `c8d3c9d5` | no | 5 success, 1 skipped | 2026-09-27 02:58 CDT | +| [#120](https://github.com/SamGu-NRX/house-scanning/pull/120) | `t3/41-walk-task-hold` | `8487a02f` | no | 7 success, 1 skipped | 2026-09-27 03:07 CDT | +| [#119](https://github.com/SamGu-NRX/house-scanning/pull/119) | `t3/41-anchored-marks` | `f141a621` | no | 6 success, 1 failure, 1 pending, 2 skipped; failing: iOS build | 2026-09-27 03:05 CDT | +| [#117](https://github.com/SamGu-NRX/house-scanning/pull/117) | `t3/41-target-ring` | `8b42e13a` | yes | 2 success, 2 skipped | 2026-09-27 03:25 CDT | +| [#109](https://github.com/SamGu-NRX/house-scanning/pull/109) | `t3/41-gap-loop-result` | `a44780a2` | no | 6 success, 2 skipped | 2026-09-27 03:05 CDT | +| [#108](https://github.com/SamGu-NRX/house-scanning/pull/108) | `t3/41-ground-and-meter-tap` | `9c2a5676` | no | 4 success, 1 failure, 2 skipped; failing: iOS build | 2026-09-27 03:05 CDT | +| [#104](https://github.com/SamGu-NRX/house-scanning/pull/104) | `t3/41-gate-coaching` | `633c26d7` | no | 6 success, 1 failure, 1 pending, 2 skipped; failing: iOS build | 2026-09-27 03:06 CDT | +| [#102](https://github.com/SamGu-NRX/house-scanning/pull/102) | `t3/41-card-replies` | `98d96caa` | no | 6 success, 2 skipped | 2026-09-27 03:05 CDT | +| [#100](https://github.com/SamGu-NRX/house-scanning/pull/100) | `t3/41-ar-canvas-fallback` | `41291e82` | no | 5 success, 1 skipped | 2026-09-27 03:04 CDT | +| [#99](https://github.com/SamGu-NRX/house-scanning/pull/99) | `t3/41-end-preview` | `b5006fca` | no | 6 success, 1 skipped | 2026-09-27 02:55 CDT | | [#91](https://github.com/SamGu-NRX/house-scanning/pull/91) | `t3/experience-result` | `9583b129` | no | 5 success | 2026-09-27 01:46 CDT | | [#90](https://github.com/SamGu-NRX/house-scanning/pull/90) | `t3/check-experiments` | `9d66b31c` | no | 5 success | 2026-09-26 23:39 CDT | -| [#89](https://github.com/SamGu-NRX/house-scanning/pull/89) | `t3/meter-brand` | `0a3a05e2` | no | 5 success | 2026-09-27 01:18 CDT | | [#88](https://github.com/SamGu-NRX/house-scanning/pull/88) | `t3/autodetect-lab` | `538f60e1` | yes | 2 success | 2026-09-26 23:00 CDT | -| [#87](https://github.com/SamGu-NRX/house-scanning/pull/87) | `t3/41-zip-deflate` | `5150321a` | yes | 2 success, 1 skipped | 2026-09-27 01:43 CDT | +| [#87](https://github.com/SamGu-NRX/house-scanning/pull/87) | `t3/41-zip-deflate` | `5150321a` | no | 6 success, 1 skipped | 2026-09-27 02:55 CDT | | [#86](https://github.com/SamGu-NRX/house-scanning/pull/86) | `t3/experience` | `b442f0a0` | yes | 2 success, 1 skipped | 2026-09-27 00:00 CDT | | [#61](https://github.com/SamGu-NRX/house-scanning/pull/61) | `t3/app-review-docs` | `c0b628b6` | no | 4 success | 2026-09-27 01:56 CDT | -| [#60](https://github.com/SamGu-NRX/house-scanning/pull/60) | `t3/device-test-4.1-analysis` | `c566490c` | no | 2 success | 2026-09-27 02:01 CDT | +| [#60](https://github.com/SamGu-NRX/house-scanning/pull/60) | `t3/device-test-4.1-analysis` | `c566490c` | no | 2 success | 2026-09-27 02:20 CDT | | [#59](https://github.com/SamGu-NRX/house-scanning/pull/59) | `t3/signals-lab` | `0cde05e3` | yes | 2 success | 2026-09-26 21:38 CDT | | [#58](https://github.com/SamGu-NRX/house-scanning/pull/58) | `t3/ios-ci-no-preboot` | `3c1298ab` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 22:18 CDT | | [#56](https://github.com/SamGu-NRX/house-scanning/pull/56) | `t3/walk-guidance-gaps` | `cae4f426` | no | 3 success | 2026-09-26 23:57 CDT | @@ -33,64 +36,61 @@ Generated 2026-09-27 02:12 CDT by `make scoreboard` (`uv run python -m hsverify. | [#52](https://github.com/SamGu-NRX/house-scanning/pull/52) | `t3/battery-mark` | `68a66a82` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 23:57 CDT | | [#49](https://github.com/SamGu-NRX/house-scanning/pull/49) | `t3/gap-skip-continues` | `ce901122` | no | 3 success | 2026-09-26 23:57 CDT | | [#46](https://github.com/SamGu-NRX/house-scanning/pull/46) | `t3/field-test-run2` | `afd02c82` | no | 2 success | 2026-09-26 17:36 CDT | -| [#23](https://github.com/SamGu-NRX/house-scanning/pull/23) | `t3/device-field-test` | `321ea7e2` | no | 2 success | 2026-09-26 19:39 CDT | -| [#22](https://github.com/SamGu-NRX/house-scanning/pull/22) | `t3/packet` | `89372a0a` | no | 4 success | 2026-09-27 01:05 CDT | -| [#21](https://github.com/SamGu-NRX/house-scanning/pull/21) | `t3/ios-map3d` | `c77577b7` | yes | 2 success, 2 skipped | 2026-09-27 00:28 CDT | ## S1 Evals on real data -`origin/t3/evals` at [`a7f68ba1`](https://github.com/SamGu-NRX/house-scanning/tree/a7f68ba15f7ffc589e28270d3fe84968e9d3becb). 1 met, 2 partial, 1 gap, 1 not run. +`origin/t3/evals` at [`17e338d9`](https://github.com/SamGu-NRX/house-scanning/tree/17e338d9b3e048dddeec51627b7616852db83f19). 1 met, 2 partial, 1 gap, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S1-1 | One command regenerates every number from a clean checkout | partial | make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac. | | S1-2 | Dataset URLs and sha256 recorded | gap | none found in 1 files; missing ADVIO, ETH3D | | S1-3 | Metric code tested on hand-computed cases | not run | slow; run with --slow | -| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/METHODS.md](https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/METHODS.md), [experiments/evals/field/FIELD_SHEET.md](https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/field/FIELD_SHEET.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/results/advio_drift.md) | -| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/METHODS.md](https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/METHODS.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/results/eth3d_visibility_sensitivity.md) | +| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/METHODS.md](https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/METHODS.md), [experiments/evals/field/FIELD_SHEET.md](https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/field/FIELD_SHEET.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/results/advio_drift.md) | +| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/METHODS.md](https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/METHODS.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/results/eth3d_visibility_sensitivity.md) | ## S2 Server and solver -`origin/t3/server` at [`4a9fa44f`](https://github.com/SamGu-NRX/house-scanning/tree/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf). 6 met, 1 partial, 1 gap. +`origin/t3/server` at [`86b2b4d6`](https://github.com/SamGu-NRX/house-scanning/tree/86b2b4d6912317486762b420e6b8972a67e2b83c). 7 met, 1 partial. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S2-1 | pytest green | met | exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 352 passed, 1 warning in 89.34s (0:01:29) | -| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_golden_public.py) | -| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, equality unsure, mirror; missing larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_properties.py), [server/tests/test_final_review.py](https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_final_review.py) | -| S2-4 | Schemas published | met | present at 4a9fa44f [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/schemas/result.schema.json) | -| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_existing_battery.py](https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_existing_battery.py), [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_schemas.py) | -| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 561.4 (want < 1000) at sha 4a9fa44f: ~/house-scanning-data/reports/e2e/20260927-020953-4a9fa44f/report.json | -| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 33ms / All checks passed! / 32 files already formatted | -| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | gap | contract_ok = false (want True) at sha 4a9fa44f: ~/house-scanning-data/reports/e2e/20260927-020953-4a9fa44f/report.json | +| S2-1 | pytest green | met | exit 0: return lib.intersection_prec(a, b, grid_size, **kwargs) / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 366 passed, 21 warnings in 59.86s | +| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_golden_public.py) | +| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, equality unsure, mirror; missing larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_properties.py), [server/tests/test_final_review.py](https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_final_review.py) | +| S2-4 | Schemas published | met | present at 86b2b4d6 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/schemas/result.schema.json) | +| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_existing_battery.py](https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_existing_battery.py), [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_schemas.py) | +| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 516.7 (want < 1000) at sha 86b2b4d6: ~/house-scanning-data/reports/e2e/20260927-032543-86b2b4d6/report.json | +| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 33ms / All checks passed! / 34 files already formatted | +| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | met | contract_ok = true (want True) at sha 86b2b4d6: ~/house-scanning-data/reports/e2e/20260927-032543-86b2b4d6/report.json | ## S3 iOS capture app -`origin/t3/ios-mvf` at [`f14947eb`](https://github.com/SamGu-NRX/house-scanning/tree/f14947eb9bff5b707660c8d873b72e60ba056631). 3 met, 3 partial, 1 gap, 1 stale, 2 no evidence yet. +`origin/t3/ios-mvf` at [`e4268831`](https://github.com/SamGu-NRX/house-scanning/tree/e4268831762b10acbf0117aacaa7e5e28cad92ea). 1 met, 3 partial, 1 gap, 3 stale, 2 no evidence yet. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S3-1 | CI build green | no evidence yet | no open PR for t3/ios-mvf | -| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/project.yml) | -| S3-3 | Zero build warnings | met | build.warnings = length 0 (want 0) at sha f14947eb: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | -| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage, guidance; missing auto-capture [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift) | -| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/SourceRestartUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/SourceRestartUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/AccessibilityAudit.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/AccessibilityAudit.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at f14947eb: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | +| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/project.yml) | +| S3-3 | Zero build warnings | stale | report for sha f14947eb, ref at e4268831: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | +| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage, guidance; missing auto-capture [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift) | +| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/SourceRestartUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/SourceRestartUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/AccessibilityAudit.swift](https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/AccessibilityAudit.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at e4268831: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | | S3-7 | Screenshots of each state in the PR | no evidence yet | no open PR for t3/ios-mvf | -| S3-8 | Exported scene.json validates against C1 | met | app_export_scene_valid = true (want True) at app_sha f14947eb: ~/house-scanning-data/reports/e2e/20260927-020953-4a9fa44f/report.json | +| S3-8 | Exported scene.json validates against C1 | stale | report for app_sha f14947eb, ref at e4268831: ~/house-scanning-data/reports/e2e/20260927-032543-86b2b4d6/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/ios-mvf` at [`f14947eb`](https://github.com/SamGu-NRX/house-scanning/tree/f14947eb9bff5b707660c8d873b72e60ba056631), `origin/t3/verification` at [`64d70d1f`](https://github.com/SamGu-NRX/house-scanning/tree/64d70d1f34396e822adcb5d502fd3d8c55b0dca1). 3 met, 1 partial. +`origin/t3/ios-mvf` at [`e4268831`](https://github.com/SamGu-NRX/house-scanning/tree/e4268831762b10acbf0117aacaa7e5e28cad92ea), `origin/t3/verification` at [`7761447b`](https://github.com/SamGu-NRX/house-scanning/tree/7761447b7c54ed950c843222418375cdab8daf80). 2 met, 2 stale. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S4-1 | End to end from a real replay to a placement result passes | partial | app_export_passed = true (want True) at app_sha f14947eb: ~/house-scanning-data/reports/e2e/20260927-020953-4a9fa44f/report.json (met on the synthetic replay against the hosted server at f14947e (12 states, one server gap request); the real ADVIO replay is blocked by its data (the walk never covers the ground by the meter), last real run 194f2eb) | -| S4-2 | Screenshot report per app state | met | states = length 12 (want at least 9) at sha f14947eb: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | +| S4-1 | End to end from a real replay to a placement result passes | stale | report for app_sha f14947eb, ref at e4268831: ~/house-scanning-data/reports/e2e/20260927-032543-86b2b4d6/report.json | +| S4-2 | Screenshot report per app state | stale | report for sha f14947eb, ref at e4268831: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | | S4-3 | Product description written from the code and checked against screenshots | met | docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state. | -| S4-4 | Scoreboard of every PR | met | present at 64d70d1f [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/64d70d1f34396e822adcb5d502fd3d8c55b0dca1/verification/SCOREBOARD.md) | +| S4-4 | Scoreboard of every PR | met | present at 7761447b [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/7761447b7c54ed950c843222418375cdab8daf80/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index 3a4291479..81215b946 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,36 +1,108 @@ { - "generated_at": "2026-09-27T02:12:37-05:00", + "generated_at": "2026-09-27T03:27:46-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": [ "S2-1", "S2-7" ], "prs": [ + { + "number": 137, + "title": "ios: review follow-ups for #132 (AR double draw, result chevron, aim legend contrast)", + "branch": "t3/41-review-followups-2", + "head": "5c83ad49efd108874102b75327798ecd4d654f82", + "draft": true, + "updated": "2026-09-27T08:12:57Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 2 + }, + "failing": [], + "text": "2 success, 2 skipped" + } + }, + { + "number": 136, + "title": "fix(ios): end a side at the phone; confirm and back out of the next wall (#71, #70)", + "branch": "t3/41-ends-corner", + "head": "32fc075482bc528eb5a1d14a21334a1e4c30fa17", + "draft": true, + "updated": "2026-09-27T08:22:49Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 2 + }, + "failing": [], + "text": "2 success, 2 skipped" + } + }, + { + "number": 135, + "title": "ios: confirm before a repeated Can't get there ends the scan; no spot for a wall never walked", + "branch": "t3/41-walk-exit", + "head": "057cfd421ecee5872b8c527bddbbc77efe486a27", + "draft": true, + "updated": "2026-09-27T08:11:44Z", + "ci": { + "counts": { + "success": 2, + "failure": 0, + "pending": 0, + "skipped": 2 + }, + "failing": [], + "text": "2 success, 2 skipped" + } + }, + { + "number": 134, + "title": "docs: rebuild the README around the demo, with quick start, data and limits", + "branch": "shrey/readme", + "head": "9ccb79e20dd092708e2a56f421f8223c40498898", + "draft": false, + "updated": "2026-09-27T08:25:33Z", + "ci": { + "counts": { + "success": 4, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "4 success" + } + }, { "number": 124, "title": "tools: viewer that films a capture reaching the server, as a labeled replay or read-only live view", "branch": "t3/integration-viewer", - "head": "980cee892d79e27b33ac1a4a7f5696da9755abdb", + "head": "d966a09ff7eda4c6295f70a8ed6c3f9ef9174b30", "draft": false, - "updated": "2026-09-27T07:10:18Z", + "updated": "2026-09-27T08:17:45Z", "ci": { "counts": { - "success": 3, + "success": 4, "failure": 0, - "pending": 1, + "pending": 0, "skipped": 0 }, "failing": [], - "text": "3 success, 1 pending" + "text": "4 success" } }, { "number": 123, "title": "ios: House Scan Integration records each scan as a capture packet and, with consent, uploads it while scanning", "branch": "t3/hunter-integration", - "head": "9a2368c86a172e8840e3fb56299d4dfb58163416", + "head": "f9d0afa00d86bee6263250f89bda08fe2f10a3cb", "draft": true, - "updated": "2026-09-27T07:08:17Z", + "updated": "2026-09-27T07:30:21Z", "ci": { "counts": { "success": 2, @@ -47,17 +119,17 @@ "title": "server: stop requests at the wall's ends, keep placeholder distances out of them, name things without ids", "branch": "t3/41-server-requests-text", "head": "c8d3c9d5d4575e8e52ef86052fdb9c48987fbf94", - "draft": true, - "updated": "2026-09-27T06:55:58Z", + "draft": false, + "updated": "2026-09-27T07:58:22Z", "ci": { "counts": { - "success": 3, + "success": 5, "failure": 0, "pending": 0, "skipped": 1 }, "failing": [], - "text": "3 success, 1 skipped" + "text": "5 success, 1 skipped" } }, { @@ -65,17 +137,17 @@ "title": "ios: hold a walk aim until it is met or stalls, and ask for the meter ground only near the meter", "branch": "t3/41-walk-task-hold", "head": "8487a02f811b12fd5e8eda3bd9f79e5b84089f1e", - "draft": true, - "updated": "2026-09-27T07:07:24Z", + "draft": false, + "updated": "2026-09-27T08:07:51Z", "ci": { "counts": { - "success": 2, + "success": 7, "failure": 0, - "pending": 1, + "pending": 0, "skipped": 1 }, "failing": [], - "text": "2 success, 1 pending, 1 skipped" + "text": "7 success, 1 skipped" } }, { @@ -83,26 +155,28 @@ "title": "ios: log meter-anchor corrections so marks can be checked against drift", "branch": "t3/41-anchored-marks", "head": "f141a6212f3bba8c26695bf3cb5cb40ef0f1cf9d", - "draft": true, - "updated": "2026-09-27T06:55:12Z", + "draft": false, + "updated": "2026-09-27T08:05:25Z", "ci": { "counts": { - "success": 2, - "failure": 0, - "pending": 0, + "success": 6, + "failure": 1, + "pending": 1, "skipped": 2 }, - "failing": [], - "text": "2 success, 2 skipped" + "failing": [ + "iOS build" + ], + "text": "6 success, 1 failure, 1 pending, 2 skipped; failing: iOS build" } }, { "number": 117, "title": "ios: aim ring fills as its stretch is captured, with a legend and an unambiguous arrow", "branch": "t3/41-target-ring", - "head": "6d82e62a9c1d08a2247694d36e0b3be7ae05673c", + "head": "8b42e13a532f5c3fc72b6b78bcec8824a3715721", "draft": true, - "updated": "2026-09-27T07:04:02Z", + "updated": "2026-09-27T08:25:01Z", "ci": { "counts": { "success": 2, @@ -119,17 +193,17 @@ "title": "ios: gap-loop follow-ups, upload ticks, the nearer unseen end and request stretches (#39, #31, #83, #75)", "branch": "t3/41-gap-loop-result", "head": "a44780a21d021b65515c61f93b6b84c1a3de97be", - "draft": true, - "updated": "2026-09-27T06:59:30Z", + "draft": false, + "updated": "2026-09-27T08:05:22Z", "ci": { "counts": { - "success": 2, + "success": 6, "failure": 0, "pending": 0, "skipped": 2 }, "failing": [], - "text": "2 success, 2 skipped" + "text": "6 success, 2 skipped" } }, { @@ -137,17 +211,19 @@ "title": "ios: ground needs a plausible meter height and never rises; estimated meter taps are checked and re-fitted to the detected wall", "branch": "t3/41-ground-and-meter-tap", "head": "9c2a5676fff6b31b1eb1e4bde63a5473ae61b5f3", - "draft": true, - "updated": "2026-09-27T06:49:26Z", + "draft": false, + "updated": "2026-09-27T08:05:00Z", "ci": { "counts": { - "success": 2, - "failure": 0, + "success": 4, + "failure": 1, "pending": 0, "skipped": 2 }, - "failing": [], - "text": "2 success, 2 skipped" + "failing": [ + "iOS build" + ], + "text": "4 success, 1 failure, 2 skipped; failing: iOS build" } }, { @@ -155,17 +231,19 @@ "title": "ios: gate coaching keeps the task card, holds through dark flicker, and says \"Turn more slowly\" for turns", "branch": "t3/41-gate-coaching", "head": "633c26d751fc24ecf8968756a29139fa6b969126", - "draft": true, - "updated": "2026-09-27T07:03:52Z", + "draft": false, + "updated": "2026-09-27T08:06:36Z", "ci": { "counts": { - "success": 2, - "failure": 0, - "pending": 0, + "success": 6, + "failure": 1, + "pending": 1, "skipped": 2 }, - "failing": [], - "text": "2 success, 2 skipped" + "failing": [ + "iOS build" + ], + "text": "6 success, 1 failure, 1 pending, 2 skipped; failing: iOS build" } }, { @@ -173,17 +251,17 @@ "title": "ios: card replies name the task, stay under gate coaching, and lock after a new card", "branch": "t3/41-card-replies", "head": "98d96caa41b24dc3140ca5a5fbaec4a0c610205f", - "draft": true, - "updated": "2026-09-27T06:59:29Z", + "draft": false, + "updated": "2026-09-27T08:05:16Z", "ci": { "counts": { - "success": 2, + "success": 6, "failure": 0, "pending": 0, "skipped": 2 }, "failing": [], - "text": "2 success, 2 skipped" + "text": "6 success, 2 skipped" } }, { @@ -191,17 +269,17 @@ "title": "ios: draw the AR result with the Canvas until RealityKit is seen drawing it", "branch": "t3/41-ar-canvas-fallback", "head": "41291e8287bd7a2452c6b53b36120670c3210317", - "draft": true, - "updated": "2026-09-27T06:47:25Z", + "draft": false, + "updated": "2026-09-27T08:04:58Z", "ci": { "counts": { - "success": 2, + "success": 5, "failure": 0, "pending": 0, - "skipped": 2 + "skipped": 1 }, "failing": [], - "text": "2 success, 2 skipped" + "text": "5 success, 1 skipped" } }, { @@ -209,17 +287,17 @@ "title": "ios: \"leaves out N ft you walked\" only on walk tasks, and on the end question", "branch": "t3/41-end-preview", "head": "b5006fca782aa280caa767e476fd5c62b496a9e2", - "draft": true, - "updated": "2026-09-27T06:44:16Z", + "draft": false, + "updated": "2026-09-27T07:55:03Z", "ci": { "counts": { - "success": 2, + "success": 6, "failure": 0, "pending": 0, - "skipped": 2 + "skipped": 1 }, "failing": [], - "text": "2 success, 2 skipped" + "text": "6 success, 1 skipped" } }, { @@ -258,24 +336,6 @@ "text": "5 success" } }, - { - "number": 89, - "title": "ios: show the meter's brand beside the number candidates", - "branch": "t3/meter-brand", - "head": "0a3a05e200fa42daf002d5fc77687c898935cffa", - "draft": false, - "updated": "2026-09-27T06:18:33Z", - "ci": { - "counts": { - "success": 5, - "failure": 0, - "pending": 0, - "skipped": 0 - }, - "failing": [], - "text": "5 success" - } - }, { "number": 88, "title": "experiments: auto-detect lab for wall objects and the meter brand", @@ -299,17 +359,17 @@ "title": "ios: deflate the text entries in the shared scan zip", "branch": "t3/41-zip-deflate", "head": "5150321a22f540e669755270dd76caa23957d8a4", - "draft": true, - "updated": "2026-09-27T06:43:53Z", + "draft": false, + "updated": "2026-09-27T07:55:06Z", "ci": { "counts": { - "success": 2, + "success": 6, "failure": 0, "pending": 0, "skipped": 1 }, "failing": [], - "text": "2 success, 1 skipped" + "text": "6 success, 1 skipped" } }, { @@ -354,7 +414,7 @@ "branch": "t3/device-test-4.1-analysis", "head": "c566490c5e3fa0d45803895bb463cb260d11a906", "draft": false, - "updated": "2026-09-27T07:01:07Z", + "updated": "2026-09-27T07:20:15Z", "ci": { "counts": { "success": 2, @@ -497,60 +557,6 @@ "failing": [], "text": "2 success" } - }, - { - "number": 23, - "title": "experiments: device field test runbook and run 1 of the TestFlight build", - "branch": "t3/device-field-test", - "head": "321ea7e2705b80f61eaac3b4c7c3ddbb6503c675", - "draft": false, - "updated": "2026-09-27T00:39:14Z", - "ci": { - "counts": { - "success": 2, - "failure": 0, - "pending": 0, - "skipped": 0 - }, - "failing": [], - "text": "2 success" - } - }, - { - "number": 22, - "title": "packet: point to the server's capture packet 0.4 and propose 0.5 additions", - "branch": "t3/packet", - "head": "89372a0a4143918f0d0e67c7e2bbae5ac60e8023", - "draft": false, - "updated": "2026-09-27T06:05:00Z", - "ci": { - "counts": { - "success": 4, - "failure": 0, - "pending": 0, - "skipped": 0 - }, - "failing": [], - "text": "4 success" - } - }, - { - "number": 21, - "title": "ios: live 3D map as the coverage model where depth exists", - "branch": "t3/ios-map3d", - "head": "c77577b7bbf24d927925654df61b776157ca4101", - "draft": true, - "updated": "2026-09-27T05:28:18Z", - "ci": { - "counts": { - "success": 2, - "failure": 0, - "pending": 0, - "skipped": 2 - }, - "failing": [], - "text": "2 success, 2 skipped" - } } ], "pr_error": null, @@ -561,7 +567,7 @@ "refs": [ { "ref": "origin/t3/evals", - "sha": "a7f68ba15f7ffc589e28270d3fe84968e9d3becb" + "sha": "17e338d9b3e048dddeec51627b7616852db83f19" } ], "metrics": [ @@ -569,7 +575,7 @@ "id": "S1-1", "text": "One command regenerates every number from a clean checkout", "ref": "origin/t3/evals", - "sha": "a7f68ba15f7ffc589e28270d3fe84968e9d3becb", + "sha": "17e338d9b3e048dddeec51627b7616852db83f19", "status": "partial", "evidence": "make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac.", "links": [] @@ -578,7 +584,7 @@ "id": "S1-2", "text": "Dataset URLs and sha256 recorded", "ref": "origin/t3/evals", - "sha": "a7f68ba15f7ffc589e28270d3fe84968e9d3becb", + "sha": "17e338d9b3e048dddeec51627b7616852db83f19", "status": "gap", "evidence": "none found in 1 files; missing ADVIO, ETH3D", "links": [] @@ -587,7 +593,7 @@ "id": "S1-3", "text": "Metric code tested on hand-computed cases", "ref": "origin/t3/evals", - "sha": "a7f68ba15f7ffc589e28270d3fe84968e9d3becb", + "sha": "17e338d9b3e048dddeec51627b7616852db83f19", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -596,21 +602,21 @@ "id": "S1-4", "text": "ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft", "ref": "origin/t3/evals", - "sha": "a7f68ba15f7ffc589e28270d3fe84968e9d3becb", + "sha": "17e338d9b3e048dddeec51627b7616852db83f19", "status": "met", "evidence": "all 5 found", "links": [ [ "experiments/evals/METHODS.md", - "https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/METHODS.md" + "https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/METHODS.md" ], [ "experiments/evals/field/FIELD_SHEET.md", - "https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/field/FIELD_SHEET.md" + "https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/field/FIELD_SHEET.md" ], [ "experiments/evals/results/advio_drift.md", - "https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/results/advio_drift.md" + "https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/results/advio_drift.md" ] ] }, @@ -618,21 +624,21 @@ "id": "S1-5", "text": "ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question", "ref": "origin/t3/evals", - "sha": "a7f68ba15f7ffc589e28270d3fe84968e9d3becb", + "sha": "17e338d9b3e048dddeec51627b7616852db83f19", "status": "partial", "evidence": "found 1-3 m row, 3-10 m row, scale error; missing verdict", "links": [ [ "experiments/evals/METHODS.md", - "https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/METHODS.md" + "https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/METHODS.md" ], [ "experiments/evals/results/eth3d_recon.md", - "https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/results/eth3d_recon.md" + "https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/results/eth3d_recon.md" ], [ "experiments/evals/results/eth3d_visibility_sensitivity.md", - "https://github.com/SamGu-NRX/house-scanning/blob/a7f68ba15f7ffc589e28270d3fe84968e9d3becb/experiments/evals/results/eth3d_visibility_sensitivity.md" + "https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/results/eth3d_visibility_sensitivity.md" ] ] } @@ -644,7 +650,7 @@ "refs": [ { "ref": "origin/t3/server", - "sha": "4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf" + "sha": "86b2b4d6912317486762b420e6b8972a67e2b83c" } ], "metrics": [ @@ -652,22 +658,22 @@ "id": "S2-1", "text": "pytest green", "ref": "origin/t3/server", - "sha": "4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf", + "sha": "86b2b4d6912317486762b420e6b8972a67e2b83c", "status": "met", - "evidence": "exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 352 passed, 1 warning in 89.34s (0:01:29)", + "evidence": "exit 0: return lib.intersection_prec(a, b, grid_size, **kwargs) / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 366 passed, 21 warnings in 59.86s", "links": [] }, { "id": "S2-2", "text": "Every public golden test 01-14 of docs/research/t3-lane-c-review.md present", "ref": "origin/t3/server", - "sha": "4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf", + "sha": "86b2b4d6912317486762b420e6b8972a67e2b83c", "status": "met", "evidence": "all 14 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_golden_public.py" ] ] }, @@ -675,21 +681,21 @@ "id": "S2-3", "text": "Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail", "ref": "origin/t3/server", - "sha": "4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf", + "sha": "86b2b4d6912317486762b420e6b8972a67e2b83c", "status": "partial", "evidence": "found missing coverage, equality unsure, mirror; missing larger clearance", "links": [ [ "server/tests/test_coderabbit_review.py", - "https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_coderabbit_review.py" + "https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_coderabbit_review.py" ], [ "server/tests/test_properties.py", - "https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_properties.py" + "https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_properties.py" ], [ "server/tests/test_final_review.py", - "https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_final_review.py" + "https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_final_review.py" ] ] }, @@ -697,17 +703,17 @@ "id": "S2-4", "text": "Schemas published", "ref": "origin/t3/server", - "sha": "4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf", + "sha": "86b2b4d6912317486762b420e6b8972a67e2b83c", "status": "met", - "evidence": "present at 4a9fa44f", + "evidence": "present at 86b2b4d6", "links": [ [ "server/schemas/scene.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/schemas/scene.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/schemas/scene.schema.json" ], [ "server/schemas/result.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/schemas/result.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/schemas/result.schema.json" ] ] }, @@ -715,21 +721,21 @@ "id": "S2-5", "text": "Schemas validated in tests", "ref": "origin/t3/server", - "sha": "4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf", + "sha": "86b2b4d6912317486762b420e6b8972a67e2b83c", "status": "met", "evidence": "all 2 found", "links": [ [ "server/tests/test_existing_battery.py", - "https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_existing_battery.py" + "https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_existing_battery.py" ], [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_golden_public.py" ], [ "server/tests/test_schemas.py", - "https://github.com/SamGu-NRX/house-scanning/blob/4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf/server/tests/test_schemas.py" + "https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_schemas.py" ] ] }, @@ -737,27 +743,27 @@ "id": "S2-6", "text": "A real-derived scene returns a result in under 1 s", "ref": "origin/t3/server", - "sha": "4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf", + "sha": "86b2b4d6912317486762b420e6b8972a67e2b83c", "status": "met", - "evidence": "latency_ms = 561.4 (want < 1000) at sha 4a9fa44f: ~/house-scanning-data/reports/e2e/20260927-020953-4a9fa44f/report.json", + "evidence": "latency_ms = 516.7 (want < 1000) at sha 86b2b4d6: ~/house-scanning-data/reports/e2e/20260927-032543-86b2b4d6/report.json", "links": [] }, { "id": "S2-7", "text": "ruff clean", "ref": "origin/t3/server", - "sha": "4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf", + "sha": "86b2b4d6912317486762b420e6b8972a67e2b83c", "status": "met", - "evidence": "exit 0: Installed 33 packages in 33ms / All checks passed! / 32 files already formatted", + "evidence": "exit 0: Installed 33 packages in 33ms / All checks passed! / 34 files already formatted", "links": [] }, { "id": "S2-8", "text": "Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time", "ref": "origin/t3/server", - "sha": "4a9fa44f2cadf8aacace12cce7b3b4335ad61fcf", - "status": "gap", - "evidence": "contract_ok = false (want True) at sha 4a9fa44f: ~/house-scanning-data/reports/e2e/20260927-020953-4a9fa44f/report.json", + "sha": "86b2b4d6912317486762b420e6b8972a67e2b83c", + "status": "met", + "evidence": "contract_ok = true (want True) at sha 86b2b4d6: ~/house-scanning-data/reports/e2e/20260927-032543-86b2b4d6/report.json", "links": [] } ] @@ -768,7 +774,7 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "f14947eb9bff5b707660c8d873b72e60ba056631" + "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea" } ], "metrics": [ @@ -776,7 +782,7 @@ "id": "S3-1", "text": "CI build green", "ref": "origin/t3/ios-mvf", - "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", + "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", "status": "no evidence yet", "evidence": "no open PR for t3/ios-mvf", "links": [] @@ -785,13 +791,13 @@ "id": "S3-2", "text": "Built as Swift 6 with strict concurrency", "ref": "origin/t3/ios-mvf", - "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", + "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", "status": "met", "evidence": "all 2 found", "links": [ [ "ios/project.yml", - "https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/project.yml" + "https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/project.yml" ] ] }, @@ -799,30 +805,30 @@ "id": "S3-3", "text": "Zero build warnings", "ref": "origin/t3/ios-mvf", - "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", - "status": "met", - "evidence": "build.warnings = length 0 (want 0) at sha f14947eb: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", + "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", + "status": "stale", + "evidence": "report for sha f14947eb, ref at e4268831: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", "links": [] }, { "id": "S3-4", "text": "Unit tests for coverage, auto-capture and guidance", "ref": "origin/t3/ios-mvf", - "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", + "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", "status": "partial", "evidence": "found coverage, guidance; missing auto-capture", "links": [ [ "ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift" ] ] }, @@ -830,21 +836,21 @@ "id": "S3-5", "text": "UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop", "ref": "origin/t3/ios-mvf", - "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", + "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", "status": "partial", "evidence": "all 3 found (the tests exist; passing is S3-1)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/FullFlowUITests.swift" ], [ "ios/HouseScanUITests/SourceRestartUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/SourceRestartUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/SourceRestartUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -852,17 +858,17 @@ "id": "S3-6", "text": "Accessibility audit on every screen", "ref": "origin/t3/ios-mvf", - "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", + "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", "status": "partial", "evidence": "3 matches in 2 files (an audit exists; which screens it covers needs a read)", "links": [ [ "ios/HouseScanUITests/AccessibilityAudit.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/AccessibilityAudit.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/AccessibilityAudit.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/f14947eb9bff5b707660c8d873b72e60ba056631/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -870,16 +876,16 @@ "id": "S3-6b", "text": "Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues", "ref": "origin/t3/ios-mvf", - "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", + "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", "status": "stale", - "evidence": "report for sha a39d0a50, ref at f14947eb: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", + "evidence": "report for sha a39d0a50, ref at e4268831: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", "links": [] }, { "id": "S3-7", "text": "Screenshots of each state in the PR", "ref": "origin/t3/ios-mvf", - "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", + "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", "status": "no evidence yet", "evidence": "no open PR for t3/ios-mvf", "links": [] @@ -888,16 +894,16 @@ "id": "S3-8", "text": "Exported scene.json validates against C1", "ref": "origin/t3/ios-mvf", - "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", - "status": "met", - "evidence": "app_export_scene_valid = true (want True) at app_sha f14947eb: ~/house-scanning-data/reports/e2e/20260927-020953-4a9fa44f/report.json", + "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", + "status": "stale", + "evidence": "report for app_sha f14947eb, ref at e4268831: ~/house-scanning-data/reports/e2e/20260927-032543-86b2b4d6/report.json", "links": [] }, { "id": "S3-9", "text": "Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design", "ref": "origin/t3/ios-mvf", - "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", + "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -910,11 +916,11 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "f14947eb9bff5b707660c8d873b72e60ba056631" + "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea" }, { "ref": "origin/t3/verification", - "sha": "64d70d1f34396e822adcb5d502fd3d8c55b0dca1" + "sha": "7761447b7c54ed950c843222418375cdab8daf80" } ], "metrics": [ @@ -922,25 +928,25 @@ "id": "S4-1", "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/ios-mvf", - "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", - "status": "partial", - "evidence": "app_export_passed = true (want True) at app_sha f14947eb: ~/house-scanning-data/reports/e2e/20260927-020953-4a9fa44f/report.json (met on the synthetic replay against the hosted server at f14947e (12 states, one server gap request); the real ADVIO replay is blocked by its data (the walk never covers the ground by the meter), last real run 194f2eb)", + "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", + "status": "stale", + "evidence": "report for app_sha f14947eb, ref at e4268831: ~/house-scanning-data/reports/e2e/20260927-032543-86b2b4d6/report.json", "links": [] }, { "id": "S4-2", "text": "Screenshot report per app state", "ref": "origin/t3/ios-mvf", - "sha": "f14947eb9bff5b707660c8d873b72e60ba056631", - "status": "met", - "evidence": "states = length 12 (want at least 9) at sha f14947eb: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", + "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", + "status": "stale", + "evidence": "report for sha f14947eb, ref at e4268831: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", "links": [] }, { "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "64d70d1f34396e822adcb5d502fd3d8c55b0dca1", + "sha": "7761447b7c54ed950c843222418375cdab8daf80", "status": "met", "evidence": "docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state.", "links": [] @@ -949,13 +955,13 @@ "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "64d70d1f34396e822adcb5d502fd3d8c55b0dca1", + "sha": "7761447b7c54ed950c843222418375cdab8daf80", "status": "met", - "evidence": "present at 64d70d1f", + "evidence": "present at 7761447b", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/64d70d1f34396e822adcb5d502fd3d8c55b0dca1/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/7761447b7c54ed950c843222418375cdab8daf80/verification/SCOREBOARD.md" ] ] } From 21c8832408e8484d64f4e54b4ca903ddb6c5af4a Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 04:31:20 -0500 Subject: [PATCH 55/59] verification: a reject needs every start failing; a footprint needs its wall; pin to a726199 Two false certifications from Codex's review of 371c512, each with a test built from the exact faulty response that fails at 371c512: - 4114676165: a reject with pass or unsure starts in its sweep passed the invariants. A reject now needs every start to fail. - 4114676173: a passing spot at start 11, 31 in wide, on a wall ending at 12 passed. Every passing or unsure spot and sweep start must lie on one straight stretch of scanned wall, walls meeting in a straight line joined as the server joins them, within 0.05 ft. A first version picked the stretch ending where a start began and flagged three correct scenes; a start on a boundary now belongs to the stretch it runs onto. e2e pinned to a726199: 50 of 50, contract_ok. Two further Codex findings are follow-up #153. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/README.md | 4 +- verification/SCOREBOARD.md | 68 +++--- verification/hsverify/resultcheck.py | 68 ++++++ verification/scoreboard/scoreboard.json | 308 ++++++++++++------------ verification/tests/test_resultcheck.py | 75 ++++++ 5 files changed, 334 insertions(+), 189 deletions(-) diff --git a/verification/README.md b/verification/README.md index c4857b26f..c3b7f1e59 100644 --- a/verification/README.md +++ b/verification/README.md @@ -19,8 +19,8 @@ Reports go to `~/house-scanning-data/reports/`, outside git, because replay fram non-commercial datasets. What each check asserts is in the docstrings of `hsverify/e2e.py`, `resultcheck.py` and `simrun.py`. -**Result.** At `t3/server` `86b2b4d`, with the app's upload from `f14947e` (synthetic replay, hosted -server), all 50 scenes pass (45 answered and checked; slowest real scene 0.52 s). +**Result.** At `t3/server` `a726199`, with the app's upload from `f14947e` (synthetic replay, hosted +server), all 50 scenes pass (45 answered and checked; slowest real scene 0.78 s at load average 68). **What it changed.** S2 fixed seven defects it found: - unseen ground passing the pool check; diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index 68ffed5a3..f968153e4 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,28 +1,30 @@ # Scoreboard -Generated 2026-09-27 03:27 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran for S2-1, S2-7 only. Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-27 04:31 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran for S2-1, S2-7 only. Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | -| [#137](https://github.com/SamGu-NRX/house-scanning/pull/137) | `t3/41-review-followups-2` | `5c83ad49` | yes | 2 success, 2 skipped | 2026-09-27 03:12 CDT | -| [#136](https://github.com/SamGu-NRX/house-scanning/pull/136) | `t3/41-ends-corner` | `32fc0754` | yes | 2 success, 2 skipped | 2026-09-27 03:22 CDT | -| [#135](https://github.com/SamGu-NRX/house-scanning/pull/135) | `t3/41-walk-exit` | `057cfd42` | yes | 2 success, 2 skipped | 2026-09-27 03:11 CDT | -| [#134](https://github.com/SamGu-NRX/house-scanning/pull/134) | `shrey/readme` | `9ccb79e2` | no | 4 success | 2026-09-27 03:25 CDT | -| [#124](https://github.com/SamGu-NRX/house-scanning/pull/124) | `t3/integration-viewer` | `d966a09f` | no | 4 success | 2026-09-27 03:17 CDT | -| [#123](https://github.com/SamGu-NRX/house-scanning/pull/123) | `t3/hunter-integration` | `f9d0afa0` | yes | 2 success, 2 skipped | 2026-09-27 02:30 CDT | +| [#152](https://github.com/SamGu-NRX/house-scanning/pull/152) | `t3/85-onboarding-moves` | `e085372d` | yes | 2 success, 2 skipped | 2026-09-27 04:24 CDT | +| [#151](https://github.com/SamGu-NRX/house-scanning/pull/151) | `t3/140-object-tap-reach` | `fd50357d` | no | 4 success, 1 pending, 2 skipped | 2026-09-27 04:26 CDT | +| [#149](https://github.com/SamGu-NRX/house-scanning/pull/149) | `t3/126-closeup-reduce-motion-recovery` | `15cbb77b` | no | 4 success, 1 pending, 2 skipped | 2026-09-27 04:22 CDT | +| [#147](https://github.com/SamGu-NRX/house-scanning/pull/147) | `t3/41-ends-corner-2` | `86da28da` | yes | 2 success, 2 skipped | 2026-09-27 04:05 CDT | +| [#146](https://github.com/SamGu-NRX/house-scanning/pull/146) | `t3/110-result-details-followups` | `6e8403fd` | no | 4 success, 1 pending, 2 skipped | 2026-09-27 04:26 CDT | +| [#143](https://github.com/SamGu-NRX/house-scanning/pull/143) | `t3/practice-meter` | `10b04e89` | no | 3 success, 1 skipped | 2026-09-27 04:18 CDT | +| [#135](https://github.com/SamGu-NRX/house-scanning/pull/135) | `t3/41-walk-exit` | `88912e50` | yes | 2 success, 2 skipped | 2026-09-27 04:24 CDT | +| [#134](https://github.com/SamGu-NRX/house-scanning/pull/134) | `shrey/readme` | `8a78c822` | no | 4 success | 2026-09-27 04:04 CDT | +| [#124](https://github.com/SamGu-NRX/house-scanning/pull/124) | `t3/integration-viewer` | `9c0680b6` | no | 4 success | 2026-09-27 03:45 CDT | +| [#123](https://github.com/SamGu-NRX/house-scanning/pull/123) | `t3/hunter-integration` | `5cc51246` | yes | 2 success, 2 skipped | 2026-09-27 03:56 CDT | | [#122](https://github.com/SamGu-NRX/house-scanning/pull/122) | `t3/41-server-requests-text` | `c8d3c9d5` | no | 5 success, 1 skipped | 2026-09-27 02:58 CDT | | [#120](https://github.com/SamGu-NRX/house-scanning/pull/120) | `t3/41-walk-task-hold` | `8487a02f` | no | 7 success, 1 skipped | 2026-09-27 03:07 CDT | -| [#119](https://github.com/SamGu-NRX/house-scanning/pull/119) | `t3/41-anchored-marks` | `f141a621` | no | 6 success, 1 failure, 1 pending, 2 skipped; failing: iOS build | 2026-09-27 03:05 CDT | -| [#117](https://github.com/SamGu-NRX/house-scanning/pull/117) | `t3/41-target-ring` | `8b42e13a` | yes | 2 success, 2 skipped | 2026-09-27 03:25 CDT | +| [#119](https://github.com/SamGu-NRX/house-scanning/pull/119) | `t3/41-anchored-marks` | `f141a621` | no | 6 success, 2 failure, 2 skipped; failing: iOS build, iOS build | 2026-09-27 03:05 CDT | | [#109](https://github.com/SamGu-NRX/house-scanning/pull/109) | `t3/41-gap-loop-result` | `a44780a2` | no | 6 success, 2 skipped | 2026-09-27 03:05 CDT | | [#108](https://github.com/SamGu-NRX/house-scanning/pull/108) | `t3/41-ground-and-meter-tap` | `9c2a5676` | no | 4 success, 1 failure, 2 skipped; failing: iOS build | 2026-09-27 03:05 CDT | -| [#104](https://github.com/SamGu-NRX/house-scanning/pull/104) | `t3/41-gate-coaching` | `633c26d7` | no | 6 success, 1 failure, 1 pending, 2 skipped; failing: iOS build | 2026-09-27 03:06 CDT | +| [#104](https://github.com/SamGu-NRX/house-scanning/pull/104) | `t3/41-gate-coaching` | `633c26d7` | no | 6 success, 2 failure, 2 skipped; failing: iOS build, iOS build | 2026-09-27 03:06 CDT | | [#102](https://github.com/SamGu-NRX/house-scanning/pull/102) | `t3/41-card-replies` | `98d96caa` | no | 6 success, 2 skipped | 2026-09-27 03:05 CDT | | [#100](https://github.com/SamGu-NRX/house-scanning/pull/100) | `t3/41-ar-canvas-fallback` | `41291e82` | no | 5 success, 1 skipped | 2026-09-27 03:04 CDT | | [#99](https://github.com/SamGu-NRX/house-scanning/pull/99) | `t3/41-end-preview` | `b5006fca` | no | 6 success, 1 skipped | 2026-09-27 02:55 CDT | -| [#91](https://github.com/SamGu-NRX/house-scanning/pull/91) | `t3/experience-result` | `9583b129` | no | 5 success | 2026-09-27 01:46 CDT | | [#90](https://github.com/SamGu-NRX/house-scanning/pull/90) | `t3/check-experiments` | `9d66b31c` | no | 5 success | 2026-09-26 23:39 CDT | | [#88](https://github.com/SamGu-NRX/house-scanning/pull/88) | `t3/autodetect-lab` | `538f60e1` | yes | 2 success | 2026-09-26 23:00 CDT | | [#87](https://github.com/SamGu-NRX/house-scanning/pull/87) | `t3/41-zip-deflate` | `5150321a` | no | 6 success, 1 skipped | 2026-09-27 02:55 CDT | @@ -34,8 +36,6 @@ Generated 2026-09-27 03:27 CDT by `make scoreboard` (`uv run python -m hsverify. | [#56](https://github.com/SamGu-NRX/house-scanning/pull/56) | `t3/walk-guidance-gaps` | `cae4f426` | no | 3 success | 2026-09-26 23:57 CDT | | [#55](https://github.com/SamGu-NRX/house-scanning/pull/55) | `t3/motion-permission` | `9607a481` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 23:57 CDT | | [#52](https://github.com/SamGu-NRX/house-scanning/pull/52) | `t3/battery-mark` | `68a66a82` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 23:57 CDT | -| [#49](https://github.com/SamGu-NRX/house-scanning/pull/49) | `t3/gap-skip-continues` | `ce901122` | no | 3 success | 2026-09-26 23:57 CDT | -| [#46](https://github.com/SamGu-NRX/house-scanning/pull/46) | `t3/field-test-run2` | `afd02c82` | no | 2 success | 2026-09-26 17:36 CDT | ## S1 Evals on real data @@ -51,46 +51,46 @@ Generated 2026-09-27 03:27 CDT by `make scoreboard` (`uv run python -m hsverify. ## S2 Server and solver -`origin/t3/server` at [`86b2b4d6`](https://github.com/SamGu-NRX/house-scanning/tree/86b2b4d6912317486762b420e6b8972a67e2b83c). 7 met, 1 partial. +`origin/t3/server` at [`a7261993`](https://github.com/SamGu-NRX/house-scanning/tree/a72619933ceb2966c7f1451fcba77f2d8c09ca50). 7 met, 1 partial. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S2-1 | pytest green | met | exit 0: return lib.intersection_prec(a, b, grid_size, **kwargs) / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 366 passed, 21 warnings in 59.86s | -| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_golden_public.py) | -| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, equality unsure, mirror; missing larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_properties.py), [server/tests/test_final_review.py](https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_final_review.py) | -| S2-4 | Schemas published | met | present at 86b2b4d6 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/schemas/result.schema.json) | -| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_existing_battery.py](https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_existing_battery.py), [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_schemas.py) | -| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 516.7 (want < 1000) at sha 86b2b4d6: ~/house-scanning-data/reports/e2e/20260927-032543-86b2b4d6/report.json | -| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 33ms / All checks passed! / 34 files already formatted | -| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | met | contract_ok = true (want True) at sha 86b2b4d6: ~/house-scanning-data/reports/e2e/20260927-032543-86b2b4d6/report.json | +| S2-1 | pytest green | met | exit 0: return lib.intersection_prec(a, b, grid_size, **kwargs) / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 371 passed, 1 xfailed, 3 warnings in 132.95s (0:02:12) | +| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_golden_public.py) | +| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, equality unsure, mirror; missing larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_properties.py), [server/tests/test_final_review.py](https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_final_review.py) | +| S2-4 | Schemas published | met | present at a7261993 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/schemas/result.schema.json) | +| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_existing_battery.py](https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_existing_battery.py), [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_schemas.py) | +| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 776.7 (want < 1000) at sha a7261993: ~/house-scanning-data/reports/e2e/20260927-042649-a7261993/report.json | +| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 71ms / All checks passed! / 35 files already formatted | +| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | met | contract_ok = true (want True) at sha a7261993: ~/house-scanning-data/reports/e2e/20260927-042649-a7261993/report.json | ## S3 iOS capture app -`origin/t3/ios-mvf` at [`e4268831`](https://github.com/SamGu-NRX/house-scanning/tree/e4268831762b10acbf0117aacaa7e5e28cad92ea). 1 met, 3 partial, 1 gap, 3 stale, 2 no evidence yet. +`origin/t3/ios-mvf` at [`b90fb8e4`](https://github.com/SamGu-NRX/house-scanning/tree/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6). 1 met, 3 partial, 1 gap, 3 stale, 2 no evidence yet. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S3-1 | CI build green | no evidence yet | no open PR for t3/ios-mvf | -| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/project.yml) | -| S3-3 | Zero build warnings | stale | report for sha f14947eb, ref at e4268831: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | -| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage, guidance; missing auto-capture [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift) | -| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/SourceRestartUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/SourceRestartUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/AccessibilityAudit.swift](https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/AccessibilityAudit.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at e4268831: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | +| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/project.yml) | +| S3-3 | Zero build warnings | stale | report for sha f14947eb, ref at b90fb8e4: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | +| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage, guidance; missing auto-capture [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift) | +| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/SourceRestartUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/SourceRestartUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/AccessibilityAudit.swift](https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/AccessibilityAudit.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at b90fb8e4: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | | S3-7 | Screenshots of each state in the PR | no evidence yet | no open PR for t3/ios-mvf | -| S3-8 | Exported scene.json validates against C1 | stale | report for app_sha f14947eb, ref at e4268831: ~/house-scanning-data/reports/e2e/20260927-032543-86b2b4d6/report.json | +| S3-8 | Exported scene.json validates against C1 | stale | report for app_sha f14947eb, ref at b90fb8e4: ~/house-scanning-data/reports/e2e/20260927-042649-a7261993/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/ios-mvf` at [`e4268831`](https://github.com/SamGu-NRX/house-scanning/tree/e4268831762b10acbf0117aacaa7e5e28cad92ea), `origin/t3/verification` at [`7761447b`](https://github.com/SamGu-NRX/house-scanning/tree/7761447b7c54ed950c843222418375cdab8daf80). 2 met, 2 stale. +`origin/t3/ios-mvf` at [`b90fb8e4`](https://github.com/SamGu-NRX/house-scanning/tree/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6), `origin/t3/verification` at [`371c5123`](https://github.com/SamGu-NRX/house-scanning/tree/371c5123c97124ddfd1f78aa809fc3170abd1139). 2 met, 2 stale. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S4-1 | End to end from a real replay to a placement result passes | stale | report for app_sha f14947eb, ref at e4268831: ~/house-scanning-data/reports/e2e/20260927-032543-86b2b4d6/report.json | -| S4-2 | Screenshot report per app state | stale | report for sha f14947eb, ref at e4268831: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | +| S4-1 | End to end from a real replay to a placement result passes | stale | report for app_sha f14947eb, ref at b90fb8e4: ~/house-scanning-data/reports/e2e/20260927-042649-a7261993/report.json | +| S4-2 | Screenshot report per app state | stale | report for sha f14947eb, ref at b90fb8e4: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | | S4-3 | Product description written from the code and checked against screenshots | met | docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state. | -| S4-4 | Scoreboard of every PR | met | present at 7761447b [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/7761447b7c54ed950c843222418375cdab8daf80/verification/SCOREBOARD.md) | +| S4-4 | Scoreboard of every PR | met | present at 371c5123 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/371c5123c97124ddfd1f78aa809fc3170abd1139/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 9d3fe603d..5c71a1246 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -282,6 +282,11 @@ def invariant_problems( if decision == "reject": if spot is not None: problems.append("decision reject but a spot is given") + # A reject says no start can pass or be settled by a person: every start fails. + for outcome in ("pass", "unsure"): + runs = [r["start_ft"] for r in result.get("sweep", []) if r["outcome"] == outcome] + if runs: + problems.append(f"decision reject but the sweep has {outcome} starts {runs}") if result.get("missing_evidence"): problems.append("decision reject while asking for more evidence") if decision == "manual_review" and not result.get("reasons"): @@ -316,6 +321,8 @@ def invariant_problems( problems.append(f"spot.meter_offset_ft {got} != centre - meter {want}") problems += spot_sweep_problems(result) + if rules is not None: + problems += footprint_problems(scene, result, rules) problems += missing_evidence_problems(scene, result, rules) if rules is not None: problems += coverage_problems(scene, result, rules) @@ -323,6 +330,67 @@ def invariant_problems( return problems +# Walls and baseline points within this of one straight line are one straight stretch +# (server/scene.py COLLINEAR_FT and _join_straight_walls at a726199). Also the slack on a +# footprint's ends, since s here and the server's s can differ by a merged point's offset. +COLLINEAR_FT = 0.05 + + +def straight_pieces(scene: dict, rules: RuleSet) -> list[tuple[float, float]]: + """The s-extent of each straight stretch of scanned wall a battery can back onto: baseline + segments joined where the next one starts at the previous one's end and continues its line, + within a wall or across two walls that meet.""" + pieces: list[list] = [] # [s0, s1, start point, end point] + for _, s0, s1, a, b in segments_s(scene, rules) or []: + if pieces: + prev = pieces[-1] + length = math.dist(prev[2], prev[3]) + if length > EPS and math.dist(prev[3], a) <= COLLINEAR_FT: + ux, uz = (prev[3][0] - prev[2][0]) / length, (prev[3][1] - prev[2][1]) / length + off = abs((b[0] - prev[2][0]) * uz - (b[1] - prev[2][1]) * ux) + ahead = (b[0] - prev[3][0]) * ux + (b[1] - prev[3][1]) * uz > 0 + if off <= COLLINEAR_FT and ahead and abs(s0 - prev[1]) <= COLLINEAR_FT: + prev[1], prev[3] = s1, b + continue + pieces.append([s0, s1, a, b]) + return [(p[0], p[1]) for p in pieces] + + +def footprint_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: + """A battery that passes or is left to a person backs onto one straight stretch of scanned + wall along its whole width; a footprint past the stretch's end (a corner, the end of the + wall, a gap) can't sit flush, so the server fails it (server/solver.py check_backing).""" + pieces = straight_pieces(scene, rules) + if not pieces: + return [] + + def off_wall(lo: float, hi: float) -> str | None: + # The stretch it starts on, or else ends on: the one it would back onto. A start on a + # boundary belongs to the stretch it runs onto, an end to the one it runs off. + piece = next((p for p in pieces if p[0] - EPS <= lo < p[1] - EPS), None) or next( + (p for p in pieces if p[0] + EPS < hi <= p[1] + EPS), None + ) + if piece is None: + return "is not on any scanned wall" + if lo < piece[0] - COLLINEAR_FT or hi > piece[1] + COLLINEAR_FT: + return f"runs past its straight wall [{piece[0]:.2f}, {piece[1]:.2f}]" + return None + + problems = [] + spot = result.get("spot") + if spot is not None and spot["outcome"] in ("pass", "unsure"): + lo, hi = spot["span_ft"] + if (why := off_wall(lo, hi)) is not None: + problems.append(f"spot [{lo:.2f}, {hi:.2f}] {why}") + for run in result.get("sweep", []): + if run["outcome"] not in ("pass", "unsure"): + continue + for start in dict.fromkeys(run["start_ft"]): # a run's two ends bound all its starts + if (why := off_wall(start, start + rules.width_ft)) is not None: + problems.append(f"sweep {run['outcome']} start {start:.2f} {why}") + return problems + + def spot_sweep_problems(result: dict) -> list[str]: """The chosen spot is one of the evaluated starts, so the sweep must give its start the spot's own outcome, and a passing spot must lie in a passing run. A result could otherwise diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index 81215b946..3bdf982bf 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-09-27T03:27:46-05:00", + "generated_at": "2026-09-27T04:31:04-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": [ "S2-1", @@ -7,12 +7,12 @@ ], "prs": [ { - "number": 137, - "title": "ios: review follow-ups for #132 (AR double draw, result chevron, aim legend contrast)", - "branch": "t3/41-review-followups-2", - "head": "5c83ad49efd108874102b75327798ecd4d654f82", + "number": 152, + "title": "ios: preview the walk's moves in onboarding", + "branch": "t3/85-onboarding-moves", + "head": "e085372d9007eaeaee3e0625cbd0ab9c448d3b0f", "draft": true, - "updated": "2026-09-27T08:12:57Z", + "updated": "2026-09-27T09:24:20Z", "ci": { "counts": { "success": 2, @@ -25,12 +25,48 @@ } }, { - "number": 136, - "title": "fix(ios): end a side at the phone; confirm and back out of the next wall (#71, #70)", - "branch": "t3/41-ends-corner", - "head": "32fc075482bc528eb5a1d14a21334a1e4c30fa17", + "number": 151, + "title": "fix(ios): refuse object taps that meet the wall under the floor or far along it", + "branch": "t3/140-object-tap-reach", + "head": "fd50357db9451b4f771d3ce9feaa2fb00d26bee2", + "draft": false, + "updated": "2026-09-27T09:26:07Z", + "ci": { + "counts": { + "success": 4, + "failure": 0, + "pending": 1, + "skipped": 2 + }, + "failing": [], + "text": "4 success, 1 pending, 2 skipped" + } + }, + { + "number": 149, + "title": "Recheck camera access on return, keep rejected meter close-ups unlinked, honour Reduce Motion on Not ", + "branch": "t3/126-closeup-reduce-motion-recovery", + "head": "15cbb77b84078f0505d82786a0f37a843565f7d8", + "draft": false, + "updated": "2026-09-27T09:22:54Z", + "ci": { + "counts": { + "success": 4, + "failure": 0, + "pending": 1, + "skipped": 2 + }, + "failing": [], + "text": "4 success, 1 pending, 2 skipped" + } + }, + { + "number": 147, + "title": "fix(ios): bound an end at the phone by the evidence; settle tracking; say 'you saw'", + "branch": "t3/41-ends-corner-2", + "head": "86da28da0615101497e57eb817630a4bb8dabc9a", "draft": true, - "updated": "2026-09-27T08:22:49Z", + "updated": "2026-09-27T09:05:01Z", "ci": { "counts": { "success": 2, @@ -42,13 +78,49 @@ "text": "2 success, 2 skipped" } }, + { + "number": 146, + "title": "Draw unclean AR spots as outlines and face the closest spot in the result model", + "branch": "t3/110-result-details-followups", + "head": "6e8403fd185cde9a5aa336f869c0ecfe6d62595b", + "draft": false, + "updated": "2026-09-27T09:26:24Z", + "ci": { + "counts": { + "success": 4, + "failure": 0, + "pending": 1, + "skipped": 2 + }, + "failing": [], + "text": "4 success, 1 pending, 2 skipped" + } + }, + { + "number": 143, + "title": "ios: practice meter, a drawn sample meter for scanning without one", + "branch": "t3/practice-meter", + "head": "10b04e89819c4e450a722f9c41511bbd7a84f6aa", + "draft": false, + "updated": "2026-09-27T09:18:54Z", + "ci": { + "counts": { + "success": 3, + "failure": 0, + "pending": 0, + "skipped": 1 + }, + "failing": [], + "text": "3 success, 1 skipped" + } + }, { "number": 135, "title": "ios: confirm before a repeated Can't get there ends the scan; no spot for a wall never walked", "branch": "t3/41-walk-exit", - "head": "057cfd421ecee5872b8c527bddbbc77efe486a27", + "head": "88912e50a9ece658e74fc148ba2ba2f1b6df0f75", "draft": true, - "updated": "2026-09-27T08:11:44Z", + "updated": "2026-09-27T09:24:19Z", "ci": { "counts": { "success": 2, @@ -64,9 +136,9 @@ "number": 134, "title": "docs: rebuild the README around the demo, with quick start, data and limits", "branch": "shrey/readme", - "head": "9ccb79e20dd092708e2a56f421f8223c40498898", + "head": "8a78c822d6fb7eca4e91f4b110bbc81ae9f78b81", "draft": false, - "updated": "2026-09-27T08:25:33Z", + "updated": "2026-09-27T09:04:37Z", "ci": { "counts": { "success": 4, @@ -82,9 +154,9 @@ "number": 124, "title": "tools: viewer that films a capture reaching the server, as a labeled replay or read-only live view", "branch": "t3/integration-viewer", - "head": "d966a09ff7eda4c6295f70a8ed6c3f9ef9174b30", + "head": "9c0680b664183d9414bb21f612d536f9da02575d", "draft": false, - "updated": "2026-09-27T08:17:45Z", + "updated": "2026-09-27T08:45:52Z", "ci": { "counts": { "success": 4, @@ -100,9 +172,9 @@ "number": 123, "title": "ios: House Scan Integration records each scan as a capture packet and, with consent, uploads it while scanning", "branch": "t3/hunter-integration", - "head": "f9d0afa00d86bee6263250f89bda08fe2f10a3cb", + "head": "5cc51246ff69b1560d1638be3521daec8278f5ab", "draft": true, - "updated": "2026-09-27T07:30:21Z", + "updated": "2026-09-27T08:56:11Z", "ci": { "counts": { "success": 2, @@ -160,32 +232,15 @@ "ci": { "counts": { "success": 6, - "failure": 1, - "pending": 1, + "failure": 2, + "pending": 0, "skipped": 2 }, "failing": [ + "iOS build", "iOS build" ], - "text": "6 success, 1 failure, 1 pending, 2 skipped; failing: iOS build" - } - }, - { - "number": 117, - "title": "ios: aim ring fills as its stretch is captured, with a legend and an unambiguous arrow", - "branch": "t3/41-target-ring", - "head": "8b42e13a532f5c3fc72b6b78bcec8824a3715721", - "draft": true, - "updated": "2026-09-27T08:25:01Z", - "ci": { - "counts": { - "success": 2, - "failure": 0, - "pending": 0, - "skipped": 2 - }, - "failing": [], - "text": "2 success, 2 skipped" + "text": "6 success, 2 failure, 2 skipped; failing: iOS build, iOS build" } }, { @@ -236,14 +291,15 @@ "ci": { "counts": { "success": 6, - "failure": 1, - "pending": 1, + "failure": 2, + "pending": 0, "skipped": 2 }, "failing": [ + "iOS build", "iOS build" ], - "text": "6 success, 1 failure, 1 pending, 2 skipped; failing: iOS build" + "text": "6 success, 2 failure, 2 skipped; failing: iOS build, iOS build" } }, { @@ -300,24 +356,6 @@ "text": "6 success, 1 skipped" } }, - { - "number": 91, - "title": "ios: result screen leads with the answer, shows the nearest rejected spot and links each unsure check to its view", - "branch": "t3/experience-result", - "head": "9583b1294882f40c963f5d21af9b053a6d3d367d", - "draft": false, - "updated": "2026-09-27T06:46:57Z", - "ci": { - "counts": { - "success": 5, - "failure": 0, - "pending": 0, - "skipped": 0 - }, - "failing": [], - "text": "5 success" - } - }, { "number": 90, "title": "build: include available experiment suites in root checks", @@ -521,42 +559,6 @@ ], "text": "2 success, 1 failure; failing: iOS build" } - }, - { - "number": 49, - "title": "ios: 'I can't get there' moves on to the next requested view (#39)", - "branch": "t3/gap-skip-continues", - "head": "ce90112202265f211509e5f73cdbe178f1894506", - "draft": false, - "updated": "2026-09-27T04:57:20Z", - "ci": { - "counts": { - "success": 3, - "failure": 0, - "pending": 0, - "skipped": 0 - }, - "failing": [], - "text": "3 success" - } - }, - { - "number": 46, - "title": "experiments: field test run 2 on build 3.1", - "branch": "t3/field-test-run2", - "head": "afd02c820bd025b1c077d8e238d3f3421e5d9f3a", - "draft": false, - "updated": "2026-09-26T22:36:21Z", - "ci": { - "counts": { - "success": 2, - "failure": 0, - "pending": 0, - "skipped": 0 - }, - "failing": [], - "text": "2 success" - } } ], "pr_error": null, @@ -650,7 +652,7 @@ "refs": [ { "ref": "origin/t3/server", - "sha": "86b2b4d6912317486762b420e6b8972a67e2b83c" + "sha": "a72619933ceb2966c7f1451fcba77f2d8c09ca50" } ], "metrics": [ @@ -658,22 +660,22 @@ "id": "S2-1", "text": "pytest green", "ref": "origin/t3/server", - "sha": "86b2b4d6912317486762b420e6b8972a67e2b83c", + "sha": "a72619933ceb2966c7f1451fcba77f2d8c09ca50", "status": "met", - "evidence": "exit 0: return lib.intersection_prec(a, b, grid_size, **kwargs) / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 366 passed, 21 warnings in 59.86s", + "evidence": "exit 0: return lib.intersection_prec(a, b, grid_size, **kwargs) / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 371 passed, 1 xfailed, 3 warnings in 132.95s (0:02:12)", "links": [] }, { "id": "S2-2", "text": "Every public golden test 01-14 of docs/research/t3-lane-c-review.md present", "ref": "origin/t3/server", - "sha": "86b2b4d6912317486762b420e6b8972a67e2b83c", + "sha": "a72619933ceb2966c7f1451fcba77f2d8c09ca50", "status": "met", "evidence": "all 14 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_golden_public.py" ] ] }, @@ -681,21 +683,21 @@ "id": "S2-3", "text": "Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail", "ref": "origin/t3/server", - "sha": "86b2b4d6912317486762b420e6b8972a67e2b83c", + "sha": "a72619933ceb2966c7f1451fcba77f2d8c09ca50", "status": "partial", "evidence": "found missing coverage, equality unsure, mirror; missing larger clearance", "links": [ [ "server/tests/test_coderabbit_review.py", - "https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_coderabbit_review.py" + "https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_coderabbit_review.py" ], [ "server/tests/test_properties.py", - "https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_properties.py" + "https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_properties.py" ], [ "server/tests/test_final_review.py", - "https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_final_review.py" + "https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_final_review.py" ] ] }, @@ -703,17 +705,17 @@ "id": "S2-4", "text": "Schemas published", "ref": "origin/t3/server", - "sha": "86b2b4d6912317486762b420e6b8972a67e2b83c", + "sha": "a72619933ceb2966c7f1451fcba77f2d8c09ca50", "status": "met", - "evidence": "present at 86b2b4d6", + "evidence": "present at a7261993", "links": [ [ "server/schemas/scene.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/schemas/scene.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/schemas/scene.schema.json" ], [ "server/schemas/result.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/schemas/result.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/schemas/result.schema.json" ] ] }, @@ -721,21 +723,21 @@ "id": "S2-5", "text": "Schemas validated in tests", "ref": "origin/t3/server", - "sha": "86b2b4d6912317486762b420e6b8972a67e2b83c", + "sha": "a72619933ceb2966c7f1451fcba77f2d8c09ca50", "status": "met", "evidence": "all 2 found", "links": [ [ "server/tests/test_existing_battery.py", - "https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_existing_battery.py" + "https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_existing_battery.py" ], [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_golden_public.py" ], [ "server/tests/test_schemas.py", - "https://github.com/SamGu-NRX/house-scanning/blob/86b2b4d6912317486762b420e6b8972a67e2b83c/server/tests/test_schemas.py" + "https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_schemas.py" ] ] }, @@ -743,27 +745,27 @@ "id": "S2-6", "text": "A real-derived scene returns a result in under 1 s", "ref": "origin/t3/server", - "sha": "86b2b4d6912317486762b420e6b8972a67e2b83c", + "sha": "a72619933ceb2966c7f1451fcba77f2d8c09ca50", "status": "met", - "evidence": "latency_ms = 516.7 (want < 1000) at sha 86b2b4d6: ~/house-scanning-data/reports/e2e/20260927-032543-86b2b4d6/report.json", + "evidence": "latency_ms = 776.7 (want < 1000) at sha a7261993: ~/house-scanning-data/reports/e2e/20260927-042649-a7261993/report.json", "links": [] }, { "id": "S2-7", "text": "ruff clean", "ref": "origin/t3/server", - "sha": "86b2b4d6912317486762b420e6b8972a67e2b83c", + "sha": "a72619933ceb2966c7f1451fcba77f2d8c09ca50", "status": "met", - "evidence": "exit 0: Installed 33 packages in 33ms / All checks passed! / 34 files already formatted", + "evidence": "exit 0: Installed 33 packages in 71ms / All checks passed! / 35 files already formatted", "links": [] }, { "id": "S2-8", "text": "Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time", "ref": "origin/t3/server", - "sha": "86b2b4d6912317486762b420e6b8972a67e2b83c", + "sha": "a72619933ceb2966c7f1451fcba77f2d8c09ca50", "status": "met", - "evidence": "contract_ok = true (want True) at sha 86b2b4d6: ~/house-scanning-data/reports/e2e/20260927-032543-86b2b4d6/report.json", + "evidence": "contract_ok = true (want True) at sha a7261993: ~/house-scanning-data/reports/e2e/20260927-042649-a7261993/report.json", "links": [] } ] @@ -774,7 +776,7 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea" + "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6" } ], "metrics": [ @@ -782,7 +784,7 @@ "id": "S3-1", "text": "CI build green", "ref": "origin/t3/ios-mvf", - "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", + "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", "status": "no evidence yet", "evidence": "no open PR for t3/ios-mvf", "links": [] @@ -791,13 +793,13 @@ "id": "S3-2", "text": "Built as Swift 6 with strict concurrency", "ref": "origin/t3/ios-mvf", - "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", + "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", "status": "met", "evidence": "all 2 found", "links": [ [ "ios/project.yml", - "https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/project.yml" + "https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/project.yml" ] ] }, @@ -805,30 +807,30 @@ "id": "S3-3", "text": "Zero build warnings", "ref": "origin/t3/ios-mvf", - "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", + "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", "status": "stale", - "evidence": "report for sha f14947eb, ref at e4268831: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", + "evidence": "report for sha f14947eb, ref at b90fb8e4: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", "links": [] }, { "id": "S3-4", "text": "Unit tests for coverage, auto-capture and guidance", "ref": "origin/t3/ios-mvf", - "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", + "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", "status": "partial", "evidence": "found coverage, guidance; missing auto-capture", "links": [ [ "ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift" ] ] }, @@ -836,21 +838,21 @@ "id": "S3-5", "text": "UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop", "ref": "origin/t3/ios-mvf", - "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", + "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", "status": "partial", "evidence": "all 3 found (the tests exist; passing is S3-1)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/FullFlowUITests.swift" ], [ "ios/HouseScanUITests/SourceRestartUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/SourceRestartUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/SourceRestartUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -858,17 +860,17 @@ "id": "S3-6", "text": "Accessibility audit on every screen", "ref": "origin/t3/ios-mvf", - "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", + "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", "status": "partial", "evidence": "3 matches in 2 files (an audit exists; which screens it covers needs a read)", "links": [ [ "ios/HouseScanUITests/AccessibilityAudit.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/AccessibilityAudit.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/AccessibilityAudit.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/e4268831762b10acbf0117aacaa7e5e28cad92ea/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -876,16 +878,16 @@ "id": "S3-6b", "text": "Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues", "ref": "origin/t3/ios-mvf", - "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", + "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", "status": "stale", - "evidence": "report for sha a39d0a50, ref at e4268831: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", + "evidence": "report for sha a39d0a50, ref at b90fb8e4: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", "links": [] }, { "id": "S3-7", "text": "Screenshots of each state in the PR", "ref": "origin/t3/ios-mvf", - "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", + "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", "status": "no evidence yet", "evidence": "no open PR for t3/ios-mvf", "links": [] @@ -894,16 +896,16 @@ "id": "S3-8", "text": "Exported scene.json validates against C1", "ref": "origin/t3/ios-mvf", - "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", + "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", "status": "stale", - "evidence": "report for app_sha f14947eb, ref at e4268831: ~/house-scanning-data/reports/e2e/20260927-032543-86b2b4d6/report.json", + "evidence": "report for app_sha f14947eb, ref at b90fb8e4: ~/house-scanning-data/reports/e2e/20260927-042649-a7261993/report.json", "links": [] }, { "id": "S3-9", "text": "Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design", "ref": "origin/t3/ios-mvf", - "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", + "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -916,11 +918,11 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea" + "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6" }, { "ref": "origin/t3/verification", - "sha": "7761447b7c54ed950c843222418375cdab8daf80" + "sha": "371c5123c97124ddfd1f78aa809fc3170abd1139" } ], "metrics": [ @@ -928,25 +930,25 @@ "id": "S4-1", "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/ios-mvf", - "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", + "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", "status": "stale", - "evidence": "report for app_sha f14947eb, ref at e4268831: ~/house-scanning-data/reports/e2e/20260927-032543-86b2b4d6/report.json", + "evidence": "report for app_sha f14947eb, ref at b90fb8e4: ~/house-scanning-data/reports/e2e/20260927-042649-a7261993/report.json", "links": [] }, { "id": "S4-2", "text": "Screenshot report per app state", "ref": "origin/t3/ios-mvf", - "sha": "e4268831762b10acbf0117aacaa7e5e28cad92ea", + "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", "status": "stale", - "evidence": "report for sha f14947eb, ref at e4268831: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", + "evidence": "report for sha f14947eb, ref at b90fb8e4: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", "links": [] }, { "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "7761447b7c54ed950c843222418375cdab8daf80", + "sha": "371c5123c97124ddfd1f78aa809fc3170abd1139", "status": "met", "evidence": "docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state.", "links": [] @@ -955,13 +957,13 @@ "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "7761447b7c54ed950c843222418375cdab8daf80", + "sha": "371c5123c97124ddfd1f78aa809fc3170abd1139", "status": "met", - "evidence": "present at 7761447b", + "evidence": "present at 371c5123", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/7761447b7c54ed950c843222418375cdab8daf80/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/371c5123c97124ddfd1f78aa809fc3170abd1139/verification/SCOREBOARD.md" ] ] } diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index d138519fd..de0509e9a 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -16,6 +16,7 @@ coverage_problems, declared_height_problems, expectation_problems, + footprint_problems, invariant_problems, less_coverage_problems, margin_problem, @@ -26,6 +27,7 @@ outcome_at, outcome_lengths, run_starts, + straight_pieces, tape_to_tap, with_ground_short_of, with_less_coverage, @@ -1128,3 +1130,76 @@ def test_a_ground_gap_narrower_than_the_sampling_step_is_not_counted_seen(): assert GroundPlan(scene, rules).seen((7.5095, 1.0)) is False scene["coverage"]["observed"][2]["span_ft"] = [7.5, 40.0] assert coverage_problems(scene, r, rules) == [] + + +@pytest.mark.parametrize("outcome", ["pass", "unsure"]) +def test_a_reject_must_have_every_start_failing(outcome): + # Review 4114676165: decision reject, no spot, nothing asked, but the sweep has a start + # that is not failing and stats that agree with it. + run = { + "wall_id": "w1", + "start_ft": [1.0, 1.0], + "outcome": outcome, + "failing": [], + "unsure": [] if outcome == "pass" else ["gas_clearance"], + } + r = result(decision="reject", spot=False, checks=[check("fail", 2.0)], sweep=[run]) + r["stats"] |= { + "candidates": 1, + "pass": int(outcome == "pass"), + "unsure": int(outcome == "unsure"), + "fail": 0, + } + assert any( + f"decision reject but the sweep has {outcome} starts" in m + for m in invariant_problems(SCENE, r, rules=RULES) + ) + + +def broadly_seen() -> dict: + scene = copy.deepcopy(SCENE) + scene["coverage"]["ends"] = {"left": {"kind": "limit"}, "right": {"kind": "limit"}} + scene["coverage"]["observed"] = [ + {"band": "wall", "span_ft": [-100.0, 100.0]}, + {"band": "ground", "span_ft": [-100.0, 100.0], "out_ft": 30.0}, + ] + return scene + + +def test_a_passing_spot_must_stand_on_the_scanned_wall(): + # Review 4114676173: a pass at start 11, 31 in wide, on a wall that ends at s = 12. + r = result( + decision="pass", + sweep=[ + { + "wall_id": "w1", + "start_ft": [11.0, 11.0], + "outcome": "pass", + "failing": [], + "unsure": [], + } + ], + ) + r["spot"]["span_ft"] = [11.0, 11.0 + 31 / 12] + msgs = invariant_problems(broadly_seen(), r, rules=RULES) + assert any("spot [11.00, 13.58] runs past its straight wall [-12.00, 12.00]" in m for m in msgs) + assert any("sweep pass start 11.00 runs past its straight wall" in m for m in msgs) + + +def test_walls_meeting_in_a_straight_line_are_one_stretch_and_a_corner_is_not(): + scene = broadly_seen() + scene["walls"] = [ + {"id": "w1", "baseline": [[-12.0, 0.0], [5.0, 0.0]], "plus_minus_ft": 0.0}, + {"id": "w2", "baseline": [[5.0, 0.0], [12.0, 0.0]], "plus_minus_ft": 0.0}, + ] + assert straight_pieces(scene, RULES) == [(-12.0, 12.0)] + r = result(sweep=[]) + r["spot"]["span_ft"] = [4.0, 4.0 + 31 / 12] # across the join + assert footprint_problems(scene, r, RULES) == [] + scene["walls"][1]["baseline"] = [[5.0, 0.0], [5.0, 7.0]] # a corner at s = 5 + assert straight_pieces(scene, RULES) == [(-12.0, 5.0), (5.0, 12.0)] + assert footprint_problems(scene, r, RULES) == [ + "spot [4.00, 6.58] runs past its straight wall [-12.00, 5.00]" + ] + r["spot"]["span_ft"] = [5.0, 5.0 + 31 / 12] # starts exactly at the corner, on the next wall + assert footprint_problems(scene, r, RULES) == [] From 255bc848679e7327fecc77c0348d089dd1a3f334 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 07:59:21 -0500 Subject: [PATCH 56/59] verification: pin to server 3baa338; accept a pass by the smallest reported margin At 3baa338 the checker flagged three scenes; both causes were the checker's: - meter_working_space 1.444446 +/- 1.444445 against 0 was called unsure. The server rounds to 6 decimals, rule lines sit on that grid, and rounding keeps order, so any positive reported margin proves the pass. The margin rule accepted only margins above 1e-6. - g09-reach-20p2-pm03 and the two drift cases encoded the old symmetric route error. From 3baa338 the meter's error lengthens a run but can't shorten it while the meter stands on its wall's line (server README, route_length). Their expectations are re-derived from that rule: the drift fail line moves from 25.016 to 24.659, and the 20.2 +/- 0.3 spot moves to 19.9. S2-3's probe missed the monotonicity property's name (test_more_space.py, since 22292a8); it now matches obstacle_away. All server rows are met at 3baa338. e2e: 50 of 50, no problems. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/README.md | 4 +- verification/SCOREBOARD.md | 110 ++-- verification/e2e/cases/README.md | 18 +- .../e2e/cases/d-reach-drift-left.json | 16 +- .../e2e/cases/d-reach-drift-right.json | 16 +- .../e2e/cases/g09-reach-20p2-pm03.json | 12 +- verification/e2e/cases/generate.py | 70 ++- verification/hsverify/resultcheck.py | 11 + verification/scoreboard/metrics.yaml | 3 +- verification/scoreboard/scoreboard.json | 542 +++++++++--------- verification/tests/test_resultcheck.py | 12 + 11 files changed, 430 insertions(+), 384 deletions(-) diff --git a/verification/README.md b/verification/README.md index c3b7f1e59..6bbe879cf 100644 --- a/verification/README.md +++ b/verification/README.md @@ -19,8 +19,8 @@ Reports go to `~/house-scanning-data/reports/`, outside git, because replay fram non-commercial datasets. What each check asserts is in the docstrings of `hsverify/e2e.py`, `resultcheck.py` and `simrun.py`. -**Result.** At `t3/server` `a726199`, with the app's upload from `f14947e` (synthetic replay, hosted -server), all 50 scenes pass (45 answered and checked; slowest real scene 0.78 s at load average 68). +**Result.** At `t3/server` `3baa338`, with the app's upload from `f14947e` (synthetic replay, hosted +server), all 50 scenes pass (45 answered and checked; slowest real scene 0.62 s). **What it changed.** S2 fixed seven defects it found: - unseen ground passing the pool check; diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index f968153e4..17035084f 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,96 +1,96 @@ # Scoreboard -Generated 2026-09-27 04:31 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran for S2-1, S2-7 only. Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-27 07:58 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran for S2-1, S2-7 only. Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | -| [#152](https://github.com/SamGu-NRX/house-scanning/pull/152) | `t3/85-onboarding-moves` | `e085372d` | yes | 2 success, 2 skipped | 2026-09-27 04:24 CDT | -| [#151](https://github.com/SamGu-NRX/house-scanning/pull/151) | `t3/140-object-tap-reach` | `fd50357d` | no | 4 success, 1 pending, 2 skipped | 2026-09-27 04:26 CDT | -| [#149](https://github.com/SamGu-NRX/house-scanning/pull/149) | `t3/126-closeup-reduce-motion-recovery` | `15cbb77b` | no | 4 success, 1 pending, 2 skipped | 2026-09-27 04:22 CDT | -| [#147](https://github.com/SamGu-NRX/house-scanning/pull/147) | `t3/41-ends-corner-2` | `86da28da` | yes | 2 success, 2 skipped | 2026-09-27 04:05 CDT | -| [#146](https://github.com/SamGu-NRX/house-scanning/pull/146) | `t3/110-result-details-followups` | `6e8403fd` | no | 4 success, 1 pending, 2 skipped | 2026-09-27 04:26 CDT | -| [#143](https://github.com/SamGu-NRX/house-scanning/pull/143) | `t3/practice-meter` | `10b04e89` | no | 3 success, 1 skipped | 2026-09-27 04:18 CDT | -| [#135](https://github.com/SamGu-NRX/house-scanning/pull/135) | `t3/41-walk-exit` | `88912e50` | yes | 2 success, 2 skipped | 2026-09-27 04:24 CDT | -| [#134](https://github.com/SamGu-NRX/house-scanning/pull/134) | `shrey/readme` | `8a78c822` | no | 4 success | 2026-09-27 04:04 CDT | -| [#124](https://github.com/SamGu-NRX/house-scanning/pull/124) | `t3/integration-viewer` | `9c0680b6` | no | 4 success | 2026-09-27 03:45 CDT | -| [#123](https://github.com/SamGu-NRX/house-scanning/pull/123) | `t3/hunter-integration` | `5cc51246` | yes | 2 success, 2 skipped | 2026-09-27 03:56 CDT | -| [#122](https://github.com/SamGu-NRX/house-scanning/pull/122) | `t3/41-server-requests-text` | `c8d3c9d5` | no | 5 success, 1 skipped | 2026-09-27 02:58 CDT | -| [#120](https://github.com/SamGu-NRX/house-scanning/pull/120) | `t3/41-walk-task-hold` | `8487a02f` | no | 7 success, 1 skipped | 2026-09-27 03:07 CDT | -| [#119](https://github.com/SamGu-NRX/house-scanning/pull/119) | `t3/41-anchored-marks` | `f141a621` | no | 6 success, 2 failure, 2 skipped; failing: iOS build, iOS build | 2026-09-27 03:05 CDT | -| [#109](https://github.com/SamGu-NRX/house-scanning/pull/109) | `t3/41-gap-loop-result` | `a44780a2` | no | 6 success, 2 skipped | 2026-09-27 03:05 CDT | -| [#108](https://github.com/SamGu-NRX/house-scanning/pull/108) | `t3/41-ground-and-meter-tap` | `9c2a5676` | no | 4 success, 1 failure, 2 skipped; failing: iOS build | 2026-09-27 03:05 CDT | -| [#104](https://github.com/SamGu-NRX/house-scanning/pull/104) | `t3/41-gate-coaching` | `633c26d7` | no | 6 success, 2 failure, 2 skipped; failing: iOS build, iOS build | 2026-09-27 03:06 CDT | -| [#102](https://github.com/SamGu-NRX/house-scanning/pull/102) | `t3/41-card-replies` | `98d96caa` | no | 6 success, 2 skipped | 2026-09-27 03:05 CDT | -| [#100](https://github.com/SamGu-NRX/house-scanning/pull/100) | `t3/41-ar-canvas-fallback` | `41291e82` | no | 5 success, 1 skipped | 2026-09-27 03:04 CDT | -| [#99](https://github.com/SamGu-NRX/house-scanning/pull/99) | `t3/41-end-preview` | `b5006fca` | no | 6 success, 1 skipped | 2026-09-27 02:55 CDT | -| [#90](https://github.com/SamGu-NRX/house-scanning/pull/90) | `t3/check-experiments` | `9d66b31c` | no | 5 success | 2026-09-26 23:39 CDT | -| [#88](https://github.com/SamGu-NRX/house-scanning/pull/88) | `t3/autodetect-lab` | `538f60e1` | yes | 2 success | 2026-09-26 23:00 CDT | -| [#87](https://github.com/SamGu-NRX/house-scanning/pull/87) | `t3/41-zip-deflate` | `5150321a` | no | 6 success, 1 skipped | 2026-09-27 02:55 CDT | -| [#86](https://github.com/SamGu-NRX/house-scanning/pull/86) | `t3/experience` | `b442f0a0` | yes | 2 success, 1 skipped | 2026-09-27 00:00 CDT | -| [#61](https://github.com/SamGu-NRX/house-scanning/pull/61) | `t3/app-review-docs` | `c0b628b6` | no | 4 success | 2026-09-27 01:56 CDT | -| [#60](https://github.com/SamGu-NRX/house-scanning/pull/60) | `t3/device-test-4.1-analysis` | `c566490c` | no | 2 success | 2026-09-27 02:20 CDT | -| [#59](https://github.com/SamGu-NRX/house-scanning/pull/59) | `t3/signals-lab` | `0cde05e3` | yes | 2 success | 2026-09-26 21:38 CDT | -| [#58](https://github.com/SamGu-NRX/house-scanning/pull/58) | `t3/ios-ci-no-preboot` | `3c1298ab` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 22:18 CDT | -| [#56](https://github.com/SamGu-NRX/house-scanning/pull/56) | `t3/walk-guidance-gaps` | `cae4f426` | no | 3 success | 2026-09-26 23:57 CDT | -| [#55](https://github.com/SamGu-NRX/house-scanning/pull/55) | `t3/motion-permission` | `9607a481` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 23:57 CDT | -| [#52](https://github.com/SamGu-NRX/house-scanning/pull/52) | `t3/battery-mark` | `68a66a82` | no | 2 success, 1 failure; failing: iOS build | 2026-09-26 23:57 CDT | +| [#169](https://github.com/SamGu-NRX/house-scanning/pull/169) | `t3/ci-flake-fixes-beta` | `9fc60195` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 07:40 CDT | +| [#168](https://github.com/SamGu-NRX/house-scanning/pull/168) | `t3/164-space-ends` | `6b21236f` | yes | 2 success, 2 skipped | 2026-09-27 07:22 CDT | +| [#165](https://github.com/SamGu-NRX/house-scanning/pull/165) | `t3/ci-flake-fixes` | `ee8cdcc1` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 07:40 CDT | +| [#158](https://github.com/SamGu-NRX/house-scanning/pull/158) | `t3/live-fog` | `f57310ca` | no | 5 success, 1 failure, 1 skipped; failing: iOS build | 2026-09-27 06:07 CDT | +| [#143](https://github.com/SamGu-NRX/house-scanning/pull/143) | `t3/practice-meter` | `aac574f8` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 06:31 CDT | +| [#134](https://github.com/SamGu-NRX/house-scanning/pull/134) | `shrey/readme` | `b0c5829c` | no | 4 success | 2026-09-27 07:54 CDT | +| [#124](https://github.com/SamGu-NRX/house-scanning/pull/124) | `t3/integration-viewer` | `9c0680b6` | no | 5 success | 2026-09-27 05:47 CDT | +| [#123](https://github.com/SamGu-NRX/house-scanning/pull/123) | `t3/hunter-integration` | `17b53baa` | yes | 2 success, 1 skipped | 2026-09-27 07:17 CDT | +| [#122](https://github.com/SamGu-NRX/house-scanning/pull/122) | `t3/41-server-requests-text` | `ee882e48` | no | 4 success, 1 skipped | 2026-09-27 07:40 CDT | +| [#120](https://github.com/SamGu-NRX/house-scanning/pull/120) | `t3/41-walk-task-hold` | `9d4d12e5` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 06:49 CDT | +| [#119](https://github.com/SamGu-NRX/house-scanning/pull/119) | `t3/41-anchored-marks` | `f03430fe` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 07:49 CDT | +| [#109](https://github.com/SamGu-NRX/house-scanning/pull/109) | `t3/41-gap-loop-result` | `f2309cb3` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 06:48 CDT | +| [#108](https://github.com/SamGu-NRX/house-scanning/pull/108) | `t3/41-ground-and-meter-tap` | `92c8f92f` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 07:49 CDT | +| [#104](https://github.com/SamGu-NRX/house-scanning/pull/104) | `t3/41-gate-coaching` | `040d9a61` | no | 4 success, 1 skipped | 2026-09-27 07:13 CDT | +| [#102](https://github.com/SamGu-NRX/house-scanning/pull/102) | `t3/41-card-replies` | `fd8f9b30` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 07:20 CDT | +| [#100](https://github.com/SamGu-NRX/house-scanning/pull/100) | `t3/41-ar-canvas-fallback` | `0ada11b9` | no | 4 success, 1 pending | 2026-09-27 07:42 CDT | +| [#99](https://github.com/SamGu-NRX/house-scanning/pull/99) | `t3/41-end-preview` | `f5a27c64` | no | 4 success, 1 pending | 2026-09-27 07:08 CDT | +| [#90](https://github.com/SamGu-NRX/house-scanning/pull/90) | `t3/check-experiments` | `9d66b31c` | no | 6 success | 2026-09-27 05:47 CDT | +| [#88](https://github.com/SamGu-NRX/house-scanning/pull/88) | `t3/autodetect-lab` | `538f60e1` | yes | 3 success | 2026-09-27 05:49 CDT | +| [#87](https://github.com/SamGu-NRX/house-scanning/pull/87) | `t3/41-zip-deflate` | `8d2a6a13` | no | 5 success | 2026-09-27 06:15 CDT | +| [#86](https://github.com/SamGu-NRX/house-scanning/pull/86) | `t3/experience` | `b442f0a0` | yes | 3 success, 1 skipped | 2026-09-27 05:48 CDT | +| [#61](https://github.com/SamGu-NRX/house-scanning/pull/61) | `t3/app-review-docs` | `c0b628b6` | no | 5 success | 2026-09-27 05:48 CDT | +| [#60](https://github.com/SamGu-NRX/house-scanning/pull/60) | `t3/device-test-4.1-analysis` | `c566490c` | no | 3 success | 2026-09-27 05:55 CDT | +| [#59](https://github.com/SamGu-NRX/house-scanning/pull/59) | `t3/signals-lab` | `0cde05e3` | yes | 3 success | 2026-09-27 05:50 CDT | +| [#58](https://github.com/SamGu-NRX/house-scanning/pull/58) | `t3/ios-ci-no-preboot` | `7f1037e0` | no | 4 success, 1 skipped | 2026-09-27 07:02 CDT | +| [#56](https://github.com/SamGu-NRX/house-scanning/pull/56) | `t3/walk-guidance-gaps` | `a9e23c61` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 06:11 CDT | +| [#55](https://github.com/SamGu-NRX/house-scanning/pull/55) | `t3/motion-permission` | `b289c7df` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 07:13 CDT | +| [#52](https://github.com/SamGu-NRX/house-scanning/pull/52) | `t3/battery-mark` | `086ad94b` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 07:23 CDT | +| [#49](https://github.com/SamGu-NRX/house-scanning/pull/49) | `t3/gap-skip-continues` | `e5761901` | no | 4 success, 1 skipped | 2026-09-27 07:22 CDT | +| [#46](https://github.com/SamGu-NRX/house-scanning/pull/46) | `t3/field-test-run2` | `afd02c82` | no | 3 success | 2026-09-27 05:48 CDT | ## S1 Evals on real data -`origin/t3/evals` at [`17e338d9`](https://github.com/SamGu-NRX/house-scanning/tree/17e338d9b3e048dddeec51627b7616852db83f19). 1 met, 2 partial, 1 gap, 1 not run. +`origin/t3/evals` at [`4f52036f`](https://github.com/SamGu-NRX/house-scanning/tree/4f52036f78c7ab45350004fda241da3125d2bb5e). 1 met, 2 partial, 1 gap, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S1-1 | One command regenerates every number from a clean checkout | partial | make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac. | | S1-2 | Dataset URLs and sha256 recorded | gap | none found in 1 files; missing ADVIO, ETH3D | | S1-3 | Metric code tested on hand-computed cases | not run | slow; run with --slow | -| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/METHODS.md](https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/METHODS.md), [experiments/evals/field/FIELD_SHEET.md](https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/field/FIELD_SHEET.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/results/advio_drift.md) | -| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/METHODS.md](https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/METHODS.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/results/eth3d_visibility_sensitivity.md) | +| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/METHODS.md](https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/METHODS.md), [experiments/evals/field/FIELD_SHEET.md](https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/field/FIELD_SHEET.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/results/advio_drift.md) | +| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/METHODS.md](https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/METHODS.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/results/eth3d_visibility_sensitivity.md) | ## S2 Server and solver -`origin/t3/server` at [`a7261993`](https://github.com/SamGu-NRX/house-scanning/tree/a72619933ceb2966c7f1451fcba77f2d8c09ca50). 7 met, 1 partial. +`origin/t3/server` at [`3baa3387`](https://github.com/SamGu-NRX/house-scanning/tree/3baa3387a6877961345afa7785f1530c1aeb2b1b). 8 met. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S2-1 | pytest green | met | exit 0: return lib.intersection_prec(a, b, grid_size, **kwargs) / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 371 passed, 1 xfailed, 3 warnings in 132.95s (0:02:12) | -| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_golden_public.py) | -| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | partial | found missing coverage, equality unsure, mirror; missing larger clearance [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_properties.py), [server/tests/test_final_review.py](https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_final_review.py) | -| S2-4 | Schemas published | met | present at a7261993 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/schemas/result.schema.json) | -| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_existing_battery.py](https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_existing_battery.py), [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_schemas.py) | -| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 776.7 (want < 1000) at sha a7261993: ~/house-scanning-data/reports/e2e/20260927-042649-a7261993/report.json | -| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 71ms / All checks passed! / 35 files already formatted | -| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | met | contract_ok = true (want True) at sha a7261993: ~/house-scanning-data/reports/e2e/20260927-042649-a7261993/report.json | +| S2-1 | pytest green | met | exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 380 passed, 1 xfailed, 1 warning in 80.06s (0:01:20) | +| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_golden_public.py) | +| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | met | all 4 found [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_properties.py), [server/tests/test_final_review.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_final_review.py) | +| S2-4 | Schemas published | met | present at 3baa3387 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/schemas/result.schema.json) | +| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_existing_battery.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_existing_battery.py), [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_schemas.py) | +| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 616.8 (want < 1000) at sha 3baa3387: ~/house-scanning-data/reports/e2e/20260927-075636-3baa3387/report.json | +| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 37ms / All checks passed! / 36 files already formatted | +| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | met | contract_ok = true (want True) at sha 3baa3387: ~/house-scanning-data/reports/e2e/20260927-075636-3baa3387/report.json | ## S3 iOS capture app -`origin/t3/ios-mvf` at [`b90fb8e4`](https://github.com/SamGu-NRX/house-scanning/tree/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6). 1 met, 3 partial, 1 gap, 3 stale, 2 no evidence yet. +`origin/t3/ios-mvf` at [`6be087df`](https://github.com/SamGu-NRX/house-scanning/tree/6be087df207351fe3149b1924f78e91a92d2c3eb). 1 met, 3 partial, 1 gap, 3 stale, 2 no evidence yet. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S3-1 | CI build green | no evidence yet | no open PR for t3/ios-mvf | -| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/project.yml) | -| S3-3 | Zero build warnings | stale | report for sha f14947eb, ref at b90fb8e4: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | -| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage, guidance; missing auto-capture [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift) | -| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/SourceRestartUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/SourceRestartUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/AccessibilityAudit.swift](https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/AccessibilityAudit.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at b90fb8e4: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | +| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/project.yml) | +| S3-3 | Zero build warnings | stale | report for sha f14947eb, ref at 6be087df: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | +| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage, guidance; missing auto-capture [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift) | +| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/SourceRestartUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/SourceRestartUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/AccessibilityAudit.swift](https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/AccessibilityAudit.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at 6be087df: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | | S3-7 | Screenshots of each state in the PR | no evidence yet | no open PR for t3/ios-mvf | -| S3-8 | Exported scene.json validates against C1 | stale | report for app_sha f14947eb, ref at b90fb8e4: ~/house-scanning-data/reports/e2e/20260927-042649-a7261993/report.json | +| S3-8 | Exported scene.json validates against C1 | stale | report for app_sha f14947eb, ref at 6be087df: ~/house-scanning-data/reports/e2e/20260927-075636-3baa3387/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/ios-mvf` at [`b90fb8e4`](https://github.com/SamGu-NRX/house-scanning/tree/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6), `origin/t3/verification` at [`371c5123`](https://github.com/SamGu-NRX/house-scanning/tree/371c5123c97124ddfd1f78aa809fc3170abd1139). 2 met, 2 stale. +`origin/t3/ios-mvf` at [`6be087df`](https://github.com/SamGu-NRX/house-scanning/tree/6be087df207351fe3149b1924f78e91a92d2c3eb), `origin/t3/verification` at [`21c88324`](https://github.com/SamGu-NRX/house-scanning/tree/21c8832408e8484d64f4e54b4ca903ddb6c5af4a). 2 met, 2 stale. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S4-1 | End to end from a real replay to a placement result passes | stale | report for app_sha f14947eb, ref at b90fb8e4: ~/house-scanning-data/reports/e2e/20260927-042649-a7261993/report.json | -| S4-2 | Screenshot report per app state | stale | report for sha f14947eb, ref at b90fb8e4: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | +| S4-1 | End to end from a real replay to a placement result passes | stale | report for app_sha f14947eb, ref at 6be087df: ~/house-scanning-data/reports/e2e/20260927-075636-3baa3387/report.json | +| S4-2 | Screenshot report per app state | stale | report for sha f14947eb, ref at 6be087df: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | | S4-3 | Product description written from the code and checked against screenshots | met | docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state. | -| S4-4 | Scoreboard of every PR | met | present at 371c5123 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/371c5123c97124ddfd1f78aa809fc3170abd1139/verification/SCOREBOARD.md) | +| S4-4 | Scoreboard of every PR | met | present at 21c88324 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/21c8832408e8484d64f4e54b4ca903ddb6c5af4a/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/e2e/cases/README.md b/verification/e2e/cases/README.md index cdc9b6f2f..698333c6f 100644 --- a/verification/e2e/cases/README.md +++ b/verification/e2e/cases/README.md @@ -122,7 +122,7 @@ From `rules.yaml` at 2c9348f, rechecked at f2705dd: | g07-headroom-unobserved | 07 | headroom unsure (unobserved); same intervals unsure; `missing_evidence` non-empty | No overheads, and the overhead band is left out of coverage. | | g07-headroom-partial-span | 07 | starts in (5.5, 8.4) fail with `headroom`; starts in (1.3, 5.3) and (8.6, 12.4) pass | Overhead 5.5 ± 0.5 over s = [8, 8.5] only. A battery [a, a + W] overlaps it when 8 − W < a < 8.5, i.e. 5.417 < a < 8.5, even if it covers only a few inches of the battery. The schema's overheads have no depth out from the wall, so the golden's "front strip only" is tested along the wall instead. | | g09-reach-14, -15, -16, -20, -21 | 09 | `route_length` measured L, ±0: pass at 14, unsure at 15, 16 and 20, fail at 21. For pass and unsure the spot is [L, L + W] and the sweep at start L has that outcome. For fail the spot is null and the sweep at L fails with `route_length`. No photo request | The wall runs s = [−1, L + W], all ground before L is deck and [L, L + W] is concrete. Every start before L fails `ground_surface`, starts left of the meter also fail the working space, and L is the last start. So the route is L with no corner. 14 < 15, pass. 15 is not inside 15. 16 is past 15 but below 20. For 20: 20 + 0 is not < 20 and 20 − 0 is not > 20, so unsure. For 21: 21 − 0 > 20, fail. | -| g09-reach-20p2-pm03 | 09 | `route_length` unsure, measured 20.2, ±0.3; spot at L | Meter at ±0.3, wall exact. 20.2 − 0.3 = 19.9 is not > 20, and 20.5 is not < 20. | +| g09-reach-20p2-pm03 | 09 | `route_length` unsure, measured 20.05, ±0.15; spot at L − 0.3; the start at L fails | Meter at ±0.3, wall exact. The meter's error lengthens the run but can't shorten it while the meter stands on its wall's line (server README at 3baa338), so a start s routes over [s, s + 0.3]. The battery moves with the meter against the ground patches, so the first start that may stand on the concrete is 19.9: route 19.9 to 20.2, unsure. The start at 20.2 routes 20.2 to 20.5, above 20, so it fails. | | g09-reach-20p4-pm03 | 09 | spot null; the sweep at L fails with `route_length` | 20.4 − 0.3 = 20.1 > 20. | | g09-vertical-run | 09 | `route_length` unsure, measured 16, ±0; `route_path` pass; spot at [11, 11 + W] | Pad at 15 − 4 = 11. An `elec_box` at s = [4, 5], 0 to 3.5 ft tall, sits across the 1 ft cable run, and elec boxes detour. Over: 2 × (3.5 − 1) = 5. Routed 11 + 5 = 16 = 15 + 1, past the confident reach. Measuring along the wall alone (11) would pass. | | c5-no-coverage | C5, scene schema | decision manual_review; `missing_evidence` non-empty; starts in (−6, −3.9) and (1.3, 7.4) unsure, starts in (−3.8, 1.2) fail with `meter_working_space` | Clean wall s = [−6, 10] with ground and facing data but no `coverage`. "Absent means nothing is known to be observed" makes every start unsure except where the working space, which needs no observation, fails it. Absent ends default to unexplored, which rules out reject. | @@ -151,18 +151,20 @@ move. ### d-reach-drift-right: cable reach, `at_most` 20 with review line 15 -Wall s = [−1, 29], meter at 0. For a start a > 0 the route is a (straight wall, no corners) and its -error is 0.3 (meter) + 0.3 + 0.16(a + W) (wall at the far edge) = 0.6 + 0.16(a + W). +Wall s = [−1, 29], meter at 0. For a start a > 0 the route is a (straight wall, no corners). The +wall's error at the far edge, e_w = 0.3 + 0.16(a + W), moves it either way; the meter's 0.3 only +lengthens it, since the meter stands on its wall's line (server README, `route_length`, at +3baa338). So the route runs from a − e_w to a + 0.3 + e_w. | Line | Condition | Start | | --- | --- | --- | | pass / unsure | a + 0.6 + 0.16(a + W) < 15 | a < (14.4 − 0.16W) / 1.16 = 12.0575 | -| unsure / fail | a − 0.6 − 0.16(a + W) > 20 | a > (20.6 + 0.16W) / 0.84 = 25.0159 | +| unsure / fail | a − 0.3 − 0.16(a + W) > 20 | a > (20.3 + 0.16W) / 0.84 = 24.6587 | -Asserted starts: pass at 5 and 11.8575. Unsure at 12.2575, 18 and 24.8159. Fail at 25.2159 and +Asserted starts: pass at 5 and 11.8575. Unsure at 12.2575, 18 and 24.4587. Fail at 24.8587 and 25.5. Each boundary is bracketed 0.2 ft inside on both sides, which is more than one 2 in sweep step, so each point falls inside a run rather than between two. A cutoff that takes drift at the -near edge sits 0.16W / 0.84 = 0.492 ft lower, at 24.5238, and fails the start at 24.8159. +near edge sits 0.16W / 0.84 = 0.492 ft lower, at 24.1667, and fails the start at 24.4587. ### d-reach-drift-left: the same, left of the meter @@ -198,8 +200,8 @@ tell the detour and leaves every route past it unsure. - **Walked distance for an object with a span.** For a tapped object spanning [s0, s1], it could be s0, s1 or the nearest end. d-gas-drift uses a zero-length span to avoid the question. - **Whether the meter's error enters the route.** The README says drift is not added to the - meter. Whether its 0.3 base enters the route error is inferred from 2c9348f, where both - defaults gave a route at ±0.6. + meter. Until 3baa338 its 0.3 entered the route error both ways (inferred from 2c9348f, where + both defaults gave a route at ±0.6); from 3baa338 the README states it lengthens the run only. - **How errors combine.** Linear sum is assumed, as elsewhere in this suite. Root-sum-square would move every line. - **Whether drift enters checks with explicit inputs.** Facing gaps and the working space involve diff --git a/verification/e2e/cases/d-reach-drift-left.json b/verification/e2e/cases/d-reach-drift-left.json index 401e22f08..1f015c85b 100644 --- a/verification/e2e/cases/d-reach-drift-left.json +++ b/verification/e2e/cases/d-reach-drift-left.json @@ -1,8 +1,8 @@ { "generated_by": "e2e/cases/generate.py", "id": "d-reach-drift-left", - "source": "rules.yaml errors.drift_per_ft and route.* at origin/t3/server f2705dd; C5 margin rule", - "description": "Mirror of d-reach-drift-right. Left of the meter the route runs to the battery's right edge b = a + W, so the boundaries sit at a = -12.058 - W and a = -25.016 - W.", + "source": "rules.yaml errors.drift_per_ft and route.* at origin/t3/server f2705dd; route range at 3baa338; C5 margin rule", + "description": "Mirror of d-reach-drift-right. Left of the meter the route runs to the battery's right edge b = a + W, so the boundaries sit at a = -12.058 - W and a = -24.659 - W.", "scene": { "schema_version": "1.0", "meter": { @@ -133,25 +133,25 @@ "wall_id": "w1", "start_ft": -20.5833, "outcome": "unsure", - "why": "18 + 3.89 > 15; 18 - 3.89 < 20 (b = a + W)" + "why": "18 + 3.89 > 15; 18 - 3.59 < 20 (b = a + W)" }, { "wall_id": "w1", - "start_ft": -27.3992, + "start_ft": -27.0421, "outcome": "unsure", - "why": "just inside a - 0.6 - 0.16(a + W) <= 20 (b = a + W)" + "why": "just inside a - 0.3 - 0.16(a + W) <= 20 (b = a + W)" }, { "wall_id": "w1", - "start_ft": -27.7992, + "start_ft": -27.4421, "outcome": "fail", - "why": "just past a - 0.6 - 0.16(a + W) > 20 (b = a + W)" + "why": "just past a - 0.3 - 0.16(a + W) > 20 (b = a + W)" }, { "wall_id": "w1", "start_ft": -28.0833, "outcome": "fail", - "why": "25.5 - 5.09 = 20.41 > 20 (b = a + W)" + "why": "25.5 - 4.79 = 20.71 > 20 (b = a + W)" } ] } diff --git a/verification/e2e/cases/d-reach-drift-right.json b/verification/e2e/cases/d-reach-drift-right.json index 06bd1abb8..826109a2e 100644 --- a/verification/e2e/cases/d-reach-drift-right.json +++ b/verification/e2e/cases/d-reach-drift-right.json @@ -1,8 +1,8 @@ { "generated_by": "e2e/cases/generate.py", "id": "d-reach-drift-right", - "source": "rules.yaml errors.drift_per_ft and route.* at origin/t3/server f2705dd; C5 margin rule", - "description": "Straight wall with default (drifting) errors. The route to a start a right of the meter is a +/- (0.6 + 0.16(a + W)): pass below a = 12.058, fail above a = 25.016, unsure between.", + "source": "rules.yaml errors.drift_per_ft and route.* at origin/t3/server f2705dd; route range at 3baa338; C5 margin rule", + "description": "Straight wall with default (drifting) errors. The route to a start a right of the meter is a - (0.3 + 0.16(a + W)) to a + 0.6 + 0.16(a + W): pass below a = 12.058, fail above a = 24.659, unsure between.", "scene": { "schema_version": "1.0", "meter": { @@ -133,25 +133,25 @@ "wall_id": "w1", "start_ft": 18.0, "outcome": "unsure", - "why": "18 + 3.89 > 15; 18 - 3.89 < 20" + "why": "18 + 3.89 > 15; 18 - 3.59 < 20" }, { "wall_id": "w1", - "start_ft": 24.8159, + "start_ft": 24.4587, "outcome": "unsure", - "why": "just inside a - 0.6 - 0.16(a + W) <= 20" + "why": "just inside a - 0.3 - 0.16(a + W) <= 20" }, { "wall_id": "w1", - "start_ft": 25.2159, + "start_ft": 24.8587, "outcome": "fail", - "why": "just past a - 0.6 - 0.16(a + W) > 20" + "why": "just past a - 0.3 - 0.16(a + W) > 20" }, { "wall_id": "w1", "start_ft": 25.5, "outcome": "fail", - "why": "25.5 - 5.09 = 20.41 > 20" + "why": "25.5 - 4.79 = 20.71 > 20" } ] } diff --git a/verification/e2e/cases/g09-reach-20p2-pm03.json b/verification/e2e/cases/g09-reach-20p2-pm03.json index 60625b5f5..b47ac4c51 100644 --- a/verification/e2e/cases/g09-reach-20p2-pm03.json +++ b/verification/e2e/cases/g09-reach-20p2-pm03.json @@ -2,7 +2,7 @@ "generated_by": "e2e/cases/generate.py", "id": "g09-reach-20p2-pm03", "source": "t3-lane-c-review.md golden 09 at origin/t3/research a7d91f1", - "description": "Route 20.2 +/- 0.3 ft (meter tap error): 20.2 - 0.3 = 19.9 is not above 20, so unsure.", + "description": "Meter at +/- 0.3 ft. The spot is the first start that may stand on the concrete, L - 0.3 = 19.9, whose route is 19.9 to 20.2: not above 20, so unsure. The start at 20.2 routes 20.2 to 20.5, above 20, so it fails.", "scene": { "schema_version": "1.0", "meter": { @@ -138,16 +138,16 @@ "spot": { "wall_id": "w1", "span_within": [ - 20.2, - 22.78333333333333 + 19.9, + 22.48333333333333 ] }, "checks": [ { "match": "route_length", "outcome": "unsure", - "measured_ft": 20.2, - "plus_minus_ft": 0.3 + "measured_ft": 20.05, + "plus_minus_ft": 0.15 } ], "sweep_runs": [ @@ -157,7 +157,7 @@ 20.199, 20.201 ], - "outcome": "unsure" + "outcome": "fail" } ], "decision_not": [ diff --git a/verification/e2e/cases/generate.py b/verification/e2e/cases/generate.py index 57ebdba8d..e30079fec 100644 --- a/verification/e2e/cases/generate.py +++ b/verification/e2e/cases/generate.py @@ -752,7 +752,25 @@ def reach_scene(L, meter_pm=0.0, objects=()): return scene -def reach_case(cid, L, route_ft, e, outcome, description, objects=(), extra_checks=()): +def reach_case( + cid, + L, + route_ft, + e, + outcome, + description, + objects=(), + extra_checks=(), + meter_pm=None, + spot_start=None, + at_L_outcome=None, +): + """`e` is the route check's expected error; the meter's error is `e` unless `meter_pm` says + otherwise. The spot starts at L unless `spot_start` says otherwise, and the start at L has + the spot's outcome unless `at_L_outcome` says otherwise.""" + meter_pm = e if meter_pm is None else meter_pm + spot_start = L if spot_start is None else spot_start + at_L_outcome = outcome if at_L_outcome is None else at_L_outcome at_L = [round(L - 0.001, 3), round(L + 0.001, 3)] check = { "match": "route_length", @@ -767,16 +785,16 @@ def reach_case(cid, L, route_ft, e, outcome, description, objects=(), extra_chec {"wall_id": "w1", "start_ft": at_L, "outcome": "fail", "failing_match": "route_length"} ] else: - expect["spot"] = {"wall_id": "w1", "span_within": [L, L + W]} + expect["spot"] = {"wall_id": "w1", "span_within": [spot_start, spot_start + W]} expect["checks"] = [check, *extra_checks] - expect["sweep_runs"] = [{"wall_id": "w1", "start_ft": at_L, "outcome": outcome}] + expect["sweep_runs"] = [{"wall_id": "w1", "start_ft": at_L, "outcome": at_L_outcome}] if outcome != "pass": expect["decision_not"] = ["pass"] case( cid, golden("09"), description, - reach_scene(L, e, objects), + reach_scene(L, meter_pm, objects), expect, {"route_length": MAX_ROUTE}, ) @@ -823,13 +841,22 @@ def reach_case(cid, L, route_ft, e, outcome, description, objects=(), extra_chec "fail", "Route 21 ft, 1 ft past the 20 ft maximum: the only start on usable ground fails.", ) +# The meter's error can lengthen the run, since the meter may stand off the wall, but not +# shorten it while it stands on its wall's line (server README, route_length, at 3baa338): a +# start s routes over [s, s + 0.3]. The battery moves with the meter against the ground patches, +# so starts from L - 0.3 may stand on the concrete and are unsure; the nearest is the spot. reach_case( "g09-reach-20p2-pm03", 20.2, - 20.2, - 0.3, + 20.2 - 0.3 / 2, + 0.3 / 2, "unsure", - "Route 20.2 +/- 0.3 ft (meter tap error): 20.2 - 0.3 = 19.9 is not above 20, so unsure.", + "Meter at +/- 0.3 ft. The spot is the first start that may stand on the concrete, " + "L - 0.3 = 19.9, whose route is 19.9 to 20.2: not above 20, so unsure. The start at 20.2 " + "routes 20.2 to 20.5, above 20, so it fails.", + meter_pm=0.3, + spot_start=20.2 - 0.3, + at_L_outcome="fail", ) reach_case( "g09-reach-20p4-pm03", @@ -892,25 +919,29 @@ def starts(points): ] -# Cable reach, right of the meter. Route = a (near edge), error = meter 0.3 + wall at the far -# edge (0.3 + 0.16(a + W)) = 0.6 + 0.16(a + W). Pass: a + e < 15 (review line) -> -# a < (14.4 - 0.16W)/1.16. Fail: a - e > 20 -> a > (20.6 + 0.16W)/0.84. +# Cable reach, right of the meter. Route = a (near edge). The wall's error at the far edge, +# e_w = 0.3 + 0.16(a + W), moves it either way; the meter's 0.3 only lengthens it, since the +# meter stands on its wall's line (server README, route_length, at 3baa338). Pass: +# a + 0.3 + e_w < 15 (review line) -> a < (14.4 - 0.16W)/1.16. Fail: a - e_w > 20 -> +# a > (20.3 + 0.16W)/0.84. R_PASS = (CONFIDENT - METER_E - WALL_E - DRIFT * W) / (1 + DRIFT) -R_FAIL = (MAX_ROUTE + METER_E + WALL_E + DRIFT * W) / (1 - DRIFT) +R_FAIL = (MAX_ROUTE + WALL_E + DRIFT * W) / (1 - DRIFT) reach_points = [ (5.0, "pass", "route 5 + 1.81 = 6.81 < 15"), (R_PASS - PT, "pass", "just inside a + 0.6 + 0.16(a + W) < 15"), (R_PASS + PT, "unsure", "just past the 15 ft review line"), - (18.0, "unsure", "18 + 3.89 > 15; 18 - 3.89 < 20"), - (R_FAIL - PT, "unsure", "just inside a - 0.6 - 0.16(a + W) <= 20"), - (R_FAIL + PT, "fail", "just past a - 0.6 - 0.16(a + W) > 20"), - (25.5, "fail", "25.5 - 5.09 = 20.41 > 20"), + (18.0, "unsure", "18 + 3.89 > 15; 18 - 3.59 < 20"), + (R_FAIL - PT, "unsure", "just inside a - 0.3 - 0.16(a + W) <= 20"), + (R_FAIL + PT, "fail", "just past a - 0.3 - 0.16(a + W) > 20"), + (25.5, "fail", "25.5 - 4.79 = 20.71 > 20"), ] case( "d-reach-drift-right", - "rules.yaml errors.drift_per_ft and route.* at origin/t3/server f2705dd; C5 margin rule", + "rules.yaml errors.drift_per_ft and route.* at origin/t3/server f2705dd; route range at " + "3baa338; C5 margin rule", "Straight wall with default (drifting) errors. The route to a start a right of the meter is " - "a +/- (0.6 + 0.16(a + W)): pass below a = 12.058, fail above a = 25.016, unsure between.", + "a - (0.3 + 0.16(a + W)) to a + 0.6 + 0.16(a + W): pass below a = 12.058, fail above " + "a = 24.659, unsure between.", drift_scene(-1, 29), {"start_outcomes": starts(reach_points)}, {"route_length": MAX_ROUTE}, @@ -919,9 +950,10 @@ def starts(points): # the far edge is a. case( "d-reach-drift-left", - "rules.yaml errors.drift_per_ft and route.* at origin/t3/server f2705dd; C5 margin rule", + "rules.yaml errors.drift_per_ft and route.* at origin/t3/server f2705dd; route range at " + "3baa338; C5 margin rule", "Mirror of d-reach-drift-right. Left of the meter the route runs to the battery's right edge " - "b = a + W, so the boundaries sit at a = -12.058 - W and a = -25.016 - W.", + "b = a + W, so the boundaries sit at a = -12.058 - W and a = -24.659 - W.", drift_scene(-29, 1), { "start_outcomes": starts([(-s - W, o, why + " (b = a + W)") for s, o, why in reach_points]), diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 5c71a1246..0783cc98e 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -254,6 +254,17 @@ def margin_problem(check: dict) -> str | None: other_cause = outcome == "unsure" and cause not in (None, "margin") if outcome == expected or (other_cause and not clear_fail): return None + # The EPS above allows for float noise; a PASS may also clear its line by less. The server + # rounds each number to 6 decimals and rule lines sit on that grid, and rounding keeps order: + # a true margin of zero or less can't come out positive. So any positive reported margin + # proves the pass (3baa338 places spots at the edge of a pass, a margin of 1e-6). + if outcome == "pass" and expected == "unsure": + if cmp == "at_least": + proven = all(m - e - x > SERVER_EPS for x in lines) + else: + proven = all(x - (m + e) > SERVER_EPS for x in lines) + if proven: + return None because = f" ({cause})" if cause else "" return ( f"check {check['id']}: measured {m} ± {e}{band} should be {expected}, " diff --git a/verification/scoreboard/metrics.yaml b/verification/scoreboard/metrics.yaml index 6a2c4673c..7dce509de 100644 --- a/verification/scoreboard/metrics.yaml +++ b/verification/scoreboard/metrics.yaml @@ -109,7 +109,8 @@ metrics: missing coverage: 'coverage|unobserved|unseen' equality unsure: 'equal' mirror: 'mirror|symmetr' - larger clearance: 'monoton|larger_clearance|clearance' + # S2 names it for moving an obstacle away (test_more_space.py, since 22292a8). + larger clearance: 'monoton|larger_clearance|clearance|obstacle_away' - id: S2-4 workstream: S2 diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index 3bdf982bf..e01d7b358 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-09-27T04:31:04-05:00", + "generated_at": "2026-09-27T07:58:56-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": [ "S2-1", @@ -7,66 +7,30 @@ ], "prs": [ { - "number": 152, - "title": "ios: preview the walk's moves in onboarding", - "branch": "t3/85-onboarding-moves", - "head": "e085372d9007eaeaee3e0625cbd0ab9c448d3b0f", - "draft": true, - "updated": "2026-09-27T09:24:20Z", - "ci": { - "counts": { - "success": 2, - "failure": 0, - "pending": 0, - "skipped": 2 - }, - "failing": [], - "text": "2 success, 2 skipped" - } - }, - { - "number": 151, - "title": "fix(ios): refuse object taps that meet the wall under the floor or far along it", - "branch": "t3/140-object-tap-reach", - "head": "fd50357db9451b4f771d3ce9feaa2fb00d26bee2", + "number": 169, + "title": "ios: bring the t3/ios-mvf UI test flake fixes to beta", + "branch": "t3/ci-flake-fixes-beta", + "head": "9fc6019504c0da23fa6edae02df993f5caa00298", "draft": false, - "updated": "2026-09-27T09:26:07Z", + "updated": "2026-09-27T12:40:26Z", "ci": { "counts": { - "success": 4, - "failure": 0, - "pending": 1, - "skipped": 2 - }, - "failing": [], - "text": "4 success, 1 pending, 2 skipped" - } - }, - { - "number": 149, - "title": "Recheck camera access on return, keep rejected meter close-ups unlinked, honour Reduce Motion on Not ", - "branch": "t3/126-closeup-reduce-motion-recovery", - "head": "15cbb77b84078f0505d82786a0f37a843565f7d8", - "draft": false, - "updated": "2026-09-27T09:22:54Z", - "ci": { - "counts": { - "success": 4, + "success": 3, "failure": 0, "pending": 1, - "skipped": 2 + "skipped": 1 }, "failing": [], - "text": "4 success, 1 pending, 2 skipped" + "text": "3 success, 1 pending, 1 skipped" } }, { - "number": 147, - "title": "fix(ios): bound an end at the phone by the evidence; settle tracking; say 'you saw'", - "branch": "t3/41-ends-corner-2", - "head": "86da28da0615101497e57eb817630a4bb8dabc9a", + "number": 168, + "title": "ios: treat a corridor's far wall or a side yard's fence as where the space ends", + "branch": "t3/164-space-ends", + "head": "6b21236f0ec21b1262212807dd9713145ed315b2", "draft": true, - "updated": "2026-09-27T09:05:01Z", + "updated": "2026-09-27T12:22:46Z", "ci": { "counts": { "success": 2, @@ -79,66 +43,68 @@ } }, { - "number": 146, - "title": "Draw unclean AR spots as outlines and face the closest spot in the result model", - "branch": "t3/110-result-details-followups", - "head": "6e8403fd185cde9a5aa336f869c0ecfe6d62595b", + "number": 165, + "title": "ios tests: audit the walk once the autopilot holds it; triage of the recurring UI test failures", + "branch": "t3/ci-flake-fixes", + "head": "ee8cdcc1fd21bc39d16fd02ea299d57957fc9e1d", "draft": false, - "updated": "2026-09-27T09:26:24Z", + "updated": "2026-09-27T12:40:18Z", "ci": { "counts": { - "success": 4, + "success": 3, "failure": 0, "pending": 1, - "skipped": 2 + "skipped": 1 }, "failing": [], - "text": "4 success, 1 pending, 2 skipped" + "text": "3 success, 1 pending, 1 skipped" } }, { - "number": 143, - "title": "ios: practice meter, a drawn sample meter for scanning without one", - "branch": "t3/practice-meter", - "head": "10b04e89819c4e450a722f9c41511bbd7a84f6aa", + "number": 158, + "title": "Draw the prototype's fog and live dots over the walk", + "branch": "t3/live-fog", + "head": "f57310cac5577fef61baadca86729c0b4b43b080", "draft": false, - "updated": "2026-09-27T09:18:54Z", + "updated": "2026-09-27T11:07:27Z", "ci": { "counts": { - "success": 3, - "failure": 0, + "success": 5, + "failure": 1, "pending": 0, "skipped": 1 }, - "failing": [], - "text": "3 success, 1 skipped" + "failing": [ + "iOS build" + ], + "text": "5 success, 1 failure, 1 skipped; failing: iOS build" } }, { - "number": 135, - "title": "ios: confirm before a repeated Can't get there ends the scan; no spot for a wall never walked", - "branch": "t3/41-walk-exit", - "head": "88912e50a9ece658e74fc148ba2ba2f1b6df0f75", - "draft": true, - "updated": "2026-09-27T09:24:19Z", + "number": 143, + "title": "ios: practice meter, a drawn sample meter for scanning without one", + "branch": "t3/practice-meter", + "head": "aac574f8c3ab6094a9d4947eb61e5c4ac5022f22", + "draft": false, + "updated": "2026-09-27T11:31:45Z", "ci": { "counts": { - "success": 2, + "success": 3, "failure": 0, - "pending": 0, - "skipped": 2 + "pending": 1, + "skipped": 1 }, "failing": [], - "text": "2 success, 2 skipped" + "text": "3 success, 1 pending, 1 skipped" } }, { "number": 134, "title": "docs: rebuild the README around the demo, with quick start, data and limits", "branch": "shrey/readme", - "head": "8a78c822d6fb7eca4e91f4b110bbc81ae9f78b81", + "head": "b0c5829c1273158aabdc0ea2ec7c931ebead48f6", "draft": false, - "updated": "2026-09-27T09:04:37Z", + "updated": "2026-09-27T12:54:49Z", "ci": { "counts": { "success": 4, @@ -156,204 +122,196 @@ "branch": "t3/integration-viewer", "head": "9c0680b664183d9414bb21f612d536f9da02575d", "draft": false, - "updated": "2026-09-27T08:45:52Z", + "updated": "2026-09-27T10:47:39Z", "ci": { "counts": { - "success": 4, + "success": 5, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "4 success" + "text": "5 success" } }, { "number": 123, - "title": "ios: House Scan Integration records each scan as a capture packet and, with consent, uploads it while scanning", + "title": "ios: House Scan Integration records each scan as a capture packet, uploads it during the scan with per-scan consent, and shows the test server's answer", "branch": "t3/hunter-integration", - "head": "5cc51246ff69b1560d1638be3521daec8278f5ab", + "head": "17b53baa21d6d7e4f513a65e8e91d3bc3f580ac8", "draft": true, - "updated": "2026-09-27T08:56:11Z", + "updated": "2026-09-27T12:17:47Z", "ci": { "counts": { "success": 2, "failure": 0, "pending": 0, - "skipped": 2 + "skipped": 1 }, "failing": [], - "text": "2 success, 2 skipped" + "text": "2 success, 1 skipped" } }, { "number": 122, "title": "server: stop requests at the wall's ends, keep placeholder distances out of them, name things without ids", "branch": "t3/41-server-requests-text", - "head": "c8d3c9d5d4575e8e52ef86052fdb9c48987fbf94", + "head": "ee882e4884e9ee5a818549ecd7e707da8485229e", "draft": false, - "updated": "2026-09-27T07:58:22Z", + "updated": "2026-09-27T12:40:57Z", "ci": { "counts": { - "success": 5, + "success": 4, "failure": 0, "pending": 0, "skipped": 1 }, "failing": [], - "text": "5 success, 1 skipped" + "text": "4 success, 1 skipped" } }, { "number": 120, "title": "ios: hold a walk aim until it is met or stalls, and ask for the meter ground only near the meter", "branch": "t3/41-walk-task-hold", - "head": "8487a02f811b12fd5e8eda3bd9f79e5b84089f1e", + "head": "9d4d12e55bb0b762d1a1fd57f41ea95dbd25a345", "draft": false, - "updated": "2026-09-27T08:07:51Z", + "updated": "2026-09-27T11:49:01Z", "ci": { "counts": { - "success": 7, + "success": 3, "failure": 0, - "pending": 0, + "pending": 1, "skipped": 1 }, "failing": [], - "text": "7 success, 1 skipped" + "text": "3 success, 1 pending, 1 skipped" } }, { "number": 119, "title": "ios: log meter-anchor corrections so marks can be checked against drift", "branch": "t3/41-anchored-marks", - "head": "f141a6212f3bba8c26695bf3cb5cb40ef0f1cf9d", + "head": "f03430fef6159fdfd230e9df248a3cdfc0ed58e4", "draft": false, - "updated": "2026-09-27T08:05:25Z", + "updated": "2026-09-27T12:49:53Z", "ci": { "counts": { - "success": 6, - "failure": 2, - "pending": 0, - "skipped": 2 + "success": 3, + "failure": 0, + "pending": 1, + "skipped": 1 }, - "failing": [ - "iOS build", - "iOS build" - ], - "text": "6 success, 2 failure, 2 skipped; failing: iOS build, iOS build" + "failing": [], + "text": "3 success, 1 pending, 1 skipped" } }, { "number": 109, "title": "ios: gap-loop follow-ups, upload ticks, the nearer unseen end and request stretches (#39, #31, #83, #75)", "branch": "t3/41-gap-loop-result", - "head": "a44780a21d021b65515c61f93b6b84c1a3de97be", + "head": "f2309cb37c7b7d23de8f46f5d89229f0452a8605", "draft": false, - "updated": "2026-09-27T08:05:22Z", + "updated": "2026-09-27T11:48:59Z", "ci": { "counts": { - "success": 6, + "success": 3, "failure": 0, - "pending": 0, - "skipped": 2 + "pending": 1, + "skipped": 1 }, "failing": [], - "text": "6 success, 2 skipped" + "text": "3 success, 1 pending, 1 skipped" } }, { "number": 108, "title": "ios: ground needs a plausible meter height and never rises; estimated meter taps are checked and re-fitted to the detected wall", "branch": "t3/41-ground-and-meter-tap", - "head": "9c2a5676fff6b31b1eb1e4bde63a5473ae61b5f3", + "head": "92c8f92f821dba2167045ba9810853a724a787fa", "draft": false, - "updated": "2026-09-27T08:05:00Z", + "updated": "2026-09-27T12:49:02Z", "ci": { "counts": { - "success": 4, - "failure": 1, - "pending": 0, - "skipped": 2 + "success": 3, + "failure": 0, + "pending": 1, + "skipped": 1 }, - "failing": [ - "iOS build" - ], - "text": "4 success, 1 failure, 2 skipped; failing: iOS build" + "failing": [], + "text": "3 success, 1 pending, 1 skipped" } }, { "number": 104, "title": "ios: gate coaching keeps the task card, holds through dark flicker, and says \"Turn more slowly\" for turns", "branch": "t3/41-gate-coaching", - "head": "633c26d751fc24ecf8968756a29139fa6b969126", + "head": "040d9a61cd8c6fd43784c3d7575c00fd991cf182", "draft": false, - "updated": "2026-09-27T08:06:36Z", + "updated": "2026-09-27T12:13:22Z", "ci": { "counts": { - "success": 6, - "failure": 2, + "success": 4, + "failure": 0, "pending": 0, - "skipped": 2 + "skipped": 1 }, - "failing": [ - "iOS build", - "iOS build" - ], - "text": "6 success, 2 failure, 2 skipped; failing: iOS build, iOS build" + "failing": [], + "text": "4 success, 1 skipped" } }, { "number": 102, "title": "ios: card replies name the task, stay under gate coaching, and lock after a new card", "branch": "t3/41-card-replies", - "head": "98d96caa41b24dc3140ca5a5fbaec4a0c610205f", + "head": "fd8f9b30e491c8175fd2390310e30ec81bf5c85d", "draft": false, - "updated": "2026-09-27T08:05:16Z", + "updated": "2026-09-27T12:20:00Z", "ci": { "counts": { - "success": 6, + "success": 3, "failure": 0, - "pending": 0, - "skipped": 2 + "pending": 1, + "skipped": 1 }, "failing": [], - "text": "6 success, 2 skipped" + "text": "3 success, 1 pending, 1 skipped" } }, { "number": 100, "title": "ios: draw the AR result with the Canvas until RealityKit is seen drawing it", "branch": "t3/41-ar-canvas-fallback", - "head": "41291e8287bd7a2452c6b53b36120670c3210317", + "head": "0ada11b9dcb67452e311ee4e4e8e9ffb50807bbb", "draft": false, - "updated": "2026-09-27T08:04:58Z", + "updated": "2026-09-27T12:42:15Z", "ci": { "counts": { - "success": 5, + "success": 4, "failure": 0, - "pending": 0, - "skipped": 1 + "pending": 1, + "skipped": 0 }, "failing": [], - "text": "5 success, 1 skipped" + "text": "4 success, 1 pending" } }, { "number": 99, "title": "ios: \"leaves out N ft you walked\" only on walk tasks, and on the end question", "branch": "t3/41-end-preview", - "head": "b5006fca782aa280caa767e476fd5c62b496a9e2", + "head": "f5a27c6428ba7fae05a06c2807d603bc24d9e7c1", "draft": false, - "updated": "2026-09-27T07:55:03Z", + "updated": "2026-09-27T12:08:03Z", "ci": { "counts": { - "success": 6, + "success": 4, "failure": 0, - "pending": 0, - "skipped": 1 + "pending": 1, + "skipped": 0 }, "failing": [], - "text": "6 success, 1 skipped" + "text": "4 success, 1 pending" } }, { @@ -362,16 +320,16 @@ "branch": "t3/check-experiments", "head": "9d66b31c40c940c5bab7baea0aef5c5a87be3c10", "draft": false, - "updated": "2026-09-27T04:39:38Z", + "updated": "2026-09-27T10:47:33Z", "ci": { "counts": { - "success": 5, + "success": 6, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "5 success" + "text": "6 success" } }, { @@ -380,34 +338,34 @@ "branch": "t3/autodetect-lab", "head": "538f60e1e9796b26ab80bec50004bcc2eb04d38e", "draft": true, - "updated": "2026-09-27T04:00:03Z", + "updated": "2026-09-27T10:49:21Z", "ci": { "counts": { - "success": 2, + "success": 3, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "2 success" + "text": "3 success" } }, { "number": 87, "title": "ios: deflate the text entries in the shared scan zip", "branch": "t3/41-zip-deflate", - "head": "5150321a22f540e669755270dd76caa23957d8a4", + "head": "8d2a6a13abf1f6abb4d328a0b5c74620b0ee295f", "draft": false, - "updated": "2026-09-27T07:55:06Z", + "updated": "2026-09-27T11:15:29Z", "ci": { "counts": { - "success": 6, + "success": 5, "failure": 0, "pending": 0, - "skipped": 1 + "skipped": 0 }, "failing": [], - "text": "6 success, 1 skipped" + "text": "5 success" } }, { @@ -416,16 +374,16 @@ "branch": "t3/experience", "head": "b442f0a0436daf36f1f929d578a63791e1186637", "draft": true, - "updated": "2026-09-27T05:00:35Z", + "updated": "2026-09-27T10:48:07Z", "ci": { "counts": { - "success": 2, + "success": 3, "failure": 0, "pending": 0, "skipped": 1 }, "failing": [], - "text": "2 success, 1 skipped" + "text": "3 success, 1 skipped" } }, { @@ -434,16 +392,16 @@ "branch": "t3/app-review-docs", "head": "c0b628b6b9ec2b314ae418db956a27bc579ee642", "draft": false, - "updated": "2026-09-27T06:56:32Z", + "updated": "2026-09-27T10:48:27Z", "ci": { "counts": { - "success": 4, + "success": 5, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "4 success" + "text": "5 success" } }, { @@ -452,16 +410,16 @@ "branch": "t3/device-test-4.1-analysis", "head": "c566490c5e3fa0d45803895bb463cb260d11a906", "draft": false, - "updated": "2026-09-27T07:20:15Z", + "updated": "2026-09-27T10:55:55Z", "ci": { "counts": { - "success": 2, + "success": 3, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "2 success" + "text": "3 success" } }, { @@ -470,94 +428,124 @@ "branch": "t3/signals-lab", "head": "0cde05e30a7d07b01d8a458a0eeefbf2695ab2da", "draft": true, - "updated": "2026-09-27T02:38:16Z", + "updated": "2026-09-27T10:50:43Z", "ci": { "counts": { - "success": 2, + "success": 3, "failure": 0, "pending": 0, "skipped": 0 }, "failing": [], - "text": "2 success" + "text": "3 success" } }, { "number": 58, "title": "ci(ios): drop the background Simulator boot and the duplicate t3/ios-mvf push run", "branch": "t3/ios-ci-no-preboot", - "head": "3c1298ab46869730418c5cafc6b4d60eafe0e6fb", + "head": "7f1037e0cb2f32be1c52f0394a8ca2e47337d00b", "draft": false, - "updated": "2026-09-27T03:18:03Z", + "updated": "2026-09-27T12:02:46Z", "ci": { "counts": { - "success": 2, - "failure": 1, + "success": 4, + "failure": 0, "pending": 0, - "skipped": 0 + "skipped": 1 }, - "failing": [ - "iOS build" - ], - "text": "2 success, 1 failure; failing: iOS build" + "failing": [], + "text": "4 success, 1 skipped" } }, { "number": 56, - "title": "Tests for #24, #28 and #29: replay field runs 1 and 2 against the walk guidance", + "title": "Tests for #24, #28 and #29: synthetic walk-guidance cases modeled on field runs 1 and 2", "branch": "t3/walk-guidance-gaps", - "head": "cae4f426f1f5e5a0f4ba4dde97347d04df724d40", + "head": "a9e23c612326fe9b25a85eae28bdac450ace18b8", "draft": false, - "updated": "2026-09-27T04:57:46Z", + "updated": "2026-09-27T11:11:28Z", "ci": { "counts": { "success": 3, "failure": 0, - "pending": 0, - "skipped": 0 + "pending": 1, + "skipped": 1 }, "failing": [], - "text": "3 success" + "text": "3 success, 1 pending, 1 skipped" } }, { "number": 55, "title": "Ask for Motion & Fitness on the onboarding, where it is explained (#43)", "branch": "t3/motion-permission", - "head": "9607a481359897e405fc4e2acfe7149c4599dd8d", + "head": "b289c7dfa90c8760e0c34b96e5383a8c27a2bb8e", "draft": false, - "updated": "2026-09-27T04:57:34Z", + "updated": "2026-09-27T12:13:49Z", "ci": { "counts": { - "success": 2, - "failure": 1, - "pending": 0, - "skipped": 0 + "success": 3, + "failure": 0, + "pending": 1, + "skipped": 1 }, - "failing": [ - "iOS build" - ], - "text": "2 success, 1 failure; failing: iOS build" + "failing": [], + "text": "3 success, 1 pending, 1 skipped" } }, { "number": 52, "title": "Mark an existing battery or another box on the wall (#27)", "branch": "t3/battery-mark", - "head": "68a66a827c6fd6a56e18f0ffea25387f0cf23396", + "head": "086ad94b7fdfccef09e9ddadffac908b63f974f4", "draft": false, - "updated": "2026-09-27T04:57:26Z", + "updated": "2026-09-27T12:23:52Z", "ci": { "counts": { - "success": 2, - "failure": 1, + "success": 3, + "failure": 0, + "pending": 1, + "skipped": 1 + }, + "failing": [], + "text": "3 success, 1 pending, 1 skipped" + } + }, + { + "number": 49, + "title": "ios: 'I can't get there' moves on to the next requested view (#39)", + "branch": "t3/gap-skip-continues", + "head": "e57619015a11c995f3581f145ac6b544450c1eb7", + "draft": false, + "updated": "2026-09-27T12:22:12Z", + "ci": { + "counts": { + "success": 4, + "failure": 0, + "pending": 0, + "skipped": 1 + }, + "failing": [], + "text": "4 success, 1 skipped" + } + }, + { + "number": 46, + "title": "experiments: field test run 2 on build 3.1", + "branch": "t3/field-test-run2", + "head": "afd02c820bd025b1c077d8e238d3f3421e5d9f3a", + "draft": false, + "updated": "2026-09-27T10:48:15Z", + "ci": { + "counts": { + "success": 3, + "failure": 0, "pending": 0, "skipped": 0 }, - "failing": [ - "iOS build" - ], - "text": "2 success, 1 failure; failing: iOS build" + "failing": [], + "text": "3 success" } } ], @@ -569,7 +557,7 @@ "refs": [ { "ref": "origin/t3/evals", - "sha": "17e338d9b3e048dddeec51627b7616852db83f19" + "sha": "4f52036f78c7ab45350004fda241da3125d2bb5e" } ], "metrics": [ @@ -577,7 +565,7 @@ "id": "S1-1", "text": "One command regenerates every number from a clean checkout", "ref": "origin/t3/evals", - "sha": "17e338d9b3e048dddeec51627b7616852db83f19", + "sha": "4f52036f78c7ab45350004fda241da3125d2bb5e", "status": "partial", "evidence": "make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac.", "links": [] @@ -586,7 +574,7 @@ "id": "S1-2", "text": "Dataset URLs and sha256 recorded", "ref": "origin/t3/evals", - "sha": "17e338d9b3e048dddeec51627b7616852db83f19", + "sha": "4f52036f78c7ab45350004fda241da3125d2bb5e", "status": "gap", "evidence": "none found in 1 files; missing ADVIO, ETH3D", "links": [] @@ -595,7 +583,7 @@ "id": "S1-3", "text": "Metric code tested on hand-computed cases", "ref": "origin/t3/evals", - "sha": "17e338d9b3e048dddeec51627b7616852db83f19", + "sha": "4f52036f78c7ab45350004fda241da3125d2bb5e", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -604,21 +592,21 @@ "id": "S1-4", "text": "ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft", "ref": "origin/t3/evals", - "sha": "17e338d9b3e048dddeec51627b7616852db83f19", + "sha": "4f52036f78c7ab45350004fda241da3125d2bb5e", "status": "met", "evidence": "all 5 found", "links": [ [ "experiments/evals/METHODS.md", - "https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/METHODS.md" + "https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/METHODS.md" ], [ "experiments/evals/field/FIELD_SHEET.md", - "https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/field/FIELD_SHEET.md" + "https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/field/FIELD_SHEET.md" ], [ "experiments/evals/results/advio_drift.md", - "https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/results/advio_drift.md" + "https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/results/advio_drift.md" ] ] }, @@ -626,21 +614,21 @@ "id": "S1-5", "text": "ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question", "ref": "origin/t3/evals", - "sha": "17e338d9b3e048dddeec51627b7616852db83f19", + "sha": "4f52036f78c7ab45350004fda241da3125d2bb5e", "status": "partial", "evidence": "found 1-3 m row, 3-10 m row, scale error; missing verdict", "links": [ [ "experiments/evals/METHODS.md", - "https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/METHODS.md" + "https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/METHODS.md" ], [ "experiments/evals/results/eth3d_recon.md", - "https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/results/eth3d_recon.md" + "https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/results/eth3d_recon.md" ], [ "experiments/evals/results/eth3d_visibility_sensitivity.md", - "https://github.com/SamGu-NRX/house-scanning/blob/17e338d9b3e048dddeec51627b7616852db83f19/experiments/evals/results/eth3d_visibility_sensitivity.md" + "https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/results/eth3d_visibility_sensitivity.md" ] ] } @@ -652,7 +640,7 @@ "refs": [ { "ref": "origin/t3/server", - "sha": "a72619933ceb2966c7f1451fcba77f2d8c09ca50" + "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b" } ], "metrics": [ @@ -660,22 +648,22 @@ "id": "S2-1", "text": "pytest green", "ref": "origin/t3/server", - "sha": "a72619933ceb2966c7f1451fcba77f2d8c09ca50", + "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", "status": "met", - "evidence": "exit 0: return lib.intersection_prec(a, b, grid_size, **kwargs) / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 371 passed, 1 xfailed, 3 warnings in 132.95s (0:02:12)", + "evidence": "exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 380 passed, 1 xfailed, 1 warning in 80.06s (0:01:20)", "links": [] }, { "id": "S2-2", "text": "Every public golden test 01-14 of docs/research/t3-lane-c-review.md present", "ref": "origin/t3/server", - "sha": "a72619933ceb2966c7f1451fcba77f2d8c09ca50", + "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", "status": "met", "evidence": "all 14 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_golden_public.py" ] ] }, @@ -683,21 +671,21 @@ "id": "S2-3", "text": "Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail", "ref": "origin/t3/server", - "sha": "a72619933ceb2966c7f1451fcba77f2d8c09ca50", - "status": "partial", - "evidence": "found missing coverage, equality unsure, mirror; missing larger clearance", + "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", + "status": "met", + "evidence": "all 4 found", "links": [ [ "server/tests/test_coderabbit_review.py", - "https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_coderabbit_review.py" + "https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_coderabbit_review.py" ], [ "server/tests/test_properties.py", - "https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_properties.py" + "https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_properties.py" ], [ "server/tests/test_final_review.py", - "https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_final_review.py" + "https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_final_review.py" ] ] }, @@ -705,17 +693,17 @@ "id": "S2-4", "text": "Schemas published", "ref": "origin/t3/server", - "sha": "a72619933ceb2966c7f1451fcba77f2d8c09ca50", + "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", "status": "met", - "evidence": "present at a7261993", + "evidence": "present at 3baa3387", "links": [ [ "server/schemas/scene.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/schemas/scene.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/schemas/scene.schema.json" ], [ "server/schemas/result.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/schemas/result.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/schemas/result.schema.json" ] ] }, @@ -723,21 +711,21 @@ "id": "S2-5", "text": "Schemas validated in tests", "ref": "origin/t3/server", - "sha": "a72619933ceb2966c7f1451fcba77f2d8c09ca50", + "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", "status": "met", "evidence": "all 2 found", "links": [ [ "server/tests/test_existing_battery.py", - "https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_existing_battery.py" + "https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_existing_battery.py" ], [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_golden_public.py" ], [ "server/tests/test_schemas.py", - "https://github.com/SamGu-NRX/house-scanning/blob/a72619933ceb2966c7f1451fcba77f2d8c09ca50/server/tests/test_schemas.py" + "https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_schemas.py" ] ] }, @@ -745,27 +733,27 @@ "id": "S2-6", "text": "A real-derived scene returns a result in under 1 s", "ref": "origin/t3/server", - "sha": "a72619933ceb2966c7f1451fcba77f2d8c09ca50", + "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", "status": "met", - "evidence": "latency_ms = 776.7 (want < 1000) at sha a7261993: ~/house-scanning-data/reports/e2e/20260927-042649-a7261993/report.json", + "evidence": "latency_ms = 616.8 (want < 1000) at sha 3baa3387: ~/house-scanning-data/reports/e2e/20260927-075636-3baa3387/report.json", "links": [] }, { "id": "S2-7", "text": "ruff clean", "ref": "origin/t3/server", - "sha": "a72619933ceb2966c7f1451fcba77f2d8c09ca50", + "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", "status": "met", - "evidence": "exit 0: Installed 33 packages in 71ms / All checks passed! / 35 files already formatted", + "evidence": "exit 0: Installed 33 packages in 37ms / All checks passed! / 36 files already formatted", "links": [] }, { "id": "S2-8", "text": "Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time", "ref": "origin/t3/server", - "sha": "a72619933ceb2966c7f1451fcba77f2d8c09ca50", + "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", "status": "met", - "evidence": "contract_ok = true (want True) at sha a7261993: ~/house-scanning-data/reports/e2e/20260927-042649-a7261993/report.json", + "evidence": "contract_ok = true (want True) at sha 3baa3387: ~/house-scanning-data/reports/e2e/20260927-075636-3baa3387/report.json", "links": [] } ] @@ -776,7 +764,7 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6" + "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb" } ], "metrics": [ @@ -784,7 +772,7 @@ "id": "S3-1", "text": "CI build green", "ref": "origin/t3/ios-mvf", - "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", + "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", "status": "no evidence yet", "evidence": "no open PR for t3/ios-mvf", "links": [] @@ -793,13 +781,13 @@ "id": "S3-2", "text": "Built as Swift 6 with strict concurrency", "ref": "origin/t3/ios-mvf", - "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", + "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", "status": "met", "evidence": "all 2 found", "links": [ [ "ios/project.yml", - "https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/project.yml" + "https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/project.yml" ] ] }, @@ -807,30 +795,30 @@ "id": "S3-3", "text": "Zero build warnings", "ref": "origin/t3/ios-mvf", - "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", + "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", "status": "stale", - "evidence": "report for sha f14947eb, ref at b90fb8e4: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", + "evidence": "report for sha f14947eb, ref at 6be087df: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", "links": [] }, { "id": "S3-4", "text": "Unit tests for coverage, auto-capture and guidance", "ref": "origin/t3/ios-mvf", - "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", + "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", "status": "partial", "evidence": "found coverage, guidance; missing auto-capture", "links": [ [ "ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift" ] ] }, @@ -838,21 +826,21 @@ "id": "S3-5", "text": "UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop", "ref": "origin/t3/ios-mvf", - "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", + "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", "status": "partial", "evidence": "all 3 found (the tests exist; passing is S3-1)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/FullFlowUITests.swift" ], [ "ios/HouseScanUITests/SourceRestartUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/SourceRestartUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/SourceRestartUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -860,17 +848,17 @@ "id": "S3-6", "text": "Accessibility audit on every screen", "ref": "origin/t3/ios-mvf", - "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", + "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", "status": "partial", "evidence": "3 matches in 2 files (an audit exists; which screens it covers needs a read)", "links": [ [ "ios/HouseScanUITests/AccessibilityAudit.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/AccessibilityAudit.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/AccessibilityAudit.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/b90fb8e401860d21448495b3d5cb3a2eb0a98ef6/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -878,16 +866,16 @@ "id": "S3-6b", "text": "Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues", "ref": "origin/t3/ios-mvf", - "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", + "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", "status": "stale", - "evidence": "report for sha a39d0a50, ref at b90fb8e4: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", + "evidence": "report for sha a39d0a50, ref at 6be087df: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", "links": [] }, { "id": "S3-7", "text": "Screenshots of each state in the PR", "ref": "origin/t3/ios-mvf", - "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", + "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", "status": "no evidence yet", "evidence": "no open PR for t3/ios-mvf", "links": [] @@ -896,16 +884,16 @@ "id": "S3-8", "text": "Exported scene.json validates against C1", "ref": "origin/t3/ios-mvf", - "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", + "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", "status": "stale", - "evidence": "report for app_sha f14947eb, ref at b90fb8e4: ~/house-scanning-data/reports/e2e/20260927-042649-a7261993/report.json", + "evidence": "report for app_sha f14947eb, ref at 6be087df: ~/house-scanning-data/reports/e2e/20260927-075636-3baa3387/report.json", "links": [] }, { "id": "S3-9", "text": "Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design", "ref": "origin/t3/ios-mvf", - "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", + "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -918,11 +906,11 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6" + "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb" }, { "ref": "origin/t3/verification", - "sha": "371c5123c97124ddfd1f78aa809fc3170abd1139" + "sha": "21c8832408e8484d64f4e54b4ca903ddb6c5af4a" } ], "metrics": [ @@ -930,25 +918,25 @@ "id": "S4-1", "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/ios-mvf", - "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", + "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", "status": "stale", - "evidence": "report for app_sha f14947eb, ref at b90fb8e4: ~/house-scanning-data/reports/e2e/20260927-042649-a7261993/report.json", + "evidence": "report for app_sha f14947eb, ref at 6be087df: ~/house-scanning-data/reports/e2e/20260927-075636-3baa3387/report.json", "links": [] }, { "id": "S4-2", "text": "Screenshot report per app state", "ref": "origin/t3/ios-mvf", - "sha": "b90fb8e401860d21448495b3d5cb3a2eb0a98ef6", + "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", "status": "stale", - "evidence": "report for sha f14947eb, ref at b90fb8e4: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", + "evidence": "report for sha f14947eb, ref at 6be087df: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", "links": [] }, { "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "371c5123c97124ddfd1f78aa809fc3170abd1139", + "sha": "21c8832408e8484d64f4e54b4ca903ddb6c5af4a", "status": "met", "evidence": "docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state.", "links": [] @@ -957,13 +945,13 @@ "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "371c5123c97124ddfd1f78aa809fc3170abd1139", + "sha": "21c8832408e8484d64f4e54b4ca903ddb6c5af4a", "status": "met", - "evidence": "present at 371c5123", + "evidence": "present at 21c88324", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/371c5123c97124ddfd1f78aa809fc3170abd1139/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/21c8832408e8484d64f4e54b4ca903ddb6c5af4a/verification/SCOREBOARD.md" ] ] } diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index de0509e9a..6c10260fa 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -1203,3 +1203,15 @@ def test_walls_meeting_in_a_straight_line_are_one_stretch_and_a_corner_is_not(): ] r["spot"]["span_ft"] = [5.0, 5.0 + 31 / 12] # starts exactly at the corner, on the next wall assert footprint_problems(scene, r, RULES) == [] + + +def test_a_pass_by_the_smallest_reported_margin_is_a_pass(): + # 3baa338 puts a spot at the edge of its working-space pass: 1.444446 +/- 1.444445 against + # 0. The server rounds to 6 decimals, which keeps order, so a reported margin above the + # error proves the true one is; equal reported numbers do not. + c = check("pass", 1.444446, 1.444445, 0.0) | {"id": "meter_working_space"} + assert margin_problem(c) is None + assert "should be unsure" in margin_problem(c | {"measured_ft": 1.444445}) + at_most = check("pass", 19.999999, 0.0, 20.0, cmp="at_most") + assert margin_problem(at_most) is None + assert "should be unsure" in margin_problem(at_most | {"measured_ft": 20.0}) From 379e5ad0a124689fdb508df595f9f8b47c4b7b64 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 08:59:30 -0500 Subject: [PATCH 57/59] verification: match server 3baa338's coverage, from the final Opus review of 255bc84 Findings 1 to 5, each with a test from its probe that fails at 255bc84: 1. The battery's error adds the larger of the meter's plan error and its slide along the walls to the wall's error (server _errors). Spans measured along the walls (facing, headroom, wall_equipment_above, now widened too) take the along-the-walls error only. 2. Past the battery's ends, ground is needed out to depth + sqrt(R^2 - d^2), the straight-line reach from the footprint, not R - d. 3. A passing footprint keeps its error clear of its straight wall's ends (check_backing); with the 0.05 ft slack widened to 0.5 a test now fails. 4. Coverage is needed for every check the rules define for the scene, and a result that leaves one out is flagged. battery_clearance counts when a battery is marked or its clearance exceeds gas's, as the server evaluates it. 5. The route's wall is needed past the meter by its error. Running the stricter checker at 3baa338 flagged four scenes; three were the checker's and are aligned with the server's documented ground model: spans take the along error (the app scan's headroom), seen ground grows by 0.0049 ft (d-gas-drift, 0.0007 ft short), and ground past a corner behind the next wall is house, not yard (g05). The plan check also tests each sample at its own depth. The fourth is a server issue: ETH3D after its one capture (see the PR). Also: the README says hostile inputs may be answered with a valid result (finding 8), and S2-1 runs the server tests with CI's fixed Hypothesis examples (finding 7). e2e at 3baa338: 49 of 50; contract_ok false on ETH3D only. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/README.md | 7 +- verification/SCOREBOARD.md | 72 +++--- verification/hsverify/resultcheck.py | 199 +++++++++++++--- verification/scoreboard/metrics.yaml | 7 +- verification/scoreboard/scoreboard.json | 192 ++++++++-------- verification/tests/test_resultcheck.py | 291 ++++++++++++++++++++---- 6 files changed, 564 insertions(+), 204 deletions(-) diff --git a/verification/README.md b/verification/README.md index 6bbe879cf..8e601d5a0 100644 --- a/verification/README.md +++ b/verification/README.md @@ -3,8 +3,8 @@ **Question.** Do the app and server branches do what the plan says? Pass: the app at `t3/ios-mvf`'s head reaches a result from the real ADVIO replay and keeps its scan; every server answer at `t3/server`'s head validates against the result schema and never passes without the -coverage its checks need; hostile inputs are refused within 10 s and 500 MB; each plan metric -has evidence at the current head. +coverage its checks need; hostile inputs are refused, or answered with a valid result, within +10 s and 500 MB of server growth; each plan metric has evidence at the current head. **Run**, from `verification/`, with uv (and Xcode 26 with an iOS Simulator for `sim-app`): @@ -20,7 +20,8 @@ non-commercial datasets. What each check asserts is in the docstrings of `hsveri `resultcheck.py` and `simrun.py`. **Result.** At `t3/server` `3baa338`, with the app's upload from `f14947e` (synthetic replay, hosted -server), all 50 scenes pass (45 answered and checked; slowest real scene 0.62 s). +server), 49 of 50 scenes pass (45 answered and checked; slowest real scene 0.42 s). The ETH3D scene +fails: after its one requested capture the server leaves pool unsure with no request or reason. **What it changed.** S2 fixed seven defects it found: - unseen ground passing the pool check; diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index 17035084f..5ef26f2bf 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,96 +1,96 @@ # Scoreboard -Generated 2026-09-27 07:58 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran for S2-1, S2-7 only. Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-27 08:59 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran for S2-1, S2-7 only. Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | -| [#169](https://github.com/SamGu-NRX/house-scanning/pull/169) | `t3/ci-flake-fixes-beta` | `9fc60195` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 07:40 CDT | -| [#168](https://github.com/SamGu-NRX/house-scanning/pull/168) | `t3/164-space-ends` | `6b21236f` | yes | 2 success, 2 skipped | 2026-09-27 07:22 CDT | -| [#165](https://github.com/SamGu-NRX/house-scanning/pull/165) | `t3/ci-flake-fixes` | `ee8cdcc1` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 07:40 CDT | +| [#172](https://github.com/SamGu-NRX/house-scanning/pull/172) | `t3/ci-flake-fixes-beta-2` | `c3aeabea` | no | 5 success, 1 skipped | 2026-09-27 08:28 CDT | +| [#168](https://github.com/SamGu-NRX/house-scanning/pull/168) | `t3/164-space-ends` | `381d0d3a` | yes | 2 success, 2 skipped | 2026-09-27 08:35 CDT | +| [#165](https://github.com/SamGu-NRX/house-scanning/pull/165) | `t3/ci-flake-fixes` | `4925f37f` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 08:58 CDT | | [#158](https://github.com/SamGu-NRX/house-scanning/pull/158) | `t3/live-fog` | `f57310ca` | no | 5 success, 1 failure, 1 skipped; failing: iOS build | 2026-09-27 06:07 CDT | -| [#143](https://github.com/SamGu-NRX/house-scanning/pull/143) | `t3/practice-meter` | `aac574f8` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 06:31 CDT | -| [#134](https://github.com/SamGu-NRX/house-scanning/pull/134) | `shrey/readme` | `b0c5829c` | no | 4 success | 2026-09-27 07:54 CDT | +| [#143](https://github.com/SamGu-NRX/house-scanning/pull/143) | `t3/practice-meter` | `d298dfbe` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 08:01 CDT | +| [#134](https://github.com/SamGu-NRX/house-scanning/pull/134) | `shrey/readme` | `2bd6a2a2` | no | 4 success | 2026-09-27 08:19 CDT | | [#124](https://github.com/SamGu-NRX/house-scanning/pull/124) | `t3/integration-viewer` | `9c0680b6` | no | 5 success | 2026-09-27 05:47 CDT | | [#123](https://github.com/SamGu-NRX/house-scanning/pull/123) | `t3/hunter-integration` | `17b53baa` | yes | 2 success, 1 skipped | 2026-09-27 07:17 CDT | -| [#122](https://github.com/SamGu-NRX/house-scanning/pull/122) | `t3/41-server-requests-text` | `ee882e48` | no | 4 success, 1 skipped | 2026-09-27 07:40 CDT | -| [#120](https://github.com/SamGu-NRX/house-scanning/pull/120) | `t3/41-walk-task-hold` | `9d4d12e5` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 06:49 CDT | +| [#122](https://github.com/SamGu-NRX/house-scanning/pull/122) | `t3/41-server-requests-text` | `7046c93b` | no | 4 success, 1 skipped | 2026-09-27 08:10 CDT | +| [#120](https://github.com/SamGu-NRX/house-scanning/pull/120) | `t3/41-walk-task-hold` | `30ba9b39` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 08:24 CDT | | [#119](https://github.com/SamGu-NRX/house-scanning/pull/119) | `t3/41-anchored-marks` | `f03430fe` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 07:49 CDT | -| [#109](https://github.com/SamGu-NRX/house-scanning/pull/109) | `t3/41-gap-loop-result` | `f2309cb3` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 06:48 CDT | +| [#109](https://github.com/SamGu-NRX/house-scanning/pull/109) | `t3/41-gap-loop-result` | `ecc7a576` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 08:36 CDT | | [#108](https://github.com/SamGu-NRX/house-scanning/pull/108) | `t3/41-ground-and-meter-tap` | `92c8f92f` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 07:49 CDT | -| [#104](https://github.com/SamGu-NRX/house-scanning/pull/104) | `t3/41-gate-coaching` | `040d9a61` | no | 4 success, 1 skipped | 2026-09-27 07:13 CDT | -| [#102](https://github.com/SamGu-NRX/house-scanning/pull/102) | `t3/41-card-replies` | `fd8f9b30` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 07:20 CDT | -| [#100](https://github.com/SamGu-NRX/house-scanning/pull/100) | `t3/41-ar-canvas-fallback` | `0ada11b9` | no | 4 success, 1 pending | 2026-09-27 07:42 CDT | +| [#104](https://github.com/SamGu-NRX/house-scanning/pull/104) | `t3/41-gate-coaching` | `5f8f8777` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 08:53 CDT | +| [#102](https://github.com/SamGu-NRX/house-scanning/pull/102) | `t3/41-card-replies` | `4183f61b` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 08:52 CDT | +| [#100](https://github.com/SamGu-NRX/house-scanning/pull/100) | `t3/41-ar-canvas-fallback` | `0ada11b9` | no | 4 success, 1 pending | 2026-09-27 08:29 CDT | | [#99](https://github.com/SamGu-NRX/house-scanning/pull/99) | `t3/41-end-preview` | `f5a27c64` | no | 4 success, 1 pending | 2026-09-27 07:08 CDT | | [#90](https://github.com/SamGu-NRX/house-scanning/pull/90) | `t3/check-experiments` | `9d66b31c` | no | 6 success | 2026-09-27 05:47 CDT | | [#88](https://github.com/SamGu-NRX/house-scanning/pull/88) | `t3/autodetect-lab` | `538f60e1` | yes | 3 success | 2026-09-27 05:49 CDT | -| [#87](https://github.com/SamGu-NRX/house-scanning/pull/87) | `t3/41-zip-deflate` | `8d2a6a13` | no | 5 success | 2026-09-27 06:15 CDT | +| [#87](https://github.com/SamGu-NRX/house-scanning/pull/87) | `t3/41-zip-deflate` | `aa5999e0` | no | 4 success, 1 pending | 2026-09-27 08:24 CDT | | [#86](https://github.com/SamGu-NRX/house-scanning/pull/86) | `t3/experience` | `b442f0a0` | yes | 3 success, 1 skipped | 2026-09-27 05:48 CDT | | [#61](https://github.com/SamGu-NRX/house-scanning/pull/61) | `t3/app-review-docs` | `c0b628b6` | no | 5 success | 2026-09-27 05:48 CDT | | [#60](https://github.com/SamGu-NRX/house-scanning/pull/60) | `t3/device-test-4.1-analysis` | `c566490c` | no | 3 success | 2026-09-27 05:55 CDT | | [#59](https://github.com/SamGu-NRX/house-scanning/pull/59) | `t3/signals-lab` | `0cde05e3` | yes | 3 success | 2026-09-27 05:50 CDT | | [#58](https://github.com/SamGu-NRX/house-scanning/pull/58) | `t3/ios-ci-no-preboot` | `7f1037e0` | no | 4 success, 1 skipped | 2026-09-27 07:02 CDT | -| [#56](https://github.com/SamGu-NRX/house-scanning/pull/56) | `t3/walk-guidance-gaps` | `a9e23c61` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 06:11 CDT | -| [#55](https://github.com/SamGu-NRX/house-scanning/pull/55) | `t3/motion-permission` | `b289c7df` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 07:13 CDT | +| [#56](https://github.com/SamGu-NRX/house-scanning/pull/56) | `t3/walk-guidance-gaps` | `16b1c249` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 08:24 CDT | +| [#55](https://github.com/SamGu-NRX/house-scanning/pull/55) | `t3/motion-permission` | `b289c7df` | no | 4 success, 1 skipped | 2026-09-27 07:58 CDT | | [#52](https://github.com/SamGu-NRX/house-scanning/pull/52) | `t3/battery-mark` | `086ad94b` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 07:23 CDT | -| [#49](https://github.com/SamGu-NRX/house-scanning/pull/49) | `t3/gap-skip-continues` | `e5761901` | no | 4 success, 1 skipped | 2026-09-27 07:22 CDT | +| [#49](https://github.com/SamGu-NRX/house-scanning/pull/49) | `t3/gap-skip-continues` | `6c3ac625` | no | 4 success, 1 skipped | 2026-09-27 08:47 CDT | | [#46](https://github.com/SamGu-NRX/house-scanning/pull/46) | `t3/field-test-run2` | `afd02c82` | no | 3 success | 2026-09-27 05:48 CDT | ## S1 Evals on real data -`origin/t3/evals` at [`4f52036f`](https://github.com/SamGu-NRX/house-scanning/tree/4f52036f78c7ab45350004fda241da3125d2bb5e). 1 met, 2 partial, 1 gap, 1 not run. +`origin/t3/evals` at [`b4601ca5`](https://github.com/SamGu-NRX/house-scanning/tree/b4601ca59f2122a54d91c6981cdc74507248d6b1). 1 met, 2 partial, 1 gap, 1 not run. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S1-1 | One command regenerates every number from a clean checkout | partial | make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac. | | S1-2 | Dataset URLs and sha256 recorded | gap | none found in 1 files; missing ADVIO, ETH3D | | S1-3 | Metric code tested on hand-computed cases | not run | slow; run with --slow | -| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/METHODS.md](https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/METHODS.md), [experiments/evals/field/FIELD_SHEET.md](https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/field/FIELD_SHEET.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/results/advio_drift.md) | -| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/METHODS.md](https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/METHODS.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/results/eth3d_visibility_sensitivity.md) | +| S1-4 | ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft | met | all 5 found [experiments/evals/METHODS.md](https://github.com/SamGu-NRX/house-scanning/blob/b4601ca59f2122a54d91c6981cdc74507248d6b1/experiments/evals/METHODS.md), [experiments/evals/field/FIELD_SHEET.md](https://github.com/SamGu-NRX/house-scanning/blob/b4601ca59f2122a54d91c6981cdc74507248d6b1/experiments/evals/field/FIELD_SHEET.md), [experiments/evals/results/advio_drift.md](https://github.com/SamGu-NRX/house-scanning/blob/b4601ca59f2122a54d91c6981cdc74507248d6b1/experiments/evals/results/advio_drift.md) | +| S1-5 | ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question | partial | found 1-3 m row, 3-10 m row, scale error; missing verdict [experiments/evals/METHODS.md](https://github.com/SamGu-NRX/house-scanning/blob/b4601ca59f2122a54d91c6981cdc74507248d6b1/experiments/evals/METHODS.md), [experiments/evals/results/eth3d_recon.md](https://github.com/SamGu-NRX/house-scanning/blob/b4601ca59f2122a54d91c6981cdc74507248d6b1/experiments/evals/results/eth3d_recon.md), [experiments/evals/results/eth3d_visibility_sensitivity.md](https://github.com/SamGu-NRX/house-scanning/blob/b4601ca59f2122a54d91c6981cdc74507248d6b1/experiments/evals/results/eth3d_visibility_sensitivity.md) | ## S2 Server and solver -`origin/t3/server` at [`3baa3387`](https://github.com/SamGu-NRX/house-scanning/tree/3baa3387a6877961345afa7785f1530c1aeb2b1b). 8 met. +`origin/t3/server` at [`3baa3387`](https://github.com/SamGu-NRX/house-scanning/tree/3baa3387a6877961345afa7785f1530c1aeb2b1b). 7 met, 1 gap. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S2-1 | pytest green | met | exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 380 passed, 1 xfailed, 1 warning in 80.06s (0:01:20) | +| S2-1 | pytest green, the property tests under CI's fixed Hypothesis examples | met | exit 0: return lib.difference_prec(a, b, grid_size, **kwargs) / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 380 passed, 1 xfailed, 5 warnings in 61.63s (0:01:01) | | S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_golden_public.py) | | S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | met | all 4 found [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_properties.py), [server/tests/test_final_review.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_final_review.py) | | S2-4 | Schemas published | met | present at 3baa3387 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/schemas/result.schema.json) | | S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_existing_battery.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_existing_battery.py), [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_schemas.py) | -| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 616.8 (want < 1000) at sha 3baa3387: ~/house-scanning-data/reports/e2e/20260927-075636-3baa3387/report.json | -| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 37ms / All checks passed! / 36 files already formatted | -| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | met | contract_ok = true (want True) at sha 3baa3387: ~/house-scanning-data/reports/e2e/20260927-075636-3baa3387/report.json | +| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 423.1 (want < 1000) at sha 3baa3387: ~/house-scanning-data/reports/e2e/20260927-085738-3baa3387/report.json | +| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 27ms / All checks passed! / 36 files already formatted | +| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | gap | contract_ok = false (want True) at sha 3baa3387: ~/house-scanning-data/reports/e2e/20260927-085738-3baa3387/report.json | ## S3 iOS capture app -`origin/t3/ios-mvf` at [`6be087df`](https://github.com/SamGu-NRX/house-scanning/tree/6be087df207351fe3149b1924f78e91a92d2c3eb). 1 met, 3 partial, 1 gap, 3 stale, 2 no evidence yet. +`origin/t3/ios-mvf` at [`c820cf15`](https://github.com/SamGu-NRX/house-scanning/tree/c820cf150a73cb97527428d580c330f631253f9a). 1 met, 3 partial, 1 gap, 3 stale, 2 no evidence yet. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S3-1 | CI build green | no evidence yet | no open PR for t3/ios-mvf | -| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/project.yml) | -| S3-3 | Zero build warnings | stale | report for sha f14947eb, ref at 6be087df: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | -| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage, guidance; missing auto-capture [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift) | -| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/SourceRestartUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/SourceRestartUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/AccessibilityAudit.swift](https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/AccessibilityAudit.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/ScreenStatesUITests.swift) | -| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at 6be087df: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | +| S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/project.yml) | +| S3-3 | Zero build warnings | stale | report for sha f14947eb, ref at c820cf15: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | +| S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage, guidance; missing auto-capture [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift) | +| S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/SourceRestartUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanUITests/SourceRestartUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/AccessibilityAudit.swift](https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanUITests/AccessibilityAudit.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanUITests/ScreenStatesUITests.swift) | +| S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at c820cf15: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | | S3-7 | Screenshots of each state in the PR | no evidence yet | no open PR for t3/ios-mvf | -| S3-8 | Exported scene.json validates against C1 | stale | report for app_sha f14947eb, ref at 6be087df: ~/house-scanning-data/reports/e2e/20260927-075636-3baa3387/report.json | +| S3-8 | Exported scene.json validates against C1 | stale | report for app_sha f14947eb, ref at c820cf15: ~/house-scanning-data/reports/e2e/20260927-085738-3baa3387/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/ios-mvf` at [`6be087df`](https://github.com/SamGu-NRX/house-scanning/tree/6be087df207351fe3149b1924f78e91a92d2c3eb), `origin/t3/verification` at [`21c88324`](https://github.com/SamGu-NRX/house-scanning/tree/21c8832408e8484d64f4e54b4ca903ddb6c5af4a). 2 met, 2 stale. +`origin/t3/ios-mvf` at [`c820cf15`](https://github.com/SamGu-NRX/house-scanning/tree/c820cf150a73cb97527428d580c330f631253f9a), `origin/t3/verification` at [`255bc848`](https://github.com/SamGu-NRX/house-scanning/tree/255bc848679e7327fecc77c0348d089dd1a3f334). 2 met, 2 stale. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S4-1 | End to end from a real replay to a placement result passes | stale | report for app_sha f14947eb, ref at 6be087df: ~/house-scanning-data/reports/e2e/20260927-075636-3baa3387/report.json | -| S4-2 | Screenshot report per app state | stale | report for sha f14947eb, ref at 6be087df: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | +| S4-1 | End to end from a real replay to a placement result passes | stale | report for app_sha f14947eb, ref at c820cf15: ~/house-scanning-data/reports/e2e/20260927-085738-3baa3387/report.json | +| S4-2 | Screenshot report per app state | stale | report for sha f14947eb, ref at c820cf15: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | | S4-3 | Product description written from the code and checked against screenshots | met | docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state. | -| S4-4 | Scoreboard of every PR | met | present at 21c88324 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/21c8832408e8484d64f4e54b4ca903ddb6c5af4a/verification/SCOREBOARD.md) | +| S4-4 | Scoreboard of every PR | met | present at 255bc848 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/255bc848679e7327fecc77c0348d089dd1a3f334/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 0783cc98e..706b55ee3 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -29,6 +29,7 @@ import hashlib import itertools import math +from collections.abc import Callable from dataclasses import dataclass from typing import Any @@ -59,6 +60,9 @@ class Need: radius: float = 0.0 height: float = 0.0 widen: bool = True + # Widened by the battery's error along the walls only (along_error), not in plan: spans + # measured in s move with the meter as the battery does (server 3baa338, along_err). + along: bool = False # Scene lists whose entries settle facing_gap and headroom where they cover the battery. @@ -126,9 +130,10 @@ def reach(key: str) -> Need: "drive_clearance": (reach("drive_ft"),), "pool_clearance": (reach("pool_ft"),), "opening_clearance": (Need("wall", r("opening_ft"), opening_height),), - "wall_equipment_above": (Need("wall", r("wall_equipment_ft"), headroom, widen=False),), - "facing_gap": (Need("facing", 0.0, depth + value("facing", "min_ft")),), - "headroom": (Need("overhead", 0.0, headroom),), + # Widened by the battery's error from 3baa338 (server README, wall_equipment_above). + "wall_equipment_above": (Need("wall", r("wall_equipment_ft"), headroom, along=True),), + "facing_gap": (Need("facing", 0.0, depth + value("facing", "min_ft"), along=True),), + "headroom": (Need("overhead", 0.0, headroom, along=True),), } errors = { name: value("errors", f"{name}_ft") @@ -338,9 +343,14 @@ def invariant_problems( if rules is not None: problems += coverage_problems(scene, result, rules) problems += declared_height_problems(scene, result, rules) + problems += dropped_check_problems(scene, result, rules) return problems +# Seen ground grows by this in every direction before it is compared, closing gaps under the +# 0.01 ft coverage tolerance (server/scene.py SEEN_GROWTH_FT at 3baa338). +SEEN_GROWTH_FT = 0.005 - 1e-4 + # Walls and baseline points within this of one straight line are one straight stretch # (server/scene.py COLLINEAR_FT and _join_straight_walls at a726199). Also the slack on a # footprint's ends, since s here and the server's s can differ by a merged point's offset. @@ -375,7 +385,7 @@ def footprint_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: if not pieces: return [] - def off_wall(lo: float, hi: float) -> str | None: + def off_wall(lo: float, hi: float, outcome: str, wall_id: str) -> str | None: # The stretch it starts on, or else ends on: the one it would back onto. A start on a # boundary belongs to the stretch it runs onto, an end to the one it runs off. piece = next((p for p in pieces if p[0] - EPS <= lo < p[1] - EPS), None) or next( @@ -385,19 +395,28 @@ def off_wall(lo: float, hi: float) -> str | None: return "is not on any scanned wall" if lo < piece[0] - COLLINEAR_FT or hi > piece[1] + COLLINEAR_FT: return f"runs past its straight wall [{piece[0]:.2f}, {piece[1]:.2f}]" + if outcome != "pass": + return None + # A pass also keeps its position error clear of the stretch's ends: within it the + # battery may not sit flush, and the server leaves it unsure (check_backing). + e = battery_error(scene, rules, wall_id, lo, hi) + for end, clear in ((piece[0], lo - e - piece[0]), (piece[1], piece[1] - hi - e)): + if clear < -COLLINEAR_FT: + return f"ends within its error ({e:.2f} ft) of its straight wall's end at {end:.2f}" return None problems = [] spot = result.get("spot") if spot is not None and spot["outcome"] in ("pass", "unsure"): lo, hi = spot["span_ft"] - if (why := off_wall(lo, hi)) is not None: + if (why := off_wall(lo, hi, spot["outcome"], spot["wall_id"])) is not None: problems.append(f"spot [{lo:.2f}, {hi:.2f}] {why}") for run in result.get("sweep", []): if run["outcome"] not in ("pass", "unsure"): continue for start in dict.fromkeys(run["start_ft"]): # a run's two ends bound all its starts - if (why := off_wall(start, start + rules.width_ft)) is not None: + battery = (start, start + rules.width_ft) + if (why := off_wall(*battery, run["outcome"], run["wall_id"])) is not None: problems.append(f"sweep {run['outcome']} start {start:.2f} {why}") return problems @@ -431,15 +450,75 @@ def required_span(lo: float, hi: float, radius: float) -> tuple[float, float]: def battery_error(scene: dict, rules: RuleSet, wall_id: str, lo: float, hi: float) -> float: - """Position error of a battery over [lo, hi] on `wall_id`: the wall's explicit error, or the - default for its `source` (tap, mesh, plane) plus drift at the battery's edge further from the - meter (S2 70ab0b0).""" + """Position error of a battery over [lo, hi] on `wall_id`, as server 3baa338 takes it for + its checks (solver.py _errors): its wall's error at its far edge from the meter, plus the + larger of the meter's error in plan and the battery's slide along the walls (slide).""" wall = next((w for w in scene["walls"] if w["id"] == wall_id), None) if wall is None: raise ValueError(f"the result names wall {wall_id!r}, which the scene does not have") + return wall_error_at(wall, rules, max(abs(lo), abs(hi))) + max( + meter_error(scene, rules), slide(scene, rules, lo, hi) + ) + + +def along_error(scene: dict, rules: RuleSet, wall_id: str, lo: float, hi: float) -> float: + """The battery's error against spans measured along the walls: its wall's at its far edge + plus its slide, without the meter's plan error (server 3baa338 _errors, along_err).""" + wall = next(w for w in scene["walls"] if w["id"] == wall_id) + return wall_error_at(wall, rules, max(abs(lo), abs(hi))) + slide(scene, rules, lo, hi) + + +def wall_error_at(wall: dict, rules: RuleSet, s: float) -> float: + """A wall's error at distance |s| along the walls from the meter: its explicit one, or its + source's default plus drift.""" if "plus_minus_ft" in wall: return wall["plus_minus_ft"] - return wall_error(wall, rules) + rules.errors["drift_per_ft"] * max(abs(lo), abs(hi)) + return wall_error(wall, rules) + rules.errors["drift_per_ft"] * abs(s) + + +def meter_error(scene: dict, rules: RuleSet) -> float: + """The meter's position error: its explicit one or rules.yaml's errors.meter_ft (no drift).""" + return scene.get("meter", {}).get("plus_minus_ft", rules.errors["meter"]) + + +def slide(scene: dict, rules: RuleSet, lo: float, hi: float) -> float: + """How far a battery over [lo, hi] can move along the walls against spans measured in s when + the meter or a corner between them moves within its error (server 3baa338 _slide): the + meter's error times the difference between its wall's direction and the battery's, plus each + other wall between the meter and the battery's near edge, at its far end, times the same + difference for that wall. 0 on the meter's own wall.""" + segments = segments_s(scene, rules) + if not segments: + return 0.0 + walls = {w["id"]: w for w in scene["walls"]} + + def along(seg) -> Point: + _, _, _, p, q = seg + n = math.dist(p, q) + return ((q[0] - p[0]) / n, (q[1] - p[1]) / n) + + def under(s: float): + return min(segments, key=lambda g: max(g[1] - s, s - g[2], 0.0)) + + battery = along(under((lo + hi) / 2)) + + def turn(seg) -> float: + a = along(seg) + return math.hypot(battery[0] - a[0], battery[1] - a[1]) + + meter_wall = scene["meter"]["wall_id"] + meter_seg = min( + (g for g in segments if g[0] == meter_wall), key=lambda g: max(g[1], -g[2], 0.0) + ) + near = lo if lo > 0 else min(hi, 0.0) + a0, b0 = min(0.0, near), max(0.0, near) + total = meter_error(scene, rules) * turn(meter_seg) + for seg in segments: + a, b = max(seg[1], a0), min(seg[2], b0) + if b - a <= EPS: + continue + total += wall_error_at(walls[seg[0]], rules, max(abs(a), abs(b))) * turn(seg) + return total def wall_error(wall: dict, rules: RuleSet) -> float: @@ -460,14 +539,20 @@ def reach_gaps( - wall band: [lo - R - e, hi + R + e] along the wall, since distance along the wall is never shorter than straight-line distance, seen up to the check's height; - ground band: out to depth + R + e in front of the battery (a battery sits flush on one - straight segment), and at distance d past either end out to R + e - d. The ground point - there is at most d along the wall plus R + e - d out from the battery's end; + straight segment), and at distance d past either end out to depth + sqrt((R + e)^2 - d^2), + the straight-line reach from the footprint's corner; - facing and overhead bands: over [lo - e, hi + e], every place the battery may sit, beyond the check's distance unless measured. """ gaps = [] for need in rules.needs[check]: - reach = need.radius + (battery_error(scene, rules, wall_id, lo, hi) if need.widen else 0.0) + if not need.widen: + error = 0.0 + elif need.along: + error = along_error(scene, rules, wall_id, lo, hi) + else: + error = battery_error(scene, rules, wall_id, lo, hi) + reach = need.radius + error if need.band == "ground": plan = GroundPlan(scene, rules) if rules.wall_join_ft is not None else None gap = ground_gap(scene, lo, hi, rules.depth_ft, reach, plan) @@ -530,6 +615,33 @@ def run_starts(span: list[float], step_ft: float | None) -> list[float]: return [a + k * step_ft for k in range(count)] + [b] +# Checks every result lists besides those with coverage needs (server README, "What settles +# each check", at 3baa338). +ALWAYS_CHECKS = frozenset({"meter_working_space", "route_path", "route_length"}) + + +def expected_checks(scene: dict, rules: RuleSet) -> set[str]: + """The checks the rules define for this scene. battery_clearance only when the scene marks + an existing battery or the rules set its clearance above gas's (server 3baa338).""" + ids = set(rules.needs) + battery = rules.needs.get("battery_clearance") + gas = rules.needs.get("gas_clearance") + marked = any(o.get("type") == "battery" for o in scene.get("objects", [])) + if battery and not marked and not (gas and battery[0].radius > gas[0].radius): + ids.discard("battery_clearance") + return ids + + +def dropped_check_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: + listed = {c["id"] for c in result.get("checks", [])} + if not listed: + return [] + wanted = expected_checks(scene, rules) | ALWAYS_CHECKS + return [ + f"the result leaves out check {c}, which the rules define" for c in sorted(wanted - listed) + ] + + def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: """Missing coverage is never a pass (C5), to the reach each check's rule looks out to (see reach_gaps). A sweep run that passes needs, at each start sampled from it (run_starts), every @@ -540,9 +652,9 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: route_wall = observed( scene, "wall", rules.route_height_ft, beyond=True, wall_default_ft=rules.headroom_ft ) - # A passing run passes every check the server evaluated; a check it did not evaluate (a - # battery clearance with no battery in the scene, say) needs nothing. - evaluated = {c["id"] for c in result.get("checks", [])} or set(rules.needs) + # Coverage is needed for every check the rules define for this scene, whether or not the + # result lists it: a result that drops a check must not skip its coverage. + evaluated = expected_checks(scene, rules) if "coverage" not in scene and result["decision"] == "pass": problems.append("decision pass for a scene with no coverage at all") @@ -550,7 +662,12 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: if run["outcome"] != "pass": continue lo, hi = run["start_ft"][0], run["start_ft"][1] + rules.width_ft - route_lo, route_hi = min(0.0, lo), max(0.0, hi) + # The route starts at the meter, which may stand its error either side of s = 0 + # (server README, route_path), within the scanned chain. + m = meter_error(scene, rules) + ends = chain_ends_s(scene, rules) or (-math.inf, math.inf) + route_lo = max(min(lo, -m), ends[0]) + route_hi = min(max(hi, m), ends[1]) if not covers(route_wall, route_lo, route_hi): problems.append( f"sweep pass for starts {run['start_ft']} but the wall and cable route " @@ -618,11 +735,17 @@ def ground_gap( for p, q in itertools.pairwise(cuts): if q - p <= EPS: continue - inside = lo - EPS <= p and q <= hi + EPS - need = depth + radius if inside else radius - min(abs(lo - q), abs(p - hi)) + + # Past an end, a point d along the wall from the battery is within the radius of its + # footprint (depth D out from the wall) out to D + sqrt(R^2 - d^2). + def need_at(s: float) -> float: + d = max(lo - s, s - hi, 0.0) + return depth + math.sqrt(max(0.0, radius**2 - d**2)) + + need = need_at(q if q <= lo else p if p >= hi else (lo + hi) / 2) seen = max((out for x, y, out in entries if x <= p + EPS and q - EPS <= y), default=None) - if seen is None or seen + EPS < need: - if plan is not None and plan.covers((lo + hi) / 2, p, q, seen or 0.0, need): + if seen is None or seen + SEEN_GROWTH_FT < need: + if plan is not None and plan.covers((lo + hi) / 2, p, q, seen or 0.0, need_at): continue where = f"ground [{p:.2f}, {q:.2f}] observed" if seen is None: @@ -792,13 +915,33 @@ def seen(self, x: Point) -> bool: dx, dz = x[0] - origin[0], x[1] - origin[1] t = dx * along[0] + dz * along[1] n = dx * outward[0] + dz * outward[1] - if t0 - EPS <= t <= t1 + EPS and -EPS <= n <= out + SERVER_EPS: + grow = SEEN_GROWTH_FT + if t0 - grow <= t <= t1 + grow and -grow <= n <= out + grow: + return True + return False + + def required(self, x: Point) -> bool: + """Whether ground at x is yard a view must show (server 3baa338 unobserved_ground): in + front of some wall segment, or past either end of the chain. Past a corner, behind the + next wall, is neither: the house.""" + for i, (_, _, _, p, q) in enumerate(self.segments): + along, outward = self._frame(p, q) + dx, dz = x[0] - p[0], x[1] - p[1] + t = dx * along[0] + dz * along[1] + n = dx * outward[0] + dz * outward[1] + length = math.dist(p, q) + if -EPS <= t <= length + EPS and n >= -EPS: + return True + if (i == 0 and t < 0) or (i == len(self.segments) - 1 and t > length): return True return False - def covers(self, s_mid: float, p: float, q: float, out_lo: float, out_hi: float) -> bool: - """Every sampled point from s = p to q, out_lo to out_hi in front of the battery's - segment, lies in some observed ground band. + def covers( + self, s_mid: float, p: float, q: float, out_lo: float, need: Callable[[float], float] + ) -> bool: + """Every sampled point from s = p to q, deeper than out_lo (seen by the stretch's own + band) and out to need(s) in front of the battery's segment, lies in some observed ground + band. Samples are each cell's edges and its centre, in s and out. Edges alone missed a gap narrower than a cell between two bands (0.019 ft at 5a4b4f4): both its edges lie on a @@ -806,9 +949,11 @@ def covers(self, s_mid: float, p: float, q: float, out_lo: float, out_hi: float) if not self.strips: return False for s in self._samples(p, q): - for out in self._samples(out_lo, out_hi): + # Up to out_lo the stretch's own band already saw the ground. + deeper = [o for o in self._samples(out_lo, need(s)) if o > out_lo + SERVER_EPS] + for out in deeper: x = self.point(s_mid, s, out) - if x is None or not self.seen(x): + if x is None or (self.required(x) and not self.seen(x)): return False return True diff --git a/verification/scoreboard/metrics.yaml b/verification/scoreboard/metrics.yaml index 7dce509de..9c1b7c3ad 100644 --- a/verification/scoreboard/metrics.yaml +++ b/verification/scoreboard/metrics.yaml @@ -81,8 +81,11 @@ metrics: - id: S2-1 workstream: S2 ref: origin/t3/server - text: pytest green - command: {run: uv run pytest -q, cwd: server, timeout_s: 600, slow: true} + text: pytest green, the property tests under CI's fixed Hypothesis examples + # CI=1 selects the server's "ci" profile (server/tests/conftest.py: derandomize=True), as + # its CI does. A default local run draws fresh examples and can fail where CI passes: at + # 3baa338, test_capturing_exactly_what_is_requested_settles_it_in_one_round did (#11). + command: {run: CI=1 uv run pytest -q, cwd: server, timeout_s: 600, slow: true} - id: S2-2 workstream: S2 diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index e01d7b358..37c32a427 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-09-27T07:58:56-05:00", + "generated_at": "2026-09-27T08:59:19-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": [ "S2-1", @@ -7,30 +7,30 @@ ], "prs": [ { - "number": 169, - "title": "ios: bring the t3/ios-mvf UI test flake fixes to beta", - "branch": "t3/ci-flake-fixes-beta", - "head": "9fc6019504c0da23fa6edae02df993f5caa00298", + "number": 172, + "title": "ios: beta gets the legend fix and the walk's in-motion audit, report-only", + "branch": "t3/ci-flake-fixes-beta-2", + "head": "c3aeabea41a557d5a50cfedae4b3b07fd52d9687", "draft": false, - "updated": "2026-09-27T12:40:26Z", + "updated": "2026-09-27T13:28:27Z", "ci": { "counts": { - "success": 3, + "success": 5, "failure": 0, - "pending": 1, + "pending": 0, "skipped": 1 }, "failing": [], - "text": "3 success, 1 pending, 1 skipped" + "text": "5 success, 1 skipped" } }, { "number": 168, "title": "ios: treat a corridor's far wall or a side yard's fence as where the space ends", "branch": "t3/164-space-ends", - "head": "6b21236f0ec21b1262212807dd9713145ed315b2", + "head": "381d0d3a3febc4ea6f275d3815fa523144bc4e4a", "draft": true, - "updated": "2026-09-27T12:22:46Z", + "updated": "2026-09-27T13:35:12Z", "ci": { "counts": { "success": 2, @@ -44,11 +44,11 @@ }, { "number": 165, - "title": "ios tests: audit the walk once the autopilot holds it; triage of the recurring UI test failures", + "title": "ios tests: audit the walk strictly once held and in motion to report; triage of the UI test failures", "branch": "t3/ci-flake-fixes", - "head": "ee8cdcc1fd21bc39d16fd02ea299d57957fc9e1d", + "head": "4925f37fb70cb875b34b16fa03eff75218c1905f", "draft": false, - "updated": "2026-09-27T12:40:18Z", + "updated": "2026-09-27T13:58:06Z", "ci": { "counts": { "success": 3, @@ -84,9 +84,9 @@ "number": 143, "title": "ios: practice meter, a drawn sample meter for scanning without one", "branch": "t3/practice-meter", - "head": "aac574f8c3ab6094a9d4947eb61e5c4ac5022f22", + "head": "d298dfbeb66743e45d71bb58aa641f2d5fcb7662", "draft": false, - "updated": "2026-09-27T11:31:45Z", + "updated": "2026-09-27T13:01:52Z", "ci": { "counts": { "success": 3, @@ -102,9 +102,9 @@ "number": 134, "title": "docs: rebuild the README around the demo, with quick start, data and limits", "branch": "shrey/readme", - "head": "b0c5829c1273158aabdc0ea2ec7c931ebead48f6", + "head": "2bd6a2a21056d4228e9ee36bbf29a0428f8ff283", "draft": false, - "updated": "2026-09-27T12:54:49Z", + "updated": "2026-09-27T13:19:38Z", "ci": { "counts": { "success": 4, @@ -156,9 +156,9 @@ "number": 122, "title": "server: stop requests at the wall's ends, keep placeholder distances out of them, name things without ids", "branch": "t3/41-server-requests-text", - "head": "ee882e4884e9ee5a818549ecd7e707da8485229e", + "head": "7046c93b467cc04f5da0d6a9861bca1bce8e7ef3", "draft": false, - "updated": "2026-09-27T12:40:57Z", + "updated": "2026-09-27T13:10:02Z", "ci": { "counts": { "success": 4, @@ -174,9 +174,9 @@ "number": 120, "title": "ios: hold a walk aim until it is met or stalls, and ask for the meter ground only near the meter", "branch": "t3/41-walk-task-hold", - "head": "9d4d12e55bb0b762d1a1fd57f41ea95dbd25a345", + "head": "30ba9b39971ddcf86732e2011a1f06bffc7634e0", "draft": false, - "updated": "2026-09-27T11:49:01Z", + "updated": "2026-09-27T13:24:49Z", "ci": { "counts": { "success": 3, @@ -210,9 +210,9 @@ "number": 109, "title": "ios: gap-loop follow-ups, upload ticks, the nearer unseen end and request stretches (#39, #31, #83, #75)", "branch": "t3/41-gap-loop-result", - "head": "f2309cb37c7b7d23de8f46f5d89229f0452a8605", + "head": "ecc7a5768e7b8afedf66dbf8a4f65c1f5f78cd50", "draft": false, - "updated": "2026-09-27T11:48:59Z", + "updated": "2026-09-27T13:36:44Z", "ci": { "counts": { "success": 3, @@ -246,27 +246,27 @@ "number": 104, "title": "ios: gate coaching keeps the task card, holds through dark flicker, and says \"Turn more slowly\" for turns", "branch": "t3/41-gate-coaching", - "head": "040d9a61cd8c6fd43784c3d7575c00fd991cf182", + "head": "5f8f8777e1f0b03400b88a7be7dbe2b314a9def5", "draft": false, - "updated": "2026-09-27T12:13:22Z", + "updated": "2026-09-27T13:53:48Z", "ci": { "counts": { - "success": 4, + "success": 3, "failure": 0, - "pending": 0, + "pending": 1, "skipped": 1 }, "failing": [], - "text": "4 success, 1 skipped" + "text": "3 success, 1 pending, 1 skipped" } }, { "number": 102, "title": "ios: card replies name the task, stay under gate coaching, and lock after a new card", "branch": "t3/41-card-replies", - "head": "fd8f9b30e491c8175fd2390310e30ec81bf5c85d", + "head": "4183f61b516282a3e07728a857660e30f96ffc02", "draft": false, - "updated": "2026-09-27T12:20:00Z", + "updated": "2026-09-27T13:52:50Z", "ci": { "counts": { "success": 3, @@ -284,7 +284,7 @@ "branch": "t3/41-ar-canvas-fallback", "head": "0ada11b9dcb67452e311ee4e4e8e9ffb50807bbb", "draft": false, - "updated": "2026-09-27T12:42:15Z", + "updated": "2026-09-27T13:29:43Z", "ci": { "counts": { "success": 4, @@ -354,18 +354,18 @@ "number": 87, "title": "ios: deflate the text entries in the shared scan zip", "branch": "t3/41-zip-deflate", - "head": "8d2a6a13abf1f6abb4d328a0b5c74620b0ee295f", + "head": "aa5999e040b37a93ddfc91065c6954cabef59c8e", "draft": false, - "updated": "2026-09-27T11:15:29Z", + "updated": "2026-09-27T13:24:44Z", "ci": { "counts": { - "success": 5, + "success": 4, "failure": 0, - "pending": 0, + "pending": 1, "skipped": 0 }, "failing": [], - "text": "5 success" + "text": "4 success, 1 pending" } }, { @@ -462,9 +462,9 @@ "number": 56, "title": "Tests for #24, #28 and #29: synthetic walk-guidance cases modeled on field runs 1 and 2", "branch": "t3/walk-guidance-gaps", - "head": "a9e23c612326fe9b25a85eae28bdac450ace18b8", + "head": "16b1c2492b36245493120fbb066314fa14bac525", "draft": false, - "updated": "2026-09-27T11:11:28Z", + "updated": "2026-09-27T13:24:48Z", "ci": { "counts": { "success": 3, @@ -482,16 +482,16 @@ "branch": "t3/motion-permission", "head": "b289c7dfa90c8760e0c34b96e5383a8c27a2bb8e", "draft": false, - "updated": "2026-09-27T12:13:49Z", + "updated": "2026-09-27T12:58:42Z", "ci": { "counts": { - "success": 3, + "success": 4, "failure": 0, - "pending": 1, + "pending": 0, "skipped": 1 }, "failing": [], - "text": "3 success, 1 pending, 1 skipped" + "text": "4 success, 1 skipped" } }, { @@ -516,9 +516,9 @@ "number": 49, "title": "ios: 'I can't get there' moves on to the next requested view (#39)", "branch": "t3/gap-skip-continues", - "head": "e57619015a11c995f3581f145ac6b544450c1eb7", + "head": "6c3ac6252f9738e94b6ab10b16f0cfc4ee4a5d0d", "draft": false, - "updated": "2026-09-27T12:22:12Z", + "updated": "2026-09-27T13:47:11Z", "ci": { "counts": { "success": 4, @@ -557,7 +557,7 @@ "refs": [ { "ref": "origin/t3/evals", - "sha": "4f52036f78c7ab45350004fda241da3125d2bb5e" + "sha": "b4601ca59f2122a54d91c6981cdc74507248d6b1" } ], "metrics": [ @@ -565,7 +565,7 @@ "id": "S1-1", "text": "One command regenerates every number from a clean checkout", "ref": "origin/t3/evals", - "sha": "4f52036f78c7ab45350004fda241da3125d2bb5e", + "sha": "b4601ca59f2122a54d91c6981cdc74507248d6b1", "status": "partial", "evidence": "make drift at b5ca7af from a clean worktree reproduced advio_drift.md and advio_drift.json byte for byte (85 MB peak). ETH3D recon not rerun: it loads the laser scans, whose memory has not been bounded on this shared Mac.", "links": [] @@ -574,7 +574,7 @@ "id": "S1-2", "text": "Dataset URLs and sha256 recorded", "ref": "origin/t3/evals", - "sha": "4f52036f78c7ab45350004fda241da3125d2bb5e", + "sha": "b4601ca59f2122a54d91c6981cdc74507248d6b1", "status": "gap", "evidence": "none found in 1 files; missing ADVIO, ETH3D", "links": [] @@ -583,7 +583,7 @@ "id": "S1-3", "text": "Metric code tested on hand-computed cases", "ref": "origin/t3/evals", - "sha": "4f52036f78c7ab45350004fda241da3125d2bb5e", + "sha": "b4601ca59f2122a54d91c6981cdc74507248d6b1", "status": "not run", "evidence": "slow; run with --slow", "links": [] @@ -592,21 +592,21 @@ "id": "S1-4", "text": "ADVIO drift report, median and p90 inches after 3, 10, 20 and 30 ft", "ref": "origin/t3/evals", - "sha": "4f52036f78c7ab45350004fda241da3125d2bb5e", + "sha": "b4601ca59f2122a54d91c6981cdc74507248d6b1", "status": "met", "evidence": "all 5 found", "links": [ [ "experiments/evals/METHODS.md", - "https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/METHODS.md" + "https://github.com/SamGu-NRX/house-scanning/blob/b4601ca59f2122a54d91c6981cdc74507248d6b1/experiments/evals/METHODS.md" ], [ "experiments/evals/field/FIELD_SHEET.md", - "https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/field/FIELD_SHEET.md" + "https://github.com/SamGu-NRX/house-scanning/blob/b4601ca59f2122a54d91c6981cdc74507248d6b1/experiments/evals/field/FIELD_SHEET.md" ], [ "experiments/evals/results/advio_drift.md", - "https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/results/advio_drift.md" + "https://github.com/SamGu-NRX/house-scanning/blob/b4601ca59f2122a54d91c6981cdc74507248d6b1/experiments/evals/results/advio_drift.md" ] ] }, @@ -614,21 +614,21 @@ "id": "S1-5", "text": "ETH3D errors per model (median and p90 inches for 1-3 m and 3-10 m pairs, scale error %) with a verdict per question", "ref": "origin/t3/evals", - "sha": "4f52036f78c7ab45350004fda241da3125d2bb5e", + "sha": "b4601ca59f2122a54d91c6981cdc74507248d6b1", "status": "partial", "evidence": "found 1-3 m row, 3-10 m row, scale error; missing verdict", "links": [ [ "experiments/evals/METHODS.md", - "https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/METHODS.md" + "https://github.com/SamGu-NRX/house-scanning/blob/b4601ca59f2122a54d91c6981cdc74507248d6b1/experiments/evals/METHODS.md" ], [ "experiments/evals/results/eth3d_recon.md", - "https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/results/eth3d_recon.md" + "https://github.com/SamGu-NRX/house-scanning/blob/b4601ca59f2122a54d91c6981cdc74507248d6b1/experiments/evals/results/eth3d_recon.md" ], [ "experiments/evals/results/eth3d_visibility_sensitivity.md", - "https://github.com/SamGu-NRX/house-scanning/blob/4f52036f78c7ab45350004fda241da3125d2bb5e/experiments/evals/results/eth3d_visibility_sensitivity.md" + "https://github.com/SamGu-NRX/house-scanning/blob/b4601ca59f2122a54d91c6981cdc74507248d6b1/experiments/evals/results/eth3d_visibility_sensitivity.md" ] ] } @@ -646,11 +646,11 @@ "metrics": [ { "id": "S2-1", - "text": "pytest green", + "text": "pytest green, the property tests under CI's fixed Hypothesis examples", "ref": "origin/t3/server", "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", "status": "met", - "evidence": "exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 380 passed, 1 xfailed, 1 warning in 80.06s (0:01:20)", + "evidence": "exit 0: return lib.difference_prec(a, b, grid_size, **kwargs) / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 380 passed, 1 xfailed, 5 warnings in 61.63s (0:01:01)", "links": [] }, { @@ -735,7 +735,7 @@ "ref": "origin/t3/server", "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", "status": "met", - "evidence": "latency_ms = 616.8 (want < 1000) at sha 3baa3387: ~/house-scanning-data/reports/e2e/20260927-075636-3baa3387/report.json", + "evidence": "latency_ms = 423.1 (want < 1000) at sha 3baa3387: ~/house-scanning-data/reports/e2e/20260927-085738-3baa3387/report.json", "links": [] }, { @@ -744,7 +744,7 @@ "ref": "origin/t3/server", "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", "status": "met", - "evidence": "exit 0: Installed 33 packages in 37ms / All checks passed! / 36 files already formatted", + "evidence": "exit 0: Installed 33 packages in 27ms / All checks passed! / 36 files already formatted", "links": [] }, { @@ -752,8 +752,8 @@ "text": "Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time", "ref": "origin/t3/server", "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", - "status": "met", - "evidence": "contract_ok = true (want True) at sha 3baa3387: ~/house-scanning-data/reports/e2e/20260927-075636-3baa3387/report.json", + "status": "gap", + "evidence": "contract_ok = false (want True) at sha 3baa3387: ~/house-scanning-data/reports/e2e/20260927-085738-3baa3387/report.json", "links": [] } ] @@ -764,7 +764,7 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb" + "sha": "c820cf150a73cb97527428d580c330f631253f9a" } ], "metrics": [ @@ -772,7 +772,7 @@ "id": "S3-1", "text": "CI build green", "ref": "origin/t3/ios-mvf", - "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", + "sha": "c820cf150a73cb97527428d580c330f631253f9a", "status": "no evidence yet", "evidence": "no open PR for t3/ios-mvf", "links": [] @@ -781,13 +781,13 @@ "id": "S3-2", "text": "Built as Swift 6 with strict concurrency", "ref": "origin/t3/ios-mvf", - "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", + "sha": "c820cf150a73cb97527428d580c330f631253f9a", "status": "met", "evidence": "all 2 found", "links": [ [ "ios/project.yml", - "https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/project.yml" + "https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/project.yml" ] ] }, @@ -795,30 +795,30 @@ "id": "S3-3", "text": "Zero build warnings", "ref": "origin/t3/ios-mvf", - "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", + "sha": "c820cf150a73cb97527428d580c330f631253f9a", "status": "stale", - "evidence": "report for sha f14947eb, ref at 6be087df: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", + "evidence": "report for sha f14947eb, ref at c820cf15: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", "links": [] }, { "id": "S3-4", "text": "Unit tests for coverage, auto-capture and guidance", "ref": "origin/t3/ios-mvf", - "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", + "sha": "c820cf150a73cb97527428d580c330f631253f9a", "status": "partial", "evidence": "found coverage, guidance; missing auto-capture", "links": [ [ "ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift" ], [ "ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift" ] ] }, @@ -826,21 +826,21 @@ "id": "S3-5", "text": "UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop", "ref": "origin/t3/ios-mvf", - "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", + "sha": "c820cf150a73cb97527428d580c330f631253f9a", "status": "partial", "evidence": "all 3 found (the tests exist; passing is S3-1)", "links": [ [ "ios/HouseScanUITests/FullFlowUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/FullFlowUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanUITests/FullFlowUITests.swift" ], [ "ios/HouseScanUITests/SourceRestartUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/SourceRestartUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanUITests/SourceRestartUITests.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -848,17 +848,17 @@ "id": "S3-6", "text": "Accessibility audit on every screen", "ref": "origin/t3/ios-mvf", - "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", + "sha": "c820cf150a73cb97527428d580c330f631253f9a", "status": "partial", "evidence": "3 matches in 2 files (an audit exists; which screens it covers needs a read)", "links": [ [ "ios/HouseScanUITests/AccessibilityAudit.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/AccessibilityAudit.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanUITests/AccessibilityAudit.swift" ], [ "ios/HouseScanUITests/ScreenStatesUITests.swift", - "https://github.com/SamGu-NRX/house-scanning/blob/6be087df207351fe3149b1924f78e91a92d2c3eb/ios/HouseScanUITests/ScreenStatesUITests.swift" + "https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanUITests/ScreenStatesUITests.swift" ] ] }, @@ -866,16 +866,16 @@ "id": "S3-6b", "text": "Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues", "ref": "origin/t3/ios-mvf", - "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", + "sha": "c820cf150a73cb97527428d580c330f631253f9a", "status": "stale", - "evidence": "report for sha a39d0a50, ref at 6be087df: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", + "evidence": "report for sha a39d0a50, ref at c820cf15: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json", "links": [] }, { "id": "S3-7", "text": "Screenshots of each state in the PR", "ref": "origin/t3/ios-mvf", - "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", + "sha": "c820cf150a73cb97527428d580c330f631253f9a", "status": "no evidence yet", "evidence": "no open PR for t3/ios-mvf", "links": [] @@ -884,16 +884,16 @@ "id": "S3-8", "text": "Exported scene.json validates against C1", "ref": "origin/t3/ios-mvf", - "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", + "sha": "c820cf150a73cb97527428d580c330f631253f9a", "status": "stale", - "evidence": "report for app_sha f14947eb, ref at 6be087df: ~/house-scanning-data/reports/e2e/20260927-075636-3baa3387/report.json", + "evidence": "report for app_sha f14947eb, ref at c820cf15: ~/house-scanning-data/reports/e2e/20260927-085738-3baa3387/report.json", "links": [] }, { "id": "S3-9", "text": "Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design", "ref": "origin/t3/ios-mvf", - "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", + "sha": "c820cf150a73cb97527428d580c330f631253f9a", "status": "gap", "evidence": "not yet reviewed", "links": [] @@ -906,11 +906,11 @@ "refs": [ { "ref": "origin/t3/ios-mvf", - "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb" + "sha": "c820cf150a73cb97527428d580c330f631253f9a" }, { "ref": "origin/t3/verification", - "sha": "21c8832408e8484d64f4e54b4ca903ddb6c5af4a" + "sha": "255bc848679e7327fecc77c0348d089dd1a3f334" } ], "metrics": [ @@ -918,25 +918,25 @@ "id": "S4-1", "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/ios-mvf", - "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", + "sha": "c820cf150a73cb97527428d580c330f631253f9a", "status": "stale", - "evidence": "report for app_sha f14947eb, ref at 6be087df: ~/house-scanning-data/reports/e2e/20260927-075636-3baa3387/report.json", + "evidence": "report for app_sha f14947eb, ref at c820cf15: ~/house-scanning-data/reports/e2e/20260927-085738-3baa3387/report.json", "links": [] }, { "id": "S4-2", "text": "Screenshot report per app state", "ref": "origin/t3/ios-mvf", - "sha": "6be087df207351fe3149b1924f78e91a92d2c3eb", + "sha": "c820cf150a73cb97527428d580c330f631253f9a", "status": "stale", - "evidence": "report for sha f14947eb, ref at 6be087df: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", + "evidence": "report for sha f14947eb, ref at c820cf15: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", "links": [] }, { "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "21c8832408e8484d64f4e54b4ca903ddb6c5af4a", + "sha": "255bc848679e7327fecc77c0348d089dd1a3f334", "status": "met", "evidence": "docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state.", "links": [] @@ -945,13 +945,13 @@ "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "21c8832408e8484d64f4e54b4ca903ddb6c5af4a", + "sha": "255bc848679e7327fecc77c0348d089dd1a3f334", "status": "met", - "evidence": "present at 21c88324", + "evidence": "present at 255bc848", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/21c8832408e8484d64f4e54b4ca903ddb6c5af4a/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/255bc848679e7327fecc77c0348d089dd1a3f334/verification/SCOREBOARD.md" ] ] } diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 6c10260fa..5be51c441 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -16,6 +16,7 @@ coverage_problems, declared_height_problems, expectation_problems, + expected_checks, footprint_problems, invariant_problems, less_coverage_problems, @@ -26,6 +27,7 @@ observed, outcome_at, outcome_lengths, + reach_gaps, run_starts, straight_pieces, tape_to_tap, @@ -61,7 +63,7 @@ ) SCENE = { - "meter": {"pos": [0.0, 4.0, 0.0], "wall_id": "w1"}, + "meter": {"pos": [0.0, 4.0, 0.0], "wall_id": "w1", "plus_minus_ft": 0.0}, # An explicit zero error keeps the reach arithmetic in these tests to the radii alone. "walls": [{"id": "w1", "baseline": [[-12.0, 0.0], [12.0, 0.0]], "plus_minus_ft": 0.0}], "objects": [ @@ -101,7 +103,31 @@ def check(outcome="pass", measured=5.0, error=0.3, threshold=3.0, cmp="at_least" return c -def result(decision="manual_review", spot=True, checks=None, sweep=None, sent=b"{}"): +# Every check a 3baa338 result lists (battery_clearance only when the scene or rules call for it). +SERVER_CHECKS = ( + "wall_backing", + "ground_surface", + "meter_working_space", + "gas_clearance", + "ac_clearance", + "drive_clearance", + "pool_clearance", + "opening_clearance", + "wall_equipment_above", + "facing_gap", + "headroom", + "route_path", + "route_length", +) + + +def result(decision="manual_review", spot=True, checks=None, sweep=None, sent=b"{}", complete=True): + """A result listing `checks` (default: gas passing); with `complete`, every other check a + server lists is added as a pass with no numbers, so a test is about the checks it names.""" + checks = [check()] if checks is None else list(checks) + if complete: + named = {c["id"] for c in checks} + checks += [passing(i) for i in SERVER_CHECKS if i not in named] s = None if spot: s = { @@ -133,7 +159,7 @@ def result(decision="manual_review", spot=True, checks=None, sweep=None, sent=b" }, "spot": s, "route": None if s is None else {"outcome": "pass"}, - "checks": [check()] if checks is None else checks, + "checks": checks, "missing_evidence": [], "ends": {}, "sweep": sweep @@ -255,24 +281,26 @@ def with_ground(out_ft: float, span: list[float]) -> dict: def test_a_pass_needs_ground_out_to_depth_plus_each_radius(): - # Start 1 covers [1, 3.58]. Gas (radius 1) needs ground out to 1.83 + 1 = 2.83 ft. + # Start 1 covers [1, 3.58]. Gas (radius 1) needs ground out to 1.83 + 1 = 2.83 ft, in front + # and just past its ends; the first stretch short of it is [0, 1]. msgs = invariant_problems(with_ground(2.5, [-12.0, 10.0]), result(), rules=RULES) - needs = "gas_clearance needs ground [1.00, 3.58] observed out to 2.83 ft, seen 2.50 ft" + needs = "gas_clearance needs ground [0.00, 1.00] observed out to 2.83 ft, seen 2.50 ft" assert any(needs in m for m in msgs) assert invariant_problems(with_ground(2.84, [-12.0, 10.0]), result(), rules=RULES) == [] def test_past_the_battery_the_ground_needed_shrinks_with_distance(): - # d past the battery's left end at 1, gas needs ground out to 1 - d: 0.5 for [0, 0.5]. + # d past the battery's left end at 1, gas (radius 1) needs ground out to D + sqrt(1 - d^2): + # 1.83 + 0.87 = 2.70 for [0, 0.5], and D + 1 = 2.83 for [0.5, 1]. scene = with_ground(2.84, [1.0, 10.0]) scene["coverage"]["observed"] += [ - {"band": "ground", "span_ft": [0.0, 0.5], "out_ft": 0.55}, - {"band": "ground", "span_ft": [0.5, 1.0], "out_ft": 1.0}, + {"band": "ground", "span_ft": [0.0, 0.5], "out_ft": 2.71}, + {"band": "ground", "span_ft": [0.5, 1.0], "out_ft": 2.84}, ] assert invariant_problems(scene, result(), rules=RULES) == [] - scene["coverage"]["observed"][-1]["out_ft"] = 0.9 + scene["coverage"]["observed"][-1]["out_ft"] = 2.8 msgs = invariant_problems(scene, result(), rules=RULES) - assert any("ground [0.50, 1.00] observed out to 1.00 ft, seen 0.90 ft" in m for m in msgs) + assert any("ground [0.50, 1.00] observed out to 2.83 ft, seen 2.80 ft" in m for m in msgs) def test_a_pass_needs_ground_along_the_wall_to_each_radius_without_slack(): @@ -348,7 +376,7 @@ def test_the_reach_widens_by_the_battery_position_error(): scene = with_ground(3.6, [-12.0, 10.0]) del scene["walls"][0]["plus_minus_ft"] msgs = invariant_problems(scene, result(), rules=RULES) - assert any("gas_clearance needs ground [1.00, 3.58] observed out to 3.71 ft" in m for m in msgs) + assert any("gas_clearance needs ground" in m and "out to 3.71 ft" in m for m in msgs) scene["coverage"]["observed"][1]["out_ft"] = 3.72 assert invariant_problems(scene, result(), rules=RULES) == [] @@ -392,8 +420,8 @@ def test_rules_come_from_rules_yaml_and_a_missing_one_is_loud(): assert rules.needs["battery_clearance"] == (Need("ground", 3.0), Need("wall", 3.0, 6.5)) assert rules.needs["opening_clearance"] == (Need("wall", 3.0, 6.5),) assert rules.needs["wall_backing"] == (Need("wall", 0.0, 3.25),) - assert rules.needs["facing_gap"] == (Need("facing", 0.0, 4.5),) - assert rules.needs["headroom"] == (Need("overhead", 0.0, 6.5),) + assert rules.needs["facing_gap"] == (Need("facing", 0.0, 4.5, along=True),) + assert rules.needs["headroom"] == (Need("overhead", 0.0, 6.5, along=True),) assert (rules.wall_join_ft, rules.battery_height_ft, rules.headroom_ft) == (0.6, 3.25, 6.5) assert (rules.width_ft, rules.errors["plane"], rules.route_height_ft) == (2.5, 0.75, 1.0) lower = RuleSet.from_yaml( @@ -406,6 +434,7 @@ def test_rules_come_from_rules_yaml_and_a_missing_one_is_loud(): def test_expectations(): r = result( + complete=False, # no route check, which `bad` below expects sweep=[ { "wall_id": "w1", @@ -421,7 +450,7 @@ def test_expectations(): "failing": [], "unsure": [], }, - ] + ], ) ok = { "decision_not": ["pass"], @@ -600,7 +629,7 @@ def test_transforms(): default_wall = copy.deepcopy(scene) del default_wall["walls"][0]["plus_minus_ft"] assert with_more_error(default_wall, RULES)["walls"][0]["plus_minus_ft"] == pytest.approx(4.64) - assert more["meter"]["plus_minus_ft"] == 0.8 + assert more["meter"]["plus_minus_ft"] == 0.5 # SCENE's meter is exact assert [o["source"] for o in tape_to_tap(scene)["objects"]] == ["tap", "tap"] assert tape_to_tap(SCENE) is None less = with_less_coverage(scene) @@ -683,9 +712,10 @@ def test_a_request_to_see_higher_than_the_view_reached_is_not_redundant(): | { "facing_gap": (Need("facing", 0.0, 22 / 12 + 3.0),), "headroom": (Need("overhead", 0.0, 6.5),), - "battery_clearance": (Need("ground", 100.0),), }, ) +FACING_RULES = replace(RULES, needs=RULES.needs | {"facing_gap": BAND_RULES.needs["facing_gap"]}) +HEADROOM_RULES = replace(RULES, needs=RULES.needs | {"headroom": BAND_RULES.needs["headroom"]}) def test_facing_and_headroom_passes_need_their_bands(): @@ -703,32 +733,39 @@ def test_a_short_facing_view_needs_a_measurement(): r = result(checks=[check(), passing("facing_gap")]) scene = copy.deepcopy(SCENE) scene["coverage"]["observed"].append({"band": "facing", "span_ft": [0.0, 5.0], "out_ft": 3.0}) - msgs = invariant_problems(scene, r, rules=BAND_RULES) + msgs = invariant_problems(scene, r, rules=FACING_RULES) assert any("out to 4.83 ft or measured, seen 3.00 ft" in m for m in msgs) scene["facing"] = [{"wall_id": "w1", "span_ft": [0.0, 5.0], "depth_ft": 8.0}] - assert invariant_problems(scene, r, rules=BAND_RULES) == [] + assert invariant_problems(scene, r, rules=FACING_RULES) == [] def test_an_overhead_seen_clear_all_the_way_up_settles_headroom(): r = result(checks=[check(), passing("headroom")]) scene = copy.deepcopy(SCENE) scene["coverage"]["observed"].append({"band": "overhead", "span_ft": [0.0, 5.0]}) - assert invariant_problems(scene, r, rules=BAND_RULES) == [] + assert invariant_problems(scene, r, rules=HEADROOM_RULES) == [] scene["coverage"]["observed"][-1]["out_ft"] = 5.0 # clear only to 5 ft, under 6.5 - assert any("overhead [1.00, 3.58]" in m for m in invariant_problems(scene, r, rules=BAND_RULES)) - - -def test_a_check_the_server_did_not_evaluate_needs_nothing(): - evaluated = result(checks=[check()], sweep=[PASS_RUN]) - assert not any( - "battery_clearance" in m for m in invariant_problems(SCENE, evaluated, rules=BAND_RULES) - ) - with_battery = result(checks=[check(), passing("battery_clearance")], sweep=[PASS_RUN]) assert any( - "battery_clearance" in m for m in invariant_problems(SCENE, with_battery, rules=BAND_RULES) + "overhead [1.00, 3.58]" in m for m in invariant_problems(scene, r, rules=HEADROOM_RULES) ) +def test_battery_clearance_is_needed_when_its_rule_reaches_past_gas_or_a_battery_is_marked(): + # BAND_RULES set battery_clearance's radius (100) above gas's (1): the server evaluates it, + # so a result that leaves it out is flagged and its coverage is needed anyway. + wide = replace(RULES, needs=RULES.needs | {"battery_clearance": (Need("ground", 100.0),)}) + r = result(checks=[check()], sweep=[PASS_RUN]) + msgs = invariant_problems(SCENE, r, rules=wide) + assert "the result leaves out check battery_clearance, which the rules define" in msgs + assert any("battery_clearance needs ground" in m for m in msgs) + # With its radius at gas's and no battery marked, the server doesn't evaluate it. + level = replace(RULES, needs=RULES.needs | {"battery_clearance": (Need("ground", 1.0),)}) + assert expected_checks(SCENE, level) == set(level.needs) - {"battery_clearance"} + marked = copy.deepcopy(SCENE) + marked["objects"].append({"type": "battery", "wall_id": "w1", "span_ft": [8.0, 9.0]}) + assert "battery_clearance" in expected_checks(marked, level) + + @pytest.mark.parametrize( ("wall", "error"), [ @@ -740,7 +777,8 @@ def test_a_check_the_server_did_not_evaluate_needs_nothing(): ], ) def test_the_wall_source_sets_the_default_error(wall, error): - scene = {"walls": [{"id": "w1", "baseline": [[0, 0], [9, 0]], **wall}]} + meter = {"pos": [0.0, 4.0, 0.0], "wall_id": "w1", "plus_minus_ft": 0.0} + scene = {"meter": meter, "walls": [{"id": "w1", "baseline": [[0, 0], [9, 0]], **wall}]} drift = 0.0 if "plus_minus_ft" in wall else 0.16 * 1.0 # far edge 1 ft from the meter assert battery_error(scene, RULES, "w1", 0.0, 1.0) == pytest.approx(error + drift) @@ -802,9 +840,9 @@ def test_a_facing_view_that_only_reaches_the_needed_depth_is_not_beyond(): scene = copy.deepcopy(SCENE) need = 22 / 12 + 3.0 scene["coverage"]["observed"].append({"band": "facing", "span_ft": [0.0, 5.0], "out_ft": need}) - assert any("facing [1.00, 3.58]" in m for m in invariant_problems(scene, r, rules=BAND_RULES)) + assert any("facing [1.00, 3.58]" in m for m in invariant_problems(scene, r, rules=FACING_RULES)) scene["coverage"]["observed"][-1]["out_ft"] = need + 0.01 - assert invariant_problems(scene, r, rules=BAND_RULES) == [] + assert invariant_problems(scene, r, rules=FACING_RULES) == [] def test_each_start_in_a_sweep_run_needs_only_its_own_reach(): @@ -813,7 +851,7 @@ def test_each_start_in_a_sweep_run_needs_only_its_own_reach(): # uncertain. As one battery at start 10's error, the run would ask 14.15 ft in front of start 1. pool = replace(RULES, needs={"pool_clearance": (Need("ground", 10.0),)}) scene = { - "meter": {"pos": [0.0, 4.0, 0.0], "wall_id": "w1"}, + "meter": {"pos": [0.0, 4.0, 0.0], "wall_id": "w1", "plus_minus_ft": 0.0}, "walls": [{"id": "w1", "baseline": [[-40.0, 0.0], [40.0, 0.0]]}], "objects": [], "coverage": { @@ -830,7 +868,7 @@ def test_each_start_in_a_sweep_run_needs_only_its_own_reach(): assert coverage_problems(scene, r, pool) == [] scene["coverage"]["observed"][1]["out_ft"] = 12.5 assert any( - "at start 1.00 pool_clearance needs ground [1.00, 2.00] observed out to 12.71 ft" in m + "at start 1.00 pool_clearance needs ground" in m and "out to 12.71 ft" in m for m in coverage_problems(scene, r, pool) ) @@ -847,10 +885,11 @@ def test_facing_and_headroom_need_the_band_past_the_battery_by_the_wall_error(): r = result(checks=[passing("facing_gap")]) scene = copy.deepcopy(SCENE) scene["walls"][0]["plus_minus_ft"] = 0.3 + scene["coverage"]["observed"][1]["out_ft"] = 3.2 # gas: 1.83 + 1.3 in front and past ends scene["coverage"]["observed"].append({"band": "facing", "span_ft": [1.0, 3.6]}) - assert any("facing [0.70, 1.00]" in m for m in invariant_problems(scene, r, rules=BAND_RULES)) + assert any("facing [0.70, 1.00]" in m for m in invariant_problems(scene, r, rules=FACING_RULES)) scene["coverage"]["observed"][-1]["span_ft"] = [0.7, 3.9] - assert invariant_problems(scene, r, rules=BAND_RULES) == [] + assert invariant_problems(scene, r, rules=FACING_RULES) == [] def test_facing_and_headroom_widen_by_the_default_error_with_drift(): @@ -1078,7 +1117,7 @@ def test_wall_backing_needs_the_wall_seen_past_the_battery_by_its_error(): def inside_corner(w2_out: float) -> dict: # g05: w1 runs +x with its outside at +z; w2 turns up +z from its end, facing back over w1. return { - "meter": {"pos": [-6.0, 4.0, 0.0], "wall_id": "w1"}, + "meter": {"pos": [-6.0, 4.0, 0.0], "wall_id": "w1", "plus_minus_ft": 0.0}, "walls": [ {"id": "w1", "baseline": [[-7.0, 0.0], [4.0, 0.0]], "plus_minus_ft": 0.0}, {"id": "w2", "baseline": [[4.0, 0.0], [4.0, 12.0]], "plus_minus_ft": 0.0}, @@ -1097,7 +1136,8 @@ def inside_corner(w2_out: float) -> dict: def test_ground_in_front_of_one_wall_may_be_seen_from_the_next_in_an_inside_corner(): # The battery at s 4.04 (x -1.96 to 0.62) needs a 10 ft pool radius, ground out to 11.83 in - # front; w1's band reaches 10.5. From x = 4, w2's band reaching 10.66 or more covers it. + # front and within 10 ft of its corners; w1's band reaches 10.5. From x = 4, w2's band + # reaching 12 covers the rest. pool = replace(RULES, needs={"pool_clearance": (Need("ground", 10.0),)}) run = { "wall_id": "w1", @@ -1107,9 +1147,9 @@ def test_ground_in_front_of_one_wall_may_be_seen_from_the_next_in_an_inside_corn "unsure": [], } r = result(spot=False, checks=[check() | {"id": "pool_clearance"}], sweep=[run]) - assert coverage_problems(inside_corner(10.7), r, pool) == [] + assert coverage_problems(inside_corner(12.0), r, pool) == [] short = coverage_problems(inside_corner(4.0), r, pool) # w2's band stops at x = 0 - assert any("pool_clearance needs ground [4.04, 6.62] observed out to 11.83" in m for m in short) + assert any("pool_clearance needs ground" in m and "out to 11.83" in m for m in short) def test_a_ground_gap_narrower_than_the_sampling_step_is_not_counted_seen(): @@ -1215,3 +1255,174 @@ def test_a_pass_by_the_smallest_reported_margin_is_a_pass(): at_most = check("pass", 19.999999, 0.0, 20.0, cmp="at_most") assert margin_problem(at_most) is None assert "should be unsure" in margin_problem(at_most | {"measured_ft": 20.0}) + + +# Final Opus review of 255bc84 (.local/final-opus-review-probes/pr13), findings 1 to 5. + +D_FT = 22 / 12 +POOL = replace(RULES, needs={"pool_clearance": (Need("ground", 10.0),)}) + + +def open_wall(ground_out: float, meter_pm: float | None = None) -> dict: + """The probes' scene: one 60 ft wall, error 0.3, everything but the ground seen.""" + meter = {"pos": [0.0, 4.0, 0.0], "wall_id": "w1"} + if meter_pm is not None: + meter["plus_minus_ft"] = meter_pm + return { + "meter": meter, + "walls": [{"id": "w1", "baseline": [[-30.0, 0.0], [30.0, 0.0]], "plus_minus_ft": 0.3}], + "objects": [], + "coverage": { + "ends": {"left": {"kind": "limit"}, "right": {"kind": "limit"}}, + "observed": [ + {"band": "wall", "span_ft": [-30.0, 30.0]}, + {"band": "ground", "span_ft": [-30.0, 30.0], "out_ft": ground_out}, + ], + }, + } + + +def pass_run(a: float, b: float, wall_id: str = "w1") -> dict: + return {"wall_id": wall_id, "start_ft": [a, b], "outcome": "pass", "failing": [], "unsure": []} + + +def test_the_battery_error_includes_the_meters_error_in_plan(): + # meter_gap.py: ground seen to D + 10 + 0.3 + 0.15. With the meter's default 0.3 the pool + # needs D + 10 + 0.6 (server 3baa338 _errors: wall at the far edge plus the meter's). + scene = open_wall(D_FT + 10.45) + r = result(spot=False, checks=[check() | {"id": "pool_clearance"}], sweep=[pass_run(2.0, 2.0)]) + assert any( + "observed out to 12.43 ft, seen 12.28 ft" in m for m in coverage_problems(scene, r, POOL) + ) + assert battery_error(scene, RULES, "w1", 2.0, 2.0 + 31 / 12) == pytest.approx(0.6) + + +def test_the_battery_error_includes_the_slide_round_a_corner(): + # Round a right angle from the meter, the meter's 0.3 slides the battery by 0.3 x sqrt(2), + # more than its 0.3 in plan (server 3baa338 _slide). + scene = open_wall(20.0, meter_pm=0.3) + scene["walls"] = [ + {"id": "w1", "baseline": [[-30.0, 0.0], [10.0, 0.0]], "plus_minus_ft": 0.3}, + {"id": "w2", "baseline": [[10.0, 0.0], [10.0, -20.0]], "plus_minus_ft": 0.3}, + ] + assert battery_error(scene, RULES, "w2", 15.0, 17.58) == pytest.approx( + 0.3 + 0.3 * 2**0.5 + 0.3 * 2**0.5 + ) + + +def test_ground_just_past_the_battery_is_needed_out_to_the_straight_line_reach(): + # ends.py: ground in front seen to 12.5, just past the ends only to 10.4. A point d past an + # end is within R of the footprint out to D + sqrt(R^2 - d^2), not R - d. + lo, hi = 5.0, 5.0 + 31 / 12 + scene = open_wall(10.4, meter_pm=0.0) + scene["coverage"]["observed"][1:] = [ + {"band": "ground", "span_ft": [-30.0, lo], "out_ft": 10.4}, + {"band": "ground", "span_ft": [lo, hi], "out_ft": 12.5}, + {"band": "ground", "span_ft": [hi, 30.0], "out_ft": 10.4}, + ] + gaps = reach_gaps(scene, POOL, "pool_clearance", "w1", lo, hi) + assert gaps and gaps[0][1].startswith("ground [-5.30, 5.00] observed out to 12.13 ft") + + +def test_a_passing_footprint_keeps_the_wall_error_clear_of_its_stretch_end(): + # corner.py: a run whose last footprint ends at the corner s = 10 is unsure at 3baa338 + # (check_backing: within e of the segment's end); relabelled pass, it must be flagged. + scene = open_wall(20.0) + scene["walls"] = [ + {"id": "w1", "baseline": [[-30.0, 0.0], [10.0, 0.0]], "plus_minus_ft": 0.3}, + {"id": "w2", "baseline": [[10.0, 0.0], [10.0, -20.0]], "plus_minus_ft": 0.3}, + ] + r = result(spot=False, sweep=[pass_run(6.846, 10.0 - 31 / 12)]) + assert footprint_problems(scene, r, RULES) == [ + "sweep pass start 7.42 ends within its error (0.60 ft) of its straight wall's end at 10.00" + ] + r["sweep"][0]["outcome"] = "unsure" # unsure may stand that close + assert footprint_problems(scene, r, RULES) == [] + + +def test_a_footprint_past_its_stretch_by_more_than_the_slack_is_flagged(): + # With no error, a pass ending 0.2 ft past its wall's end: beyond the 0.05 ft slack. + scene = open_wall(20.0, meter_pm=0.0) + scene["walls"][0]["plus_minus_ft"] = 0.0 + r = result(spot=False, sweep=[pass_run(30.2 - 31 / 12, 30.2 - 31 / 12)]) + assert footprint_problems(scene, r, RULES) == [ + "sweep pass start 27.62 runs past its straight wall [-30.00, 30.00]" + ] + + +def test_a_result_that_drops_a_check_is_flagged_and_its_coverage_still_needed(): + # dropped.py: a PASS judged on a scene whose ground reaches drive's 5 ft but not pool's 10. + rules = replace( + RULES, + needs={ + "pool_clearance": (Need("ground", 10.0),), + "drive_clearance": (Need("ground", 5.0),), + }, + ) + scene = open_wall(D_FT + 5 + 0.6 + 0.1) + full = result(spot=False, sweep=[pass_run(2.0, 2.0)]) + dropped = full | {"checks": [c for c in full["checks"] if c["id"] != "pool_clearance"]} + msgs = invariant_problems(scene, dropped, rules=rules) + assert any("pool_clearance needs ground" in m for m in msgs) + assert "the result leaves out check pool_clearance, which the rules define" in msgs + + +def test_the_route_wall_is_needed_past_the_meter_by_its_error(): + # route_end.py: the wall seen from s = 0 only. The route starts at the meter, which may + # stand 0.3 ft either side, so the wall is needed from -0.3 (server README route_path). + scene = open_wall(20.0) + scene["coverage"]["observed"][0]["span_ft"] = [0.0, 30.0] + r = result(spot=False, checks=[], sweep=[pass_run(3.7, 14.3875)]) + assert any( + "the wall and cable route [-0.30, 16.97] were not all observed" in m + for m in coverage_problems(scene, r, RULES) + ) + + +# Flags at server 3baa338 after the review fixes, each the checker's (see the commit message). + + +def test_spans_along_the_walls_widen_by_the_along_error_not_the_meters(): + # The app scan: headroom needs the overhead band over [s0 - e, s1 + e] with e the wall's + # error plus the slide (0 on the meter's wall), not the meter's 0.3 in plan. + headroom = replace(RULES, needs={"headroom": (Need("overhead", 0.0, 6.5, along=True),)}) + scene = open_wall(20.0) # wall 0.3, meter default 0.3 + scene["coverage"]["observed"].append({"band": "overhead", "span_ft": [1.7, 4.9]}) + r = result(spot=False, sweep=[pass_run(2.0, 2.0)]) + assert coverage_problems(scene, r, headroom) == [] # needs [1.7, 4.88] + scene["coverage"]["observed"][-1]["span_ft"] = [1.8, 4.9] + assert any("overhead [1.70, 1.80]" in m for m in coverage_problems(scene, r, headroom)) + + +def test_ground_short_of_the_reach_by_less_than_the_seen_growth_is_seen(): + # d-gas-drift: ground seen to 14.50 where 14.5007 is needed. The server grows what was seen + # by 0.0049 ft before comparing (SEEN_GROWTH_FT), so a 0.0007 ft shortfall is seen. + scene = open_wall(D_FT + 10.6 - 0.0007) + r = result(spot=False, sweep=[pass_run(2.0, 2.0)]) + assert coverage_problems(scene, r, POOL) == [] + scene["coverage"]["observed"][1]["out_ft"] = D_FT + 10.6 - 0.01 + assert coverage_problems(scene, r, POOL) != [] + + +def test_ground_past_an_inside_corner_behind_the_next_wall_is_the_house(): + # g05 after a capture: past the battery's right end, the straight-line reach runs past the + # corner at x = 4 to ground behind w2, which the server's yard model leaves out. + pool = replace(RULES, needs={"pool_clearance": (Need("ground", 10.0),)}) + scene = inside_corner(10.66) + scene["coverage"]["observed"][1:] = [ + {"band": "ground", "span_ft": [-7.0, 10.0], "out_ft": 10.66}, + {"band": "ground", "span_ft": [10.0, 22.0], "out_ft": 12.0}, + ] + run = { + "wall_id": "w1", + "start_ft": [4.04, 4.04], + "outcome": "pass", + "failing": [], + "unsure": [], + } + r = result(spot=False, checks=[check() | {"id": "pool_clearance"}], sweep=[run]) + assert coverage_problems(scene, r, pool) == [] + plan = GroundPlan(scene, pool) + assert not plan.required((6.0, 5.0)) # behind w2, in front of no wall + assert plan.required((1.0, 11.5)) # in the corner's yard, in front of w1 + assert plan.required((-9.0, 3.0)) # past the chain's left end From b9f1601c0d0ea161eb346791c36157b231cfa309 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 09:08:15 -0500 Subject: [PATCH 58/59] verification: three more false certifications from Codex on 255bc84 Each with a test built from the exact faulty response: - 4114865912: a pass with checks [] got through. A pass must list checks, and a result that leaves out a check the rules define is flagged even when its list is empty. - 4114865917: spot.center and meter_offset_ft moved together by 5 ft, footprint unchanged, got through. The centre must be the footprint's centre and the middle of span_ft on its wall, half the battery's depth out. - 4114865919: manual_review with a failing spot, check and sweep got through. A manual-review spot can't fail, and a spot's checks must agree with its outcome. At 3baa338 these add no flags: 49 of 50 as before, the ETH3D server issue. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/hsverify/resultcheck.py | 44 ++++++++++++++++++++++++-- verification/tests/test_e2e.py | 30 +++++++++++++++++- verification/tests/test_resultcheck.py | 43 +++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 3 deletions(-) diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 706b55ee3..077ed609a 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -288,6 +288,8 @@ def invariant_problems( if decision == "pass": if spot is None or spot.get("outcome") != "pass": problems.append("decision pass without a passing spot") + if not checks: + problems.append("decision pass with no checks") not_passing = [c["id"] for c in checks if c["outcome"] != "pass"] if not_passing: problems.append(f"decision pass with checks not passing: {not_passing}") @@ -307,6 +309,15 @@ def invariant_problems( problems.append("decision reject while asking for more evidence") if decision == "manual_review" and not result.get("reasons"): problems.append("decision manual_review without reasons") + # A person reviews a spot that may pass; one that fails is a reject's, not theirs. + if decision == "manual_review" and spot is not None and spot["outcome"] == "fail": + problems.append("decision manual_review with a failing spot") + # A spot's outcome is its checks': a pass has every check passing, an unsure spot none failing. + if spot is not None and spot["outcome"] in ("pass", "unsure"): + for c in checks: + if c["outcome"] == "fail" or (spot["outcome"] == "pass" and c["outcome"] != "pass"): + verb = "fails" if c["outcome"] == "fail" else f"is {c['outcome']}" + problems.append(f"the spot is {spot['outcome']} but its check {c['id']} {verb}") if (spot is None) != (result.get("route") is None): problems.append("spot and route must be both null or both present") @@ -339,6 +350,7 @@ def invariant_problems( problems += spot_sweep_problems(result) if rules is not None: problems += footprint_problems(scene, result, rules) + problems += spot_position_problems(scene, result, rules) problems += missing_evidence_problems(scene, result, rules) if rules is not None: problems += coverage_problems(scene, result, rules) @@ -421,6 +433,36 @@ def off_wall(lo: float, hi: float, outcome: str, wall_id: str) -> str | None: return problems +def spot_position_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: + """Where the app places the battery must be where the result says it stands: spot.center is + its footprint's centre, and that is the middle of span_ft on the wall, half the battery's + depth out. meter_offset_ft is checked against center elsewhere, so a centre moved together + with its offset is caught here.""" + spot = result.get("spot") + if spot is None: + return [] + problems = [] + cx, cz = spot["center"] + corners = spot.get("footprint") or [] + if corners: + fx = sum(pt[0] for pt in corners) / len(corners) + fz = sum(pt[1] for pt in corners) / len(corners) + if math.dist((cx, cz), (fx, fz)) > OFFSET_TOL_FT: + problems.append( + f"spot.center [{cx:.2f}, {cz:.2f}] is not its footprint's centre " + f"[{fx:.2f}, {fz:.2f}]" + ) + lo, hi = spot["span_ft"] + plan = GroundPlan(scene, rules) + want = plan.point((lo + hi) / 2, (lo + hi) / 2, rules.depth_ft / 2) + if want is not None and math.dist((cx, cz), want) > COLLINEAR_FT: + problems.append( + f"spot.center [{cx:.2f}, {cz:.2f}] is not the centre of span [{lo:.2f}, {hi:.2f}] " + f"on its wall, [{want[0]:.2f}, {want[1]:.2f}]" + ) + return problems + + def spot_sweep_problems(result: dict) -> list[str]: """The chosen spot is one of the evaluated starts, so the sweep must give its start the spot's own outcome, and a passing spot must lie in a passing run. A result could otherwise @@ -634,8 +676,6 @@ def expected_checks(scene: dict, rules: RuleSet) -> set[str]: def dropped_check_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: listed = {c["id"] for c in result.get("checks", [])} - if not listed: - return [] wanted = expected_checks(scene, rules) | ALWAYS_CHECKS return [ f"the result leaves out check {c}, which the rules define" for c in sorted(wanted - listed) diff --git a/verification/tests/test_e2e.py b/verification/tests/test_e2e.py index a9bc642e9..13d70f552 100644 --- a/verification/tests/test_e2e.py +++ b/verification/tests/test_e2e.py @@ -202,7 +202,35 @@ def do_POST(self): }, "spot": None, "route": None, - "checks": [], + # Every check a server lists, passing with no numbers, so no check is left out. + "checks": [ + { + "id": i, + "label": i, + "outcome": "pass", + "reason": "", + "measured_ft": None, + "plus_minus_ft": None, + "threshold_ft": None, + "comparison": "at_least", + "rule": {"key": i, "source": "", "placeholder": False}, + } + for i in ( + "wall_backing", + "ground_surface", + "meter_working_space", + "gas_clearance", + "ac_clearance", + "drive_clearance", + "pool_clearance", + "opening_clearance", + "wall_equipment_above", + "facing_gap", + "headroom", + "route_path", + "route_length", + ) + ], "missing_evidence": [], "ends": {}, "sweep": [ diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 5be51c441..430067132 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -1426,3 +1426,46 @@ def test_ground_past_an_inside_corner_behind_the_next_wall_is_the_house(): assert not plan.required((6.0, 5.0)) # behind w2, in front of no wall assert plan.required((1.0, 11.5)) # in the corner's yard, in front of w1 assert plan.required((-9.0, 3.0)) # past the chain's left end + + +# Codex on 255bc84, verified by the #11 caretaker: three more false certifications. + + +def test_a_pass_with_no_checks_is_flagged(): + # 4114865912: decision pass, a passing spot and sweep, and checks []. + r = result(decision="pass", checks=[], complete=False) + r["policy"]["auto_approve"] = True + msgs = invariant_problems(broadly_seen(), r, rules=RULES) + assert "decision pass with no checks" in msgs + assert "the result leaves out check gas_clearance, which the rules define" in msgs + + +def test_the_spot_centre_must_sit_on_its_footprint_on_the_wall(): + # 4114865917: centre and meter_offset moved together by 5 ft, footprint unchanged. + r = result() + r["spot"]["center"] = [7.29, 0.92] + r["spot"]["meter_offset_ft"] = [7.29, 0.92] + msgs = invariant_problems(SCENE, r, rules=RULES) + assert any(m.startswith("spot.center [7.29, 0.92] is not its footprint's centre") for m in msgs) + assert any(m.startswith("spot.center [7.29, 0.92] is not the centre of span") for m in msgs) + assert not any("spot.center" in m for m in invariant_problems(SCENE, result(), rules=RULES)) + + +def test_manual_review_needs_a_spot_that_does_not_fail(): + # 4114865919: manual_review, a failing spot, a failing check and a matching failing run. + run = { + "wall_id": "w1", + "start_ft": [1.0, 1.0], + "outcome": "fail", + "failing": ["gas_clearance"], + "unsure": [], + } + r = result(checks=[check("fail", 2.0)], sweep=[run]) + r["spot"]["outcome"] = "fail" + r["stats"] |= {"pass": 0, "fail": 1} + msgs = invariant_problems(SCENE, r, rules=RULES) + assert "decision manual_review with a failing spot" in msgs + passing_spot = result(checks=[check("fail", 2.0)]) + assert "the spot is pass but its check gas_clearance fails" in invariant_problems( + SCENE, passing_spot, rules=RULES + ) From fb20cf77f1b4882911f0eec88863c873d22a0ee7 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 09:49:42 -0500 Subject: [PATCH 59/59] verification: app rows at #10's freeze c820cf1; band gaps under 0.01 ft are rounding - make sim-app at c820cf1 on A4EE0B20, synthetic replay, hosted public server: 17 states, a server gap request answered "can't get there", three spot checks, the result and AR. S4-2 is met there and S4-1 partial (the real ADVIO replay is blocked by its data). - The scan's less-coverage resend flagged wall_equipment_above for a 0.0037 ft sliver past the seen wall. The server's missing() drops 1D gaps under 0.01 ft (COVERAGE_TOLERANCE_FT); the wall, route, facing and overhead checks now do too. Tested at 0.0037 and 0.011 ft. e2e at 3baa338 with that scan: 49 of 50, the ETH3D server issue. #11 moved to 6c7bea1 during the run, so S2-6 and S2-8 read stale until the turn-2 re-pin. Co-Authored-By: Claude Opus 5.5 (1M context) --- verification/README.md | 7 +- verification/SCOREBOARD.md | 54 ++++----- verification/hsverify/resultcheck.py | 30 ++++- verification/scoreboard/metrics.yaml | 6 +- verification/scoreboard/scoreboard.json | 154 ++++++++++++------------ verification/tests/test_resultcheck.py | 10 ++ 6 files changed, 150 insertions(+), 111 deletions(-) diff --git a/verification/README.md b/verification/README.md index 8e601d5a0..b20d4031c 100644 --- a/verification/README.md +++ b/verification/README.md @@ -19,9 +19,10 @@ Reports go to `~/house-scanning-data/reports/`, outside git, because replay fram non-commercial datasets. What each check asserts is in the docstrings of `hsverify/e2e.py`, `resultcheck.py` and `simrun.py`. -**Result.** At `t3/server` `3baa338`, with the app's upload from `f14947e` (synthetic replay, hosted -server), 49 of 50 scenes pass (45 answered and checked; slowest real scene 0.42 s). The ETH3D scene -fails: after its one requested capture the server leaves pool unsure with no request or reason. +**Result.** At `t3/server` `3baa338`, with the app's upload from `c820cf1` (#10's freeze, synthetic +replay, hosted server), 49 of 50 scenes pass (45 answered and checked; slowest real scene 0.48 s). +The ETH3D scene fails: after its one requested capture the server leaves pool unsure with no +request or reason. **What it changed.** S2 fixed seven defects it found: - unseen ground passing the pool check; diff --git a/verification/SCOREBOARD.md b/verification/SCOREBOARD.md index 5ef26f2bf..3f5050b69 100644 --- a/verification/SCOREBOARD.md +++ b/verification/SCOREBOARD.md @@ -1,29 +1,28 @@ # Scoreboard -Generated 2026-09-27 08:59 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran for S2-1, S2-7 only. Probes are defined in `verification/scoreboard/metrics.yaml`. +Generated 2026-09-27 09:49 CDT by `make scoreboard` (`uv run python -m hsverify.scoreboard`) in `verification/`; slow probes ran for S2-1, S2-7 only. Probes are defined in `verification/scoreboard/metrics.yaml`. ## Open pull requests | # | Branch | Head | Draft | CI | Updated | | --- | --- | --- | --- | --- | --- | | [#172](https://github.com/SamGu-NRX/house-scanning/pull/172) | `t3/ci-flake-fixes-beta-2` | `c3aeabea` | no | 5 success, 1 skipped | 2026-09-27 08:28 CDT | -| [#168](https://github.com/SamGu-NRX/house-scanning/pull/168) | `t3/164-space-ends` | `381d0d3a` | yes | 2 success, 2 skipped | 2026-09-27 08:35 CDT | -| [#165](https://github.com/SamGu-NRX/house-scanning/pull/165) | `t3/ci-flake-fixes` | `4925f37f` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 08:58 CDT | +| [#165](https://github.com/SamGu-NRX/house-scanning/pull/165) | `t3/ci-flake-fixes` | `4925f37f` | no | 3 success, 1 failure, 1 skipped; failing: iOS build | 2026-09-27 08:58 CDT | | [#158](https://github.com/SamGu-NRX/house-scanning/pull/158) | `t3/live-fog` | `f57310ca` | no | 5 success, 1 failure, 1 skipped; failing: iOS build | 2026-09-27 06:07 CDT | -| [#143](https://github.com/SamGu-NRX/house-scanning/pull/143) | `t3/practice-meter` | `d298dfbe` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 08:01 CDT | -| [#134](https://github.com/SamGu-NRX/house-scanning/pull/134) | `shrey/readme` | `2bd6a2a2` | no | 4 success | 2026-09-27 08:19 CDT | +| [#143](https://github.com/SamGu-NRX/house-scanning/pull/143) | `t3/practice-meter` | `dd219a05` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 09:45 CDT | +| [#134](https://github.com/SamGu-NRX/house-scanning/pull/134) | `shrey/readme` | `9ceeece3` | no | 4 success | 2026-09-27 09:45 CDT | | [#124](https://github.com/SamGu-NRX/house-scanning/pull/124) | `t3/integration-viewer` | `9c0680b6` | no | 5 success | 2026-09-27 05:47 CDT | | [#123](https://github.com/SamGu-NRX/house-scanning/pull/123) | `t3/hunter-integration` | `17b53baa` | yes | 2 success, 1 skipped | 2026-09-27 07:17 CDT | | [#122](https://github.com/SamGu-NRX/house-scanning/pull/122) | `t3/41-server-requests-text` | `7046c93b` | no | 4 success, 1 skipped | 2026-09-27 08:10 CDT | | [#120](https://github.com/SamGu-NRX/house-scanning/pull/120) | `t3/41-walk-task-hold` | `30ba9b39` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 08:24 CDT | -| [#119](https://github.com/SamGu-NRX/house-scanning/pull/119) | `t3/41-anchored-marks` | `f03430fe` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 07:49 CDT | -| [#109](https://github.com/SamGu-NRX/house-scanning/pull/109) | `t3/41-gap-loop-result` | `ecc7a576` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 08:36 CDT | +| [#119](https://github.com/SamGu-NRX/house-scanning/pull/119) | `t3/41-anchored-marks` | `f03430fe` | no | 4 success, 1 skipped | 2026-09-27 07:49 CDT | +| [#109](https://github.com/SamGu-NRX/house-scanning/pull/109) | `t3/41-gap-loop-result` | `ecc7a576` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 09:21 CDT | | [#108](https://github.com/SamGu-NRX/house-scanning/pull/108) | `t3/41-ground-and-meter-tap` | `92c8f92f` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 07:49 CDT | | [#104](https://github.com/SamGu-NRX/house-scanning/pull/104) | `t3/41-gate-coaching` | `5f8f8777` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 08:53 CDT | | [#102](https://github.com/SamGu-NRX/house-scanning/pull/102) | `t3/41-card-replies` | `4183f61b` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 08:52 CDT | | [#100](https://github.com/SamGu-NRX/house-scanning/pull/100) | `t3/41-ar-canvas-fallback` | `0ada11b9` | no | 4 success, 1 pending | 2026-09-27 08:29 CDT | -| [#99](https://github.com/SamGu-NRX/house-scanning/pull/99) | `t3/41-end-preview` | `f5a27c64` | no | 4 success, 1 pending | 2026-09-27 07:08 CDT | -| [#90](https://github.com/SamGu-NRX/house-scanning/pull/90) | `t3/check-experiments` | `9d66b31c` | no | 6 success | 2026-09-27 05:47 CDT | +| [#99](https://github.com/SamGu-NRX/house-scanning/pull/99) | `t3/41-end-preview` | `f5a27c64` | no | 5 success | 2026-09-27 09:33 CDT | +| [#90](https://github.com/SamGu-NRX/house-scanning/pull/90) | `t3/check-experiments` | `9d66b31c` | no | 6 success | 2026-09-27 09:28 CDT | | [#88](https://github.com/SamGu-NRX/house-scanning/pull/88) | `t3/autodetect-lab` | `538f60e1` | yes | 3 success | 2026-09-27 05:49 CDT | | [#87](https://github.com/SamGu-NRX/house-scanning/pull/87) | `t3/41-zip-deflate` | `aa5999e0` | no | 4 success, 1 pending | 2026-09-27 08:24 CDT | | [#86](https://github.com/SamGu-NRX/house-scanning/pull/86) | `t3/experience` | `b442f0a0` | yes | 3 success, 1 skipped | 2026-09-27 05:48 CDT | @@ -32,10 +31,11 @@ Generated 2026-09-27 08:59 CDT by `make scoreboard` (`uv run python -m hsverify. | [#59](https://github.com/SamGu-NRX/house-scanning/pull/59) | `t3/signals-lab` | `0cde05e3` | yes | 3 success | 2026-09-27 05:50 CDT | | [#58](https://github.com/SamGu-NRX/house-scanning/pull/58) | `t3/ios-ci-no-preboot` | `7f1037e0` | no | 4 success, 1 skipped | 2026-09-27 07:02 CDT | | [#56](https://github.com/SamGu-NRX/house-scanning/pull/56) | `t3/walk-guidance-gaps` | `16b1c249` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 08:24 CDT | -| [#55](https://github.com/SamGu-NRX/house-scanning/pull/55) | `t3/motion-permission` | `b289c7df` | no | 4 success, 1 skipped | 2026-09-27 07:58 CDT | -| [#52](https://github.com/SamGu-NRX/house-scanning/pull/52) | `t3/battery-mark` | `086ad94b` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 07:23 CDT | +| [#55](https://github.com/SamGu-NRX/house-scanning/pull/55) | `t3/motion-permission` | `b289c7df` | no | 4 success, 1 skipped | 2026-09-27 09:46 CDT | +| [#52](https://github.com/SamGu-NRX/house-scanning/pull/52) | `t3/battery-mark` | `086ad94b` | no | 3 success, 1 pending, 1 skipped | 2026-09-27 09:31 CDT | | [#49](https://github.com/SamGu-NRX/house-scanning/pull/49) | `t3/gap-skip-continues` | `6c3ac625` | no | 4 success, 1 skipped | 2026-09-27 08:47 CDT | | [#46](https://github.com/SamGu-NRX/house-scanning/pull/46) | `t3/field-test-run2` | `afd02c82` | no | 3 success | 2026-09-27 05:48 CDT | +| [#23](https://github.com/SamGu-NRX/house-scanning/pull/23) | `t3/device-field-test` | `321ea7e2` | no | 3 success | 2026-09-27 05:49 CDT | ## S1 Evals on real data @@ -51,46 +51,46 @@ Generated 2026-09-27 08:59 CDT by `make scoreboard` (`uv run python -m hsverify. ## S2 Server and solver -`origin/t3/server` at [`3baa3387`](https://github.com/SamGu-NRX/house-scanning/tree/3baa3387a6877961345afa7785f1530c1aeb2b1b). 7 met, 1 gap. +`origin/t3/server` at [`6c7bea12`](https://github.com/SamGu-NRX/house-scanning/tree/6c7bea122371711284d5672410b6e1afde22b7bb). 6 met, 2 stale. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S2-1 | pytest green, the property tests under CI's fixed Hypothesis examples | met | exit 0: return lib.difference_prec(a, b, grid_size, **kwargs) / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 380 passed, 1 xfailed, 5 warnings in 61.63s (0:01:01) | -| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_golden_public.py) | -| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | met | all 4 found [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_properties.py), [server/tests/test_final_review.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_final_review.py) | -| S2-4 | Schemas published | met | present at 3baa3387 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/schemas/result.schema.json) | -| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_existing_battery.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_existing_battery.py), [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_schemas.py) | -| S2-6 | A real-derived scene returns a result in under 1 s | met | latency_ms = 423.1 (want < 1000) at sha 3baa3387: ~/house-scanning-data/reports/e2e/20260927-085738-3baa3387/report.json | -| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 27ms / All checks passed! / 36 files already formatted | -| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | gap | contract_ok = false (want True) at sha 3baa3387: ~/house-scanning-data/reports/e2e/20260927-085738-3baa3387/report.json | +| S2-1 | pytest green, the property tests under CI's fixed Hypothesis examples | met | exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 388 passed, 1 xfailed, 1 warning in 121.47s (0:02:01) | +| S2-2 | Every public golden test 01-14 of docs/research/t3-lane-c-review.md present | met | all 14 found [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7bea122371711284d5672410b6e1afde22b7bb/server/tests/test_golden_public.py) | +| S2-3 | Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail | met | all 4 found [server/tests/test_coderabbit_review.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7bea122371711284d5672410b6e1afde22b7bb/server/tests/test_coderabbit_review.py), [server/tests/test_properties.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7bea122371711284d5672410b6e1afde22b7bb/server/tests/test_properties.py), [server/tests/test_final_review.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7bea122371711284d5672410b6e1afde22b7bb/server/tests/test_final_review.py) | +| S2-4 | Schemas published | met | present at 6c7bea12 [server/schemas/scene.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/6c7bea122371711284d5672410b6e1afde22b7bb/server/schemas/scene.schema.json), [server/schemas/result.schema.json](https://github.com/SamGu-NRX/house-scanning/blob/6c7bea122371711284d5672410b6e1afde22b7bb/server/schemas/result.schema.json) | +| S2-5 | Schemas validated in tests | met | all 2 found [server/tests/test_existing_battery.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7bea122371711284d5672410b6e1afde22b7bb/server/tests/test_existing_battery.py), [server/tests/test_golden_public.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7bea122371711284d5672410b6e1afde22b7bb/server/tests/test_golden_public.py), [server/tests/test_schemas.py](https://github.com/SamGu-NRX/house-scanning/blob/6c7bea122371711284d5672410b6e1afde22b7bb/server/tests/test_schemas.py) | +| S2-6 | A real-derived scene returns a result in under 1 s | stale | report for sha 3baa3387, ref at 6c7bea12: ~/house-scanning-data/reports/e2e/20260927-094443-3baa3387/report.json | +| S2-7 | ruff clean | met | exit 0: Installed 33 packages in 40ms / All checks passed! / 37 files already formatted | +| S2-8 | Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time | stale | report for sha 3baa3387, ref at 6c7bea12: ~/house-scanning-data/reports/e2e/20260927-094443-3baa3387/report.json | ## S3 iOS capture app -`origin/t3/ios-mvf` at [`c820cf15`](https://github.com/SamGu-NRX/house-scanning/tree/c820cf150a73cb97527428d580c330f631253f9a). 1 met, 3 partial, 1 gap, 3 stale, 2 no evidence yet. +`origin/t3/ios-mvf` at [`c820cf15`](https://github.com/SamGu-NRX/house-scanning/tree/c820cf150a73cb97527428d580c330f631253f9a). 3 met, 3 partial, 1 gap, 1 stale, 2 no evidence yet. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | | S3-1 | CI build green | no evidence yet | no open PR for t3/ios-mvf | | S3-2 | Built as Swift 6 with strict concurrency | met | all 2 found [ios/project.yml](https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/project.yml) | -| S3-3 | Zero build warnings | stale | report for sha f14947eb, ref at c820cf15: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | +| S3-3 | Zero build warnings | met | build.warnings = length 0 (want 0) at sha c820cf15: ~/house-scanning-data/reports/sim/20260927-093459-t3-ios-mvf-c820cf1-c820cf15-replay/report.json | | S3-4 | Unit tests for coverage, auto-capture and guidance | partial | found coverage, guidance; missing auto-capture [ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanKit/Tests/HouseScanKitTests/GroundPatchTests.swift), [ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift](https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanKit/Tests/HouseScanKitTests/JSONSchemaValidatorTests.swift) | | S3-5 | UI tests run the whole flow from a real replay with no manual input, including one gap-instruction-recapture loop | partial | all 3 found (the tests exist; passing is S3-1) [ios/HouseScanUITests/FullFlowUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanUITests/FullFlowUITests.swift), [ios/HouseScanUITests/SourceRestartUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanUITests/SourceRestartUITests.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanUITests/ScreenStatesUITests.swift) | | S3-6 | Accessibility audit on every screen | partial | 3 matches in 2 files (an audit exists; which screens it covers needs a read) [ios/HouseScanUITests/AccessibilityAudit.swift](https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanUITests/AccessibilityAudit.swift), [ios/HouseScanUITests/ScreenStatesUITests.swift](https://github.com/SamGu-NRX/house-scanning/blob/c820cf150a73cb97527428d580c330f631253f9a/ios/HouseScanUITests/ScreenStatesUITests.swift) | | S3-6b | Independent accessibility audit (verification a11yaudit: Apple's audit on each screen the run reached, at least nine, plus symbol-name button labels) finds no issues | stale | report for sha a39d0a50, ref at c820cf15: ~/house-scanning-data/reports/a11y/20260926-071248-t3-ios-mvf-a39d0a50-head-sample/report.json | | S3-7 | Screenshots of each state in the PR | no evidence yet | no open PR for t3/ios-mvf | -| S3-8 | Exported scene.json validates against C1 | stale | report for app_sha f14947eb, ref at c820cf15: ~/house-scanning-data/reports/e2e/20260927-085738-3baa3387/report.json | +| S3-8 | Exported scene.json validates against C1 | met | app_export_scene_valid = true (want True) at app_sha c820cf15: ~/house-scanning-data/reports/e2e/20260927-094443-3baa3387/report.json | | S3-9 | Design self-review against frontend-design, emil-design-eng, holistic-ux, apple-design | gap | not yet reviewed | ## S4 Verification -`origin/t3/ios-mvf` at [`c820cf15`](https://github.com/SamGu-NRX/house-scanning/tree/c820cf150a73cb97527428d580c330f631253f9a), `origin/t3/verification` at [`255bc848`](https://github.com/SamGu-NRX/house-scanning/tree/255bc848679e7327fecc77c0348d089dd1a3f334). 2 met, 2 stale. +`origin/t3/ios-mvf` at [`c820cf15`](https://github.com/SamGu-NRX/house-scanning/tree/c820cf150a73cb97527428d580c330f631253f9a), `origin/t3/verification` at [`b9f1601c`](https://github.com/SamGu-NRX/house-scanning/tree/b9f1601c0d0ea161eb346791c36157b231cfa309). 3 met, 1 partial. | Metric | Text | Status | Evidence | | --- | --- | --- | --- | -| S4-1 | End to end from a real replay to a placement result passes | stale | report for app_sha f14947eb, ref at c820cf15: ~/house-scanning-data/reports/e2e/20260927-085738-3baa3387/report.json | -| S4-2 | Screenshot report per app state | stale | report for sha f14947eb, ref at c820cf15: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json | +| S4-1 | End to end from a real replay to a placement result passes | partial | app_export_passed = true (want True) at app_sha c820cf15: ~/house-scanning-data/reports/e2e/20260927-094443-3baa3387/report.json (met on the app's synthetic replay against the hosted server (17 states, a server gap request, three spot checks, the result and AR); the real ADVIO replay is blocked by its data (the walk never covers the ground by the meter), last real run 194f2eb) | +| S4-2 | Screenshot report per app state | met | states = length 17 (want at least 9) at sha c820cf15: ~/house-scanning-data/reports/sim/20260927-093459-t3-ios-mvf-c820cf1-c820cf15-replay/report.json | | S4-3 | Product description written from the code and checked against screenshots | met | docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state. | -| S4-4 | Scoreboard of every PR | met | present at 255bc848 [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/255bc848679e7327fecc77c0348d089dd1a3f334/verification/SCOREBOARD.md) | +| S4-4 | Scoreboard of every PR | met | present at b9f1601c [verification/SCOREBOARD.md](https://github.com/SamGu-NRX/house-scanning/blob/b9f1601c0d0ea161eb346791c36157b231cfa309/verification/SCOREBOARD.md) | ## M Maintenance diff --git a/verification/hsverify/resultcheck.py b/verification/hsverify/resultcheck.py index 077ed609a..bb276281a 100644 --- a/verification/hsverify/resultcheck.py +++ b/verification/hsverify/resultcheck.py @@ -221,6 +221,30 @@ def covers(intervals: list[tuple[float, float]], a: float, b: float, slack: floa return any(lo - slack <= a + EPS and b - EPS <= hi + slack for lo, hi in intervals) +# A gap in a 1D band narrower than this is rounding between the capture's spans and the unrolled +# walls, not an unseen stretch (server/scene.py COVERAGE_TOLERANCE_FT and missing() at 3baa338). +COVERAGE_TOLERANCE_FT = 0.01 + + +def gaps_over( + intervals: list[tuple[float, float]], a: float, b: float +) -> list[tuple[float, float]]: + """The parts of [a, b] no interval covers, as the server's missing() finds them: gaps + narrower than COVERAGE_TOLERANCE_FT don't count.""" + gaps, at = [], a + for lo, hi in sorted(intervals): + if hi <= at: + continue + if lo > at: + gaps.append((at, min(lo, b))) + at = max(at, hi) + if at >= b: + break + if at < b: + gaps.append((at, b)) + return [(x, y) for x, y in gaps if y - x >= COVERAGE_TOLERANCE_FT] + + # --- Invariants ------------------------------------------------------------------------------ @@ -603,7 +627,7 @@ def reach_gaps( wall = observed( scene, "wall", need.height, beyond=True, wall_default_ft=rules.headroom_ft ) - seen = covers(wall, a, b) + seen = not gaps_over(wall, a, b) gap = None if seen else f"wall [{a:.2f}, {b:.2f}] observed higher than {need.height} ft" else: a, b = required_span(lo, hi, reach) @@ -636,6 +660,8 @@ def measured_band_gap( seen = [out for x, y, out in entries if x <= p + EPS and q - EPS <= y] where = f"{band} [{p:.2f}, {q:.2f}] observed" if not seen: + if q - p < COVERAGE_TOLERANCE_FT: + continue return f"{where}, none seen" settled = any(x <= p + EPS and q - EPS <= y for x, y in measured) # Strictly beyond, as the server requires (server/solver.py at 6c7ca23). @@ -708,7 +734,7 @@ def coverage_problems(scene: dict, result: dict, rules: RuleSet) -> list[str]: ends = chain_ends_s(scene, rules) or (-math.inf, math.inf) route_lo = max(min(lo, -m), ends[0]) route_hi = min(max(hi, m), ends[1]) - if not covers(route_wall, route_lo, route_hi): + if gaps_over(route_wall, route_lo, route_hi): problems.append( f"sweep pass for starts {run['start_ft']} but the wall and cable route " f"[{route_lo:.2f}, {route_hi:.2f}] were not all observed higher than " diff --git a/verification/scoreboard/metrics.yaml b/verification/scoreboard/metrics.yaml index 9c1b7c3ad..c73e9cc0c 100644 --- a/verification/scoreboard/metrics.yaml +++ b/verification/scoreboard/metrics.yaml @@ -277,9 +277,9 @@ metrics: sha_key: app_sha max_status: partial cap_reason: >- - met on the synthetic replay against the hosted server at f14947e (12 states, one server - gap request); the real ADVIO replay is blocked by its data (the walk never covers the - ground by the meter), last real run 194f2eb + met on the app's synthetic replay against the hosted server (17 states, a server gap + request, three spot checks, the result and AR); the real ADVIO replay is blocked by its + data (the walk never covers the ground by the meter), last real run 194f2eb - id: S4-2 workstream: S4 diff --git a/verification/scoreboard/scoreboard.json b/verification/scoreboard/scoreboard.json index 37c32a427..421e95602 100644 --- a/verification/scoreboard/scoreboard.json +++ b/verification/scoreboard/scoreboard.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-09-27T08:59:19-05:00", + "generated_at": "2026-09-27T09:49:34-05:00", "command": "uv run python -m hsverify.scoreboard", "slow": [ "S2-1", @@ -24,24 +24,6 @@ "text": "5 success, 1 skipped" } }, - { - "number": 168, - "title": "ios: treat a corridor's far wall or a side yard's fence as where the space ends", - "branch": "t3/164-space-ends", - "head": "381d0d3a3febc4ea6f275d3815fa523144bc4e4a", - "draft": true, - "updated": "2026-09-27T13:35:12Z", - "ci": { - "counts": { - "success": 2, - "failure": 0, - "pending": 0, - "skipped": 2 - }, - "failing": [], - "text": "2 success, 2 skipped" - } - }, { "number": 165, "title": "ios tests: audit the walk strictly once held and in motion to report; triage of the UI test failures", @@ -52,12 +34,14 @@ "ci": { "counts": { "success": 3, - "failure": 0, - "pending": 1, + "failure": 1, + "pending": 0, "skipped": 1 }, - "failing": [], - "text": "3 success, 1 pending, 1 skipped" + "failing": [ + "iOS build" + ], + "text": "3 success, 1 failure, 1 skipped; failing: iOS build" } }, { @@ -84,9 +68,9 @@ "number": 143, "title": "ios: practice meter, a drawn sample meter for scanning without one", "branch": "t3/practice-meter", - "head": "d298dfbeb66743e45d71bb58aa641f2d5fcb7662", + "head": "dd219a05072cff50439f5644705246af635db39f", "draft": false, - "updated": "2026-09-27T13:01:52Z", + "updated": "2026-09-27T14:45:49Z", "ci": { "counts": { "success": 3, @@ -102,9 +86,9 @@ "number": 134, "title": "docs: rebuild the README around the demo, with quick start, data and limits", "branch": "shrey/readme", - "head": "2bd6a2a21056d4228e9ee36bbf29a0428f8ff283", + "head": "9ceeece324099a0135bdcee23b68fe0ce8a9ce14", "draft": false, - "updated": "2026-09-27T13:19:38Z", + "updated": "2026-09-27T14:45:02Z", "ci": { "counts": { "success": 4, @@ -197,13 +181,13 @@ "updated": "2026-09-27T12:49:53Z", "ci": { "counts": { - "success": 3, + "success": 4, "failure": 0, - "pending": 1, + "pending": 0, "skipped": 1 }, "failing": [], - "text": "3 success, 1 pending, 1 skipped" + "text": "4 success, 1 skipped" } }, { @@ -212,7 +196,7 @@ "branch": "t3/41-gap-loop-result", "head": "ecc7a5768e7b8afedf66dbf8a4f65c1f5f78cd50", "draft": false, - "updated": "2026-09-27T13:36:44Z", + "updated": "2026-09-27T14:21:03Z", "ci": { "counts": { "success": 3, @@ -302,16 +286,16 @@ "branch": "t3/41-end-preview", "head": "f5a27c6428ba7fae05a06c2807d603bc24d9e7c1", "draft": false, - "updated": "2026-09-27T12:08:03Z", + "updated": "2026-09-27T14:33:10Z", "ci": { "counts": { - "success": 4, + "success": 5, "failure": 0, - "pending": 1, + "pending": 0, "skipped": 0 }, "failing": [], - "text": "4 success, 1 pending" + "text": "5 success" } }, { @@ -320,7 +304,7 @@ "branch": "t3/check-experiments", "head": "9d66b31c40c940c5bab7baea0aef5c5a87be3c10", "draft": false, - "updated": "2026-09-27T10:47:33Z", + "updated": "2026-09-27T14:28:02Z", "ci": { "counts": { "success": 6, @@ -482,7 +466,7 @@ "branch": "t3/motion-permission", "head": "b289c7dfa90c8760e0c34b96e5383a8c27a2bb8e", "draft": false, - "updated": "2026-09-27T12:58:42Z", + "updated": "2026-09-27T14:46:26Z", "ci": { "counts": { "success": 4, @@ -500,7 +484,7 @@ "branch": "t3/battery-mark", "head": "086ad94b7fdfccef09e9ddadffac908b63f974f4", "draft": false, - "updated": "2026-09-27T12:23:52Z", + "updated": "2026-09-27T14:31:26Z", "ci": { "counts": { "success": 3, @@ -547,6 +531,24 @@ "failing": [], "text": "3 success" } + }, + { + "number": 23, + "title": "experiments: device field test runbook and run 1 of the TestFlight build", + "branch": "t3/device-field-test", + "head": "321ea7e2705b80f61eaac3b4c7c3ddbb6503c675", + "draft": false, + "updated": "2026-09-27T10:49:36Z", + "ci": { + "counts": { + "success": 3, + "failure": 0, + "pending": 0, + "skipped": 0 + }, + "failing": [], + "text": "3 success" + } } ], "pr_error": null, @@ -640,7 +642,7 @@ "refs": [ { "ref": "origin/t3/server", - "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b" + "sha": "6c7bea122371711284d5672410b6e1afde22b7bb" } ], "metrics": [ @@ -648,22 +650,22 @@ "id": "S2-1", "text": "pytest green, the property tests under CI's fixed Hypothesis examples", "ref": "origin/t3/server", - "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", + "sha": "6c7bea122371711284d5672410b6e1afde22b7bb", "status": "met", - "evidence": "exit 0: return lib.difference_prec(a, b, grid_size, **kwargs) / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 380 passed, 1 xfailed, 5 warnings in 61.63s (0:01:01)", + "evidence": "exit 0: from starlette.testclient import TestClient as TestClient # noqa / -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html / 388 passed, 1 xfailed, 1 warning in 121.47s (0:02:01)", "links": [] }, { "id": "S2-2", "text": "Every public golden test 01-14 of docs/research/t3-lane-c-review.md present", "ref": "origin/t3/server", - "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", + "sha": "6c7bea122371711284d5672410b6e1afde22b7bb", "status": "met", "evidence": "all 14 found", "links": [ [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/6c7bea122371711284d5672410b6e1afde22b7bb/server/tests/test_golden_public.py" ] ] }, @@ -671,21 +673,21 @@ "id": "S2-3", "text": "Property tests: missing coverage never passes, equality is unsure, left/right mirror, larger clearance never turns pass into fail", "ref": "origin/t3/server", - "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", + "sha": "6c7bea122371711284d5672410b6e1afde22b7bb", "status": "met", "evidence": "all 4 found", "links": [ [ "server/tests/test_coderabbit_review.py", - "https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_coderabbit_review.py" + "https://github.com/SamGu-NRX/house-scanning/blob/6c7bea122371711284d5672410b6e1afde22b7bb/server/tests/test_coderabbit_review.py" ], [ "server/tests/test_properties.py", - "https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_properties.py" + "https://github.com/SamGu-NRX/house-scanning/blob/6c7bea122371711284d5672410b6e1afde22b7bb/server/tests/test_properties.py" ], [ "server/tests/test_final_review.py", - "https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_final_review.py" + "https://github.com/SamGu-NRX/house-scanning/blob/6c7bea122371711284d5672410b6e1afde22b7bb/server/tests/test_final_review.py" ] ] }, @@ -693,17 +695,17 @@ "id": "S2-4", "text": "Schemas published", "ref": "origin/t3/server", - "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", + "sha": "6c7bea122371711284d5672410b6e1afde22b7bb", "status": "met", - "evidence": "present at 3baa3387", + "evidence": "present at 6c7bea12", "links": [ [ "server/schemas/scene.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/schemas/scene.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/6c7bea122371711284d5672410b6e1afde22b7bb/server/schemas/scene.schema.json" ], [ "server/schemas/result.schema.json", - "https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/schemas/result.schema.json" + "https://github.com/SamGu-NRX/house-scanning/blob/6c7bea122371711284d5672410b6e1afde22b7bb/server/schemas/result.schema.json" ] ] }, @@ -711,21 +713,21 @@ "id": "S2-5", "text": "Schemas validated in tests", "ref": "origin/t3/server", - "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", + "sha": "6c7bea122371711284d5672410b6e1afde22b7bb", "status": "met", "evidence": "all 2 found", "links": [ [ "server/tests/test_existing_battery.py", - "https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_existing_battery.py" + "https://github.com/SamGu-NRX/house-scanning/blob/6c7bea122371711284d5672410b6e1afde22b7bb/server/tests/test_existing_battery.py" ], [ "server/tests/test_golden_public.py", - "https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_golden_public.py" + "https://github.com/SamGu-NRX/house-scanning/blob/6c7bea122371711284d5672410b6e1afde22b7bb/server/tests/test_golden_public.py" ], [ "server/tests/test_schemas.py", - "https://github.com/SamGu-NRX/house-scanning/blob/3baa3387a6877961345afa7785f1530c1aeb2b1b/server/tests/test_schemas.py" + "https://github.com/SamGu-NRX/house-scanning/blob/6c7bea122371711284d5672410b6e1afde22b7bb/server/tests/test_schemas.py" ] ] }, @@ -733,27 +735,27 @@ "id": "S2-6", "text": "A real-derived scene returns a result in under 1 s", "ref": "origin/t3/server", - "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", - "status": "met", - "evidence": "latency_ms = 423.1 (want < 1000) at sha 3baa3387: ~/house-scanning-data/reports/e2e/20260927-085738-3baa3387/report.json", + "sha": "6c7bea122371711284d5672410b6e1afde22b7bb", + "status": "stale", + "evidence": "report for sha 3baa3387, ref at 6c7bea12: ~/house-scanning-data/reports/e2e/20260927-094443-3baa3387/report.json", "links": [] }, { "id": "S2-7", "text": "ruff clean", "ref": "origin/t3/server", - "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", + "sha": "6c7bea122371711284d5672410b6e1afde22b7bb", "status": "met", - "evidence": "exit 0: Installed 33 packages in 27ms / All checks passed! / 36 files already formatted", + "evidence": "exit 0: Installed 33 packages in 40ms / All checks passed! / 37 files already formatted", "links": [] }, { "id": "S2-8", "text": "Black-box (verification e2e at this SHA): results valid under C2, consistent with the decision and C5 margins, coverage to each check's required radius, missing evidence complete, ordering properties, hostile inputs refused in time", "ref": "origin/t3/server", - "sha": "3baa3387a6877961345afa7785f1530c1aeb2b1b", - "status": "gap", - "evidence": "contract_ok = false (want True) at sha 3baa3387: ~/house-scanning-data/reports/e2e/20260927-085738-3baa3387/report.json", + "sha": "6c7bea122371711284d5672410b6e1afde22b7bb", + "status": "stale", + "evidence": "report for sha 3baa3387, ref at 6c7bea12: ~/house-scanning-data/reports/e2e/20260927-094443-3baa3387/report.json", "links": [] } ] @@ -796,8 +798,8 @@ "text": "Zero build warnings", "ref": "origin/t3/ios-mvf", "sha": "c820cf150a73cb97527428d580c330f631253f9a", - "status": "stale", - "evidence": "report for sha f14947eb, ref at c820cf15: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", + "status": "met", + "evidence": "build.warnings = length 0 (want 0) at sha c820cf15: ~/house-scanning-data/reports/sim/20260927-093459-t3-ios-mvf-c820cf1-c820cf15-replay/report.json", "links": [] }, { @@ -885,8 +887,8 @@ "text": "Exported scene.json validates against C1", "ref": "origin/t3/ios-mvf", "sha": "c820cf150a73cb97527428d580c330f631253f9a", - "status": "stale", - "evidence": "report for app_sha f14947eb, ref at c820cf15: ~/house-scanning-data/reports/e2e/20260927-085738-3baa3387/report.json", + "status": "met", + "evidence": "app_export_scene_valid = true (want True) at app_sha c820cf15: ~/house-scanning-data/reports/e2e/20260927-094443-3baa3387/report.json", "links": [] }, { @@ -910,7 +912,7 @@ }, { "ref": "origin/t3/verification", - "sha": "255bc848679e7327fecc77c0348d089dd1a3f334" + "sha": "b9f1601c0d0ea161eb346791c36157b231cfa309" } ], "metrics": [ @@ -919,8 +921,8 @@ "text": "End to end from a real replay to a placement result passes", "ref": "origin/t3/ios-mvf", "sha": "c820cf150a73cb97527428d580c330f631253f9a", - "status": "stale", - "evidence": "report for app_sha f14947eb, ref at c820cf15: ~/house-scanning-data/reports/e2e/20260927-085738-3baa3387/report.json", + "status": "partial", + "evidence": "app_export_passed = true (want True) at app_sha c820cf15: ~/house-scanning-data/reports/e2e/20260927-094443-3baa3387/report.json (met on the app's synthetic replay against the hosted server (17 states, a server gap request, three spot checks, the result and AR); the real ADVIO replay is blocked by its data (the walk never covers the ground by the meter), last real run 194f2eb)", "links": [] }, { @@ -928,15 +930,15 @@ "text": "Screenshot report per app state", "ref": "origin/t3/ios-mvf", "sha": "c820cf150a73cb97527428d580c330f631253f9a", - "status": "stale", - "evidence": "report for sha f14947eb, ref at c820cf15: ~/house-scanning-data/reports/sim/20260927-013832-t3-ios-mvf-f14947eb-replay/report.json", + "status": "met", + "evidence": "states = length 17 (want at least 9) at sha c820cf15: ~/house-scanning-data/reports/sim/20260927-093459-t3-ios-mvf-c820cf1-c820cf15-replay/report.json", "links": [] }, { "id": "S4-3", "text": "Product description written from the code and checked against screenshots", "ref": "origin/t3/verification", - "sha": "255bc848679e7327fecc77c0348d089dd1a3f334", + "sha": "b9f1601c0d0ea161eb346791c36157b231cfa309", "status": "met", "evidence": "docs/app-review/product (#61) covers every screen of t3/ios-mvf at a39d0a5, checked in the Simulator there with the real server's result; its bug triage has 16 entries, 3 resolved; docs/app-review/ux/review.md judges every state.", "links": [] @@ -945,13 +947,13 @@ "id": "S4-4", "text": "Scoreboard of every PR", "ref": "origin/t3/verification", - "sha": "255bc848679e7327fecc77c0348d089dd1a3f334", + "sha": "b9f1601c0d0ea161eb346791c36157b231cfa309", "status": "met", - "evidence": "present at 255bc848", + "evidence": "present at b9f1601c", "links": [ [ "verification/SCOREBOARD.md", - "https://github.com/SamGu-NRX/house-scanning/blob/255bc848679e7327fecc77c0348d089dd1a3f334/verification/SCOREBOARD.md" + "https://github.com/SamGu-NRX/house-scanning/blob/b9f1601c0d0ea161eb346791c36157b231cfa309/verification/SCOREBOARD.md" ] ] } diff --git a/verification/tests/test_resultcheck.py b/verification/tests/test_resultcheck.py index 430067132..5846b7a9a 100644 --- a/verification/tests/test_resultcheck.py +++ b/verification/tests/test_resultcheck.py @@ -18,6 +18,7 @@ expectation_problems, expected_checks, footprint_problems, + gaps_over, invariant_problems, less_coverage_problems, margin_problem, @@ -1469,3 +1470,12 @@ def test_manual_review_needs_a_spot_that_does_not_fail(): assert "the spot is pass but its check gas_clearance fails" in invariant_problems( SCENE, passing_spot, rules=RULES ) + + +def test_a_band_gap_under_the_coverage_tolerance_is_rounding(): + # The app scan at c820cf1, less coverage: wall seen to 18.0, needed to 18.0037 for + # wall_equipment_above. The server's missing() drops gaps under 0.01 ft; 0.011 stays. + assert gaps_over([(-6.5, 18.0)], 9.94, 18.0037) == [] + assert gaps_over([(-6.5, 18.0)], 9.94, 18.011) == [(18.0, 18.011)] + assert gaps_over([(0.0, 1.0), (1.009, 3.0)], 0.5, 2.0) == [] + assert gaps_over([(0.0, 1.0), (1.011, 3.0)], 0.5, 2.0) == [(1.0, 1.011)]