diff --git a/.github/labels.json b/.github/labels.json index d972796e3..bf7807164 100644 --- a/.github/labels.json +++ b/.github/labels.json @@ -24,6 +24,11 @@ "color": "b60205", "description": "Not ready to merge. State what must change before it is ready." }, + { + "name": "full-ui", + "color": "5319e7", + "description": "iOS CI also runs the every-state accessibility audit (about 10 more minutes)." + }, { "name": "size:XS", "color": "0e8a16", diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index c7eaba1c2..b8a544293 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -2,35 +2,55 @@ name: iOS on: push: - branches: [main] + branches: [main, t3/ios-mvf] paths: - "ios/**" + - "!ios/**/*.md" - ".github/workflows/ios.yml" pull_request: + # ready_for_review: drafts are skipped below, so run once a draft is marked ready. + # labeled: adding full-ui reruns the pull request with the accessibility audit. + types: [opened, synchronize, reopened, ready_for_review, labeled] paths: - "ios/**" + - "!ios/**/*.md" - ".github/workflows/ios.yml" + # Runs every test, the audit included, on any branch. + workflow_dispatch: permissions: contents: read concurrency: - group: ios-${{ github.ref }} + # Adding any label other than full-ui gets a group of its own, so it cannot cancel a real run. + group: ios-${{ github.ref }}${{ github.event.action == 'labeled' && github.event.label.name != 'full-ui' && format('-{0}', github.run_id) || '' }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: ios-build: name: iOS build + # A free account runs 5 macOS jobs at once, shared by every open pull request, so drafts and + # labels other than full-ui don't take a slot. + if: >- + github.event_name != 'pull_request' || + (!github.event.pull_request.draft && + (github.event.action != 'labeled' || github.event.label.name == 'full-ui')) # macos-26 (arm64) is the newest GA macOS image. The xcode-27 image is still a preview. runs-on: macos-26 - timeout-minutes: 30 + timeout-minutes: 45 env: # Newest stable Xcode on macos-26 per images/macos/macos-26-arm64-Readme.md (image 20260907). DEVELOPER_DIR: /Applications/Xcode_26.6.app/Contents/Developer + # iPhone 17 on iOS 26.5 ships with Xcode 26.6 on macos-26 (images/macos/macos-26-arm64-Readme.md). + SIM_NAME: iPhone 17 + SIM_OS: "26.5" # Homebrew only installs the latest XcodeGen. The release zip gives an exact version, # and the drift check below needs the same version the committed project was made with. XCODEGEN_VERSION: 2.46.0 XCODEGEN_SHA256: 4d9e34b62172d645eed6457cac13fc222569974098ef4ee9c3368bedf0196806 + # The every-state accessibility audit relaunches the app 65 times (about 10 min). Pull + # requests skip it unless labelled full-ui; pushes and manual runs always include it. + FULL_UI: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'full-ui') }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -45,6 +65,45 @@ jobs: xcodebuild -version swift --version + # Booting takes 2 to 3 minutes and needs nothing from the build, so it runs alongside the + # steps below; the UI test step waits for it. + - name: Boot the Simulator in the background + run: | + runtime="com.apple.CoreSimulator.SimRuntime.iOS-${SIM_OS//./-}" + udid="$(xcrun simctl list devices available -j \ + | jq -r --arg runtime "$runtime" --arg name "$SIM_NAME" \ + '(.devices[$runtime] // [])[] | select(.name == $name) | .udid' | head -n 1)" + if [ -z "$udid" ]; then + echo "::error::No $SIM_NAME on iOS $SIM_OS on this runner image. Pick one from the image readme." + xcrun simctl list devices available + exit 1 + fi + echo "SIM_UDID=$udid" >> "$GITHUB_ENV" + nohup xcrun simctl bootstatus "$udid" -b > "$RUNNER_TEMP/simulator-boot.log" 2>&1 & + + # The package's drift test compares the vendored server schemas with the server's own copies: + # the checkout's files once that branch merges, else the t3/server branch, which the shallow + # checkout leaves out. One of the two is required: + # a failed fetch fails the run, and HOUSESCAN_REQUIRE_UPSTREAM makes the tests fail rather + # than skip, so CI can't go green without comparing. The checkout counts only when it holds + # every compared file: a folder alone isn't enough (main has a server/ without schemas). + - name: Fetch the server contract + run: | + fetch_unless_present() { + local branch="$1"; shift + local missing=0 + for file in "$@"; do [ -f "$file" ] || missing=1; done + if [ "$missing" = 0 ]; then + echo "$* are in the checkout; the drift tests read them." + return + fi + if ! git fetch --no-tags --depth=1 origin "+refs/heads/$branch:refs/remotes/origin/$branch"; then + echo "::error::Neither the checkout nor $branch holds $*, so the vendored copies can't be compared." + exit 1 + fi + } + fetch_unless_present t3/server server/schemas/scene.schema.json server/schemas/result.schema.json server/tests/fixtures/example-scene.json + - name: Install XcodeGen run: | curl -fsSL -o "$RUNNER_TEMP/xcodegen.zip" \ @@ -69,7 +128,45 @@ jobs: exit 1 fi + - name: Test HouseScanKit + # The capture logic is a plain Swift package; its tests run on macOS without a Simulator. + env: + HOUSESCAN_REQUIRE_UPSTREAM: "1" + run: swift test --package-path ios/HouseScanKit -Xswiftc -warnings-as-errors + + # CI builds once from clean, so the index Xcode keeps for editing is wasted work. - name: Build run: >- xcodebuild -project ios/HouseScan.xcodeproj -scheme HouseScan -configuration Debug - -destination "generic/platform=iOS" CODE_SIGNING_ALLOWED=NO build + -destination "generic/platform=iOS" CODE_SIGNING_ALLOWED=NO COMPILER_INDEX_STORE_ENABLE=NO + HOUSESCAN_GIT_COMMIT="${GITHUB_SHA::12}" build + + - name: Wait for the Simulator + timeout-minutes: 10 + run: | + xcrun simctl bootstatus "$SIM_UDID" -b + echo "Booted $SIM_NAME on iOS $SIM_OS ($SIM_UDID)." + + - name: UI test the whole flow in the Simulator + # Replays the committed synthetic wall (ios/HouseScanUITests/Fixtures/synthetic-wall) with + # -autopilot and the bundled sample result. + run: | + skip=() + if [ "$FULL_UI" != "true" ]; then + skip=(-skip-testing:HouseScanUITests/ScreenStatesUITests/testEveryStatePassesTheAccessibilityAudit) + echo "::notice::Skipped the every-state accessibility audit (about 10 min). Add the full-ui label to run it on this pull request." + fi + xcodebuild -project ios/HouseScan.xcodeproj -scheme HouseScan -configuration Debug \ + -destination "id=$SIM_UDID" \ + -resultBundlePath "$RUNNER_TEMP/HouseScanUITests.xcresult" \ + -only-testing:HouseScanUITests "${skip[@]}" \ + COMPILER_INDEX_STORE_ENABLE=NO HOUSESCAN_GIT_COMMIT="${GITHUB_SHA::12}" test + + - name: Keep UI test screenshots + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ui-test-results + path: ${{ runner.temp }}/HouseScanUITests.xcresult + if-no-files-found: ignore + retention-days: 7 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0ce41fefe..d8d9706e0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,11 +28,13 @@ The `size:*` label describes the effective diff. It is a review signal, not a me | --- | --- | --- | | Server checks | `server/` changes | `make server` | | Web checks | `web/` changes | `make web` | -| iOS build | `ios/` changes | `make ios` | +| iOS build | `ios/` changes outside Markdown, on non-draft pull requests | `make ios` | | Sync label definitions | `.github/labels.json` changes on `main` | none | | Label PR size | a pull request opens or updates | none | | TestFlight | someone runs it from the Actions tab | none | +The iOS UI tests skip the every-state accessibility audit on pull requests, because it adds about 10 minutes and macOS runners are scarce. Add the `full-ui` label when a pull request changes screens or copy; pushes to `t3/ios-mvf` and `main` always run it. + `make check` runs the three local suites. They need uv, Node 24 with pnpm, and Xcode 26 or newer; each directory's README has details. No check is required by branch rules yet. Don't call one required until the rules require its status. Keep workflows that run pull-request code away from production credentials and destructive external systems. A green CI run is evidence for the checks it ran, not proof that a capture works on a real house. diff --git a/Makefile b/Makefile index 13bca16c8..af879681c 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,8 @@ web: # CI also regenerates the project and fails on drift; run `make ios-project` for that. ios: xcodebuild -project ios/HouseScan.xcodeproj -scheme HouseScan -configuration Debug \ - -destination "generic/platform=iOS" CODE_SIGNING_ALLOWED=NO build + -destination "generic/platform=iOS" CODE_SIGNING_ALLOWED=NO \ + HOUSESCAN_GIT_COMMIT="$$(git rev-parse --short=12 HEAD 2>/dev/null || echo unknown)" build # Regenerates ios/HouseScan.xcodeproj from ios/project.yml. Point XCODEGEN at another binary # if the one on PATH is not $(XCODEGEN_VERSION): make ios-project XCODEGEN=/path/to/xcodegen diff --git a/ios/.gitignore b/ios/.gitignore index 56c060007..2e1efc1ee 100644 --- a/ios/.gitignore +++ b/ios/.gitignore @@ -8,3 +8,7 @@ xcuserdata/ # Build output when building with -derivedDataPath or into the source tree. DerivedData/ build/ + +# SwiftPM build output of the capture package. +HouseScanKit/.build/ +HouseScanKit/.swiftpm/ diff --git a/ios/Config/Info.plist b/ios/Config/Info.plist index f3aff65c1..38a4f5720 100644 --- a/ios/Config/Info.plist +++ b/ios/Config/Info.plist @@ -20,17 +20,33 @@ $(MARKETING_VERSION) CFBundleVersion $(CURRENT_PROJECT_VERSION) + HouseScanGitCommit + $(HOUSESCAN_GIT_COMMIT) + HouseScanServerURL + $(HOUSESCAN_SERVER_URL) + ITSAppUsesNonExemptEncryption + LSRequiresIPhoneOS + NSAppTransportSecurity + + NSAllowsLocalNetworking + + NSCameraUsageDescription House Scan uses the camera to measure the outside wall around your electric meter. + NSMotionUsageDescription + House Scan records your phone's motion and air pressure during the scan, so it can be measured more accurately. It stays on this phone unless you share the scan. UIApplicationSceneManifest UIApplicationSupportsMultipleScenes UILaunchScreen - + + UIColorName + Canvas + UIRequiredDeviceCapabilities arkit diff --git a/ios/Config/Shared.xcconfig b/ios/Config/Shared.xcconfig index e545d9aae..e4056d15a 100644 --- a/ios/Config/Shared.xcconfig +++ b/ios/Config/Shared.xcconfig @@ -7,4 +7,15 @@ BUNDLE_ID_PREFIX = dev.housescanning +// The placement server the app uploads to when no -serverURL launch argument is given; Info.plist +// carries it as HouseScanServerURL. xcconfig reads // as a comment, so $() splits the slashes. +// Set it empty in Local.xcconfig to fall back to the bundled sample result. +HOUSESCAN_SERVER_URL = https:/$()/house-scanning-server.vercel.app + #include? "Local.xcconfig" + +// The commit the app was built from, for the scan stamp (HouseScanGitCommit in Info.plist). A +// build setting, not a build script: the TestFlight job refuses scripts in the build while the +// signing key exists. CI and `make ios` pass the real commit (HOUSESCAN_GIT_COMMIT=); a build +// that doesn't, such as one from Xcode's Run button, says "unknown". +HOUSESCAN_GIT_COMMIT = unknown diff --git a/ios/HouseScan.xcodeproj/project.pbxproj b/ios/HouseScan.xcodeproj/project.pbxproj index 71d3ee054..f623d8f14 100644 --- a/ios/HouseScan.xcodeproj/project.pbxproj +++ b/ios/HouseScan.xcodeproj/project.pbxproj @@ -6,12 +6,292 @@ objectVersion = 77; objects = { +/* Begin PBXBuildFile section */ + A08E1583C6EBA84CB5619CD2 /* HouseScanKit in Frameworks */ = {isa = PBXBuildFile; productRef = 19E9E07688F7D0E710D21D89 /* HouseScanKit */; }; +/* End PBXBuildFile section */ + +/* Begin PBXContainerItemProxy section */ + A0652F9B03D0AB543FDB519D /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 0BC33671F32C9E42F1E3A643 /* Project object */; + proxyType = 1; + remoteGlobalIDString = C5240AAFE2CE18E751FB21F9; + remoteInfo = HouseScan; + }; +/* End PBXContainerItemProxy section */ + /* Begin PBXFileReference section */ 0C4096780A5627C9A53A29B2 /* HouseScan.app */ = {isa = PBXFileReference; includeInIndex = 0; lastKnownFileType = wrapper.application; path = HouseScan.app; sourceTree = BUILT_PRODUCTS_DIR; }; + B959F2AE1DD15E7A99C500EF /* HouseScanUITests.xctest */ = {isa = PBXFileReference; includeInIndex = 0; lastKnownFileType = wrapper.cfbundle; path = HouseScanUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; + E5448723EC7099F74C7CCBEC /* HouseScanKit */ = {isa = PBXFileReference; lastKnownFileType = folder; name = HouseScanKit; path = HouseScanKit; sourceTree = SOURCE_ROOT; }; EEAA26E1BB7E90A7AD7923C7 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; path = Info.plist; sourceTree = ""; }; F30ACBC67A4159A449077AF5 /* Shared.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Shared.xcconfig; sourceTree = ""; }; /* End PBXFileReference section */ +/* Begin PBXFileSystemSynchronizedBuildFileExceptionSet section */ + FE223665998D079375C59650 /* PBXFileSystemSynchronizedBuildFileExceptionSet */ = { + isa = PBXFileSystemSynchronizedBuildFileExceptionSet; + membershipExceptions = ( + Fixtures/results, + "Fixtures/results/no-clean-spot.json", + Fixtures/results/pass.json, + "Fixtures/results/reject-nearest.json", + "Fixtures/results/unsure-view.json", + "Fixtures/synthetic-wall", + "Fixtures/synthetic-wall-lidar", + "Fixtures/synthetic-wall-lidar/keyframes", + "Fixtures/synthetic-wall-lidar/keyframes/k00001.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00001.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00001.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00002.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00002.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00002.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00003.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00003.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00003.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00004.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00004.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00004.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00005.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00005.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00005.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00006.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00006.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00006.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00007.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00007.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00007.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00008.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00008.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00008.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00009.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00009.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00009.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00010.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00010.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00010.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00011.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00011.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00011.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00012.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00012.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00012.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00013.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00013.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00013.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00014.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00014.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00014.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00015.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00015.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00015.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00016.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00016.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00016.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00017.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00017.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00017.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00018.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00018.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00018.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00019.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00019.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00019.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00020.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00020.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00020.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00021.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00021.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00021.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00022.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00022.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00022.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00023.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00023.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00023.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00024.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00024.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00024.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00025.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00025.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00025.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00026.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00026.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00026.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00027.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00027.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00027.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00028.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00028.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00028.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00029.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00029.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00029.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00030.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00030.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00030.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00031.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00031.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00031.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00032.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00032.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00032.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00033.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00033.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00033.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00034.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00034.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00034.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00035.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00035.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00035.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00036.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00036.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00036.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00037.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00037.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00037.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00038.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00038.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00038.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00039.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00039.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00039.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00040.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00040.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00040.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00041.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00041.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00041.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00042.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00042.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00042.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00043.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00043.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00043.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00044.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00044.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00044.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00045.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00045.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00045.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00046.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00046.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00046.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00047.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00047.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00047.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00048.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00048.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00048.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00049.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00049.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00049.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00050.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00050.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00050.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00051.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00051.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00051.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00052.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00052.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00052.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00053.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00053.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00053.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00054.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00054.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00054.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00055.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00055.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00055.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00056.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00056.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00056.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00057.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00057.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00057.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00058.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00058.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00058.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00059.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00059.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00059.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00060.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00060.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00060.jpg", + "Fixtures/synthetic-wall-lidar/keyframes/k00061.confidence.u8", + "Fixtures/synthetic-wall-lidar/keyframes/k00061.depth.f32", + "Fixtures/synthetic-wall-lidar/keyframes/k00061.jpg", + "Fixtures/synthetic-wall-lidar/session.json", + "Fixtures/synthetic-wall/keyframes", + "Fixtures/synthetic-wall/keyframes/k00001.jpg", + "Fixtures/synthetic-wall/keyframes/k00002.jpg", + "Fixtures/synthetic-wall/keyframes/k00003.jpg", + "Fixtures/synthetic-wall/keyframes/k00004.jpg", + "Fixtures/synthetic-wall/keyframes/k00005.jpg", + "Fixtures/synthetic-wall/keyframes/k00006.jpg", + "Fixtures/synthetic-wall/keyframes/k00007.jpg", + "Fixtures/synthetic-wall/keyframes/k00008.jpg", + "Fixtures/synthetic-wall/keyframes/k00009.jpg", + "Fixtures/synthetic-wall/keyframes/k00010.jpg", + "Fixtures/synthetic-wall/keyframes/k00011.jpg", + "Fixtures/synthetic-wall/keyframes/k00012.jpg", + "Fixtures/synthetic-wall/keyframes/k00013.jpg", + "Fixtures/synthetic-wall/keyframes/k00014.jpg", + "Fixtures/synthetic-wall/keyframes/k00015.jpg", + "Fixtures/synthetic-wall/keyframes/k00016.jpg", + "Fixtures/synthetic-wall/keyframes/k00017.jpg", + "Fixtures/synthetic-wall/keyframes/k00018.jpg", + "Fixtures/synthetic-wall/keyframes/k00019.jpg", + "Fixtures/synthetic-wall/keyframes/k00020.jpg", + "Fixtures/synthetic-wall/keyframes/k00021.jpg", + "Fixtures/synthetic-wall/keyframes/k00022.jpg", + "Fixtures/synthetic-wall/keyframes/k00023.jpg", + "Fixtures/synthetic-wall/keyframes/k00024.jpg", + "Fixtures/synthetic-wall/keyframes/k00025.jpg", + "Fixtures/synthetic-wall/keyframes/k00026.jpg", + "Fixtures/synthetic-wall/keyframes/k00027.jpg", + "Fixtures/synthetic-wall/keyframes/k00028.jpg", + "Fixtures/synthetic-wall/keyframes/k00029.jpg", + "Fixtures/synthetic-wall/keyframes/k00030.jpg", + "Fixtures/synthetic-wall/keyframes/k00031.jpg", + "Fixtures/synthetic-wall/keyframes/k00032.jpg", + "Fixtures/synthetic-wall/keyframes/k00033.jpg", + "Fixtures/synthetic-wall/keyframes/k00034.jpg", + "Fixtures/synthetic-wall/keyframes/k00035.jpg", + "Fixtures/synthetic-wall/keyframes/k00036.jpg", + "Fixtures/synthetic-wall/keyframes/k00037.jpg", + "Fixtures/synthetic-wall/keyframes/k00038.jpg", + "Fixtures/synthetic-wall/keyframes/k00039.jpg", + "Fixtures/synthetic-wall/keyframes/k00040.jpg", + "Fixtures/synthetic-wall/keyframes/k00041.jpg", + "Fixtures/synthetic-wall/keyframes/k00042.jpg", + "Fixtures/synthetic-wall/keyframes/k00043.jpg", + "Fixtures/synthetic-wall/keyframes/k00044.jpg", + "Fixtures/synthetic-wall/keyframes/k00045.jpg", + "Fixtures/synthetic-wall/keyframes/k00046.jpg", + "Fixtures/synthetic-wall/keyframes/k00047.jpg", + "Fixtures/synthetic-wall/keyframes/k00048.jpg", + "Fixtures/synthetic-wall/keyframes/k00049.jpg", + "Fixtures/synthetic-wall/keyframes/k00050.jpg", + "Fixtures/synthetic-wall/keyframes/k00051.jpg", + "Fixtures/synthetic-wall/keyframes/k00052.jpg", + "Fixtures/synthetic-wall/keyframes/k00053.jpg", + "Fixtures/synthetic-wall/keyframes/k00054.jpg", + "Fixtures/synthetic-wall/keyframes/k00055.jpg", + "Fixtures/synthetic-wall/keyframes/k00056.jpg", + "Fixtures/synthetic-wall/keyframes/k00057.jpg", + "Fixtures/synthetic-wall/keyframes/k00058.jpg", + "Fixtures/synthetic-wall/keyframes/k00059.jpg", + "Fixtures/synthetic-wall/keyframes/k00060.jpg", + "Fixtures/synthetic-wall/keyframes/k00061.jpg", + "Fixtures/synthetic-wall/session.json", + ); + target = EF941BB37D13A698579DF2A6 /* HouseScanUITests */; + }; +/* End PBXFileSystemSynchronizedBuildFileExceptionSet section */ + /* Begin PBXFileSystemSynchronizedRootGroup section */ 050A16F154C46B951098B8C8 /* HouseScan */ = { isa = PBXFileSystemSynchronizedRootGroup; @@ -22,17 +302,49 @@ path = HouseScan; sourceTree = ""; }; + AA23F1E57E4893FBA0800AFF /* HouseScanUITests */ = { + isa = PBXFileSystemSynchronizedRootGroup; + exceptions = ( + FE223665998D079375C59650 /* PBXFileSystemSynchronizedBuildFileExceptionSet */, + ); + explicitFileTypes = { + }; + explicitFolders = ( + ); + path = HouseScanUITests; + sourceTree = ""; + }; /* End PBXFileSystemSynchronizedRootGroup section */ +/* Begin PBXFrameworksBuildPhase section */ + 52EF4397177795C26C0B600D /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + A08E1583C6EBA84CB5619CD2 /* HouseScanKit in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + /* Begin PBXGroup section */ 15857B17057201C5A1934D04 /* Products */ = { isa = PBXGroup; children = ( 0C4096780A5627C9A53A29B2 /* HouseScan.app */, + B959F2AE1DD15E7A99C500EF /* HouseScanUITests.xctest */, ); name = Products; sourceTree = ""; }; + 8885F681E8E6766935603C4F /* Packages */ = { + isa = PBXGroup; + children = ( + E5448723EC7099F74C7CCBEC /* HouseScanKit */, + ); + name = Packages; + sourceTree = ""; + }; 8AD6B37B63CD5C86504E5E20 /* Config */ = { isa = PBXGroup; children = ( @@ -47,6 +359,8 @@ children = ( 8AD6B37B63CD5C86504E5E20 /* Config */, 050A16F154C46B951098B8C8 /* HouseScan */, + AA23F1E57E4893FBA0800AFF /* HouseScanUITests */, + 8885F681E8E6766935603C4F /* Packages */, 15857B17057201C5A1934D04 /* Products */, ); sourceTree = ""; @@ -60,6 +374,7 @@ buildPhases = ( 535AA09D0DE15B25D4976AD1 /* Sources */, E009F9206BA5E923B8F3E03B /* Resources */, + 52EF4397177795C26C0B600D /* Frameworks */, ); buildRules = ( ); @@ -70,11 +385,34 @@ ); name = HouseScan; packageProductDependencies = ( + 19E9E07688F7D0E710D21D89 /* HouseScanKit */, ); productName = HouseScan; productReference = 0C4096780A5627C9A53A29B2 /* HouseScan.app */; productType = "com.apple.product-type.application"; }; + EF941BB37D13A698579DF2A6 /* HouseScanUITests */ = { + isa = PBXNativeTarget; + buildConfigurationList = CA663DA741070E6AFABA7C13 /* Build configuration list for PBXNativeTarget "HouseScanUITests" */; + buildPhases = ( + C0D00DE62C374481FAF457A5 /* Sources */, + 1B7B501579D25B4A33756562 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + 46583C8FA36D8988F43DBC2D /* PBXTargetDependency */, + ); + fileSystemSynchronizedGroups = ( + AA23F1E57E4893FBA0800AFF /* HouseScanUITests */, + ); + name = HouseScanUITests; + packageProductDependencies = ( + ); + productName = HouseScanUITests; + productReference = B959F2AE1DD15E7A99C500EF /* HouseScanUITests.xctest */; + productType = "com.apple.product-type.bundle.ui-testing"; + }; /* End PBXNativeTarget section */ /* Begin PBXProject section */ @@ -87,6 +425,10 @@ C5240AAFE2CE18E751FB21F9 = { ProvisioningStyle = Automatic; }; + EF941BB37D13A698579DF2A6 = { + ProvisioningStyle = Automatic; + TestTargetID = C5240AAFE2CE18E751FB21F9; + }; }; }; buildConfigurationList = 0274DCB965114EB4DAAAAA8C /* Build configuration list for PBXProject "HouseScan" */; @@ -98,17 +440,28 @@ ); mainGroup = F45504E255654984698A6926; minimizedProjectReferenceProxies = 1; + packageReferences = ( + 996A72D2A0E3ADCC195F64F8 /* XCLocalSwiftPackageReference "HouseScanKit" */, + ); preferredProjectObjectVersion = 77; productRefGroup = 15857B17057201C5A1934D04 /* Products */; projectDirPath = ""; projectRoot = ""; targets = ( C5240AAFE2CE18E751FB21F9 /* HouseScan */, + EF941BB37D13A698579DF2A6 /* HouseScanUITests */, ); }; /* End PBXProject section */ /* Begin PBXResourcesBuildPhase section */ + 1B7B501579D25B4A33756562 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; E009F9206BA5E923B8F3E03B /* Resources */ = { isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; @@ -126,9 +479,43 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + C0D00DE62C374481FAF457A5 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXSourcesBuildPhase section */ +/* Begin PBXTargetDependency section */ + 46583C8FA36D8988F43DBC2D /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = C5240AAFE2CE18E751FB21F9 /* HouseScan */; + targetProxy = A0652F9B03D0AB543FDB519D /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + /* Begin XCBuildConfiguration section */ + 1D9A3F7F45102D23B342FD47 /* Release */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = F30ACBC67A4159A449077AF5 /* Shared.xcconfig */; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CODE_SIGN_STYLE = Automatic; + GENERATE_INFOPLIST_FILE = YES; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + "@loader_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = "$(BUNDLE_ID_PREFIX).housescan.uitests"; + SDKROOT = iphoneos; + TARGETED_DEVICE_FAMILY = 1; + TEST_TARGET_NAME = HouseScan; + }; + name = Release; + }; AC6CD8DF3FFE7F4BCC4D6275 /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { @@ -242,6 +629,25 @@ }; name = Release; }; + EDBBEE1A17A6FC582D27DB71 /* Debug */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = F30ACBC67A4159A449077AF5 /* Shared.xcconfig */; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CODE_SIGN_STYLE = Automatic; + GENERATE_INFOPLIST_FILE = YES; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + "@loader_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = "$(BUNDLE_ID_PREFIX).housescan.uitests"; + SDKROOT = iphoneos; + TARGETED_DEVICE_FAMILY = 1; + TEST_TARGET_NAME = HouseScan; + }; + name = Debug; + }; EE18EE856B3ACBECC802FAED /* Release */ = { isa = XCBuildConfiguration; buildSettings = { @@ -321,7 +727,30 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Debug; }; + CA663DA741070E6AFABA7C13 /* Build configuration list for PBXNativeTarget "HouseScanUITests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + EDBBEE1A17A6FC582D27DB71 /* Debug */, + 1D9A3F7F45102D23B342FD47 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Debug; + }; /* End XCConfigurationList section */ + +/* Begin XCLocalSwiftPackageReference section */ + 996A72D2A0E3ADCC195F64F8 /* XCLocalSwiftPackageReference "HouseScanKit" */ = { + isa = XCLocalSwiftPackageReference; + relativePath = HouseScanKit; + }; +/* End XCLocalSwiftPackageReference section */ + +/* Begin XCSwiftPackageProductDependency section */ + 19E9E07688F7D0E710D21D89 /* HouseScanKit */ = { + isa = XCSwiftPackageProductDependency; + productName = HouseScanKit; + }; +/* End XCSwiftPackageProductDependency section */ }; rootObject = 0BC33671F32C9E42F1E3A643 /* Project object */; } diff --git a/ios/HouseScan.xcodeproj/xcshareddata/xcschemes/HouseScan.xcscheme b/ios/HouseScan.xcodeproj/xcshareddata/xcschemes/HouseScan.xcscheme index ea132a850..a25862d38 100644 --- a/ios/HouseScan.xcodeproj/xcshareddata/xcschemes/HouseScan.xcscheme +++ b/ios/HouseScan.xcodeproj/xcshareddata/xcschemes/HouseScan.xcscheme @@ -39,7 +39,20 @@ + + + + + + SessionDelegate { - SessionDelegate(model: model) - } - - func makeUIView(context: Context) -> ARView { - let view = ARView(frame: .zero, cameraMode: .ar, automaticallyConfigureSession: false) - view.session.delegateQueue = DispatchQueue(label: "HouseScan.ARSessionDelegate") - view.session.delegate = context.coordinator - - let configuration = ARWorldTrackingConfiguration() - configuration.planeDetection = [.horizontal, .vertical] - view.session.run(configuration) - return view - } - - func updateUIView(_ view: ARView, context: Context) {} - - static func dismantleUIView(_ view: ARView, coordinator: SessionDelegate) { - view.session.pause() - } -} diff --git a/ios/HouseScan/Assets.xcassets/AccentColor.colorset/Contents.json b/ios/HouseScan/Assets.xcassets/AccentColor.colorset/Contents.json new file mode 100644 index 000000000..f9b84d4b2 --- /dev/null +++ b/ios/HouseScan/Assets.xcassets/AccentColor.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors": [ + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0xF2", + "green": "0x66", + "red": "0x1F" + } + }, + "idiom": "universal" + }, + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0xFF", + "green": "0x86", + "red": "0x4A" + } + }, + "idiom": "universal", + "appearances": [ + { + "appearance": "luminosity", + "value": "dark" + } + ] + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} diff --git a/ios/HouseScan/Assets.xcassets/AppIcon.appiconset/AppIcon.png b/ios/HouseScan/Assets.xcassets/AppIcon.appiconset/AppIcon.png new file mode 100644 index 000000000..911248f87 Binary files /dev/null and b/ios/HouseScan/Assets.xcassets/AppIcon.appiconset/AppIcon.png differ diff --git a/ios/HouseScan/Assets.xcassets/AppIcon.appiconset/Contents.json b/ios/HouseScan/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 000000000..714056358 --- /dev/null +++ b/ios/HouseScan/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,14 @@ +{ + "images": [ + { + "filename": "AppIcon.png", + "idiom": "universal", + "platform": "ios", + "size": "1024x1024" + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} diff --git a/ios/HouseScan/Assets.xcassets/Canvas.colorset/Contents.json b/ios/HouseScan/Assets.xcassets/Canvas.colorset/Contents.json new file mode 100644 index 000000000..5b5227d2e --- /dev/null +++ b/ios/HouseScan/Assets.xcassets/Canvas.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors": [ + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0xEC", + "green": "0xF1", + "red": "0xF2" + } + }, + "idiom": "universal" + }, + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0x14", + "green": "0x0B", + "red": "0x07" + } + }, + "idiom": "universal", + "appearances": [ + { + "appearance": "luminosity", + "value": "dark" + } + ] + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} diff --git a/ios/HouseScan/Assets.xcassets/Caution.colorset/Contents.json b/ios/HouseScan/Assets.xcassets/Caution.colorset/Contents.json new file mode 100644 index 000000000..6523f0ed8 --- /dev/null +++ b/ios/HouseScan/Assets.xcassets/Caution.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors": [ + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0x3D", + "green": "0xB5", + "red": "0xF5" + } + }, + "idiom": "universal" + }, + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0x3D", + "green": "0xB5", + "red": "0xF5" + } + }, + "idiom": "universal", + "appearances": [ + { + "appearance": "luminosity", + "value": "dark" + } + ] + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} diff --git a/ios/HouseScan/Assets.xcassets/Chalk.colorset/Contents.json b/ios/HouseScan/Assets.xcassets/Chalk.colorset/Contents.json new file mode 100644 index 000000000..51d6327f6 --- /dev/null +++ b/ios/HouseScan/Assets.xcassets/Chalk.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors": [ + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0xEF", + "green": "0xF5", + "red": "0xF7" + } + }, + "idiom": "universal" + }, + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0xEF", + "green": "0xF5", + "red": "0xF7" + } + }, + "idiom": "universal", + "appearances": [ + { + "appearance": "luminosity", + "value": "dark" + } + ] + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} diff --git a/ios/HouseScan/Assets.xcassets/Contents.json b/ios/HouseScan/Assets.xcassets/Contents.json new file mode 100644 index 000000000..74d6a722c --- /dev/null +++ b/ios/HouseScan/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info": { + "author": "xcode", + "version": 1 + } +} diff --git a/ios/HouseScan/Assets.xcassets/CoverageCovered.colorset/Contents.json b/ios/HouseScan/Assets.xcassets/CoverageCovered.colorset/Contents.json new file mode 100644 index 000000000..32393d33c --- /dev/null +++ b/ios/HouseScan/Assets.xcassets/CoverageCovered.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors": [ + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0x73", + "green": "0xC2", + "red": "0x2F" + } + }, + "idiom": "universal" + }, + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0x73", + "green": "0xC2", + "red": "0x2F" + } + }, + "idiom": "universal", + "appearances": [ + { + "appearance": "luminosity", + "value": "dark" + } + ] + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} diff --git a/ios/HouseScan/Assets.xcassets/CoverageHidden.colorset/Contents.json b/ios/HouseScan/Assets.xcassets/CoverageHidden.colorset/Contents.json new file mode 100644 index 000000000..b87e5a162 --- /dev/null +++ b/ios/HouseScan/Assets.xcassets/CoverageHidden.colorset/Contents.json @@ -0,0 +1,20 @@ +{ + "colors": [ + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "red": "0xB4", + "green": "0x9C", + "blue": "0xFF" + } + }, + "idiom": "universal" + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} diff --git a/ios/HouseScan/Assets.xcassets/CoverageSeen.colorset/Contents.json b/ios/HouseScan/Assets.xcassets/CoverageSeen.colorset/Contents.json new file mode 100644 index 000000000..6523f0ed8 --- /dev/null +++ b/ios/HouseScan/Assets.xcassets/CoverageSeen.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors": [ + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0x3D", + "green": "0xB5", + "red": "0xF5" + } + }, + "idiom": "universal" + }, + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0x3D", + "green": "0xB5", + "red": "0xF5" + } + }, + "idiom": "universal", + "appearances": [ + { + "appearance": "luminosity", + "value": "dark" + } + ] + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} diff --git a/ios/HouseScan/Assets.xcassets/CoverageSkipped.colorset/Contents.json b/ios/HouseScan/Assets.xcassets/CoverageSkipped.colorset/Contents.json new file mode 100644 index 000000000..64489c808 --- /dev/null +++ b/ios/HouseScan/Assets.xcassets/CoverageSkipped.colorset/Contents.json @@ -0,0 +1,20 @@ +{ + "colors": [ + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "red": "0x6B", + "green": "0x73", + "blue": "0x85" + } + }, + "idiom": "universal" + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} \ No newline at end of file diff --git a/ios/HouseScan/Assets.xcassets/CoverageUnseen.colorset/Contents.json b/ios/HouseScan/Assets.xcassets/CoverageUnseen.colorset/Contents.json new file mode 100644 index 000000000..8d2368bd2 --- /dev/null +++ b/ios/HouseScan/Assets.xcassets/CoverageUnseen.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors": [ + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0xA3", + "green": "0x93", + "red": "0x8A" + } + }, + "idiom": "universal" + }, + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0xA3", + "green": "0x93", + "red": "0x8A" + } + }, + "idiom": "universal", + "appearances": [ + { + "appearance": "luminosity", + "value": "dark" + } + ] + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} diff --git a/ios/HouseScan/Assets.xcassets/Danger.colorset/Contents.json b/ios/HouseScan/Assets.xcassets/Danger.colorset/Contents.json new file mode 100644 index 000000000..a8b7aafc8 --- /dev/null +++ b/ios/HouseScan/Assets.xcassets/Danger.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors": [ + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0x4E", + "green": "0x5A", + "red": "0xFF" + } + }, + "idiom": "universal" + }, + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0x4E", + "green": "0x5A", + "red": "0xFF" + } + }, + "idiom": "universal", + "appearances": [ + { + "appearance": "luminosity", + "value": "dark" + } + ] + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} diff --git a/ios/HouseScan/Assets.xcassets/Ink.colorset/Contents.json b/ios/HouseScan/Assets.xcassets/Ink.colorset/Contents.json new file mode 100644 index 000000000..486ac48ab --- /dev/null +++ b/ios/HouseScan/Assets.xcassets/Ink.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors": [ + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0x20", + "green": "0x12", + "red": "0x0B" + } + }, + "idiom": "universal" + }, + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0x20", + "green": "0x12", + "red": "0x0B" + } + }, + "idiom": "universal", + "appearances": [ + { + "appearance": "luminosity", + "value": "dark" + } + ] + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} diff --git a/ios/HouseScan/Assets.xcassets/Muted.colorset/Contents.json b/ios/HouseScan/Assets.xcassets/Muted.colorset/Contents.json new file mode 100644 index 000000000..53fbedc54 --- /dev/null +++ b/ios/HouseScan/Assets.xcassets/Muted.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors": [ + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "red": "0x54", + "green": "0x57", + "blue": "0x5E" + } + }, + "idiom": "universal" + }, + { + "appearances": [ + { + "appearance": "luminosity", + "value": "dark" + } + ], + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "red": "0xA9", + "green": "0xAD", + "blue": "0xB5" + } + }, + "idiom": "universal" + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} \ No newline at end of file diff --git a/ios/HouseScan/Assets.xcassets/Signal.colorset/Contents.json b/ios/HouseScan/Assets.xcassets/Signal.colorset/Contents.json new file mode 100644 index 000000000..255752fb6 --- /dev/null +++ b/ios/HouseScan/Assets.xcassets/Signal.colorset/Contents.json @@ -0,0 +1,20 @@ +{ + "colors": [ + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0xF2", + "green": "0x66", + "red": "0x1F" + } + }, + "idiom": "universal" + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} \ No newline at end of file diff --git a/ios/HouseScan/Assets.xcassets/SignalText.colorset/Contents.json b/ios/HouseScan/Assets.xcassets/SignalText.colorset/Contents.json new file mode 100644 index 000000000..150db763f --- /dev/null +++ b/ios/HouseScan/Assets.xcassets/SignalText.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors": [ + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "red": "0x16", + "green": "0x50", + "blue": "0xC8" + } + }, + "idiom": "universal" + }, + { + "appearances": [ + { + "appearance": "luminosity", + "value": "dark" + } + ], + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "red": "0x8F", + "green": "0xB2", + "blue": "0xFF" + } + }, + "idiom": "universal" + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} \ No newline at end of file diff --git a/ios/HouseScan/Assets.xcassets/Surface.colorset/Contents.json b/ios/HouseScan/Assets.xcassets/Surface.colorset/Contents.json new file mode 100644 index 000000000..4e32b357b --- /dev/null +++ b/ios/HouseScan/Assets.xcassets/Surface.colorset/Contents.json @@ -0,0 +1,38 @@ +{ + "colors": [ + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0xFF", + "green": "0xFF", + "red": "0xFF" + } + }, + "idiom": "universal" + }, + { + "color": { + "color-space": "srgb", + "components": { + "alpha": "1.000", + "blue": "0x26", + "green": "0x18", + "red": "0x12" + } + }, + "idiom": "universal", + "appearances": [ + { + "appearance": "luminosity", + "value": "dark" + } + ] + } + ], + "info": { + "author": "xcode", + "version": 1 + } +} diff --git a/ios/HouseScan/CaptureScreen.swift b/ios/HouseScan/CaptureScreen.swift deleted file mode 100644 index b47a11b82..000000000 --- a/ios/HouseScan/CaptureScreen.swift +++ /dev/null @@ -1,87 +0,0 @@ -import ARKit -import SwiftUI - -/// Full-screen camera with a status panel, or a plain message when AR cannot run. -struct CaptureScreen: View { - @State private var model = CaptureSessionModel() - - var body: some View { - if !ARWorldTrackingConfiguration.isSupported { - ContentUnavailableView( - "AR isn't available", - systemImage: "arkit", - description: Text("House Scan needs an iPhone that supports ARKit world tracking. The Simulator can't run it.") - ) - } else if let failure = model.failure { - FailureView(failure: failure) - } else { - ZStack(alignment: .top) { - ARCaptureView(model: model) - .ignoresSafeArea() - TrackingStatusPanel(model: model) - .padding() - } - } - } -} - -private struct TrackingStatusPanel: View { - let model: CaptureSessionModel - - var body: some View { - VStack(alignment: .leading, spacing: 4) { - Text(model.isInterrupted ? "Camera paused" : model.trackingState.instruction) - .font(.headline) - Text("Horizontal surfaces: \(model.horizontalPlaneCount)") - Text("Vertical surfaces: \(model.verticalPlaneCount)") - } - .font(.subheadline) - .monospacedDigit() - .padding(12) - .frame(maxWidth: .infinity, alignment: .leading) - .background(.regularMaterial, in: .rect(cornerRadius: 12)) - .accessibilityElement(children: .combine) - } -} - -private struct FailureView: View { - let failure: SessionFailure - @Environment(\.openURL) private var openURL - - var body: some View { - switch failure { - case .cameraAccessDenied: - ContentUnavailableView { - Label("Camera access is off", systemImage: "camera") - } description: { - Text("House Scan needs the camera to measure your wall. Turn on Camera for House Scan in Settings.") - } actions: { - Button("Open Settings") { - if let url = URL(string: UIApplication.openSettingsURLString) { - openURL(url) - } - } - } - case .other(let message): - ContentUnavailableView( - "The camera session stopped", - systemImage: "exclamationmark.triangle", - description: Text("\(message) Close and reopen House Scan to try again.") - ) - } - } -} - -private extension TrackingState { - var instruction: String { - switch self { - case .notAvailable: "Waiting for the camera" - case .normal: "Tracking" - case .limited(.initializing): "Move your phone slowly to start" - case .limited(.excessiveMotion): "Slow down" - case .limited(.insufficientFeatures): "Move your phone slowly" - case .limited(.relocalizing): "Point back at where you were" - case .limited(.unknown): "Hold your phone steady" - } - } -} diff --git a/ios/HouseScan/CaptureSessionModel.swift b/ios/HouseScan/CaptureSessionModel.swift deleted file mode 100644 index 4f656706c..000000000 --- a/ios/HouseScan/CaptureSessionModel.swift +++ /dev/null @@ -1,120 +0,0 @@ -import ARKit -import Observation - -/// What the AR session currently reports, for the capture screen to display. -/// -/// `SessionDelegate` writes to it from ARKit callbacks; SwiftUI reads it. -@MainActor -@Observable -final class CaptureSessionModel { - private(set) var trackingState: TrackingState = .notAvailable - private(set) var isInterrupted = false - private(set) var failure: SessionFailure? - - /// Whether this device can produce LiDAR scene depth. Recorded for later capture code; - /// the session never turns depth on and the app does not require it. - let lidarAvailable: Bool - - // Plane identifiers rather than running counts, so a repeated add or remove cannot skew a count. - private var horizontalPlanes: Set = [] - private var verticalPlanes: Set = [] - - var horizontalPlaneCount: Int { horizontalPlanes.count } - var verticalPlaneCount: Int { verticalPlanes.count } - - init(lidarAvailable: Bool = ARWorldTrackingConfiguration.supportsFrameSemantics(.sceneDepth)) { - self.lidarAvailable = lidarAvailable - } - - func trackingChanged(to state: TrackingState) { - trackingState = state - } - - func planesAdded(_ planes: [DetectedPlane]) { - for plane in planes { - switch plane.orientation { - case .horizontal: horizontalPlanes.insert(plane.id) - case .vertical: verticalPlanes.insert(plane.id) - } - } - } - - func planesRemoved(_ planes: [DetectedPlane]) { - for plane in planes { - switch plane.orientation { - case .horizontal: horizontalPlanes.remove(plane.id) - case .vertical: verticalPlanes.remove(plane.id) - } - } - } - - func interruptionChanged(isInterrupted: Bool) { - self.isInterrupted = isInterrupted - } - - func sessionFailed(_ error: any Error) { - if let arError = error as? ARError, arError.code == .cameraUnauthorized { - failure = .cameraAccessDenied - } else { - failure = .other(error.localizedDescription) - } - } -} - -enum TrackingState: Equatable, Sendable { - case notAvailable - case limited(LimitedReason) - case normal - - enum LimitedReason: Equatable, Sendable { - case initializing - case excessiveMotion - case insufficientFeatures - case relocalizing - case unknown - } - - init(_ state: ARCamera.TrackingState) { - switch state { - case .notAvailable: - self = .notAvailable - case .normal: - self = .normal - case .limited(let reason): - switch reason { - case .initializing: self = .limited(.initializing) - case .excessiveMotion: self = .limited(.excessiveMotion) - case .insufficientFeatures: self = .limited(.insufficientFeatures) - case .relocalizing: self = .limited(.relocalizing) - @unknown default: self = .limited(.unknown) - } - } - } -} - -/// A plane anchor reduced to the fields the model counts, so it can cross to the main actor. -struct DetectedPlane: Sendable { - enum Orientation: Sendable { - case horizontal - case vertical - } - - let id: UUID - let orientation: Orientation - - /// Returns nil for anchors that are not planes and for alignments added after iOS 26. - init?(_ anchor: ARAnchor) { - guard let plane = anchor as? ARPlaneAnchor else { return nil } - switch plane.alignment { - case .horizontal: orientation = .horizontal - case .vertical: orientation = .vertical - @unknown default: return nil - } - id = plane.identifier - } -} - -enum SessionFailure: Equatable, Sendable { - case cameraAccessDenied - case other(String) -} diff --git a/ios/HouseScan/Contract/ResultPresentation+Reading.swift b/ios/HouseScan/Contract/ResultPresentation+Reading.swift new file mode 100644 index 000000000..87654142e --- /dev/null +++ b/ios/HouseScan/Contract/ResultPresentation+Reading.swift @@ -0,0 +1,74 @@ +import HouseScanKit + +// What the result screen leads with, read from the presentation by HouseScanKit's +// `ResultReading`, where the rules are tested. Reading the presentation rather than the server's +// answer means the engine's results and the UI demo's samples go through the same rules. + +extension ResultPresentation { + /// True when there is a spot and the meter working-space check at it, if the server ran one, + /// passed. + var spotIsClean: Bool { + ResultReading.spotIsClean(hasSpot: spot != nil, checks: readingChecks) + } + + var answer: ResultReading.Answer { + ResultReading.answer(decision: placementDecision, policyApproved: policyApproved, hasSpot: spot != nil, checks: readingChecks) + } + + /// The check lines on the result card, in the order `ResultReading.cardLines` gives. + var cardChecks: [CheckRow] { + ResultReading.cardLines(readingChecks).map { checks[$0] } + } + + /// The view that would settle `row` and that the camera can take now: only for an unsure + /// check a person needn't judge. + func viewToTake(for row: CheckRow) -> MissingEvidence? { + guard row.outcome == .unsure, !row.needsPerson, let id = row.settledBy else { return nil } + return missing.first { $0.id == id && $0.capturable } + } + + /// The first view in `missing` that settles an unsure check and can be taken now. + var firstViewToTake: MissingEvidence? { + let wanted = Set(checks.compactMap { viewToTake(for: $0)?.id }) + return missing.first { wanted.contains($0.id) } + } + + private var readingChecks: [ResultReading.Check] { + checks.map { row in + ResultReading.Check( + id: row.id, outcome: row.outcome.placementOutcome, needsPerson: row.needsPerson, + viewCapturable: viewToTake(for: row) != nil, + margin: ResultReading.margin( + measured: row.measured.map(Double.init), threshold: row.threshold.map(Double.init), + plusMinus: row.plusMinus.map(Double.init), comparison: row.comparison?.placementComparison) + ) + } + } + + private var placementDecision: PlacementDecision { + switch decision { + case .pass: .pass + case .manualReview: .manualReview + case .reject: .reject + } + } +} + +private extension CheckOutcome { + var placementOutcome: PlacementOutcome { + switch self { + case .pass: .pass + case .fail: .fail + case .unsure: .unsure + } + } +} + +private extension RuleComparison { + var placementComparison: PlacementComparison { + switch self { + case .atLeast: .atLeast + case .atMost: .atMost + } + } +} diff --git a/ios/HouseScan/Contract/ScanContract.swift b/ios/HouseScan/Contract/ScanContract.swift new file mode 100644 index 000000000..6df2b8294 --- /dev/null +++ b/ios/HouseScan/Contract/ScanContract.swift @@ -0,0 +1,890 @@ +import CoreGraphics +import Foundation +import Observation +import simd +import SwiftUI + +// The boundary between the capture engine (Runtime/) and the screens (UI/). +// +// The engine owns every decision: what phase the scan is in, what the camera has covered, when a +// keyframe is kept, what to ask for next. It writes the results into `ScanViewState`. The screens +// read that state, turn the semantic values into words, pictures and motion, and send the +// homeowner's intents back through `ScanActions`. Nothing in UI/ computes geometry or coverage, +// and nothing in Runtime/ holds user-facing copy. +// +// World coordinates are ARKit's `.gravity` frame in meters: +y is up. Wall coordinates are +// `s` (meters along the wall from the meter, negative to the left when facing the wall), +// `height` (meters above the ground at the wall) and `out` (meters from the wall toward the +// homeowner). Feet appear only in copy and in the exported scene.json. + +// MARK: - Phase + +/// Screens of the flow. The raw value is the state name the engine logs as `STATE=` +/// (OSLog subsystem `dev.housescanning.housescan`, category `state`; contract C4). +enum ScanPhase: String, Sendable, CaseIterable { + case onboarding + case findMeter + case meterCloseUp + case wallWalk + case markFeatures + case gapRequest + case uploading + /// Before the result: is anything in the area the answer's spot rests on, and what is the + /// ground where it would stand (`SpotCheck`)? + case spotConfirm + case result + case resultAR + case unsupported +} + +// MARK: - Camera + +enum TrackingQuality: Equatable, Sendable { + case notAvailable + case normal + case limited(LimitedReason) + + enum LimitedReason: Equatable, Sendable { + case initializing + case excessiveMotion + case insufficientFeatures + case relocalizing + case unknown + } + + /// The phone doesn't know where it is in the world frame the scan was measured in: ARKit is + /// relocalizing, or not tracking at all. Anything tapped into the scene has to wait. The + /// other limited states (moving fast, a plain surface) still keep the frame. + var hasLostItsPlace: Bool { + switch self { + case .notAvailable, .limited(.relocalizing): true + case .normal, .limited: false + } + } +} + +/// What the screen shows behind the overlays. +enum CameraFeed: Equatable { + /// Nothing yet, for example before camera permission. + case none + /// The live AR camera. The view comes from `ScanActions.liveCameraView()`. + case live + /// A replayed keyframe: the unrotated landscape sensor image. Draw it rotated 90° clockwise + /// with aspect fill, exactly the way `CameraProjection.viewPoint` assumes. + case still(CGImage) +} + +/// Geometry of the frame currently on screen, enough to draw world points over the feed. +/// +/// The image is the landscape sensor image; the screen shows it rotated 90° clockwise, scaled to +/// fill the portrait view, centered and cropped. That is how iOS shows the back camera held +/// upright, and how the replay draws a still. +struct CameraProjection: Equatable, Sendable { + /// Camera-to-world transform. Camera space: +x right and +y up in the sensor image, looking along -z. + var cameraToWorld: simd_float4x4 + /// fx, fy, cx, cy in pixels of the sensor image. + var intrinsics: SIMD4 + /// Sensor image width and height in pixels (landscape, width > height). + var imageSize: SIMD2 + + var cameraPosition: SIMD3 { + SIMD3(cameraToWorld.columns.3.x, cameraToWorld.columns.3.y, cameraToWorld.columns.3.z) + } + + /// Where the camera looks, in world coordinates. + var forward: SIMD3 { + -SIMD3(cameraToWorld.columns.2.x, cameraToWorld.columns.2.y, cameraToWorld.columns.2.z) + } + + /// The point in camera space: +x right, +y up in the sensor image, -z ahead. + func cameraSpace(_ world: SIMD3) -> SIMD3 { + let local = cameraToWorld.inverse * SIMD4(world, 1) + return SIMD3(local.x, local.y, local.z) + } + + /// The sensor-image pixel of a world point, or nil when it is behind the camera (or within 5 cm + /// of its plane). The pixel can fall outside the image. + func imagePixel(for world: SIMD3) -> SIMD2? { + let p = cameraSpace(world) + guard p.z < -0.05 else { return nil } + let depth = -p.z + return SIMD2(intrinsics.z + intrinsics.x * p.x / depth, intrinsics.w - intrinsics.y * p.y / depth) + } + + /// View points per sensor pixel for a portrait view of `size`. + func scale(in size: CGSize) -> CGFloat { + max(size.width / CGFloat(imageSize.y), size.height / CGFloat(imageSize.x)) + } + + /// The view point of a world point in a portrait view of `size`, or nil when it is behind the + /// camera. The point can fall outside the view. + func viewPoint(for world: SIMD3, in size: CGSize) -> CGPoint? { + guard let pixel = imagePixel(for: world) else { return nil } + return viewPoint(forImagePixel: pixel, in: size) + } + + func viewPoint(forImagePixel pixel: SIMD2, in size: CGSize) -> CGPoint { + let s = scale(in: size) + let offsetX = (size.width - CGFloat(imageSize.y) * s) / 2 + let offsetY = (size.height - CGFloat(imageSize.x) * s) / 2 + return CGPoint(x: offsetX + (CGFloat(imageSize.y) - CGFloat(pixel.y)) * s, y: offsetY + CGFloat(pixel.x) * s) + } + + func imagePixel(forViewPoint point: CGPoint, in size: CGSize) -> SIMD2 { + let s = scale(in: size) + let offsetX = (size.width - CGFloat(imageSize.y) * s) / 2 + let offsetY = (size.height - CGFloat(imageSize.x) * s) / 2 + let rotatedX = (point.x - offsetX) / s + let rotatedY = (point.y - offsetY) / s + return SIMD2(Float(rotatedY), imageSize.y - Float(rotatedX)) + } +} + +// MARK: - Wall and coverage + +/// The wall line the scan measures, anchored on the meter. The engine refreshes `meter` from the +/// meter's ARAnchor every frame, so overlays drawn from it follow ARKit's corrections. +/// +/// Where the walk followed a corner the wall goes on as another straight piece +/// (`cornerSegments`), and s runs on continuously round the corner. +struct WallGeometry: Equatable, Sendable { + var meter: SIMD3 + /// Unit, horizontal. +s runs to the right of the meter for someone facing the wall. This and + /// `outward` are the meter's piece of wall. + var along: SIMD3 + /// Unit, horizontal, from the wall toward the homeowner. + var outward: SIMD3 + /// World y of the ground at the foot of the wall. + var groundY: Float + /// Marked wall ends in meters of s, once the homeowner has marked them. + var leftEnd: Float? + var rightEnd: Float? + /// The pieces of wall past each corner the walk followed; empty for a straight wall. + var cornerSegments: [Segment] = [] + + /// A straight piece of wall past a corner. + struct Segment: Equatable, Sendable { + /// The stretch of s it covers: infinite toward the open end, the corner's s toward the meter. + var span: ClosedRange + var along: SIMD3 + var outward: SIMD3 + /// Where its line has s = `anchorS` on the ground, as an offset from the meter's foot. + var anchor: SIMD3 + var anchorS: Float + } + + func world(s: Float, height: Float, out: Float = 0) -> SIMD3 { + let foot = SIMD3(meter.x, groundY, meter.z) + guard let piece = cornerSegments.first(where: { $0.span.contains(s) }) else { + return foot + along * s + outward * out + SIMD3(0, height, 0) + } + return foot + piece.anchor + piece.along * (s - piece.anchorS) + piece.outward * out + SIMD3(0, height, 0) + } + + /// The outward direction of the piece of wall at `s`. + func outward(atS s: Float) -> SIMD3 { + cornerSegments.first { $0.span.contains(s) }?.outward ?? outward + } + + /// The along direction of the piece of wall at `s`. + func along(atS s: Float) -> SIMD3 { + cornerSegments.first { $0.span.contains(s) }?.along ?? along + } + + /// The s of the wall point nearest a world point in plan. + func s(nearest world: SIMD3) -> Float { + let d = world - SIMD3(meter.x, groundY, meter.z) + func planDistance(_ offset: SIMD3) -> Float { simd_length(SIMD2(d.x - offset.x, d.z - offset.z)) } + let meterLow = cornerSegments.map(\.span.upperBound).filter { $0 <= 0 }.max() ?? -.infinity + let meterSpan = meterLow...(cornerSegments.map(\.span.lowerBound).filter { $0 >= 0 }.min() ?? .infinity) + var best = min(max(simd_dot(d, along), meterSpan.lowerBound), meterSpan.upperBound) + var bestDistance = planDistance(along * best) + for piece in cornerSegments { + let s = min(max(piece.anchorS + simd_dot(d - piece.anchor, piece.along), piece.span.lowerBound), piece.span.upperBound) + let distance = planDistance(piece.anchor + piece.along * (s - piece.anchorS)) + if distance < bestDistance { + best = s + bestDistance = distance + } + } + return best + } + + /// Height of the meter above the ground. + var meterHeight: Float { meter.y - groundY } +} + +enum CoverageBand: String, Sendable, CaseIterable { + /// The wall face from the ground up to `CoverageStrip.wallBandHeight`. + case wall + /// The ground from the foot of the wall out to `CoverageStrip.groundBandDepth`. + case ground +} + +enum CellState: UInt8, Equatable, Sendable { + /// No kept keyframe has seen this cell. + case unseen + /// Seen, but not yet from two positions. + case seen + /// Seen from at least two positions with normal tracking. Evidence exists; this is not a pass. + case covered + /// The homeowner said they can't get there. Recorded for installer review; never evidence, + /// and drawn distinctly from both fog and covered. + case skipped + /// LiDAR phones only: the camera looked here, but depth showed something nearer in the way + /// (a bush, a bin), so the wall or ground behind it is unseen. Not evidence; drawn distinctly. + case hidden +} + +/// Coverage of the unrolled wall, in fixed-width cells along s. +struct CoverageStrip: Equatable, Sendable { + var cellWidth: Float + /// s of the left edge of cell 0. + var firstCellS: Float + var wall: [CellState] + var ground: [CellState] + var wallBandHeight: Float + var groundBandDepth: Float + /// The s range worth drawing: what has been seen plus a margin of fog ahead, clipped to the + /// marked wall ends. Cells outside it are neither fog nor evidence; don't draw them. + var visibleRange: ClosedRange + /// Increments on every change, so views can diff and animate cells that just cleared. + var revision: Int + + static let empty = CoverageStrip( + cellWidth: 0.1524, firstCellS: 0, wall: [], ground: [], + wallBandHeight: 2.4, groundBandDepth: 1.2, visibleRange: 0...0, revision: 0 + ) + + func cellRange(_ index: Int) -> ClosedRange { + let start = firstCellS + Float(index) * cellWidth + return start...(start + cellWidth) + } + + func cells(_ band: CoverageBand) -> [CellState] { + band == .wall ? wall : ground + } + + /// Fraction of cells in `range` that are covered, per band. + func coveredFraction(_ band: CoverageBand, in range: ClosedRange) -> Double { + let states = cells(band).indices.filter { range.overlaps(cellRange($0)) }.map { cells(band)[$0] } + guard !states.isEmpty else { return 0 } + return Double(states.filter { $0 == .covered }.count) / Double(states.count) + } +} + +// MARK: - Guidance + +enum WallSide: String, Equatable, Sendable { + case left + case right +} + +/// What the homeowner should do next, as a meaning. UI/ owns the words. +enum GuidanceStep: Equatable, Sendable { + /// Point the camera at the electric meter and mark it. + case findMeter + /// Marking needs a vertical surface under the reticle; step closer or aim at the wall. + case aimAtWallForMeter + /// Hold the meter in the ring until the close-up is taken. + case holdOnMeter + /// Walk along the wall toward `side`. `remaining` is meters of s still unseen on that side, + /// nil while the end is unknown. "Can't get there" and "Wall ends here" end the wall on that + /// side where `ScanViewState.endPreview` shows. + case walk(side: WallSide, remaining: Float?) + /// The walk has gone about 20 ft on this side (`GuidanceConfig.reach`); ask where the wall + /// ends, marked at the reticle (`ScanActions.markWallEnd`). + case markEnd(side: WallSide) + /// Tilt down: the ground at the foot of the wall around `s` is missing. The walk asks for the + /// ground in front of the meter (s = 0) first. + case aimAtGround(s: Float) + /// Tilt up or step back: the wall face around `s` is missing. + case aimAtWall(s: Float) + /// Too close to the wall to see enough of it. + case stepBack + /// Both ends marked and coverage is complete enough; review marked features. + case walkComplete + /// Point the phone up at the wall above this stretch (meters of s), to show what is overhead + /// where the battery would stand. + case tiltUp(span: ClosedRange) + /// The wall turns a corner on `side`: walk round it, aim at the next wall and mark it + /// (`ScanActions.markNextWall`). `refusal` is set when the last mark was refused. + case markNextWall(side: WallSide, refusal: NextWallRefusal?) + /// Show a specific gap (see `ScanViewState.gap`). + case gap + /// Something stands in front of the wall or ground around `s` (LiDAR saw it): look at that + /// part from another angle or step around the obstruction. + case seeBehind(s: Float) +} + +/// A problem that overrides guidance until it clears. UI/ owns the words. +enum Coaching: Equatable, Sendable { + case initializing + case slowDown + case needsTexture + case tooDark + case holdSteady + case relocalizing + case trackingLost + /// The phone stands past an end it can't see back from, so the walk keeps no photos: nothing + /// in them would count (`CoverageMap.unexploredEndPassed`). + case pastWallEnd +} + +/// Where a wall end would land if the homeowner ended the wall now, while ending it is on offer. +/// The strip draws it before the tap, so an end never lands somewhere the screen didn't show. +struct EndPreview: Equatable, Sendable { + var side: WallSide + /// Meters of s. During the walk, where "Wall ends here" puts it, and "Can't get there" too + /// while the walk asks to walk that way: the phone's place along the wall, kept to the stretch + /// walked on that side (`WalkedEnd`). While the walk asks for the end (`GuidanceStep.markEnd`), + /// where the reticle meets the wall. + var s: Float + /// True when `s` comes from the reticle (`ScanActions.markWallEnd`), false when from the + /// phone's place (`ScanActions.endWallHere`). + var atReticle: Bool + /// Meters of walked path past `s` on this side, set when the homeowner walked back toward the + /// meter by at least a keyframe's spacing: what the walk saw from there is left out if the wall + /// ends at `s`. + var leavesOutWalked: Float? +} + +// MARK: - Captures + +struct CaptureEvent: Identifiable, Equatable { + enum Kind: Equatable, Sendable { + case walk + case closeUp + case gap + } + + let id: Int + let kind: Kind + /// A small, upright thumbnail for the acknowledgment, when cheap to produce. + let thumbnail: CGImage? +} + +enum CloseUpState: Equatable { + /// Aiming at the meter. `hold` runs 0...1 while every gate passes; the shutter fires at 1. + case aiming(hold: Double, problem: CloseUpProblem?) + case captured(CGImage?) + /// Two tries failed, or the homeowner chose "Can't get a clear shot". Goes to review. + case skipped +} + +enum CloseUpProblem: Equatable, Sendable { + case blurry + case tooDark + case tooBright + case meterNotCentered + case tooFar + case tracking + /// The best reading's characters are too small in the photo: move closer. + case numberTooSmall + /// No number could be read at all: retake. + case noNumber +} + +/// One reading of the meter number from the close-up, for the homeowner to confirm. +struct MeterNumberCandidate: Identifiable, Equatable, Sendable { + let id: Int + var text: String + /// A barcode on the meter carries the same number. + var barcodeConfirmed: Bool +} + +/// The meter number after the close-up. The homeowner confirms it by tapping; nothing is +/// filled in automatically. +enum MeterNumberState: Equatable, Sendable { + case reading + /// The best candidates, barcode-confirmed first, at most three. + case choose([MeterNumberCandidate]) + case confirmed(String) + /// The homeowner skipped the close-up; the number goes to review. + case skipped +} + +// MARK: - Features + +enum FeatureKind: String, Equatable, Sendable, CaseIterable, Identifiable { + case gasMeter = "gas_meter" + case door + case window + case acUnit = "ac" + case driveway = "drive" + case fence + + var id: String { rawValue } + + /// How many taps a mark takes, always two: diagonal corners on the wall for a door, window or + /// gas meter (where the meter meets the wall), the two front corners on the ground for an AC + /// unit, two points along the near edge of a driveway or the foot of a fence. One tap gives + /// a place but no size, and the export sends only measured sizes (`SceneFeature`). + var tapCount: Int { 2 } +} + +struct MarkedFeature: Identifiable, Equatable, Sendable { + let id: UUID + var kind: FeatureKind + /// Along-wall extent in meters of s. + var span: ClosedRange + var bottom: Float? + var top: Float? + /// Distance out from the wall, for ground items and fences. + var out: Float? + /// World points of the taps, for drawing pins. + var points: [SIMD3] + /// Windows only: the homeowner's answer, nil until asked. + var opens: Bool? +} + +/// Marking in progress: which kind, and which tap of `kind.tapCount` comes next (0-based). +struct MarkingState: Equatable, Sendable { + var kind: FeatureKind + var step: Int + /// Set when the last tap was refused, for example no surface under it. + var refusal: MarkRefusal? +} + +/// Why a mark of the next wall round a corner was refused. +/// The ground types a scene can record (scene.schema.json `ground[].type`), in the order the +/// question lists them. +enum GroundType: String, CaseIterable, Identifiable, Sendable { + case lawn, mulch, gravel, concrete, drive, deck + var id: String { rawValue } +} + +/// The homeowner's answer to what the ground is where the battery would stand, asked in the spot +/// check. The camera can't tell mulch from soil, so this is the only source of the ground's type. +enum GroundAnswer: Equatable, Sendable { + /// Exported as a patch of this type over the footprint and its error margin, where the + /// coverage saw ground (HouseScanKit `SpotGround`). + case type(GroundType) + /// "Not sure": no patch is sent and the server reports the surface as unknown. + case notSure +} + +enum NextWallRefusal: Equatable, Sendable { + /// No wall under the circle. + case noSurface + case trackingNotReady + /// The marked wall runs nearly the same way as this one: probably the same wall. + case sameWall + /// The marked wall doesn't meet this one anywhere near where it was marked as ending. + case notAtCorner +} + +enum MarkRefusal: Equatable, Sendable { + case noSurface + case wrongSide + case tooFarFromWall + case trackingNotReady + /// A fence's second tap is round a corner from its first: each side needs its own fence. + case fenceAcrossCorner + /// An AC unit's second corner is round a corner of the wall from its first. + case unitAcrossCorner +} + +// MARK: - Gap loop + +/// One targeted request for missing evidence, from the phone's planner or the server's result. +struct GapRequest: Identifiable, Equatable, Sendable { + enum Origin: Equatable, Sendable { + case phone + case server + } + + enum Reason: Equatable, Sendable { + /// The ground under a possible battery spot has not been seen from two positions. + case groundNearCandidate + /// The wall face above a possible spot is missing. + case wallAboveCandidate + /// The server listed evidence it needs; `detail` is its plain-language text. + case server(detail: String) + /// Show the ground out to `out` meters from the wall. + case groundOut(out: Float) + /// Walk this stretch at least `out` meters from the wall: a walked path shows the space + /// in front of the wall is clear out to it. + case walkOut(out: Float) + /// Tilt up here to show what is overhead. + case overhead + } + + let id: Int + var origin: Origin + var reason: Reason + var band: CoverageBand + var span: ClosedRange + /// 0...1 of the requested cells covered so far. + var progress: Double + var isSatisfied: Bool +} + +// MARK: - Upload and result + +enum UploadState: Equatable, Sendable { + case idle + case packaging + case uploading(fraction: Double) + case analyzing + /// A network failure or a server error (5xx): sending again can work. `offline` means no + /// connection at all. + case failed(message: String, offline: Bool) + /// The server refused the scan (4xx), its answer couldn't be read, or the scan couldn't be + /// packaged. Sending again would send the same thing, so the way on is back to the review + /// or start over, never "Try again". + case rejected(message: String) + case done +} + +enum CheckOutcome: String, Equatable, Sendable { + case pass + case fail + case unsure +} + +/// Which side of a rule's limit passes: a clearance the spot must keep (at least) or a length it +/// must stay within (at most). +enum RuleComparison: Equatable, Sendable { + case atLeast + case atMost +} + +struct CheckRow: Identifiable, Equatable, Sendable { + let id: String + var title: String + var outcome: CheckOutcome + /// Plain-language reason from the server. + var reason: String + /// For an unsure check: true when a person has to judge it (a measurement inside its error + /// band, a fact the camera can't establish, or a policy that sends it to review), false when + /// more photos would settle it. + var needsPerson: Bool = false + /// The deciding measurement, the rule's limit and the measurement's error, in meters, when the + /// server gave them. A borderline result shows all three ("3 ft 2 in from the gas meter; the + /// rule is at least 3 ft and our measurement can be off by about 4 in"). + var measured: Float? = nil + var threshold: Float? = nil + var plusMinus: Float? = nil + /// Whether `threshold` is a minimum or a maximum, when the server said which. + var comparison: RuleComparison? = nil + /// The `MissingEvidence.id` of the first view that would settle this check, when the server + /// named one. + var settledBy: String? = nil +} + +struct MissingEvidence: Identifiable, Equatable, Sendable { + let id: String + var text: String + /// True when another view can resolve it now; false means it goes to installer review. + var capturable: Bool + /// The `CheckRow.id`s this view settles, as the server listed them. + var checkIDs: [String] = [] +} + +struct BatterySpot: Equatable, Sendable { + /// Along-wall extent of the footprint, meters of s. + var span: ClosedRange + /// Footprint depth out from the wall and unit height, meters (from the server's rules). + var depth: Float + var height: Float + /// Gap between the wall and the back of the unit, meters. + var offsetFromWall: Float +} + +struct ClearanceZone: Identifiable, Equatable, Sendable { + let id: String + var label: String + var outcome: CheckOutcome + /// Along-wall extent and depth out from the wall, meters. + var span: ClosedRange + var depth: Float +} + +struct ResultPresentation: Equatable, Sendable { + enum Decision: String, Equatable, Sendable { + case pass + case manualReview = "manual_review" + case reject + } + + var decision: Decision + /// The server's one-sentence summary, without `rulesNotice`. + var summary: String = "" + /// False while the server's rules hold placeholder values: every would-be pass or reject is + /// then manual_review, and the screen should say the rules aren't final. + var policyApproved: Bool = true + /// Whose rules decided, from the server ("Demo rules: ... not Base's."), to show with the + /// answer. Nil when the rules need no such label. + var rulesNotice: String? = nil + /// The first eight characters of the rules' SHA-256 (`policy.rules_sha256`), so a reviewer + /// can tell which rules answered. + var rulesHash: String? = nil + var spot: BatterySpot? + /// When there is no spot: the spot the server found closest to passing. + var nearestSpot: BatterySpot? = nil + /// The `CheckRow.id` of the first check `nearestSpot` fails. Without a spot, `checks` are the + /// checks at `nearestSpot`. + var nearestFailingCheck: String? = nil + /// Cable route as (s, height) points along the wall, meters, from the meter to the spot. + var cableRoute: [SIMD2] + var cableLength: Float? + var checks: [CheckRow] + var clearances: [ClearanceZone] + var missing: [MissingEvidence] + /// A side of the meter the walk didn't reach, so a closer spot may exist there. + var unseenSide: WallSide? = nil + /// True when no server answered and the result is the offline sample used by tests and + /// demos. The UI must say so on screen. + var isSample: Bool +} + +// MARK: - Spot check + +/// The homeowner's first answer in the spot check: what is in the marked area. +enum SpotAreaAnswer: Equatable, Sendable { + /// Nothing is there: the ground question comes next. + case clear + /// Something stands there (a bush, a bin, a step). + case somethingThere + /// A gas meter, AC unit, window or door the scan didn't mark: which one comes next. + case unmarked +} + +/// How a spot check ended. +enum SpotCheckAnswer: Equatable, Sendable { + /// Nothing is in the area, and the ground is this. A type goes to the server with the next + /// upload; "Not sure" shows the result. + case clear(GroundAnswer) + /// Something stands there: the scan stops claiming that area and is checked again. + case somethingThere + /// Unmarked equipment the phone couldn't mark now (it lost its place): the area is left out + /// as for `somethingThere`. + case unmarkedCantMark(FeatureKind) + /// No kept photo shows the whole area, so nothing could be confirmed: the area is left out. + case unconfirmed + + /// The scan stopped claiming the area. + var withdrewArea: Bool { + switch self { + case .clear: false + case .somethingThere, .unmarkedCantMark, .unconfirmed: true + } + } +} + +/// The questions asked before an answer's spot is shown as the result, on one photo: is anything +/// in the footprint or its front clearance (something standing there, or equipment the scan +/// didn't mark), and then what is the ground where the battery would stand? A photo can claim +/// wall and ground behind a bush, and a walked path can pass over something low, so the scan's +/// claims there stand only once the homeowner says the area is clear (HouseScanKit `CoverageMap`, +/// "Bounded exceptions"). Meters of s along the wall and out from it, like `BatterySpot`. +struct SpotCheck: Equatable { + enum Step: Equatable, Sendable { + /// Is anything in the marked area? + case area + /// Which unmarked thing is it? + case which + /// What is the ground where the battery would stand? + case ground + } + + /// Counts the checks of a scan. + let id: Int + /// The spot's footprint along the wall and out from it. + var spot: ClosedRange + var spotOut: ClosedRange + var spotHeight: Float + /// The whole area asked about: the footprint and its front clearance. + var area: ClosedRange + var areaDepth: Float + /// The kept photo that shows the area best; nil when none shows even half the footprint. + var photo: Photo? + /// The photo shows the whole area, so the questions can be answered. Otherwise nothing can + /// be confirmed, and the only way on leaves the area out (`SpotCheckAnswer.unconfirmed`). + var confirmable: Bool + var step: Step = .area + /// Nil until answered. The answer stays up a moment before the flow moves on. + var answer: SpotCheckAnswer? + /// Once answered: the scan goes to the server again, rather than straight to the result. + var checksAgain = false + /// The spot is the bundled sample's, not a server's (`ResultPresentation.isSample`). + var isSample: Bool + + struct Photo: Equatable { + /// The unrotated landscape sensor image. Draw it rotated 90° clockwise, as `CameraFeed.still`. + var image: CGImage + /// Where it was taken, for drawing the area over it. + var projection: CameraProjection + } +} + +// MARK: - State and intents + +/// Everything the screens read. The engine is the only writer. +@MainActor +@Observable +final class ScanViewState { + var phase: ScanPhase = .onboarding + var feed: CameraFeed = .none + var projection: CameraProjection? + var tracking: TrackingQuality = .notAvailable + var coaching: Coaching? + var guidance: GuidanceStep = .findMeter + /// A world point to aim at (ring when on screen, edge chevron when not). + var target: SIMD3? + /// A walking path on the ground from the homeowner toward the next place to stand, world points. + var path: [SIMD3] = [] + + var wall: WallGeometry? + var coverage: CoverageStrip = .empty + var closeUp: CloseUpState = .aiming(hold: 0, problem: nil) + /// Close-up attempts that ended without a usable photo. "Can't get a clear shot" appears + /// from the second one on, never earlier. + var closeUpFailedAttempts = 0 + /// Nil until the close-up photo is taken. + var meterNumber: MeterNumberState? + /// The meter's maker as read from the close-up, shown above the number candidates. It exists + /// only beside those candidates or the number confirmed from them: while `meterNumber` is + /// nil, reading or skipped it is nil, so a brand from an earlier photo can't outlive its + /// number through a retake, a skip or a new close-up. The homeowner can reject it + /// (`rejectMeterBrand`). Like the number, it stays on the phone. + var meterBrand: String? { + get { + switch meterNumber { + case .choose, .confirmed: offeredMeterBrand + case .reading, .skipped, nil: nil + } + } + set { offeredMeterBrand = newValue } + } + /// Storage for `meterBrand`, set with the candidates it was read with. Read `meterBrand`. + private var offeredMeterBrand: String? + + var captureCount = 0 + var lastCapture: CaptureEvent? + + var features: [MarkedFeature] = [] + /// Marks lying wholly past a marked end, where the scan doesn't cover them. The review says + /// so; they are still exported, since a hazard just past an end can be within clearance of a + /// spot at it. + var featuresPastEnds: Set = [] + var marking: MarkingState? + + var gap: GapRequest? + /// A wall end was just marked on this side and the engine needs to know what is there: + /// the wall turns a corner (it continues, unexplored) or something blocks it (a fence, gate + /// or property line: a real limit). Nil when nothing is being asked. + var endQuestion: WallSide? + /// Where the wall end on the side being walked would land now; nil while ending it isn't on + /// offer (a question or a mark is up, both ends are marked, or the walk is doing something + /// else). "Wall ends here" shows only while it is set. + var endPreview: EndPreview? + /// True after "Done with this wall" was refused because the ends were closer together than + /// `WallFrame.minWallLength`; the ends were cleared. False again once an end is marked. + var wallTooShort = false + /// Set after the tilt-up view: is anything overhead there (roof edge, porch, stairs)? The + /// camera can't tell open sky from an eave, so the homeowner answers. + var overheadQuestion = false + var upload: UploadState = .idle + var result: ResultPresentation? + /// The spot check of the result's spot: the question during `.spotConfirm`, and the answer + /// that stands for the spot on the result. + var spotCheck: SpotCheck? + /// True while the AR result is drawn into the live camera, where people and objects in front + /// of it hide it. The AR screen then draws no overlay of its own. + var resultInCamera = false + /// False once the camera failed after the scan was sent: the answer stays, but "See it on + /// your wall" is neither shown nor offered until a new scan starts the camera again. + var spatialResultAvailable = true + + /// True when frames come from a recorded session instead of the camera. + var isReplay = false + /// True when the autopilot is driving the intents (UI tests, demos). Show a small badge. + var isAutopilot = false + /// True when no server is configured and the result will be the bundled sample: nothing is + /// sent, and every screen that talks about the upload or shows the spot must say so. + var usesSampleResult = false + /// True when the phone has LiDAR and coverage counts only what depth confirms. + var depthAvailable = false + /// The scan's bundle (scene.json, keyframe photos and their poses) once it is packaged, for + /// "Share scan". Photos leave the phone only if the homeowner shares this. + var shareableScan: URL? + /// A camera permission or session failure the homeowner can act on. + var failure: ScanFailure? + + init() {} +} + +enum ScanFailure: Equatable, Sendable { + case cameraDenied + case arUnsupported + case sessionFailed(String) + case replayUnreadable(String) +} + +/// The homeowner's intents. View points are in the coordinate space of the full-screen camera +/// view of `viewSize`; nil means the reticle at the view's center. +@MainActor +protocol ScanActions: AnyObject { + func finishOnboarding() + func markMeter(at point: CGPoint?, viewSize: CGSize) + func skipCloseUp() + /// The homeowner's pick from `MeterNumberState.choose`; nil means "None of these", which + /// asks for a retake. + func chooseMeterNumber(_ candidate: MeterNumberCandidate?) + /// "Not " beside the number candidates: drops `ScanViewState.meterBrand`. + func rejectMeterBrand() + /// Marks a wall end where `point` meets the wall, on whichever side of the meter that is. + func markWallEnd(at point: CGPoint?, viewSize: CGSize) + /// "Wall ends here" during the walk: ends the wall on the side being walked where + /// `ScanViewState.endPreview` shows it (the phone's place), then asks what is there + /// (`ScanViewState.endQuestion`). Does nothing while `endPreview` is nil or at the reticle. + func endWallHere() + /// The answer to `ScanViewState.endQuestion`. During the walk a corner asks for the next wall + /// (`GuidanceStep.markNextWall`); a corner the walk doesn't follow exports as an unexplored + /// end, a blocked wall as a limit, and an end left unanswered stays unexplored. + func answerWallEnd(turnsCorner: Bool) + /// Marks the wall under `point` as the next wall round the corner, during + /// `GuidanceStep.markNextWall`. The walk then goes on along it; "I can't get there" + /// (`cannotAccessArea`) leaves the end as an unexplored corner instead. + func markNextWall(at point: CGPoint?, viewSize: CGSize) + /// The answer to `ScanViewState.overheadQuestion`: true when nothing is overhead. + func answerOverhead(clear: Bool) + func beginMarking(_ kind: FeatureKind) + func markFeaturePoint(at point: CGPoint?, viewSize: CGSize) + func cancelMarking() + func deleteFeature(_ id: UUID) + func setWindowOpens(_ id: UUID, opens: Bool) + /// Leave the walk for the feature review (allowed once both ends are marked). Ends closer + /// together than `WallFrame.minWallLength` are cleared instead, and `wallTooShort` is set. + func finishWalk() + /// Features confirmed; the engine runs the gap check, then uploads. + func confirmFeatures() + /// "I can't get there": the gap is recorded for installer review. + func skipGap() + /// "I can't get to this part of the wall", during the walk: the cells the guidance is asking + /// for become `.skipped` and the guidance moves on to the next task. While the walk asks to + /// walk a side, that side's end goes where `ScanViewState.endPreview` shows, as an unexplored + /// end. + func cannotAccessArea() + func retryUpload() + /// After a rejected upload: back to the feature review, keeping the scan. + func backToReview() + /// The spot check's first question (`SpotCheck.Step.area`). + func answerSpotArea(_ answer: SpotAreaAnswer) + /// Which unmarked thing is in the area (`SpotCheck.Step.which`): the camera opens to mark it, + /// or, when the phone can't mark now, the area is left out. Nil goes back to the question. + func chooseUnmarked(_ kind: FeatureKind?) + /// The ground question (`SpotCheck.Step.ground`). + func answerSpotGround(_ answer: GroundAnswer) + /// No photo shows the whole area (`SpotCheck.confirmable` false): leave it out and go on. + func continueUnconfirmed() + /// Start a capture for a server-listed missing item. + func captureMissing(_ id: String) + func showAR() + func closeAR() + func startOver() + /// The live AR camera view. Only called while `feed == .live`. + func liveCameraView() -> AnyView +} diff --git a/ios/HouseScan/HouseScanApp.swift b/ios/HouseScan/HouseScanApp.swift index d791cf645..013851423 100644 --- a/ios/HouseScan/HouseScanApp.swift +++ b/ios/HouseScan/HouseScanApp.swift @@ -4,7 +4,11 @@ import SwiftUI struct HouseScanApp: App { var body: some Scene { WindowGroup { - CaptureScreen() + if ProcessInfo.processInfo.arguments.contains("-uiDemo") { + UIDemo.makeRoot() + } else { + AppRuntime.makeRoot() + } } } } diff --git a/ios/HouseScan/Runtime/AppRuntime.swift b/ios/HouseScan/Runtime/AppRuntime.swift new file mode 100644 index 000000000..f0a1c7826 --- /dev/null +++ b/ios/HouseScan/Runtime/AppRuntime.swift @@ -0,0 +1,29 @@ +import SwiftUI + +/// Builds the engine from the launch arguments and returns `ScanRootView` bound to it. +@MainActor +enum AppRuntime { + /// One engine per process: the scene body can be re-evaluated, the scan must not restart. + static let engine = ScanEngine(options: LaunchOptions()) + + static func makeRoot() -> some View { + RuntimeRoot(engine: engine) + } +} + +private struct RuntimeRoot: View { + let engine: ScanEngine + @State private var started = false + + var body: some View { + ScanRootView(state: engine.state, actions: engine) + .task { + guard !started else { return } + started = true + engine.start() + if engine.options.autopilot { + await Autopilot(engine: engine).run() + } + } + } +} diff --git a/ios/HouseScan/Runtime/Autopilot.swift b/ios/HouseScan/Runtime/Autopilot.swift new file mode 100644 index 000000000..a3287b8c6 --- /dev/null +++ b/ios/HouseScan/Runtime/Autopilot.swift @@ -0,0 +1,553 @@ +import CoreGraphics +import Foundation +import HouseScanKit +import OSLog +import simd + +/// Drives the intents through the whole flow on a replay, for UI tests and demos (`-autopilot`). +/// +/// It calls the same `ScanActions` a homeowner's taps call. Taps are simulated at view points +/// computed by projecting chosen wall points through the shown replay frame, so the engine +/// computes the geometry exactly as it does for a finger. Each state is held for about a second +/// so screenshots show it. +@MainActor +final class Autopilot { + private let engine: ScanEngine + /// A portrait iPhone screen in points. Any size works: view points and the engine's mapping + /// back to pixels use the same size. + private let viewSize = CGSize(width: 393, height: 852) + private var hold: Double { engine.options.autopilotHold } + /// Set once a spot check was answered "Something's there" (`-autopilotSomethingThere`): the + /// checks after it are answered "It's clear", so a server that keeps moving the spot can't + /// keep the run going. + private var answeredSomethingThere = false + + init(engine: ScanEngine) { + self.engine = engine + } + + func run() async { + guard engine.options.replayFolder != nil else { + log("needs -replay; the live camera can't be driven") + return + } + guard await waitUntil(timeout: 60, { self.engine.replay != nil || self.engine.state.failure != nil }), let replay = engine.replay else { + return fail("replay did not load") + } + async let prepared: Void = replay.prepareHeldBack(endsWherePhoneStood: engine.options.autopilotCantGetThere) + await pause(hold) + await engine.waitForGate(.onboarding) + engine.finishOnboarding() + await prepared + if let window = replay.heldBack { + log("holding back frames \(window.frames.lowerBound)..<\(window.frames.upperBound) for the gap loop; gap \(window.gap.band.rawValue) \(format(window.gap.span))") + } else { + log("no frames can be held back to make a closable gap on this replay; the gap step will be skipped") + } + await pause(hold) + + await engine.waitForGate(.findMeter) + engine.markMeter(at: nil, viewSize: viewSize) + guard await waitFor(.meterCloseUp, timeout: 10) else { return fail("meter was not marked") } + await takeCloseUp() + guard await waitFor(.wallWalk, timeout: 150) else { return fail("walk did not start") } + if engine.options.autopilotCantGetThere { await skipGroundByMeterBeforeItIsSeen(replay) } + + await pause(0.5) + guard await waitUntil(timeout: 120, { !replay.isPlaying }) else { return fail("walk replay did not finish") } + await pause(1.0) + + await markFeatures(replay) + if engine.options.autopilotCantGetThere { + await endWalkByCantGetThere(replay) + } else { + await markEnds(replay) + } + await tiltUp() + await pause(1.0) + await engine.waitForGate(.wallWalk) + engine.finishWalk() + guard await waitFor(.markFeatures, timeout: 5) else { return fail("could not finish the walk (ends marked: \(engine.bothEndsMarked))") } + await pause(hold) + + await engine.waitForGate(.markFeatures) + // Confirmed after the UI test has finished with the screen, so its audit reads a still tree. + engine.confirmFeatures() + await pause(0.3) + if engine.state.phase == .gapRequest { + if engine.state.gap?.reason == .overhead { + await answerOverheadGap() + } else if replay.heldBack == nil { + await pause(hold) + log("skipping the gap: no held-back frames to show it") + await engine.waitForGate(.gapRequest) + engine.skipGap() + } else { + // The held-back frames were chosen on geometry alone; on a LiDAR replay depth can + // find part of the gap hidden in them, and then they don't settle it. + await playGapFrames(replay) + } + } + guard await waitFor(.uploading, timeout: 150) else { return fail("upload did not start") } + guard await driveToResult(replay) else { return } + writeSceneForTest() + await pause(hold) + await engine.waitForGate(.result) + engine.showAR() + guard await waitFor(.resultAR, timeout: 5) else { return fail("AR result did not open") } + await pause(hold) + await engine.waitForGate(.resultAR) + engine.closeAR() + _ = await waitFor(.result, timeout: 5) + log("done") + } + + // MARK: Steps + + /// What the close-up came to, as far as the autopilot acts on it. + private enum CloseUpOutcome { + case choose(MeterNumberCandidate) + /// The photo has to be retaken because no number could be read from it. + case retake(CloseUpProblem) + /// The flow left the close-up by itself. + case left + } + + private var closeUpOutcome: CloseUpOutcome? { + let state = engine.state + guard state.phase == .meterCloseUp else { return .left } + if case .choose(let candidates) = state.meterNumber, let first = candidates.first { return .choose(first) } + if case .aiming(_, let problem?) = state.closeUp, problem == .noNumber || problem == .numberTooSmall { return .retake(problem) } + return nil + } + + /// Picks the first meter-number candidate after the hold, as a homeowner confirming it would. + /// When the number can't be read (the synthetic fixture has none) or the close-up never + /// fires, skips the close-up instead of retaking forever. + private func takeCloseUp() async { + // 4 s for the shutter on a 3x replay, plus time for the reader. + _ = await waitUntil(timeout: 8) { self.closeUpOutcome != nil } + switch closeUpOutcome { + case .choose(let first): + await pause(hold) + engine.chooseMeterNumber(first) + log("chose meter-number candidate \(first.id)") + case .retake(let problem): + await pause(hold) + log("close-up needs a retake (\(String(describing: problem))); skipping it as a homeowner would") + await engine.waitForGate(.meterCloseUp) + engine.skipCloseUp() + case .left: + return + case nil: + log("close-up did not fire on this replay; skipping it as a homeowner would") + await engine.waitForGate(.meterCloseUp) + engine.skipCloseUp() + } + } + + /// A gas meter (one tap) and a window (two diagonal corners) on wall seen at least as high as + /// the highest tap, 1.85 m. The synthetic fixture paints them at s = -1.2 m and s = 2.0...3.0 + /// m; on other replays the nearest such wall stands in for them. + private func markFeatures(_ replay: ReplayPlayer) async { + guard let map = engine.coverage else { return } + let covered = map.wallSeenSpans().filter { $0.out >= 1.85 }.map(\.span) + guard !covered.isEmpty else { + log("no covered wall to mark features on") + return + } + let gasS = nearestCovered(-1.2, in: covered) + engine.beginMarking(.gasMeter) + // Two corners where it meets the wall, inside the fixture's 0.3 m by 0.5 m meter. + for corner in [map.wall.world(s: gasS - 0.12, height: 0.35), map.wall.world(s: gasS + 0.12, height: 0.75)] { + let tapped = await tap(corner, replay: replay) { point in + self.engine.markFeaturePoint(at: point, viewSize: self.viewSize) + } + if !tapped { log("no replay frame shows a gas meter corner") } + await pause(0.4) + } + await pause(0.8) + + let windowS = nearestCovered(2.5, in: covered) + engine.beginMarking(.window) + // Near the corners of the fixture's window (0.9 to 2.0 m high), kept inside the band a + // walking camera sees so a frame shows each tap. + for corner in [map.wall.world(s: windowS - 0.45, height: 1.85), map.wall.world(s: windowS + 0.45, height: 0.95)] { + let tapped = await tap(corner, replay: replay) { point in + self.engine.markFeaturePoint(at: point, viewSize: self.viewSize) + } + if !tapped { log("no replay frame shows a window corner") } + await pause(0.4) + } + if engine.state.marking != nil { + log("window marking incomplete (\(String(describing: self.engine.state.marking?.refusal))); cancelling it") + engine.cancelMarking() + } + if let window = engine.state.features.first(where: { $0.kind == .window }) { + engine.setWindowOpens(window.id, opens: true) + } + log("marked \(engine.state.features.count) features") + } + + /// Ends at the covered extremes (the held-back window's plan, or the walk's coverage). + private func markEnds(_ replay: ReplayPlayer) async { + guard let map = engine.coverage else { return } + let intervals = map.wallSeenSpans().map(\.span) + map.coveredIntervals(.ground) + guard let low = intervals.map(\.lowerBound).min(), let high = intervals.map(\.upperBound).max() else { + log("nothing covered; marking ends 1 m either side of the meter") + engine.setEnd(.left, at: -1, kind: .limit) + engine.setEnd(.right, at: 1, kind: .limit) + return + } + for (side, s) in [(WallSide.left, min(low, -0.2)), (.right, max(high, 0.2))] { + let point = map.wall.world(s: s, height: 0.5) + let tapped = await tap(point, replay: replay) { viewPoint in + self.engine.markWallEnd(at: viewPoint, viewSize: self.viewSize) + } + let marked = side == .left ? engine.coverage?.leftEnd : engine.coverage?.rightEnd + if !tapped || marked == nil { + log("no replay frame shows the \(side.rawValue) end; setting it directly at s=\(s)") + engine.setEnd(side, at: s, kind: .limit) + } + if let asked = engine.state.endQuestion { + // Every end counts as a corner here: a replay has no fence or gate to report. + await pause(hold) + engine.answerWallEnd(turnsCorner: true) + log("answered the \(asked.rawValue) end: turns a corner") + if case .markNextWall = engine.state.guidance { + // A replay has no live wall round the corner to mark, so it doesn't follow + // the corner, as a homeowner who can't get round it wouldn't. + await pause(hold) + engine.cannotAccessArea() + log("can't get round the \(asked.rawValue) corner on a replay; the end stays unexplored") + } + } + await pause(0.6) + } + } + + // MARK: Can't get there (-autopilotCantGetThere) + + /// The walk's first request is the ground in front of the meter. This says "Can't get there" + /// to it as soon as it shows, before the replay has shown that ground from two places: on + /// device run 1 nothing had seen it when the ends were set, and the walk has to move on. + private func skipGroundByMeterBeforeItIsSeen(_ replay: ReplayPlayer) async { + _ = await waitUntil(timeout: 20) { self.isAskingForGroundByMeter || !replay.isPlaying } + guard isAskingForGroundByMeter, let map = engine.coverage else { + log("the walk did not ask for the ground by the meter while the replay played") + return + } + let window = map.indices(overlapping: -GuidancePlanner.aimHalfWidth...GuidancePlanner.aimHalfWidth) + let covered = window.filter { map.level(.ground, $0) == .covered }.count + engine.cannotAccessArea() + log("can't get there on the ground by the meter, \(covered) of its \(window.count) cells covered; now \(ScanEngine.name(self.engine.state.guidance))") + } + + private var isAskingForGroundByMeter: Bool { + if case .aimAtGround(let s) = engine.state.guidance { return abs(s) < 0.01 } + return false + } + + /// Ends each side with "Can't get there" where the walk went farthest that way, as a + /// homeowner who reached the end of what they can walk would, instead of marking the ends. + /// The phone stands there again (that frame plays once more), and any request about the wall + /// or ground in front of it gets "Can't get there" first, until the walk asks to walk on that + /// way: that is when the tap ends the wall. The engine logs where each end went. + private func endWalkByCantGetThere(_ replay: ReplayPlayer) async { + guard let wall = engine.coverage?.wall else { return } + let walkEnd = ScanEngine.tiltUpFrames(in: replay, map: engine.coverage ?? CoverageMap(wall: wall)).lowerBound + let played = (0.. Bool { + switch engine.state.guidance { + case .walk(let walking, _), .markEnd(let walking): walking == side + default: false + } + } + + private var asksAboutTheWallHere: Bool { + switch engine.state.guidance { + case .aimAtGround, .aimAtWall, .seeBehind: true + default: false + } + } + + /// With `-autopilotGate`, leaves the scene.json of the scan the result answers in the gate + /// folder, for the UI test to check its ends and what it reports as seen, and the scene.json + /// the last upload sent as `uploaded-scene.json`. + private func writeSceneForTest() { + guard let gate = engine.options.autopilotGate else { return } + do { + if let snapshot = engine.makeUploadSnapshot(spatial: nil) { + try engine.sceneJSON(snapshot).write(to: gate.appending(path: "scene.json")) + } + try engine.spotConfirm.lastScene?.write(to: gate.appending(path: "uploaded-scene.json")) + try engine.spotConfirm.lastAnswer?.write(to: gate.appending(path: "answer.json")) + log("wrote scene.json to the gate folder") + } catch { + log("could not write scene.json to the gate folder: \(error)") + } + } + + /// The tilt-up step: plays the recording's closing tilt-up frames and answers "Open sky or + /// nothing overhead", which is what the synthetic fixture shows above its wall. A recording + /// without tilt-up frames can't answer honestly, so it gets "Can't get there" instead. + private func tiltUp() async { + guard await waitUntil(timeout: 5, { self.isTiltingUp }) else { + log("the walk did not ask to tilt up") + return + } + await pause(hold) + guard engine.playReplayTiltUp() else { + log("no tilt-up frames on this replay; skipping the step as a homeowner would") + engine.cannotAccessArea() + return + } + guard await waitUntil(timeout: 20, { self.engine.state.overheadQuestion }) else { + log("no tilt-up frame was kept; skipping the step") + if isTiltingUp { engine.cannotAccessArea() } + return + } + await pause(hold) + await answerOpenSky() + log("answered the overhead question: open sky; \(overheadSummary)") + // The rest of the tilt-up run (the synthetic replay's upper walk) plays out, so its + // views are kept before the walk ends. + guard let replay = engine.replay else { return } + _ = await waitUntil(timeout: 30) { !replay.isPlaying } + } + + /// An overhead gap request: the engine plays the recording's tilt-up frames for it, and the + /// question comes up only when one of them settles the request. The synthetic fixture shows + /// open sky above its wall, so the answer is "Open sky or nothing overhead". When no tilt-up + /// frame covers the requested span, "I can't get there", as a homeowner would. + private func answerOverheadGap() async { + guard await waitUntil(timeout: 20, { self.engine.state.overheadQuestion || self.engine.state.phase != .gapRequest }), + engine.state.overheadQuestion else { + log("no tilt-up frame covers the overhead request; skipping it") + await engine.waitForGate(.gapRequest) + engine.skipGap() + return + } + await pause(hold) + await answerOpenSky() + log("answered the overhead request: open sky; \(overheadSummary)") + } + + /// After an upload the engine raises the answer's capturable requests one at a time, uploading + /// after each, before it shows the result, and checks the answer's spot with the homeowner. + /// Drives each request and check, retries a failed upload once, and returns true once the + /// result is up. Bounded by the engine's own bound on requests, and by one refused check. + private func driveToResult(_ replay: ReplayPlayer) async -> Bool { + var retried = false + for _ in 0..<(ScanEngine.maxAutomaticGaps + 4) { + let settled = await waitUntil(timeout: 240) { + [.result, .gapRequest, .spotConfirm].contains(self.engine.state.phase) || self.uploadFailed + } + guard settled else { + fail("no result") + return false + } + switch engine.state.phase { + case .result: + return true + case .gapRequest: + await driveServerRequest(replay) + case .spotConfirm: + await answerSpotCheck() + default: + guard !retried else { + fail("no result after retry") + return false + } + retried = true + log("upload failed; retrying once") + engine.retryUpload() + } + } + fail("still no result after \(ScanEngine.maxAutomaticGaps) requests") + return false + } + + /// "It's clear" and mulch at the spot, or with `-autopilotSomethingThere` "Something's in the + /// way" the first time. When no kept photo shows the whole area there is no question, only + /// Continue, which leaves the area out. After either withdrawal the stretch withdrawn goes to + /// the gate folder as `spot-refusal.json` (the stretch's s in feet, as scene.json's spans), + /// for the UI test to check the scene against. + private func answerSpotCheck() async { + guard let check = engine.state.spotCheck, check.answer == nil else { return } + await pause(hold) + await engine.waitForGate(.spotConfirm) + if !check.confirmable { + engine.continueUnconfirmed() + log("spot check \(check.id) over \(format(check.area)): no kept photo shows the whole area; left out") + writeRefusalForTest() + } else if engine.options.autopilotSomethingThere && !answeredSomethingThere { + engine.answerSpotArea(.somethingThere) + log("spot check \(check.id) over \(format(check.area)): something's in the way") + answeredSomethingThere = true + writeRefusalForTest() + } else { + engine.answerSpotArea(.clear) + _ = await waitUntil(timeout: 5) { self.engine.state.spotCheck?.step == .ground } + await pause(hold) + engine.answerSpotGround(.type(.mulch)) + log("spot check \(check.id) over \(format(check.area)): it's clear, mulch") + } + _ = await waitUntil(timeout: 20) { self.engine.state.phase != .spotConfirm } + } + + /// The whole cells the refusal withdrew, in feet: what the scene must no longer claim. + private func writeRefusalForTest() { + guard let gate = engine.options.autopilotGate, let map = engine.coverage, + let low = map.withdrawnCells.min(), let high = map.withdrawnCells.max() else { return } + let feet = { (meters: Float) in Double(meters) * SceneUnits.feetPerMeter } + let span = [feet(map.cellRange(low).lowerBound), feet(map.cellRange(high).upperBound)] + do { + try JSONSerialization.data(withJSONObject: ["withdrawn_span_ft": span]).write(to: gate.appending(path: "spot-refusal.json")) + } catch { + log("could not write spot-refusal.json to the gate folder: \(error)") + } + } + + private var uploadFailed: Bool { + if case .failed = engine.state.upload { return engine.state.phase == .uploading } + return false + } + + /// A request the engine raised from the server's answer. The engine plays the replay's frames + /// for it; a replay shows only what it recorded, so when they don't settle the request the + /// autopilot says "I can't get there", as a homeowner would. An overhead request is answered + /// like the one before the upload. + private func driveServerRequest(_ replay: ReplayPlayer) async { + guard let request = engine.state.gap else { return } + log("server request \(request.id) over \(format(request.span))") + if request.reason == .overhead { + await answerOverheadGap() + } else { + await playGapFrames(replay) + } + // A settled request stays on screen for a moment before the next upload. + _ = await waitUntil(timeout: 10) { self.engine.state.phase != .gapRequest || self.engine.state.gap?.id != request.id } + } + + /// Waits while the engine plays the replay's frames for the current request. A replay shows + /// only what it recorded, so when they don't settle the request the autopilot says "I can't + /// get there", as a homeowner would. + private func playGapFrames(_ replay: ReplayPlayer) async { + guard let request = engine.state.gap else { return } + _ = await waitUntil(timeout: 60) { !replay.isPlaying || self.engine.state.gap?.id != request.id } + // Time for the last frame's keyframe to be stored and counted. + await pause(hold) + if engine.state.phase == .gapRequest, let gap = engine.state.gap, gap.id == request.id, !gap.isSatisfied { + log("the replay does not settle request \(request.id) (\(Int(gap.progress * 100))% of it seen); skipping it") + await engine.waitForGate(.gapRequest) + engine.skipGap() + } + } + + /// "Open sky or nothing overhead", then a wait for the view to be stored: it counts as an + /// overhead view only once its photo is on disk. + private func answerOpenSky() async { + let before = engine.coverage?.overheadCameras.count ?? 0 + engine.answerOverhead(clear: true) + _ = await waitUntil(timeout: 5) { (self.engine.coverage?.overheadCameras.count ?? 0) > before } + } + + /// The overhead views kept and the stretches they show clear, as the export will send them. + private var overheadSummary: String { + guard let map = engine.coverage else { return "no wall" } + let spans = map.overheadSpans().map { "\(format($0.span)) to \(String(format: "%.2f", $0.out)) m" } + return "\(map.overheadCameras.count) overhead view(s) kept, clear over \(spans.isEmpty ? "nothing" : spans.joined(separator: ", "))" + } + + private var isTiltingUp: Bool { + if case .tiltUp = engine.state.guidance { return true } + return false + } + + // MARK: Helpers + + /// Shows the replay frame that best shows `point`, then taps where it appears on screen. + @discardableResult + private func tap(_ point: SIMD3, replay: ReplayPlayer, action: (CGPoint) -> Void) async -> Bool { + guard let (index, viewPoint) = frameShowing(point, replay: replay) else { return false } + replay.show(index: index) + await replay.waitUntilShown(index) + await pause(0.2) + action(viewPoint) + return true + } + + private func frameShowing(_ point: SIMD3, replay: ReplayPlayer) -> (Int, CGPoint)? { + let inset: CGFloat = 40 + var best: (Int, CGPoint, CGFloat)? + for index in replay.frames.indices { + let camera = replay.camera(at: index) + let projection = CameraProjection(cameraToWorld: camera.cameraToWorld, intrinsics: camera.intrinsics, imageSize: camera.imageSize) + guard let view = projection.viewPoint(for: point, in: viewSize), + view.x > inset, view.y > inset, view.x < viewSize.width - inset, view.y < viewSize.height - inset else { continue } + let offCenter = hypot(view.x - viewSize.width / 2, view.y - viewSize.height / 2) + if offCenter < best?.2 ?? .infinity { best = (index, view, offCenter) } + } + return best.map { ($0.0, $0.1) } + } + + private func nearestCovered(_ s: Float, in intervals: [ClosedRange]) -> Float { + if intervals.contains(where: { $0.contains(s) }) { return s } + let candidates = intervals.map { interval -> Float in + let inset = min(0.5, (interval.upperBound - interval.lowerBound) / 2) + return min(max(s, interval.lowerBound + inset), interval.upperBound - inset) + } + return candidates.min { abs($0 - s) < abs($1 - s) } ?? s + } + + private func waitFor(_ phase: ScanPhase, timeout: Double) async -> Bool { + await waitUntil(timeout: timeout) { self.engine.state.phase == phase } + } + + private func waitUntil(timeout: Double, _ condition: () -> Bool) async -> Bool { + let deadline = ContinuousClock.now + .seconds(timeout) + while ContinuousClock.now < deadline { + if condition() { return true } + try? await Task.sleep(for: .milliseconds(50)) + } + return condition() + } + + private func pause(_ seconds: Double) async { + try? await Task.sleep(for: .seconds(seconds)) + } + + private func log(_ message: String) { + RuntimeLog.autopilot.info("AUTOPILOT \(message, privacy: .public)") + } + + private func fail(_ message: String) { + RuntimeLog.autopilot.error("AUTOPILOT stopped: \(message, privacy: .public) (phase \(self.engine.state.phase.rawValue, privacy: .public))") + } + + private func format(_ range: ClosedRange) -> String { + String(format: "%.2f...%.2f m", range.lowerBound, range.upperBound) + } +} diff --git a/ios/HouseScan/Runtime/CaptureRecorder.swift b/ios/HouseScan/Runtime/CaptureRecorder.swift new file mode 100644 index 000000000..4362e19b1 --- /dev/null +++ b/ios/HouseScan/Runtime/CaptureRecorder.swift @@ -0,0 +1,401 @@ +import CoreMotion +import Foundation +import HouseScanKit +import OSLog +import simd +import Synchronization + +/// The packet's sensor streams, written to disk as they arrive: the camera trajectory at every +/// ARFrame (from the AR delegate queue), Core Motion's samples (from `MotionSource`'s queue), and +/// on LiDAR phones depth frames, one file pair each, a few a second (`DepthFrameBudget`). +/// Nothing here runs on the main actor or holds an ARFrame; each sample is copied into a fixed +/// row of Doubles and appended to one raw file per stream, in 64 KB writes. +/// +/// Poses stay in ARKit's world frame on disk: the meter anchor, which defines the packet's frame, +/// is set after capture starts, and can move until the packet is written. Packaging reads the +/// rows back (`rows(_:)`) and converts them then, so samples from before the meter was marked are +/// kept too. +/// +/// One recorder is one packet session, in one world frame. `restart` starts a new session in the +/// same folder when the world frame is thrown away (a failed relocalization). +final class CaptureRecorder: Sendable { + /// The raw streams. Each row starts with `t`, seconds of device uptime. + enum Stream: Int, CaseIterable, Sendable { + /// t, tracking state code, tracking reason code, then camera-to-world as 16 floats, + /// column by column (`TrackingCode`). + case trajectory + /// t, x, y, z in g. + case accelerometer + /// t, x, y, z in rad/s. + case gyroscope + /// t, x, y, z in µT, uncalibrated. + case magnetometer + /// t, quaternion x y z w, gravity x y z (g), user acceleration x y z (g), rotation rate + /// x y z (rad/s), heading in degrees (negative: none). + case deviceMotion + /// t, pressure in kPa, relative altitude in m. + case barometer + + var width: Int { + switch self { + case .trajectory: 19 + case .accelerometer, .gyroscope, .magnetometer: 4 + case .deviceMotion: 15 + case .barometer: 3 + } + } + + var fileName: String { "\(self).f64" } + } + + /// What a session looked like when packaging flushed it. + struct Snapshot: Sendable { + var sessionID: String + /// Wall clock at the first trajectory row, from that row's uptime. + var startedAt: Date? + var firstUptime: Double? + var lastUptime: Double? + } + + /// A LiDAR depth map recorded between photos (`recordDepthFrame`). Its map and confidence are + /// on disk; this is what the packet needs to find and place them. + struct DepthFrame: Sendable { + var t: Double + var tracking: TrackingCode + var cameraToWorld: simd_float4x4 + /// [fx, fy, cx, cy] in pixels of the depth map. + var intrinsics: SIMD4 + var width: Int + var height: Int + /// Float32 meters and UInt8 confidence, relative to `directory`. + var map: String + var confidence: String + } + + private struct State { + var sessionID = UUID().uuidString + var recording = true + /// Depth frames record only while the camera is meant to be on the wall; the engine turns + /// them on for the close-up, the walk and a gap request (`setRecording`). + var recordingDepth = false + /// Rows older than this are from a world frame `restart` discarded; ARKit can deliver a + /// few after the reset. + var since: Double = 0 + var startedAt: Date? + var firstUptime: Double? + var lastUptime: Double? + var buffers: [Data] = Array(repeating: Data(), count: Stream.allCases.count) + var lastT: [Double] = Array(repeating: -.infinity, count: Stream.allCases.count) + var failed = false + var depthBudget = DepthFrameBudget() + var depthFrames: [DepthFrame] = [] + /// A failed depth-frame write stops depth frames only; the streams go on. + var depthFailed = false + } + + let directory: URL + private let state = Mutex(State()) + /// 64 KB: about 450 trajectory rows (7 s at 60 Hz) or 2000 accelerometer rows per write. + private static let flushBytes = 1 << 16 + private static let depthFolder = "depth_frames" + + init(directory: URL) { + self.directory = directory + Self.truncate(directory) + } + + /// A new session in a new world frame: the files start empty, and rows timed before now are + /// dropped. + func restart() { + state.withLock { state in + state = State() + state.since = ProcessInfo.processInfo.systemUptime + Self.truncate(directory) + } + } + + /// Off while no capture is under way (the result screens, a failed upload): rows and depth + /// frames are dropped. `depthFrames` narrows depth frames further, to the phases that aim the + /// camera at the wall. + func setRecording(_ on: Bool, depthFrames: Bool) { + state.withLock { state in + state.recording = on + state.recordingDepth = on && depthFrames + } + } + + func recordPose(t: Double, tracking: TrackingCode, cameraToWorld m: simd_float4x4) { + let columns = [m.columns.0, m.columns.1, m.columns.2, m.columns.3] + let pose = columns.flatMap { [Double($0.x), Double($0.y), Double($0.z), Double($0.w)] } + append(.trajectory, [t, Double(tracking.state), Double(tracking.reason)] + pose) + } + + /// Appends one row. A row not later than the stream's last one is dropped: every stream's `t` + /// strictly increases. + func append(_ stream: Stream, _ row: [Double]) { + precondition(row.count == stream.width, "\(stream) row has \(row.count) values, expected \(stream.width)") + let t = row[0] + state.withLock { state in + guard state.recording, !state.failed, t >= state.since, t > state.lastT[stream.rawValue] else { return } + state.lastT[stream.rawValue] = t + if stream == .trajectory { + if state.firstUptime == nil { + state.firstUptime = t + // The same instant on the wall clock: the frame is `now - t` seconds old. + state.startedAt = Date(timeIntervalSinceNow: t - ProcessInfo.processInfo.systemUptime) + } + state.lastUptime = t + } + row.withUnsafeBufferPointer { values in + for value in values { withUnsafeBytes(of: value.bitPattern.littleEndian) { state.buffers[stream.rawValue].append(contentsOf: $0) } } + } + if state.buffers[stream.rawValue].count >= Self.flushBytes { + write(stream, &state) + } + } + } + + /// Whether a depth frame at `t` would be recorded now: the AR delegate asks before copying the + /// depth map, so a frame the budget would drop costs nothing. + func wantsDepthFrame(at t: Double) -> Bool { + state.withLock { state in + state.recordingDepth && !state.depthFailed && t >= state.since && state.depthBudget.wants(t: t) + } + } + + /// Writes one LiDAR depth map with its confidence to `depth_frames/` as it arrives, so the + /// maps are never held in memory, and remembers where. At most `DepthFrameBudget.rateHz` a + /// second and `DepthFrameBudget.maxFrames` in a session; frames past either are dropped. + /// Called from the AR delegate queue: two 50 to 200 KB writes, twice a second at most. + func recordDepthFrame(t: Double, tracking: TrackingCode, cameraToWorld: simd_float4x4, intrinsics: SIMD4, depth: DepthPacket) { + guard let confidence = depth.confidence, depth.meters.count == depth.width * depth.height, confidence.count == depth.meters.count else { return } + state.withLock { state in + guard state.recordingDepth, !state.depthFailed, t >= state.since, state.depthBudget.admit(t: t) else { return } + let stem = "\(Self.depthFolder)/\(PacketDepthFrame.id(number: state.depthBudget.admitted))" + let frame = DepthFrame( + t: t, tracking: tracking, cameraToWorld: cameraToWorld, intrinsics: intrinsics, width: depth.width, height: depth.height, + map: "\(stem).f32", confidence: "\(stem).conf.u8" + ) + do { + try PacketFiles.depthData(meters: depth.meters).write(to: directory.appending(path: frame.map)) + try Data(confidence).write(to: directory.appending(path: frame.confidence)) + state.depthFrames.append(frame) + } catch { + state.depthFailed = true + RuntimeLog.capture.error("depth frames not recorded from here on: \(String(describing: error), privacy: .public)") + return + } + if state.depthBudget.isFull { + let limit = state.depthBudget.limit + RuntimeLog.capture.info("depth frames: the limit of \(limit) was reached at t=\(t); no more are recorded") + } + } + } + + /// Every depth frame recorded in this session, in time order. + func depthFrames() -> [DepthFrame] { + state.withLock { $0.depthFrames } + } + + /// A recorded depth frame's map and confidence, read back for the packet: ARKit's scene depth. + /// Nil when the files are gone or the wrong size. Call off the main actor. + func loadDepth(_ frame: DepthFrame) -> DepthPacket? { + let count = frame.width * frame.height + guard let map = try? Data(contentsOf: directory.appending(path: frame.map)), map.count == count * 4, + let confidence = try? Data(contentsOf: directory.appending(path: frame.confidence)), confidence.count == count + else { return nil } + return DepthPacket( + meters: PacketFiles.floats(littleEndian: map), width: frame.width, height: frame.height, confidence: [UInt8](confidence), + source: .arkitSceneDepth + ) + } + + /// Writes every buffered row and returns the session as it stands. + func flush() -> Snapshot { + state.withLock { state in + for stream in Stream.allCases { write(stream, &state) } + return Snapshot(sessionID: state.sessionID, startedAt: state.startedAt, firstUptime: state.firstUptime, lastUptime: state.lastUptime) + } + } + + /// Every row written for `stream`, after `flush`. Call off the main actor. + func rows(_ stream: Stream) -> [[Double]] { + guard let data = try? Data(contentsOf: directory.appending(path: stream.fileName)) else { return [] } + let values = data.withUnsafeBytes { raw in + (0..<(raw.count / 8)).map { Double(bitPattern: UInt64(littleEndian: raw.loadUnaligned(fromByteOffset: $0 * 8, as: UInt64.self))) } + } + return stride(from: 0, to: values.count - stream.width + 1, by: stream.width).map { Array(values[$0..<($0 + stream.width)]) } + } + + /// A failed write stops the recorder rather than leave a stream with a hole in it; packaging + /// then writes what reached the disk before the failure. + private func write(_ stream: Stream, _ state: inout State) { + let buffer = state.buffers[stream.rawValue] + guard !buffer.isEmpty else { return } + state.buffers[stream.rawValue] = Data() + let url = directory.appending(path: stream.fileName) + do { + let handle = try FileHandle(forWritingTo: url) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: buffer) + } catch { + state.failed = true + RuntimeLog.capture.error("stream \(String(describing: stream), privacy: .public) not recorded from here on: \(String(describing: error), privacy: .public)") + } + } + + private static func truncate(_ directory: URL) { + let files = FileManager.default + try? files.createDirectory(at: directory, withIntermediateDirectories: true) + for stream in Stream.allCases { + files.createFile(atPath: directory.appending(path: stream.fileName).path, contents: Data()) + } + let depth = directory.appending(path: depthFolder, directoryHint: .isDirectory) + try? files.removeItem(at: depth) + try? files.createDirectory(at: depth, withIntermediateDirectories: true) + } +} + +/// Tracking state and limited reason as numbers for a raw trajectory row. +struct TrackingCode: Sendable, Equatable { + /// 0 normal, 1 limited, 2 not available. + var state: Int + /// 0 none or unknown, 1 initializing, 2 relocalizing, 3 excessive motion, 4 insufficient + /// features. + var reason: Int + + init(_ tracking: TrackingQuality) { + switch tracking { + case .normal: (state, reason) = (0, 0) + case .notAvailable: (state, reason) = (2, 0) + case .limited(let why): + state = 1 + reason = switch why { + case .initializing: 1 + case .relocalizing: 2 + case .excessiveMotion: 3 + case .insufficientFeatures: 4 + case .unknown: 0 + } + } + } + + init(state: Int, reason: Int) { + self.state = state + self.reason = reason + } + + var packetTracking: PacketTracking { + let why: PacketTracking.Reason? = switch reason { + case 1: .initializing + case 2: .relocalizing + case 3: .excessiveMotion + case 4: .insufficientFeatures + default: nil + } + return switch state { + case 0: .normal + case 1: .limited(why) + default: .notAvailable + } + } +} + +/// Core Motion for the packet: accelerometer, gyroscope, magnetometer and device motion at +/// `rate`, and the barometer at whatever rate CMAltimeter delivers. Samples go straight to the +/// recorder from a private queue. Location and heading are not recorded (no consent prompt this +/// round), so device motion runs in an arbitrary-yaw frame and its heading is Core Motion's +/// negative "none". +@MainActor +final class MotionSource { + /// 100 Hz, the packet's nominal rate for the four Core Motion streams. A choice, not + /// measured: ADVIO's iPhone IMU runs at 100 Hz, and it is well under Core Motion's limit. + nonisolated static let rate: Double = 100 + + private let manager = CMMotionManager() + private let altimeter = CMAltimeter() + private let queue: OperationQueue = { + let queue = OperationQueue() + queue.name = "dev.housescanning.housescan.motion" + queue.maxConcurrentOperationCount = 1 + queue.qualityOfService = .utility + return queue + }() + private(set) var isRunning = false + + /// Which streams this phone has; the packet lists only streams with rows. + var available: Set { + var streams: Set = [] + if manager.isAccelerometerAvailable { streams.insert(.accelerometer) } + if manager.isGyroAvailable { streams.insert(.gyroscope) } + if manager.isMagnetometerAvailable { streams.insert(.magnetometer) } + if manager.isDeviceMotionAvailable { streams.insert(.deviceMotion) } + if CMAltimeter.isRelativeAltitudeAvailable() { streams.insert(.barometer) } + return streams + } + + func start(into recorder: CaptureRecorder) { + guard !isRunning else { return } + isRunning = true + Self.startUpdates(manager, altimeter, queue: queue, recorder: recorder, interval: 1 / Self.rate) + RuntimeLog.capture.info("motion recording on: \(self.available.map { "\($0)" }.sorted().joined(separator: ", "), privacy: .public)") + } + + func stop() { + guard isRunning else { return } + isRunning = false + manager.stopAccelerometerUpdates() + manager.stopGyroUpdates() + manager.stopMagnetometerUpdates() + manager.stopDeviceMotionUpdates() + altimeter.stopRelativeAltitudeUpdates() + RuntimeLog.capture.info("motion recording off") + } + + /// Nonisolated so the handlers are not main-actor closures: Core Motion calls them on `queue`. + private nonisolated static func startUpdates(_ manager: CMMotionManager, _ altimeter: CMAltimeter, queue: OperationQueue, recorder: CaptureRecorder, interval: Double) { + // Raw, as Core Motion reports it: the packet writer converts g to m/s². + if manager.isAccelerometerAvailable { + manager.accelerometerUpdateInterval = interval + manager.startAccelerometerUpdates(to: queue) { data, _ in + guard let data else { return } + let a = data.acceleration + recorder.append(.accelerometer, [data.timestamp, a.x, a.y, a.z]) + } + } + if manager.isGyroAvailable { + manager.gyroUpdateInterval = interval + manager.startGyroUpdates(to: queue) { data, _ in + guard let data else { return } + let r = data.rotationRate + recorder.append(.gyroscope, [data.timestamp, r.x, r.y, r.z]) + } + } + if manager.isMagnetometerAvailable { + manager.magnetometerUpdateInterval = interval + manager.startMagnetometerUpdates(to: queue) { data, _ in + guard let data else { return } + let m = data.magneticField + recorder.append(.magnetometer, [data.timestamp, m.x, m.y, m.z]) + } + } + if manager.isDeviceMotionAvailable { + manager.deviceMotionUpdateInterval = interval + manager.startDeviceMotionUpdates(using: .xArbitraryCorrectedZVertical, to: queue) { motion, _ in + guard let motion else { return } + let q = motion.attitude.quaternion + let g = motion.gravity + let a = motion.userAcceleration + let r = motion.rotationRate + recorder.append(.deviceMotion, [motion.timestamp, q.x, q.y, q.z, q.w, g.x, g.y, g.z, a.x, a.y, a.z, r.x, r.y, r.z, motion.heading]) + } + } + if CMAltimeter.isRelativeAltitudeAvailable() { + altimeter.startRelativeAltitudeUpdates(to: queue) { data, _ in + guard let data else { return } + recorder.append(.barometer, [data.timestamp, data.pressure.doubleValue, data.relativeAltitude.doubleValue]) + } + } + } +} diff --git a/ios/HouseScan/Runtime/GuidanceLog.swift b/ios/HouseScan/Runtime/GuidanceLog.swift new file mode 100644 index 000000000..c887c3b9b --- /dev/null +++ b/ios/HouseScan/Runtime/GuidanceLog.swift @@ -0,0 +1,89 @@ +import Foundation +import HouseScanKit + +/// Every request the homeowner was shown during capture, and how each one ended: the packet's +/// `guidance` log. The server reads it to learn what the homeowner could not reach, which +/// scene.json cannot say. +/// +/// A request is identified by its `Topic`. The engine reports what is on screen after each +/// guidance change (`show`); a new topic closes the open request with the outcome the engine +/// judges (`show`'s `closing`), unless an action already closed it with a known outcome +/// (`resolve`: "I can't get there", an answered question, a satisfied gap). +/// +/// Times are on the capture clock (`ScanEngine.captureClock`): ARFrame.timestamp live. Spans are +/// meters of s along the wall; packaging converts them to meter-frame x. +struct GuidanceLog { + typealias Kind = PacketGuidanceEntry.Kind + typealias Origin = PacketGuidanceEntry.Origin + typealias Band = PacketGuidanceEntry.Band + typealias Outcome = PacketGuidanceEntry.Outcome + + /// What a request is about. Same topic, same request: a walk's remaining distance or a gap's + /// progress changing on screen does not make a new one. + enum Topic: Equatable, Sendable { + case closeUp + case walk(WallSide) + /// Walk round the corner on this side and mark the next wall. + case nextWall(WallSide) + case markEnd(WallSide) + /// Aim at a band at the coverage cell with this index. + case aimAtGround(cell: Int) + case aimAtWall(cell: Int) + case stepBack + case tiltUp + case seeBehind(cell: Int) + /// `GapRequest.id`. + case gap(id: Int) + /// `SpotCheck.id`: the first question, then the ground question. + case spotCheck(id: Int) + case spotGround(id: Int) + } + + struct Request: Equatable, Sendable { + var topic: Topic + var kind: Kind + var origin: Origin + var message: String + var band: Band? + /// Meters of s. + var span: ClosedRange? + } + + struct Entry: Sendable { + var id: String + var request: Request + var shown: Double + var resolved: Double? + var outcome: Outcome + } + + private(set) var entries: [Entry] = [] + /// The request on screen, whether or not it is still open. + private(set) var current: Request? + private var openIndex: Int? + + /// Records what is on screen now (nil: no request, such as the review or upload screens). + /// A different request closes the open one with `closing(open, next)`. + mutating func show(_ request: Request?, at t: Double, closing: (Request, Request?) -> Outcome) { + guard request?.topic != current?.topic else { return } + if let open = openIndex { + close(open, closing(entries[open].request, request), at: t) + } + current = request + guard let request else { return } + entries.append(Entry(id: "g\(entries.count + 1)", request: request, shown: t, resolved: nil, outcome: .unresolved)) + openIndex = entries.count - 1 + } + + /// Closes the open request with a known outcome; it may stay on screen a moment longer. + mutating func resolve(_ outcome: Outcome, at t: Double) { + guard let open = openIndex else { return } + close(open, outcome, at: t) + } + + private mutating func close(_ index: Int, _ outcome: Outcome, at t: Double) { + entries[index].outcome = outcome + entries[index].resolved = max(t, entries[index].shown) + openIndex = nil + } +} diff --git a/ios/HouseScan/Runtime/KeyframeStore.swift b/ios/HouseScan/Runtime/KeyframeStore.swift new file mode 100644 index 000000000..2cd5fcb08 --- /dev/null +++ b/ios/HouseScan/Runtime/KeyframeStore.swift @@ -0,0 +1,260 @@ +import CoreGraphics +import Foundation +import HouseScanKit +import simd +import OSLog + +/// A kept photo (a keyframe, or a still such as the meter close-up): its JPEG on disk and what +/// the packet records about the frame it came from. +struct StoredKeyframe: Sendable { + /// Also the JPEG's file name without extension, for example "k00001". + let id: String + let camera: CameraFrame + /// The frame's time: ARFrame.timestamp live, the recording's time on a replay. + let t: Double + let tracking: TrackingQuality + let exposure: PhotoExposure? + /// Pixels of the stored JPEG, read from the file. + let width: Int + let height: Int + /// `laplacian_variance_luma_640` of the stored JPEG; nil when its luma could not be read. + let sharpness: Double? + /// ARKit's depth for this photo, written beside it as float32 meters (`depth/.f32`) with + /// its confidence (`depth/.conf.u8`). Live LiDAR frames only. + let depth: StoredDepth? + var fileName: String { "\(id).jpg" } + /// The camera-to-world pose ARKit reported for the photo, in its world frame at `t`, never + /// corrected. scene.json and the packet use it corrected for the meter anchor's later moves + /// (`MeterAnchorTracking.correctedPose(_:capturedAt:)`). + var rawPose: simd_float4x4 { camera.cameraToWorld } +} + +struct StoredDepth: Sendable { + let file: String + let confidenceFile: String? + let width: Int + let height: Int + /// Nil when the depth's origin is unknown (`DepthPacket.source`); live frames say ARKit's. + let source: DepthPacket.Source? +} + +/// Keyframe JPEGs and close-up stills of one scan, in Caches/Scans//. File work runs off the +/// main actor; the list of what was stored lives here on the main actor. +@MainActor +final class KeyframeStore { + let directory: URL + private(set) var keyframes: [StoredKeyframe] = [] + /// Purpose (scene.json `stills` key) to file name. + private(set) var stills: [String: String] = [:] + /// Purpose to the still's frame, for the packet's photos. + private(set) var stillFrames: [String: StoredKeyframe] = [:] + private var nextIndex = 1 + /// Bumped by `discardKeyframes`, so a write that started before it doesn't land in the list. + private var epoch = 0 + + /// Makes a new, empty scan folder and deletes every other one: only the current scan is kept + /// on the phone. Covers both a start over (the previous scan's folder) and launch (folders a + /// quit or crashed run left behind), since both make a new store. The folders to delete are + /// listed here, before a newer store can exist, and deleted later off the main actor + /// (`ScanFolderCleanup`), so a deletion that runs late can't take a newer scan's folder. + init() { + let caches = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0] + let scans = caches.appending(path: "Scans", directoryHint: .isDirectory) + directory = scans.appending(path: UUID().uuidString, directoryHint: .isDirectory) + try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + Self.delete(ScanFolderCleanup(root: scans, keeping: directory.lastPathComponent)) + } + + func nextKeyframeIndex() -> Int { + defer { nextIndex += 1 } + return nextIndex + } + + /// Writes a keyframe's JPEG unrotated, as the sensor produced it, with the pose, time and + /// camera settings of the frame those bytes came from, its sharpness, and its ARKit depth when + /// there is some. Returns whether it was stored, and an upright thumbnail for the capture + /// acknowledgment. + /// + /// Bytes that don't decode are refused before anything is written: a photo no one can open + /// is not a view, so it must not become a keyframe the coverage counts. The thumbnail is that + /// decode. (Before, an undecodable photo was stored with a gray placeholder thumbnail.) + func saveKeyframe(_ frame: SourceFrame, index: Int) async -> (stored: Bool, thumbnail: CGImage?) { + let id = String(format: "k%05d", index) + let directory = directory + let startedIn = epoch + let written = await Task.detached(priority: .utility) { () -> Result<(CGImage, StoredKeyframe), KeyframeWriteFailure> in + guard let data = Self.data(of: frame.jpeg) else { return .failure(.noPhoto) } + guard let thumbnail = ImageWork.uprightThumbnail(jpeg: data) else { return .failure(.undecodable) } + do { + try data.write(to: directory.appending(path: "\(id).jpg"), options: .atomic) + } catch { + return .failure(.writeFailed) + } + return .success((thumbnail, Self.stored(frame, id: id, jpeg: data, in: directory))) + }.value + let thumbnail: CGImage + let stored: StoredKeyframe + switch written { + case .success(let (image, entry)): + thumbnail = image + stored = entry + case .failure(let failure): + RuntimeLog.capture.error("keyframe \(id, privacy: .public) not stored: \(failure.rawValue, privacy: .public)") + return (false, nil) + } + // Taken in a world frame that was discarded while the file was being written. + guard startedIn == epoch else { + RuntimeLog.capture.info("keyframe \(id, privacy: .public) not stored: its world frame was discarded") + return (false, nil) + } + keyframes.append(stored) + keyframes.sort { $0.id < $1.id } + return (true, thumbnail) + } + + /// What the packet records about a photo written as `id`: the frame's pose, time, tracking and + /// exposure, the JPEG's own pixel size and sharpness, and its depth files, written here. + nonisolated private static func stored(_ frame: SourceFrame, id: String, jpeg: Data, in directory: URL) -> StoredKeyframe { + let luma = ImageWork.luma(jpeg: jpeg) + let sharpness = luma.map { PacketSharpness.laplacianVarianceLuma640(luma: $0.pixels, width: $0.width, height: $0.height) } + return StoredKeyframe( + id: id, camera: frame.camera, t: frame.timestamp, tracking: frame.tracking, exposure: frame.exposure, + width: luma?.width ?? Int(frame.camera.imageSize.x), height: luma?.height ?? Int(frame.camera.imageSize.y), + sharpness: sharpness, depth: frame.sensorDepth.flatMap { writeDepth($0, id: id, in: directory) } + ) + } + + /// Writes ARKit's depth for a photo into the scan folder: float32 little-endian meters, + /// `depth/.f32`, and its confidence, `depth/.conf.u8`. A failed write loses only the + /// depth: the photo is the keyframe, and the packet then lists no depth for it. + nonisolated private static func writeDepth(_ depth: DepthPacket, id: String, in directory: URL) -> StoredDepth? { + let file = "depth/\(id).f32" + let confidenceFile = depth.confidence == nil ? nil : "depth/\(id).conf.u8" + do { + try FileManager.default.createDirectory(at: directory.appending(path: "depth", directoryHint: .isDirectory), withIntermediateDirectories: true) + var map = Data(capacity: depth.meters.count * 4) + for value in depth.meters { withUnsafeBytes(of: value.bitPattern.littleEndian) { map.append(contentsOf: $0) } } + try map.write(to: directory.appending(path: file), options: .atomic) + if let confidence = depth.confidence, let confidenceFile { + try Data(confidence).write(to: directory.appending(path: confidenceFile), options: .atomic) + } + return StoredDepth(file: file, confidenceFile: confidenceFile, width: depth.width, height: depth.height, source: depth.source) + } catch { + RuntimeLog.capture.error("keyframe \(id, privacy: .public) depth not stored: \(String(describing: error), privacy: .public)") + return nil + } + } + + /// A stored photo's depth, read back for the packet. Nil when the files are gone or the wrong + /// size. + nonisolated static func loadDepth(_ depth: StoredDepth, in directory: URL) -> DepthPacket? { + guard let map = try? Data(contentsOf: directory.appending(path: depth.file)), map.count == depth.width * depth.height * 4 else { return nil } + var confidence: [UInt8]? + if let file = depth.confidenceFile { + guard let data = try? Data(contentsOf: directory.appending(path: file)), data.count == depth.width * depth.height else { return nil } + confidence = [UInt8](data) + } + return DepthPacket(meters: PacketFiles.floats(littleEndian: map), width: depth.width, height: depth.height, confidence: confidence, source: depth.source) + } + + enum KeyframeWriteFailure: String, Error { + case noPhoto = "no photo data" + case undecodable = "photo does not decode" + case writeFailed = "write failed" + } + + /// Writes a still such as the meter close-up from `frame`. Returns false when there was + /// nothing to write. + func saveStill(_ frame: SourceFrame, name: String) async -> Bool { + let directory = directory + let id = Self.purpose(of: name) + let startedIn = epoch + let written = await Task.detached(priority: .utility) { () -> StoredKeyframe? in + guard let data = Self.data(of: frame.jpeg), (try? data.write(to: directory.appending(path: name), options: .atomic)) != nil else { return nil } + return Self.stored(frame, id: id, jpeg: data, in: directory) + }.value + guard let written else { return false } + // Taken in a world frame that was discarded while the file was being written. The file is + // left alone: a close-up in the new frame may already have written the same name, and + // nothing reads a still that `stills` doesn't list. + guard startedIn == epoch else { + RuntimeLog.capture.info("still \(name, privacy: .public) not kept: its world frame was discarded") + return false + } + stills[id] = name + stillFrames[id] = written + return true + } + + func thumbnail(ofStill name: String) async -> CGImage? { + let url = directory.appending(path: name) + return await Task.detached(priority: .utility) { () -> CGImage? in + guard let data = try? Data(contentsOf: url) else { return nil } + return ImageWork.uprightThumbnail(jpeg: data) + }.value + } + + /// Forgets keyframes and stills taken in a world frame that no longer exists (after a failed + /// relocalization), the meter close-up included: export and the packet read only what + /// `keyframes` and `stills` list, so a close-up skipped in the new frame exports none. Keyframe + /// files stay until the scan is discarded; still files go now, since the next close-up + /// reuses the name (`meter_close.jpg`). + func discardKeyframes() { + epoch += 1 + keyframes = [] + for name in stills.values { + try? FileManager.default.removeItem(at: directory.appending(path: name)) + } + stills = [:] + stillFrames = [:] + } + + /// Where the packet is assembled, and the zip Share scan offers. + var packetFolder: URL { directory.appending(path: "packet", directoryHint: .isDirectory) } + var bundleURL: URL { directory.appending(path: "scan.zip") } + + /// Zips the packet folder into `scan.zip`, manifest.json at the zip's root, and removes the + /// folder. The zip is what "Share scan" offers; the upload sends scene.json alone. It stays on + /// the phone until the next scan unless the homeowner shares it. + /// + /// Streamed to disk one file at a time: a long walk's JPEGs held in memory twice (the entries + /// and the archive) is what an in-memory build cost. + nonisolated static func zipPacket(_ folder: URL, to url: URL) throws { + let files = FileManager.default + guard let walker = files.enumerator(at: folder, includingPropertiesForKeys: [.isRegularFileKey]) else { + throw CocoaError(.fileReadNoSuchFile, userInfo: [NSFilePathErrorKey: folder.path]) + } + let root = folder.standardizedFileURL.pathComponents.count + var names: [String] = [] + for case let file as URL in walker where (try? file.resourceValues(forKeys: [.isRegularFileKey]).isRegularFile) == true { + names.append(file.standardizedFileURL.pathComponents.dropFirst(root).joined(separator: "/")) + } + // manifest.json first, so a reader streaming the zip meets it before the files it names. + names.sort { ($0 == "manifest.json" ? 0 : 1, $0) < ($1 == "manifest.json" ? 0 : 1, $1) } + let entries = names.map { name in (name: name, load: { () throws -> Data in try Data(contentsOf: folder.appending(path: name)) }) } + try ZipWriter.write(entries, to: url, modified: Date()) + try? files.removeItem(at: folder) + } + + /// Off the main actor. A write still in flight for a deleted folder fails, because its + /// directory is gone, and is dropped like any failed write. + nonisolated private static func delete(_ cleanup: ScanFolderCleanup) { + Task.detached(priority: .utility) { + for (url, error) in cleanup.run() { + RuntimeLog.engine.error("could not delete old scan \(url.lastPathComponent, privacy: .public): \(String(describing: error), privacy: .public)") + } + } + } + + nonisolated private static func data(of payload: JPEGPayload) -> Data? { + switch payload { + case .data(let data): data + case .file(let url): try? Data(contentsOf: url) + case .none: nil + } + } + + nonisolated private static func purpose(of name: String) -> String { + (name as NSString).deletingPathExtension + } +} diff --git a/ios/HouseScan/Runtime/LaunchOptions.swift b/ios/HouseScan/Runtime/LaunchOptions.swift new file mode 100644 index 000000000..e5deeb9c6 --- /dev/null +++ b/ios/HouseScan/Runtime/LaunchOptions.swift @@ -0,0 +1,72 @@ +import Foundation +import OSLog + +/// Verification hooks from the launch arguments (contract C4). +/// +/// - `-replay `: play a recorded measure-lab-session v2 folder instead of the camera. +/// - `-autopilot`: drive the intents automatically (UI tests, demos). +/// - `-serverURL `: the placement server to upload to. Without it the app uses the build's +/// default, Info.plist `HouseScanServerURL` (set from `HOUSESCAN_SERVER_URL` in +/// Config/Shared.xcconfig); with neither, uploads answer with the bundled sample. +/// - `-sampleResult`: answer uploads with the bundled sample result, flagged as a sample, even +/// when a server is configured. UI tests pass it to stay offline and deterministic. +/// - `-autopilotHold `: how long the autopilot leaves each screen up (default 1.2 s). +/// UI tests raise it so each screen stays long enough to screenshot and audit. +/// - `-autopilotGate `: before the flow leaves a screen, wait until a file named after +/// that phase exists in the folder. UI tests write it once they have screenshotted and audited +/// the screen, so a slow audit can never miss a screen. After the result shows, the autopilot +/// also writes the scan's scene.json there, for the test to check. +/// - `-autopilotCantGetThere`: the autopilot ends the walk with "Can't get there" instead of +/// marking the ends (`Autopilot.endWalkByCantGetThere`). +/// - `-autopilotSomethingThere`: the autopilot answers the first spot check "Something's there" +/// instead of "It's clear", so the scan is checked again without that area. +struct LaunchOptions: Equatable { + var replayFolder: URL? + var autopilot = false + var serverURL: URL? + var sampleResult = false + var autopilotHold: Double = 1.2 + var autopilotGate: URL? + var autopilotCantGetThere = false + var autopilotSomethingThere = false + + init( + arguments: [String] = ProcessInfo.processInfo.arguments, + defaultServerURL: String? = Bundle.main.object(forInfoDictionaryKey: "HouseScanServerURL") as? String + ) { + func value(after flag: String) -> String? { + guard let index = arguments.firstIndex(of: flag), index + 1 < arguments.count else { return nil } + return arguments[index + 1] + } + if let path = value(after: "-replay") { + replayFolder = URL(fileURLWithPath: (path as NSString).expandingTildeInPath, isDirectory: true) + } + autopilot = arguments.contains("-autopilot") + autopilotCantGetThere = arguments.contains("-autopilotCantGetThere") + autopilotSomethingThere = arguments.contains("-autopilotSomethingThere") + serverURL = (value(after: "-serverURL") ?? defaultServerURL).flatMap(Self.serverURL) + sampleResult = arguments.contains("-sampleResult") + if let gate = value(after: "-autopilotGate") { autopilotGate = URL(fileURLWithPath: gate, isDirectory: true) } + if let hold = value(after: "-autopilotHold").flatMap(Double.init), hold > 0 { autopilotHold = hold } + } + + /// An http(s) URL with a host, or nil. An empty build setting leaves the plist value empty, + /// and an unexpanded "$(HOUSESCAN_SERVER_URL)" has no scheme; both mean "no server". + private static func serverURL(_ text: String) -> URL? { + guard let url = URL(string: text.trimmingCharacters(in: .whitespaces)), + let scheme = url.scheme?.lowercased(), scheme == "https" || scheme == "http", + url.host() != nil else { return nil } + return url + } +} + +/// Log channels. STATE lines are parsed by verification/hsverify/statelog.py, which needs them public. +enum RuntimeLog { + static let state = Logger(subsystem: "dev.housescanning.housescan", category: "state") + static let guidance = Logger(subsystem: "dev.housescanning.housescan", category: "guidance") + static let engine = Logger(subsystem: "dev.housescanning.housescan", category: "engine") + static let autopilot = Logger(subsystem: "dev.housescanning.housescan", category: "autopilot") + /// Tracking, relocalization and every capture-gate decision, for reading a real session back. + /// Only enum-valued reasons, frame ids and counts are public; never meter numbers or images. + static let capture = Logger(subsystem: "dev.housescanning.housescan", category: "capture") +} diff --git a/ios/HouseScan/Runtime/LiveCapture.swift b/ios/HouseScan/Runtime/LiveCapture.swift new file mode 100644 index 000000000..1a0dcf4c3 --- /dev/null +++ b/ios/HouseScan/Runtime/LiveCapture.swift @@ -0,0 +1,670 @@ +import ARKit +import CoreImage +import Foundation +import HouseScanKit +import ImageIO +import RealityKit +import Synchronization +import SwiftUI + +/// What the AR delegate should prepare with each sampled frame. +enum LiveMode: Sendable { + /// Pose and quality only. + case idle + /// Encode a JPEG about three times a second while the phone is held on the meter. + case closeUp + /// Encode a JPEG when the phone has moved or turned enough that the frame could be kept. + case walk +} + +enum LiveEvent: Sendable { + /// With the time of the last frame the session delivered before it: the event reaches the + /// main actor on its own path, and can arrive before frames captured earlier. + case interrupted(lastFrameTime: Double?) + case interruptionEnded + case cameraDenied + case failed(String) +} + +struct VerticalPlaneHit { + let position: SIMD3 + let normal: SIMD3 + let transform: simd_float4x4 + /// Which kind of plane the ray hit; the export widens the meter's error for an estimated one. + let source: MeterPlaneSource +} + +/// The live ARKit source: RealityKit's ARView running world tracking with `.gravity` alignment and +/// horizontal and vertical plane detection. On an iPhone with LiDAR it also turns on per-frame +/// scene depth, which coverage uses to tell a wall from a bush in front of it and the packet +/// records with each photo and a few times a second between photos, and the scene mesh with +/// per-face classification, which the packet carries; without LiDAR it runs without either. +@MainActor +final class LiveCapture { + let arView: ARView + private let delegate: LiveSessionDelegate + + /// True when this phone gives per-frame LiDAR depth. + static var supportsDepth: Bool { ARWorldTrackingConfiguration.supportsFrameSemantics(.sceneDepth) } + /// True when this phone builds a LiDAR mesh of the scene. + static var supportsMesh: Bool { ARWorldTrackingConfiguration.supportsSceneReconstruction(.mesh) } + /// True when the mesh can carry ARKit's per-face classification (wall, floor, door, ...), + /// which the packet's mesh records. + static var supportsClassifiedMesh: Bool { ARWorldTrackingConfiguration.supportsSceneReconstruction(.meshWithClassification) } + /// True when this phone can hide virtual content behind a person standing in front of it. + static var supportsPeopleOcclusion: Bool { ARWorldTrackingConfiguration.supportsFrameSemantics(.personSegmentationWithDepth) } + + /// The AR result in the camera view, while one is shown (`showResult`). + private var result: AnchorEntity? + private var occludesPeople = false + + init(onFrame: @escaping @MainActor @Sendable (SourceFrame) -> Void, onEvent: @escaping @MainActor @Sendable (LiveEvent) -> Void) { + arView = ARView(frame: .zero, cameraMode: .ar, automaticallyConfigureSession: false) + delegate = LiveSessionDelegate(onFrame: onFrame, onEvent: onEvent) + arView.session.delegateQueue = DispatchQueue(label: "dev.housescanning.housescan.ar-delegate", qos: .userInitiated) + arView.session.delegate = delegate + arView.renderOptions.insert(.disableMotionBlur) + } + + /// People occlusion runs only while the AR result is on screen: the capture has no virtual + /// content for it to hide. + private static func configuration(occludingPeople: Bool = false) -> ARWorldTrackingConfiguration { + let configuration = ARWorldTrackingConfiguration() + // .gravity, not .gravityAndHeading: compass heading drifts near a house's metal and wiring. + configuration.worldAlignment = .gravity + configuration.planeDetection = [.horizontal, .vertical] + if supportsClassifiedMesh { + configuration.sceneReconstruction = .meshWithClassification + } else if supportsMesh { + configuration.sceneReconstruction = .mesh + } + if supportsDepth { configuration.frameSemantics.insert(.sceneDepth) } + if occludingPeople, supportsPeopleOcclusion { configuration.frameSemantics.insert(.personSegmentationWithDepth) } + return configuration + } + + /// What the running session was configured with, for the packet's `session.device`. + struct Settings: Sendable { + var sceneDepth: Bool + var mesh: Bool + /// The mesh carries ARKit's per-face classification: `sceneReconstruction` included + /// `.meshWithClassification`, not only `.mesh`. + var meshClassification: Bool + /// The camera's frame rate: the trajectory's nominal rate. + var framesPerSecond: Int + } + + var settings: Settings { + let running = arView.session.configuration as? ARWorldTrackingConfiguration ?? Self.configuration() + return Settings( + sceneDepth: running.frameSemantics.contains(.sceneDepth), + mesh: running.sceneReconstruction.contains(.mesh), + meshClassification: running.sceneReconstruction.contains(.meshWithClassification), + framesPerSecond: running.videoFormat.framesPerSecond + ) + } + + /// Where every ARFrame's pose goes as it arrives (`CaptureRecorder.recordPose`). + func setRecorder(_ recorder: CaptureRecorder?) { + delegate.shared.withLock { $0.recorder = recorder } + } + + func start() { + RuntimeLog.engine.info("LiDAR: scene depth \(Self.supportsDepth ? "on" : "not available", privacy: .public), mesh \(Self.supportsMesh ? "on" : "not available", privacy: .public)") + arView.session.run(Self.configuration(occludingPeople: occludesPeople)) + } + + func pause() { + arView.session.pause() + } + + /// Starts world tracking over with a fresh map, after relocalization failed. The old mesh goes + /// with the old anchors. + func restart() { + arView.session.run(Self.configuration(occludingPeople: occludesPeople), options: [.resetTracking, .removeExistingAnchors]) + delegate.shared.withLock { $0.meterAnchorID = nil } + } + + /// Shows `model` in the camera view, hung on the meter's anchor so it follows ARKit's + /// corrections to it. The model is in world axes with the meter at its origin, as they stood + /// when the anchor was at `pose` (`MeterAnchorTracking.pose`, the pose the engine's wall + /// agrees with). It is attached in that pose's frame, so a turn ARKit has made to the anchor + /// since carries the model with it. Cancelling the anchor's current turn instead drew the + /// wall as it stood before the turn. People and, on a phone with LiDAR, the scene mesh hide + /// it where they stand in front of it. False when the meter has no anchor in the session; + /// nothing is shown then. + func showResult(_ model: Entity, builtFor pose: simd_float4x4) -> Bool { + removeResult() + guard let id = delegate.shared.withLock({ $0.meterAnchorID }), + arView.session.currentFrame?.anchors.contains(where: { $0.identifier == id }) == true else { return false } + let holder = AnchorEntity(.anchor(identifier: id)) + model.orientation = simd_quatf(pose).inverse + holder.addChild(model) + arView.scene.addAnchor(holder) + result = holder + setOcclusion(true) + return true + } + + func hideResult() { + removeResult() + setOcclusion(false) + } + + /// Hides the shown result without taking it out, while the anchor's pose can't be trusted. + func setResultVisible(_ visible: Bool) { + result?.isEnabled = visible + } + + private func removeResult() { + result?.removeFromParent() + result = nil + } + + private func setOcclusion(_ on: Bool) { + if Self.supportsMesh { + if on { + arView.environment.sceneUnderstanding.options.insert(.occlusion) + } else { + arView.environment.sceneUnderstanding.options.remove(.occlusion) + } + } + guard Self.supportsPeopleOcclusion, occludesPeople != on else { return } + occludesPeople = on + // Without reset options the session keeps its tracking, anchors and mesh. + arView.session.run(Self.configuration(occludingPeople: on)) + } + + /// ARKit's mesh in world meters, with one `ARMeshClassification` raw value per triangle (0, + /// none, when the session runs without classification). + struct MeshSnapshot: Sendable { + var mesh: TriangleMesh + var classification: [UInt8] + } + + /// The LiDAR mesh ARKit has built so far, in world meters, or nil when there is none (no + /// LiDAR, or nothing reconstructed yet). Each anchor's vertices are read out of its Metal + /// buffer, never written, and moved into world coordinates by the anchor's transform. + /// Read from `frame`'s own anchors, the frame the engine takes its snapshot on + /// (`SourceFrame.spatial`), never from whatever frame the session holds when asked. + nonisolated static func meshSnapshot(of frame: ARFrame) -> MeshSnapshot? { + let anchors = frame.anchors.compactMap { $0 as? ARMeshAnchor } + guard !anchors.isEmpty else { return nil } + var vertices: [SIMD3] = [] + var indices: [UInt32] = [] + var classification: [UInt8] = [] + for anchor in anchors { + let source = anchor.geometry.vertices + let faces = anchor.geometry.faces + guard source.format == .float3, faces.indexCountPerPrimitive == 3, faces.bytesPerIndex == 4 || faces.bytesPerIndex == 2 else { + RuntimeLog.engine.error("mesh anchor skipped: vertex format \(source.format.rawValue), \(faces.indexCountPerPrimitive) indices per face, \(faces.bytesPerIndex) bytes per index") + continue + } + let first = UInt32(vertices.count) + let points = UnsafeRawPointer(source.buffer.contents()).advanced(by: source.offset) + for i in 0..( + p.loadUnaligned(as: Float.self), p.loadUnaligned(fromByteOffset: 4, as: Float.self), + p.loadUnaligned(fromByteOffset: 8, as: Float.self), 1 + ) + let world = anchor.transform * local + vertices.append(SIMD3(world.x, world.y, world.z)) + } + let raw = UnsafeRawPointer(faces.buffer.contents()) + for i in 0..<(faces.count * 3) { + let index = faces.bytesPerIndex == 4 + ? raw.loadUnaligned(fromByteOffset: i * 4, as: UInt32.self) + : UInt32(raw.loadUnaligned(fromByteOffset: i * 2, as: UInt16.self)) + indices.append(first + index) + } + classification += Self.faceClasses(anchor.geometry) + } + return vertices.isEmpty ? nil : MeshSnapshot(mesh: TriangleMesh(vertices: vertices, indices: indices), classification: classification) + } + + /// One class byte per face of `geometry`: its classification source when it has one in the + /// layout ARKit documents (one uchar per face), else 0 (none) for every face. + nonisolated private static func faceClasses(_ geometry: ARMeshGeometry) -> [UInt8] { + let faces = geometry.faces.count + guard let source = geometry.classification, source.format == .uchar, source.count == faces, source.componentsPerVector == 1 else { + return [UInt8](repeating: 0, count: faces) + } + let base = UnsafeRawPointer(source.buffer.contents()).advanced(by: source.offset) + return (0.. + /// `ARPlaneExtent.rotationOnYAxis`, radians. + var rotationOnYAxis: Float + /// `ARPlaneExtent.width` and `height`. + var extent: SIMD2 + /// `ARPlaneGeometry.boundaryVertices`, anchor coordinates. + var boundary: [SIMD3] + } + + /// Every plane ARKit had detected on `frame`. + nonisolated static func planeSnapshot(of frame: ARFrame) -> [PlaneSnapshot] { + frame.anchors.compactMap { $0 as? ARPlaneAnchor }.map { plane in + PlaneSnapshot( + id: plane.identifier.uuidString, vertical: plane.alignment == .vertical, + classification: Self.name(plane.classification), anchorToWorld: plane.transform, center: plane.center, + rotationOnYAxis: plane.planeExtent.rotationOnYAxis, + extent: SIMD2(plane.planeExtent.width, plane.planeExtent.height), boundary: plane.geometry.boundaryVertices + ) + } + } + + nonisolated private static func name(_ classification: ARPlaneAnchor.Classification) -> PacketPlane.Classification? { + switch classification { + case .none: PacketPlane.Classification.none + case .wall: .wall + case .floor: .floor + case .ceiling: .ceiling + case .table: .table + case .seat: .seat + case .window: .window + case .door: .door + @unknown default: nil + } + } + + func setMode(_ mode: LiveMode) { + delegate.shared.withLock { $0.mode = mode } + } + + /// Asks for the next sampled frame to carry its mesh and planes (`SourceFrame.spatial`). + func requestSpatialCapture() { + delegate.shared.withLock { $0.wantsSpatial = true } + } + + /// Raycast from a view point to a vertical surface: a detected plane's extent first, then a + /// vertical plane ARKit estimates from feature points at the tap. Never `.existingPlaneInfinite`, + /// which extends a fence's or another wall's plane past its edges, so a tap beside it lands on + /// a surface that isn't there. The hit says which kind it was. + func raycastVerticalPlane(from point: CGPoint) -> VerticalPlaneHit? { + let targets: [(ARRaycastQuery.Target, MeterPlaneSource)] = [(.existingPlaneGeometry, .detectedPlane), (.estimatedPlane, .estimatedPlane)] + for (target, source) in targets { + guard let result = arView.raycast(from: point, allowing: target, alignment: .vertical).first else { continue } + // An estimated plane has no anchor; the result's own y axis is the surface normal. + let transform = result.worldTransform + let planeTransform = result.anchor?.transform ?? transform + let normal = simd_normalize(SIMD3(planeTransform.columns.1.x, planeTransform.columns.1.y, planeTransform.columns.1.z)) + return VerticalPlaneHit( + position: SIMD3(transform.columns.3.x, transform.columns.3.y, transform.columns.3.z), + normal: normal, transform: transform, source: source + ) + } + return nil + } + + /// Anchors the meter so ARKit keeps refining its position; returns the anchor's id. + func addMeterAnchor(at transform: simd_float4x4) -> UUID { + let anchor = ARAnchor(name: "meter", transform: transform) + arView.session.add(anchor: anchor) + delegate.shared.withLock { $0.meterAnchorID = anchor.identifier } + return anchor.identifier + } + + func removeAnchor(_ id: UUID) { + if let anchor = arView.session.currentFrame?.anchors.first(where: { $0.identifier == id }) { + arView.session.remove(anchor: anchor) + } + delegate.shared.withLock { $0.meterAnchorID = nil } + } +} + +/// Shared between the main actor and the AR delegate queue. +struct LiveShared: Sendable { + var mode: LiveMode = .idle + var meterAnchorID: UUID? + var recorder: CaptureRecorder? + /// Set by `requestSpatialCapture`: the next sampled frame also carries its mesh and planes. + var wantsSpatial = false +} + +/// The mesh and planes of one delivered frame, read from that frame's own anchors, for the +/// upload's snapshot (`ScanEngine.takeUploadSnapshot`). +struct FrameSpatialCapture: Sendable { + /// The frame's time: the snapshot takes this only on the frame with the same time. + var time: Double + var mesh: LiveCapture.MeshSnapshot? + var planes: [LiveCapture.PlaneSnapshot] +} + +/// Receives ARSession callbacks on a private serial queue. Each sampled frame is reduced to a +/// Sendable `SourceFrame` there (pose, intrinsics, a luma quality measure and, when it could be +/// kept, a JPEG of the unrotated sensor image) and then sent to the main actor. ARKit objects +/// never leave the delegate queue, except the camera image of the one frame being encoded. +/// +/// JPEG encoding runs on its own queue, one frame at a time. Encoding on the delegate queue held +/// it for the length of an encode, so ARKit's next frames queued up, each holding a camera buffer +/// from ARKit's small pool. A keyframe candidate that arrives while an encode is running is sent +/// without a photo instead of waiting: it can't be kept, and the next sampled frame can be. +final class LiveSessionDelegate: NSObject, ARSessionDelegate, Sendable { + let shared = Mutex(LiveShared()) + private let onFrame: @MainActor @Sendable (SourceFrame) -> Void + private let onEvent: @MainActor @Sendable (LiveEvent) -> Void + private let queueState = Mutex(QueueState()) + private let context = CIContext(options: [.cacheIntermediates: false]) + /// Serial, and every frame the main actor gets passes through it, pose-only ones too, then + /// through the main queue (`deliver`), which is FIFO: frames reach the engine in capture + /// order even when a sampled frame waited on its JPEG. A pose-only frame used to go straight + /// to the main actor and overtake the sampled frame before it; the engine then saw time run + /// back. The cost is that a pose may wait behind one encode. + private let encodeQueue = DispatchQueue(label: "dev.housescanning.housescan.jpeg", qos: .userInitiated) + + private struct QueueState { + var frameCount = 0 + /// The latest ARFrame's time, for stamping an interruption. + var lastFrameTime: Double? + var lastEncode: (time: Double, camera: CameraFrame)? + var encoding = false + } + + /// The camera image handed to the encode queue. CVPixelBuffer is not Sendable; ARKit doesn't + /// write to a delivered frame's image, and only the encode queue reads it. + private struct PixelBufferBox: @unchecked Sendable { + let buffer: CVPixelBuffer + } + + /// Every sixth frame (about 10 per second at 60 fps) is sampled for capture; more adds cost, + /// not coverage. Every second frame carries the pose, so overlays drawn over the 60 fps camera + /// view move at 30 fps instead of 10. + private static let sampleEvery = 6 + private static let poseEvery = 2 + /// A walk JPEG is encoded once the phone moved 0.15 m or turned 5° since the last one: well + /// under auto-capture's 0.5 m / 15° spacing, so no keepable frame lacks an image. + private static let encodeMove: Float = 0.15 + private static let encodeTurn: Float = 5 * .pi / 180 + private static let encodeInterval = 0.3 + /// A walk JPEG is also encoded after this long without one, moved or not. The overhead + /// question is only asked on a frame with a photo (the answer stores it), and a phone held + /// still while tilted up would otherwise never get one. A guess: one encode a second while + /// standing still costs little; not measured on a phone. + private static let encodeStill = 1.0 + + init(onFrame: @escaping @MainActor @Sendable (SourceFrame) -> Void, onEvent: @escaping @MainActor @Sendable (LiveEvent) -> Void) { + self.onFrame = onFrame + self.onEvent = onEvent + } + + func session(_ session: ARSession, didUpdate frame: ARFrame) { + let count = queueState.withLock { state -> Int in + state.frameCount += 1 + state.lastFrameTime = frame.timestamp + return state.frameCount + } + let shared = shared.withLock { $0 } + let tracking = Self.tracking(frame.camera.trackingState) + // Every frame's pose goes to the packet's trajectory, before any sampling. + shared.recorder?.recordPose(t: frame.timestamp, tracking: TrackingCode(tracking), cameraToWorld: frame.camera.transform) + let intrinsics = frame.camera.intrinsics + let resolution = frame.camera.imageResolution + let camera = CameraFrame( + cameraToWorld: frame.camera.transform, + intrinsics: SIMD4(intrinsics.columns.0.x, intrinsics.columns.1.y, intrinsics.columns.2.x, intrinsics.columns.2.y), + imageSize: SIMD2(Float(resolution.width), Float(resolution.height)) + ) + if let recorder = shared.recorder, tracking == .normal { + Self.recordDepthFrame(frame, camera: camera, into: recorder) + } + guard count % Self.poseEvery == 0 else { return } + let meterAnchor = shared.meterAnchorID.flatMap { id in frame.anchors.first { $0.identifier == id }?.transform } + guard count % Self.sampleEvery == 0 else { + let pose = SourceFrame( + id: "live-\(count)", timestamp: frame.timestamp, camera: camera, tracking: tracking, + quality: nil, jpeg: .none, still: nil, meterAnchor: meterAnchor, isPoseOnly: true + ) + encodeQueue.async { [self, pose] in deliver(pose) } + return + } + let quality = Self.quality(frame.capturedImage) + let ground = frame.anchors.compactMap { $0 as? ARPlaneAnchor } + .filter { $0.alignment == .horizontal } + .map(Self.groundEvidence) + var snapshot = SourceFrame( + id: "live-\(count)", timestamp: frame.timestamp, camera: camera, tracking: tracking, + quality: quality, jpeg: .none, still: nil, meterAnchor: meterAnchor, groundPlanes: ground + ) + let wantsSpatial = self.shared.withLock { state in + defer { state.wantsSpatial = false } + return state.wantsSpatial + } + if wantsSpatial { + snapshot.spatial = FrameSpatialCapture( + time: frame.timestamp, mesh: LiveCapture.meshSnapshot(of: frame), planes: LiveCapture.planeSnapshot(of: frame)) + } + let image = tracking == .normal && shouldEncode(mode: shared.mode, time: frame.timestamp, camera: camera) + ? PixelBufferBox(buffer: frame.capturedImage) : nil + // Depth and camera settings go with the photo: a frame without one can't be kept. Copied + // here, so the ARFrame is not held past this callback. + if image != nil { + snapshot.exposure = Self.exposure(frame) + if let depth = frame.sceneDepth, let copied = Self.depthCopy(depth) { + // The packet takes ARKit's depth only with its confidence (required from 1.1); + // coverage uses the map either way. + snapshot.sensorDepth = copied.confidence == nil ? nil : copied + snapshot.depth = DepthImage( + meters: copied.meters, width: copied.width, height: copied.height, confidence: copied.confidence, + intrinsics: DepthImage.intrinsics(scaling: camera.intrinsics, from: camera.imageSize, toWidth: copied.width, height: copied.height) + ) + } + } + encodeQueue.async { [self, snapshot] in + var delivered = snapshot + if let image { + if let data = encode(image.buffer) { delivered.jpeg = .data(data) } + queueState.withLock { $0.encoding = false } + } + deliver(delivered) + } + } + + /// Hands a frame to the engine on the main queue. Called only from `encodeQueue`, in order; + /// `DispatchQueue.main` runs blocks in the order they were queued, where separate `Task`s + /// aimed at the main actor carry no such promise. + private func deliver(_ frame: SourceFrame) { + DispatchQueue.main.async { [onFrame] in + MainActor.assumeIsolated { onFrame(frame) } + } + } + + /// A depth frame for the packet: LiDAR depth between photos, a few a second, from frames with + /// normal tracking, so each sits on the trajectory with a pose worth fusing. The recorder's + /// budget decides first, so a frame it would drop is never copied. The intrinsics are the + /// camera's scaled to the depth map's grid, which covers the same view. + private static func recordDepthFrame(_ frame: ARFrame, camera: CameraFrame, into recorder: CaptureRecorder) { + guard let depth = frame.sceneDepth, recorder.wantsDepthFrame(at: frame.timestamp), + let copied = depthCopy(depth), copied.confidence != nil else { return } + recorder.recordDepthFrame( + t: frame.timestamp, tracking: TrackingCode(.normal), cameraToWorld: frame.camera.transform, + intrinsics: DepthImage.intrinsics(scaling: camera.intrinsics, from: camera.imageSize, toWidth: copied.width, height: copied.height), + depth: copied + ) + } + + /// Whether to encode this frame; claims the single encode slot when it says yes. + private func shouldEncode(mode: LiveMode, time: Double, camera: CameraFrame) -> Bool { + queueState.withLock { state in + guard !state.encoding else { return false } + let decision: Bool + switch mode { + case .idle: + decision = false + case .closeUp: + decision = time - (state.lastEncode?.time ?? -.infinity) >= Self.encodeInterval + case .walk: + if let last = state.lastEncode { + let moved = simd_distance(camera.position, last.camera.position) >= Self.encodeMove + || camera.rotationAngle(to: last.camera) >= Self.encodeTurn + decision = (moved && time - last.time >= Self.encodeInterval) || time - last.time >= Self.encodeStill + } else { + decision = true + } + } + if decision { + state.lastEncode = (time, camera) + state.encoding = true + } + return decision + } + } + + /// JPEG of the sensor image as captured: landscape, unrotated, matching the intrinsics. + private func encode(_ buffer: CVPixelBuffer) -> Data? { + let image = CIImage(cvPixelBuffer: buffer) + let space = CGColorSpace(name: CGColorSpace.sRGB) ?? CGColorSpaceCreateDeviceRGB() + let options = [CIImageRepresentationOption(rawValue: kCGImageDestinationLossyCompressionQuality as String): 0.8] + return context.jpegRepresentation(of: image, colorSpace: space, options: options) + } + + /// Sharpness and exposure from the luma plane, sampling every 8th pixel of every 8th row + /// (240 x 180 for a 1920 x 1440 buffer). + private static func quality(_ buffer: CVPixelBuffer) -> FrameQuality? { + CVPixelBufferLockBaseAddress(buffer, .readOnly) + defer { CVPixelBufferUnlockBaseAddress(buffer, .readOnly) } + guard CVPixelBufferGetPlaneCount(buffer) >= 1, let base = CVPixelBufferGetBaseAddressOfPlane(buffer, 0) else { return nil } + let width = CVPixelBufferGetWidthOfPlane(buffer, 0) + let height = CVPixelBufferGetHeightOfPlane(buffer, 0) + let bytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(buffer, 0) + guard width >= 8, height >= 8 else { return nil } + let raw = UnsafeRawBufferPointer(start: base, count: bytesPerRow * height) + return FrameQuality(LumaImage(sampling: raw, width: width, height: height, bytesPerRow: bytesPerRow, step: 8)) + } + + /// The camera settings of the frame: exposure time and offset from ARCamera, and ISO, focal + /// length and f-number from the frame's EXIF (`kCGImagePropertyExif*` keys) when present. + private static func exposure(_ frame: ARFrame) -> PhotoExposure { + let exif = frame.exifData + func number(_ key: CFString) -> Double? { + (exif[key as String] as? NSNumber)?.doubleValue ?? (exif[key as String] as? [NSNumber])?.first?.doubleValue + } + let duration = frame.camera.exposureDuration + return PhotoExposure( + durationS: duration > 0 ? duration : nil, + offsetEV: Double(frame.camera.exposureOffset), + iso: number(kCGImagePropertyExifISOSpeedRatings).flatMap { $0 > 0 ? $0 : nil }, + focalLengthMM: number(kCGImagePropertyExifFocalLength).flatMap { $0 > 0 ? $0 : nil }, + fNumber: number(kCGImagePropertyExifFNumber).flatMap { $0 > 0 ? $0 : nil } + ) + } + + /// The LiDAR depth map (Float32 meters along the camera's -z, 256 x 192 on current iPhones) + /// with its confidence, copied row by row. The depth map covers the same view as the camera + /// image at a lower resolution, so coverage's copy takes the camera's intrinsics scaled by the + /// size ratio on each axis (`DepthImage.intrinsics(scaling:...)`). ARKit writes no value it + /// could not measure as NaN; those become 0, "no measurement" in the packet. + private static func depthCopy(_ data: ARDepthData) -> DepthPacket? { + let map = data.depthMap + CVPixelBufferLockBaseAddress(map, .readOnly) + defer { CVPixelBufferUnlockBaseAddress(map, .readOnly) } + guard CVPixelBufferGetPixelFormatType(map) == kCVPixelFormatType_DepthFloat32, let base = CVPixelBufferGetBaseAddress(map) else { return nil } + let width = CVPixelBufferGetWidth(map) + let height = CVPixelBufferGetHeight(map) + let rowBytes = CVPixelBufferGetBytesPerRow(map) + guard width > 0, height > 0 else { return nil } + var meters = [Float](repeating: 0, count: width * height) + for y in 0.. 0 ? value : 0 + } + } + var confidence: [UInt8]? + if let levels = data.confidenceMap { + CVPixelBufferLockBaseAddress(levels, .readOnly) + defer { CVPixelBufferUnlockBaseAddress(levels, .readOnly) } + if CVPixelBufferGetPixelFormatType(levels) == kCVPixelFormatType_OneComponent8, + CVPixelBufferGetWidth(levels) == width, CVPixelBufferGetHeight(levels) == height, + let levelBase = CVPixelBufferGetBaseAddress(levels) { + let levelRowBytes = CVPixelBufferGetBytesPerRow(levels) + var values = [UInt8](repeating: 0, count: width * height) + for y in 0.. TrackingQuality { + switch state { + case .notAvailable: .notAvailable + case .normal: .normal + case .limited(let reason): + switch reason { + case .initializing: .limited(.initializing) + case .excessiveMotion: .limited(.excessiveMotion) + case .insufficientFeatures: .limited(.insufficientFeatures) + case .relocalizing: .limited(.relocalizing) + @unknown default: .limited(.unknown) + } + } + } + + /// A horizontal plane as the ground choice reads it (`GroundPlaneChoice`): its height, its + /// class and its outline in plan, world meters. + private static func groundEvidence(_ plane: ARPlaneAnchor) -> GroundPlaneEvidence { + let center = plane.transform * SIMD4(plane.center, 1) + let kind: GroundPlaneEvidence.Kind = switch plane.classification { + case .floor: .floor + case .table, .seat: .furniture + case .none: .unclassified + default: .other + } + let boundary = plane.geometry.boundaryVertices.map { vertex -> SIMD2 in + let world = plane.transform * SIMD4(vertex, 1) + return SIMD2(world.x, world.z) + } + return GroundPlaneEvidence(y: center.y, kind: kind, boundary: boundary) + } + + // MARK: Session events + + func sessionShouldAttemptRelocalization(_ session: ARSession) -> Bool { + // Resume the walk in the same world frame after an interruption (checklist R4). + true + } + + func sessionWasInterrupted(_ session: ARSession) { + let last = queueState.withLock { $0.lastFrameTime } + Task { @MainActor [onEvent] in onEvent(.interrupted(lastFrameTime: last)) } + } + + func sessionInterruptionEnded(_ session: ARSession) { + Task { @MainActor [onEvent] in onEvent(.interruptionEnded) } + } + + func session(_ session: ARSession, didFailWithError error: any Error) { + let event: LiveEvent + if let arError = error as? ARError, arError.code == .cameraUnauthorized { + event = .cameraDenied + } else { + event = .failed(error.localizedDescription) + } + Task { @MainActor [onEvent] in onEvent(event) } + } +} + +/// The ARView as a SwiftUI view. +struct LiveCameraView: UIViewRepresentable { + let arView: ARView + + func makeUIView(context: Context) -> ARView { arView } + func updateUIView(_ view: ARView, context: Context) {} +} diff --git a/ios/HouseScan/Runtime/MeterNumberReading.swift b/ios/HouseScan/Runtime/MeterNumberReading.swift new file mode 100644 index 000000000..30e3117d4 --- /dev/null +++ b/ios/HouseScan/Runtime/MeterNumberReading.swift @@ -0,0 +1,49 @@ +import Foundation + +// The boundary between the engine's close-up flow and the meter-number reader. The checks and +// their thresholds come from the close-up eval (experiments/meter-closeup/README.md on +// t3/meter-closeup at 944cbe1, "For the app: what to port"). + +/// What one close-up photo yielded. +struct MeterReadout: Sendable, Equatable { + /// The best candidates, barcode-confirmed first, at most three. Empty when the photo must be + /// retaken. + var candidates: [MeterNumberCandidate] + /// Why to retake (`.blurry` from the focus check, `.noNumber` when nothing was read); nil when + /// there are candidates to show. + var retake: CloseUpProblem? + /// The top candidate's characters are small in the photo, so after "None of these" the right + /// advice is "move closer" (`.numberTooSmall`) rather than a plain retake. + var numberTooSmall: Bool + /// The photo decoded and passed the focus check, whether or not a number was read in it. Only + /// such a photo's view goes into coverage (`CloseUpCredit`). + var photoPassedChecks: Bool + /// The maker named on the meter (`MeterBrand.read`), or nil when no line names one. Shown + /// beside the candidates; it counts only when the homeowner confirms a number without + /// rejecting it. + var brand: String? = nil +} + +/// Reads the meter number from the close-up JPEG. Runs off the main actor. +protocol MeterNumberReader: Sendable { + func read(jpeg: Data) async -> MeterReadout +} + +/// Reads nothing: every photo asks for a retake with `.noNumber`. Stands in until the Vision +/// reader lands. +struct UnavailableMeterNumberReader: MeterNumberReader { + func read(jpeg: Data) async -> MeterReadout { + // No check ran, so nothing vouches for the photo. + MeterReadout(candidates: [], retake: .noNumber, numberTooSmall: false, photoPassedChecks: false) + } +} + +/// The reader the engine uses. +enum MeterNumberReaders { + static func make() -> any MeterNumberReader { + // The close-up JPEG holds the landscape sensor image unrotated (LiveCapture.encode) and the + // app is portrait-only, so the upright photo is the stored one turned 90° clockwise, as + // ImageWork.uprightThumbnail shows it: EXIF `.right`. + VisionMeterNumberReader(orientation: .right) + } +} diff --git a/ios/HouseScan/Runtime/MeterReader/VisionMeterNumberReader.swift b/ios/HouseScan/Runtime/MeterReader/VisionMeterNumberReader.swift new file mode 100644 index 000000000..6a1c9c495 --- /dev/null +++ b/ios/HouseScan/Runtime/MeterReader/VisionMeterNumberReader.swift @@ -0,0 +1,99 @@ +import CoreGraphics +import Foundation +import HouseScanKit +import ImageIO +import Vision + +/// Reads the meter number from a close-up with Apple Vision, configured as the close-up eval +/// measured it (experiments/meter-closeup/README.md on t3/meter-closeup at 944cbe1, "For the app: +/// what to port"): focus check first, then text and barcodes, then HouseScanKit's ranking and size +/// check. +struct VisionMeterNumberReader: MeterNumberReader { + /// How the stored pixels turn upright, as an EXIF orientation. Vision reads and the size check + /// measures the upright photo. + let orientation: CGImagePropertyOrientation + + @concurrent func read(jpeg: Data) async -> MeterReadout { + Self.readout(jpeg: jpeg, orientation: orientation) + } + + /// Nothing read in a photo that decoded and is in focus. + private static let retakeNoNumber = MeterReadout(candidates: [], retake: .noNumber, numberTooSmall: false, photoPassedChecks: true) + private static let undecodable = MeterReadout(candidates: [], retake: .noNumber, numberTooSmall: false, photoPassedChecks: false) + + private static func readout(jpeg: Data, orientation: CGImagePropertyOrientation) -> MeterReadout { + guard let source = CGImageSourceCreateWithData(jpeg as CFData, nil), + let image = CGImageSourceCreateImageAtIndex(source, 0, nil), + let gray = luma(of: image) + else { return undecodable } + + // The eval's flow blocks a blurry photo before showing candidates, so Vision need not run. + if MeterPhotoChecks.isOutOfFocus(sharpness: MeterPhotoChecks.wholePhotoSharpness(gray)) { + return MeterReadout(candidates: [], retake: .blurry, numberTooSmall: false, photoPassedChecks: false) + } + + let text = VNRecognizeTextRequest() + text.revision = VNRecognizeTextRequestRevision3 + // The eval read 97% of meter numbers at .accurate and 75% at .fast; language correction + // added nothing. + text.recognitionLevel = .accurate + text.recognitionLanguages = ["en-US"] + text.usesLanguageCorrection = false + // 0 is the default (full resolution); set so a changed default cannot drop small labels. + text.minimumTextHeight = 0 + let barcodes = VNDetectBarcodesRequest() + barcodes.revision = VNDetectBarcodesRequestRevision4 + do { + try VNImageRequestHandler(cgImage: image, orientation: orientation).perform([text, barcodes]) + } catch { + return retakeNoNumber + } + + let lines = (text.results ?? []).compactMap { observation -> MeterTextLine? in + guard let best = observation.topCandidates(1).first else { return nil } + return MeterTextLine(text: best.string, box: topLeftBox(observation.boundingBox)) + } + let payloads = (barcodes.results ?? []).compactMap(\.payloadStringValue) + let found = MeterNumberRanking.candidates(lines: lines, barcodePayloads: payloads) + let turned: Set = [.left, .leftMirrored, .right, .rightMirrored] + let uprightHeight = turned.contains(orientation) ? image.width : image.height + guard let choices = MeterPhotoChecks.choices(from: found, photoHeight: uprightHeight) else { + return retakeNoNumber + } + return MeterReadout( + candidates: choices.candidates.enumerated().map { index, candidate in + MeterNumberCandidate(id: index, text: candidate.core, barcodeConfirmed: candidate.barcodeConfirmed) + }, + retake: nil, + numberTooSmall: choices.numberTooSmall, + photoPassedChecks: true, + brand: MeterBrand.read(lines) + ) + } + + /// Vision's normalized box has a bottom-left origin; the ranking uses a top-left one. + private static func topLeftBox(_ box: CGRect) -> MeterBox { + MeterBox(x: box.minX, y: 1 - box.maxY, width: box.width, height: box.height) + } + + /// 8-bit luma of the stored pixels. Drawn in the image's own RGB color space when it has one, + /// so the values are the file's, as PIL reads them, not color-matched. + private static func luma(of image: CGImage) -> MeterGrayImage? { + let width = image.width + let height = image.height + let space = image.colorSpace.flatMap { $0.model == .rgb ? $0 : nil } + ?? CGColorSpace(name: CGColorSpace.sRGB) ?? CGColorSpaceCreateDeviceRGB() + let bytesPerRow = 4 * width + var bytes = [UInt8](repeating: 0, count: bytesPerRow * height) + let drawn = bytes.withUnsafeMutableBytes { buffer -> Bool in + guard let context = CGContext( + data: buffer.baseAddress, width: width, height: height, bitsPerComponent: 8, + bytesPerRow: bytesPerRow, space: space, bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue + ) else { return false } + context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height)) + return true + } + guard drawn else { return nil } + return bytes.withUnsafeBytes { MeterGrayImage(rgbx: $0, width: width, height: height, bytesPerRow: bytesPerRow) } + } +} diff --git a/ios/HouseScan/Runtime/ReplayPlayer.swift b/ios/HouseScan/Runtime/ReplayPlayer.swift new file mode 100644 index 000000000..47b18dcfa --- /dev/null +++ b/ios/HouseScan/Runtime/ReplayPlayer.swift @@ -0,0 +1,191 @@ +import CoreGraphics +import Foundation +import HouseScanKit +import OSLog +import simd + +/// Plays a recorded measure-lab-session v2 folder (contract C3) as if it were the camera. +/// +/// Frames are delivered at their recorded timing divided by `speed`, each with its own pose and +/// intrinsics, and its LiDAR depth when the session recorded some (`keyframes[].depth`, read by +/// `ReplaySession.loadDepth`), through the same engine path as live frames. JPEGs and depth are +/// decoded off the main actor. +@MainActor +final class ReplayPlayer { + let folder: URL + let session: ReplaySession + var frames: [ReplayFrame] { session.frames } + let planned: [PlannedFrame] + /// The wall the replay measures: the recorded one, or one assumed from the trajectory. + let wall: WallFrame + let wallDescription: String + /// True when the ground height comes from the recording's wall taps; an assumed wall puts + /// the ground 1.4 m under the mean camera height, a guess. + let groundMeasured: Bool + /// Frames the autopilot holds back from the walk for the gap loop, once prepared. + private(set) var heldBack: ReplayPlanning.HeldBackWindow? + private(set) var isPlaying = false + private(set) var shownIndex: Int? + + private let onFrame: @MainActor (SourceFrame) -> Void + private var task: Task? + + /// What loading a replay produces: the session, its frames as cameras, and its wall. + struct Loaded: Sendable { + let session: ReplaySession + let planned: [PlannedFrame] + let wall: WallFrame + let wallDescription: String + let groundMeasured: Bool + } + + /// Reads session.json and settles the wall. Deriving an assumed wall projects every frame + /// against many candidate walls, so call this off the main actor. + nonisolated static func load(folder: URL) throws -> Loaded { + let session = try ReplaySession.load(folder: folder) + let planned = session.frames.map { + PlannedFrame( + camera: CameraFrame(cameraToWorld: $0.cameraToWorld, intrinsics: $0.intrinsics, imageSize: SIMD2(Float($0.width), Float($0.height))), + timestamp: $0.timestamp, trackingNormal: $0.trackingNormal + ) + } + if let declared = session.declaredWall, let frame = WallFrame(meter: declared.meter, outward: declared.outward, groundY: declared.groundY) { + return Loaded(session: session, planned: planned, wall: frame, wallDescription: "recorded (wall taps in session.json)", groundMeasured: true) + } + guard let assumed = ReplayPlanning.assumedWall(frames: planned) else { throw ReplayError.noFrames } + let description = String( + format: "assumed from the trajectory: parallel to the walk, %.2f m to the side the camera faces, meter where the walk covers most, %d cells covered within 20 ft of it with every frame; not a measured wall", + assumed.offset, assumed.coveredCells + ) + return Loaded(session: session, planned: planned, wall: assumed.wall, wallDescription: description, groundMeasured: false) + } + + /// One frame's depth, read off the main actor; nil when the session recorded none for it. A + /// frame whose depth files are missing or the wrong size plays without depth, logged. + nonisolated private static func loadDepth(_ frame: ReplayFrame, folder: URL) -> DepthImage? { + do { + return try ReplaySession.loadDepth(for: frame, folder: folder) + } catch { + RuntimeLog.engine.error("replay depth for \(frame.id, privacy: .public) unreadable: \(String(describing: error), privacy: .public)") + return nil + } + } + + init(folder: URL, loaded: Loaded, onFrame: @escaping @MainActor (SourceFrame) -> Void) { + self.folder = folder + self.onFrame = onFrame + session = loaded.session + planned = loaded.planned + wall = loaded.wall + wallDescription = loaded.wallDescription + groundMeasured = loaded.groundMeasured + } + + /// Finds the frames to hold back for the autopilot's gap loop. Heavy, so it runs off the + /// main actor; call once before the walk. + /// Planned over the walk's frames only: the recording's closing tilt-up run is played for the + /// tilt-up step and for requests above the walk (`ScanEngine.tiltUpFrames`), never in the walk. + /// With `endsWherePhoneStood`, against the ends a walk ended by "Can't get there" sets, + /// 1 m inside them (the cells skipped there are within 0.5 m of the phone). + func prepareHeldBack(endsWherePhoneStood: Bool = false) async { + let walkEnd = ScanEngine.tiltUpFrames(in: self, map: CoverageMap(wall: wall)).lowerBound + let planned = Array(planned[.. CameraFrame { planned[index].camera } + + // MARK: Playback + + /// Plays `range` in order, skipping `excluding`, at `speed` times the recorded pace. + func play(range: Range, excluding: Range? = nil, speed: Double) { + stop() + let indices = range.filter { !(excluding?.contains($0) ?? false) } + guard !indices.isEmpty else { return } + isPlaying = true + let frames = frames + let folder = folder + task = Task { [weak self] in + var previous: Double? + for index in indices { + let frame = frames[index] + let url = folder.appending(path: frame.imagePath) + let started = ContinuousClock.now + let (decoded, frameDepth) = await Task.detached(priority: .userInitiated) { + (ImageWork.decode(url), Self.loadDepth(frame, folder: folder)) + }.value + if let previous { + let wait = Duration.seconds(max(0, (frame.timestamp - previous) / speed)) - (ContinuousClock.now - started) + if wait > .zero { try? await Task.sleep(for: wait) } + } + previous = frame.timestamp + guard !Task.isCancelled, let self else { return } + self.deliver(index: index, decoded: decoded, depth: frameDepth, isReview: false) + } + self?.isPlaying = false + } + } + + /// Shows one frame for tapping or review, outside auto-capture. + func show(index: Int) { + stop() + guard frames.indices.contains(index) else { return } + let url = folder.appending(path: frames[index].imagePath) + task = Task { [weak self] in + let decoded = await Task.detached(priority: .userInitiated) { ImageWork.decode(url) }.value + guard !Task.isCancelled else { return } + self?.deliver(index: index, decoded: decoded, depth: nil, isReview: true) + } + } + + /// Waits until the frame being shown is `index` (after `show`). + func waitUntilShown(_ index: Int) async { + for _ in 0..<100 where shownIndex != index { + try? await Task.sleep(for: .milliseconds(20)) + } + } + + func stop() { + task?.cancel() + task = nil + isPlaying = false + } + + /// A frame shown for review is never kept, so it goes without depth. + private func deliver(index: Int, decoded: (CGImage, FrameQuality)?, depth: DepthImage?, isReview: Bool) { + let frame = frames[index] + shownIndex = index + onFrame(SourceFrame( + id: frame.id, + timestamp: frame.timestamp, + camera: planned[index].camera, + tracking: frame.trackingNormal ? .normal : .limited(.unknown), + quality: decoded?.1, + jpeg: .file(folder.appending(path: frame.imagePath)), + still: decoded?.0, + meterAnchor: nil, + depth: depth, + isReview: isReview + )) + } + + // MARK: Choosing frames + + /// The frame that shows `point` best: in front of the camera, inside the image with the + /// largest margin, preferring nearer cameras. + func bestFrame(showing point: SIMD3) -> Int? { + var best: (index: Int, score: Float)? + for index in planned.indices { + let camera = planned[index].camera + guard let pixel = camera.pixel(of: point), camera.contains(pixel: pixel, margin: 0.1) else { continue } + let centered = simd_length((pixel - camera.imageSize / 2) / camera.imageSize) + let score = centered + 0.05 * simd_distance(camera.position, point) + if score < best?.score ?? .infinity { best = (index, score) } + } + return best?.index + } +} diff --git a/ios/HouseScan/Runtime/ResultClient.swift b/ios/HouseScan/Runtime/ResultClient.swift new file mode 100644 index 000000000..5b8f2ac6a --- /dev/null +++ b/ios/HouseScan/Runtime/ResultClient.swift @@ -0,0 +1,178 @@ +import Foundation +import HouseScanKit + +/// Sends a scan's scene.json to the placement server and returns the result JSON (contract C2). +@MainActor +protocol ResultClient: AnyObject { + /// True when the answer is the bundled sample, not a server's analysis. + var isSample: Bool { get } + func submit(scene: Data, progress: @escaping @Sendable (Double) -> Void) async throws -> Data +} + +/// Posts scene.json to the placement server: `POST {serverURL}/v1/placements` with the JSON as +/// the body and `Content-Type: application/json`. Only the JSON goes: the server's solver reads no +/// photos and skips the image check for a bare scene.json, so the keyframes stay on the phone +/// (and in the scan folder's `scan.zip`, which leaves only if the homeowner shares it). The +/// response body is the result JSON. +@MainActor +final class HTTPResultClient: ResultClient { + let serverURL: URL + let isSample = false + + init(serverURL: URL) { + self.serverURL = serverURL + } + + nonisolated static func makeRequest(serverURL: URL) -> URLRequest { + var request = URLRequest(url: serverURL.appending(path: "v1/placements")) + request.httpMethod = "POST" + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.setValue("application/json", forHTTPHeaderField: "Accept") + request.timeoutInterval = 120 + return request + } + + func submit(scene: Data, progress: @escaping @Sendable (Double) -> Void) async throws -> Data { + let request = Self.makeRequest(serverURL: serverURL) + let delegate = UploadProgressDelegate(progress: progress) + let (data, response) = try await URLSession.shared.upload(for: request, from: scene, delegate: delegate) + guard let http = response as? HTTPURLResponse else { throw UploadError.notHTTP } + guard (200..<300).contains(http.statusCode) else { + throw UploadError.server( + status: http.statusCode, body: String(decoding: data.prefix(300), as: UTF8.self), + retryAfter: http.value(forHTTPHeaderField: "Retry-After")) + } + progress(1) + return data + } +} + +enum UploadError: Error, CustomStringConvertible { + case notHTTP + case server(status: Int, body: String, retryAfter: String? = nil) + case missingSample + + var description: String { + switch self { + case .notHTTP: "The server's answer was not HTTP." + case .server(let status, let body, _): "The server answered \(status): \(body)" + case .missingSample: "SampleResult.json is missing from the app bundle." + } + } +} + +/// What the homeowner reads about a failed upload. Which failures are worth sending again is +/// decided (and tested) in HouseScanKit's `UploadFailureKind`; the technical detail goes to the +/// log, and the screen never shows raw error text. +enum UploadFailure { + /// A failure while sending or reading the answer (not while packaging the scan). + static func state(for error: any Error) -> UploadState { + let kind: UploadFailureKind = if case .server(let status, _, let retryAfter) = error as? UploadError { + UploadFailureKind.classify(httpStatus: status, retryAfter: retryAfter) + } else { + UploadFailureKind.classify(error) + } + return switch kind { + case .offline: + .failed(message: "Your phone isn't connected to the internet. Your scan is saved on this phone.", offline: true) + case .unreachable: + .failed(message: "We couldn't reach the House Scan server. Your scan is saved on this phone, so you can try again.", offline: false) + case .serverError: + .failed(message: "The House Scan server had a problem. Your scan is saved on this phone, so you can try again.", offline: false) + case .busy(let retryAfter): + .failed(message: busyMessage(retryAfter), offline: false) + case .refused: + .rejected(message: "The server couldn't use this scan. Go back to the review to check your marks, or start over.") + case .unreadableAnswer: + .rejected(message: "We couldn't read the server's answer. Go back to the review and send it again, or start over.") + } + } + + /// A busy server: when it said how long to wait (at most a day, `UploadFailureKind`), the + /// homeowner hears that, rounded up to a minute past 90 seconds; either way the scan is kept + /// and "Try again" is offered at once. + static func busyMessage(_ retryAfter: Int?) -> String { + let wait: String? = retryAfter.map { raw in + let seconds = min(max(raw, 1), UploadFailureKind.maxRetryAfterSeconds) + let minutes = seconds / 60 + (seconds % 60 == 0 ? 0 : 1) + return seconds <= 90 ? "about \(seconds) seconds" : "about \(minutes) minutes" + } + let when = wait.map { "Try again in \($0)." } ?? "Try again in a moment." + return "The House Scan server is busy. Your scan is saved on this phone. \(when)" + } + + /// The answer came back about a wall or ground the phone had corrected since, twice + /// (`AnswerFreshness.stillChanging`): it isn't shown, and "Try again" sends a fresh scan. + static let stillChanging: UploadState = .failed( + message: "Your phone was still adjusting the scan when the answer came back. Hold it steady and try again.", offline: false) + + /// The scan couldn't be turned into scene.json. A driveway or fence that has to be marked + /// again says which, so the homeowner knows what to fix in the review. + static func packaging(_ error: any Error) -> UploadState { + switch error as? SceneExportError { + case .fenceAcrossCorner?: + // A fence marked before the wall was followed round a corner. + return .rejected(message: "A fence runs round a corner of your wall. Go back to the review and mark it on each side of the corner as its own fence.") + case .objectAcrossCorner?: + return .rejected(message: "An AC unit's corners are on two sides of a corner of your wall. Go back to the review and mark it again.") + default: + break + } + switch error as? ScanEngine.ExportError { + case .markCollapsed(.driveway)?: + return .rejected(message: "Mark the driveway again: its two points came out on top of each other.") + case .markCollapsed(.fence)?: + return .rejected(message: "Mark the fence again: its two points came out on top of each other.") + case .markCollapsed(.gasMeter)?: + return .rejected(message: "Mark the gas meter again: its two corners came out on top of each other.") + case .markCollapsed(.acUnit)?: + return .rejected(message: "Mark the AC unit again: its two corners came out on top of each other.") + default: + return .rejected(message: "This scan couldn't be prepared for sending. Go back to the review to check your marks, or start over.") + } + } +} + +final class UploadProgressDelegate: NSObject, URLSessionTaskDelegate, Sendable { + private let progress: @Sendable (Double) -> Void + + init(progress: @escaping @Sendable (Double) -> Void) { + self.progress = progress + } + + /// Once the whole body is sent the server is working out the spot in the same request, so + /// report 1 and let the screen move on to "Check clearances". Before that, stop at 0.99 so + /// rounding never shows 100%. Holding 0.99 until the answer left the screen on "Sending + /// measurements, 99%" for the whole analysis (field test run 1). + func urlSession(_ session: URLSession, task: URLSessionTask, didSendBodyData bytesSent: Int64, totalBytesSent: Int64, totalBytesExpectedToSend: Int64) { + guard totalBytesExpectedToSend > 0 else { return } + if totalBytesSent >= totalBytesExpectedToSend { + progress(1) + } else { + progress(min(0.99, Double(totalBytesSent) / Double(totalBytesExpectedToSend))) + } + } +} + +/// Answers with the bundled SampleResult.json, for tests and demos without a server. The result +/// is flagged `isSample` so the screen says it is not a real analysis. +@MainActor +final class SampleResultClient: ResultClient { + let isSample = true + /// Seconds the fake upload takes, so each upload state is visible in demos and UI tests. + let pace: Double + + init(pace: Double) { + self.pace = pace + } + + func submit(scene: Data, progress: @escaping @Sendable (Double) -> Void) async throws -> Data { + for step in 1...4 { + try await Task.sleep(for: .seconds(pace / 5)) + progress(Double(step) / 4) + } + try await Task.sleep(for: .seconds(pace)) + guard let url = Bundle.main.url(forResource: "SampleResult", withExtension: "json") else { throw UploadError.missingSample } + return try Data(contentsOf: url) + } +} diff --git a/ios/HouseScan/Runtime/SampleResult.json b/ios/HouseScan/Runtime/SampleResult.json new file mode 100644 index 000000000..075ec5855 --- /dev/null +++ b/ios/HouseScan/Runtime/SampleResult.json @@ -0,0 +1,126 @@ +{ + "schema_version": "1.0", + "decision": "manual_review", + "summary": "Sample result for offline demos, not a real analysis: the spot size (2.5 ft wide, 1.0 ft deep, 4.5 ft tall) and every threshold here are illustrative values, not rules.", + "reasons": [ + {"code": "policy_not_approved", "message": "Sample: no approved policy is loaded, so a person makes the final call."}, + {"code": "unsure_checks", "message": "Sample: two checks at this spot are unsure.", "checks": ["window_clearance", "front_clearance"]}, + {"code": "unobserved_area", "message": "Sample: the ground in front of the spot was not seen.", "checks": ["front_clearance"]} + ], + "policy": { + "id": null, + "version": null, + "auto_approve": false, + "sources": ["public"], + "rules_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + }, + "spot": { + "outcome": "unsure", + "wall_id": "wall", + "segment": 0, + "span_ft": [3.0, 5.5], + "width_ft": 2.5, + "depth_ft": 1.0, + "height_ft": 4.5, + "footprint": [[3.0, 0.0], [5.5, 0.0], [5.5, 1.0], [3.0, 1.0]], + "center": [4.25, 0.5], + "along": [1.0, 0.0], + "outward": [0.0, 1.0], + "meter_offset_ft": [4.25, 0.5], + "route_length_ft": 3.0 + }, + "route": { + "outcome": "pass", + "length_ft": 3.0, + "plus_minus_ft": 0.3, + "height_ft": 1.5, + "polyline": [[0.0, 0.0], [3.0, 0.0]], + "detours": [], + "crossings": [] + }, + "checks": [ + { + "id": "gas_clearance", + "label": "Distance from the gas meter", + "outcome": "pass", + "reason": "Sample: the gas meter is well clear of the spot.", + "measured_ft": 6.2, + "plus_minus_ft": 0.3, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": "objects[0] gas_meter", + "rule": {"key": "clearances.gas_ft", "source": "Sample value, not a rule citation", "placeholder": true} + }, + { + "id": "window_clearance", + "label": "Distance from the window", + "outcome": "unsure", + "unsure_cause": "margin", + "reason": "Sample: the gap is within the measurement error of the limit, so a person must judge it.", + "measured_ft": 3.1, + "plus_minus_ft": 0.3, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": "objects[1] window", + "rule": {"key": "clearances.opening_ft", "source": "Sample value, not a rule citation", "placeholder": true} + }, + { + "id": "front_clearance", + "label": "Clear space in front", + "outcome": "unsure", + "unsure_cause": "unobserved", + "reason": "Sample: the ground in front of the spot was not seen.", + "measured_ft": null, + "plus_minus_ft": null, + "threshold_ft": 3.0, + "comparison": "at_least", + "subject": null, + "rule": {"key": "facing.min_ft", "source": "Sample value, not a rule citation", "placeholder": true} + }, + { + "id": "route_length", + "label": "Cable run from the meter", + "outcome": "pass", + "reason": "Sample: the cable run is short.", + "measured_ft": 3.0, + "plus_minus_ft": 0.3, + "threshold_ft": 20.0, + "comparison": "at_most", + "subject": null, + "rule": {"key": "route.max_ft", "source": "Sample value, not a rule citation", "placeholder": true} + } + ], + "nearest_considered": null, + "missing_evidence": [ + { + "kind": "band", + "band": "ground", + "span_ft": [3.0, 5.5], + "checks": ["front_clearance"], + "message": "Sample: film the ground in front of the spot." + }, + { + "kind": "past_end", + "side": "left", + "message": "Sample: walk past the left end of the wall to see whether it continues." + } + ], + "ends": { + "left": {"kind": "unexplored", "s_ft": -8.0, "point": [-8.0, 0.0]}, + "right": {"kind": "limit", "s_ft": 12.0, "point": [12.0, 0.0]} + }, + "sweep": [ + {"wall_id": "wall", "start_ft": [-8.0, -1.5], "outcome": "fail", "failing": ["gas_clearance"], "unsure": []}, + {"wall_id": "wall", "start_ft": [-1.0, 2.5], "outcome": "fail", "failing": ["meter_clearance"], "unsure": []}, + {"wall_id": "wall", "start_ft": [3.0, 6.0], "outcome": "unsure", "failing": [], "unsure": ["window_clearance", "front_clearance"]}, + {"wall_id": "wall", "start_ft": [6.5, 9.5], "outcome": "unsure", "failing": [], "unsure": ["front_clearance"]} + ], + "stats": { + "candidates": 36, + "pass": 0, + "unsure": 14, + "fail": 22, + "elapsed_ms": 0, + "input_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + } +} diff --git a/ios/HouseScan/Runtime/ScanEngine+Actions.swift b/ios/HouseScan/Runtime/ScanEngine+Actions.swift new file mode 100644 index 000000000..32cc593b1 --- /dev/null +++ b/ios/HouseScan/Runtime/ScanEngine+Actions.swift @@ -0,0 +1,470 @@ +import CoreGraphics +import Foundation +import HouseScanKit +import OSLog +import simd +import SwiftUI + +extension ScanEngine: ScanActions { + func finishOnboarding() { + guard state.phase == .onboarding else { return } + go(.findMeter) + } + + func markMeter(at point: CGPoint?, viewSize: CGSize) { + guard state.phase == .findMeter else { return } + if let replay { + // A replay has no live surfaces to raycast; its wall comes from the recording (or is + // assumed from the trajectory, see ReplayPlayer.wallDescription). + let wall = replay.wall + guard setWall(meter: wall.meter, outward: wall.outward, groundY: wall.groundY, groundMeasured: replay.groundMeasured) else { return } + markTimes[MarkKey.meter] = captureClock + go(.meterCloseUp) + return + } + guard let live = liveCapture, let frame = currentFrame else { return } + guard frame.tracking == .normal else { + state.guidance = .aimAtWallForMeter + return + } + let viewPoint = point ?? CGPoint(x: viewSize.width / 2, y: viewSize.height / 2) + // A detected vertical plane's own geometry first; failing that, ARKit's estimated vertical + // plane, whose wider error the export records. Never an infinite plane, which extends a + // fence or another wall past its edges. + guard let hit = live.raycastVerticalPlane(from: viewPoint) else { + state.guidance = .aimAtWallForMeter + RuntimeLog.engine.info("meter tap refused: no vertical plane") + return + } + meterPlaneSource = hit.source + var outward = SIMD3(hit.normal.x, 0, hit.normal.z) + if simd_dot(outward, frame.camera.position - hit.position) < 0 { outward = -outward } + // Until a horizontal plane shows up below the wall, the ground is a guess: a phone held at + // chest height, 1.4 m above it. `refineGround` replaces the guess as planes arrive. + let measured = groundBelow(hit.position, along: simd_normalize(simd_cross(-outward, SIMD3(0, 1, 0)))) + guard setWall(meter: hit.position, outward: outward, groundY: measured ?? frame.camera.position.y - 1.4, groundMeasured: measured != nil) else { + state.guidance = .aimAtWallForMeter + return + } + // The map carries the line's source: the export writes it and the walked clearance + // takes the server's error for it. + updateCoverage { $0.setWallLineSource(meterLineSource) } + setMeterAnchor(live.addMeterAnchor(at: hit.transform), pose: hit.transform) + markTimes[MarkKey.meter] = captureClock + go(.meterCloseUp) + } + + /// The ground at the wall of the meter at `meter`, running along `along`: a detected plane + /// that reaches the wall's foot by the meter and isn't furniture, floor-classified first, + /// then the lowest (`GroundPlaneChoice`). Nil when none does; the ground stays a guess. + func groundBelow(_ meter: SIMD3, along: SIMD3) -> Float? { + GroundPlaneChoice.groundY(meter: meter, along: along, planes: detectedGroundPlanes) + } + + func skipCloseUp() { + guard state.phase == .meterCloseUp else { return } + state.closeUp = .skipped + state.meterNumber = .skipped + RuntimeLog.engine.info("close-up skipped after \(self.state.closeUpFailedAttempts) failed attempts") + observeCloseUpView() + go(.wallWalk) + } + + /// The homeowner's pick of the meter number. The number stays on the phone, in + /// `state.meterNumber`: scene.json has no field for it and its schema allows no extra + /// properties, so the server never receives it. + func chooseMeterNumber(_ candidate: MeterNumberCandidate?) { + guard state.phase == .meterCloseUp, case .choose(let candidates) = state.meterNumber else { return } + guard let candidate else { + // "None of these": small characters mean the phone was too far for a clear read. + retakeCloseUp(currentMeterReadout?.numberTooSmall == true ? .numberTooSmall : .noNumber) + return + } + guard let chosen = candidates.first(where: { $0.id == candidate.id }) else { return } + state.meterNumber = .confirmed(chosen.text) + RuntimeLog.engine.info("meter number confirmed (\(chosen.barcodeConfirmed ? "barcode-confirmed" : "text only", privacy: .public)), brand \(self.state.meterBrand == nil ? "none" : "kept", privacy: .public)") + observeCloseUpView() + finishCloseUp() + } + + func rejectMeterBrand() { + guard state.phase == .meterCloseUp, case .choose = state.meterNumber else { return } + state.meterBrand = nil + RuntimeLog.engine.info("meter brand rejected") + } + + /// Puts the close-up photo's view into coverage, under the same rules as a walk keyframe: a + /// cell counts once seen from two positions at least 0.25 m apart, and the view is replayed + /// with the kept keyframes whenever the wall moves (`CoverageMap.observedCameras`). + /// + /// Only when the close-up step ends (number confirmed, or skipped), because the photo on disk + /// is final then, and only the view `closeUpCredit` holds: the shot whose photo is on disk, + /// after the reader found it decodable and in focus. A skip after a photo the reader + /// rejected as blurry, or while a retake's photo is still saving or being read, adds nothing. + /// A photo in focus where only the number couldn't be read still counts. The map always + /// exists here: the close-up follows the meter mark, which sets the wall (`setWall`). No time + /// is passed, so the pose never joins the walked path: nothing is kept between the close-up + /// and the walk's first frame. + private func observeCloseUpView() { + guard let view = closeUpCredit.take() else { return } + // Corrected for anchor moves made while the photo was saved and read. + let camera = view.time.map { correctedPose(view.camera.cameraToWorld, capturedAt: $0) } + .map { CameraFrame(cameraToWorld: $0, intrinsics: view.camera.intrinsics, imageSize: view.camera.imageSize) } ?? view.camera + var delta: CoverageMap.Delta? + updateCoverage { delta = $0.observe(camera, trackingNormal: true, depth: view.depth) } + RuntimeLog.capture.info("close-up view in coverage: \(delta?.newlySeen ?? 0) cells newly seen, \(delta?.newlyCovered ?? 0) newly covered") + } + + /// Marks a wall end during the walk, or, during a server past_end request, marks that + /// request's end again at wherever the wall is now seen to stop. + func markWallEnd(at point: CGPoint?, viewSize: CGSize) { + guard state.phase == .wallWalk || (state.phase == .gapRequest && pastEndSide != nil), + let wall = coverage?.wall, let frame = currentFrame else { return } + guard let hit = wallHit(point, viewSize: viewSize, frame: frame, wall: wall) else { return } + let side: WallSide = hit.s < 0 ? .left : .right + if state.phase == .gapRequest, side != pastEndSide { return } + // Unexplored until the homeowner says something blocks the wall there: an unanswered + // question must not tell the server the usable wall stops at this point. + state.endQuestion = side + setEnd(side, at: hit.s, kind: .unexplored) + } + + func answerWallEnd(turnsCorner: Bool) { + guard let side = state.endQuestion else { return } + state.endQuestion = nil + if wallEndKinds[side] != nil { setEndKind(side, turnsCorner ? .unexplored : .limit) } + // During the walk a corner is followed: the next wall is marked, and the walk goes on + // along it. Until then the end stays marked, as an unexplored corner. + if turnsCorner, state.phase == .wallWalk, wallEndKinds[side] != nil { + nextWallSide = side + nextWallRefusal = nil + } + if state.phase == .gapRequest, side == pastEndSide { settlePastEnd() } + if state.phase == .wallWalk, let frame = currentFrame { + resetGuidanceAfterSkip(camera: frame.camera, time: frame.timestamp) + } + } + + /// The wall round the corner: a raycast on a vertical plane under `point`, the same one the + /// meter tap uses. Where its line meets the current wall's line on the ground is the corner; + /// the wall chain turns there, the end on that side opens again and the walk goes on. + func markNextWall(at point: CGPoint?, viewSize: CGSize) { + guard state.phase == .wallWalk, let side = nextWallSide, coverage != nil, let frame = currentFrame else { return } + func refuse(_ refusal: NextWallRefusal, _ reason: String) { + nextWallRefusal = refusal + state.guidance = .markNextWall(side: side, refusal: refusal) + RuntimeLog.engine.info("next wall refused: \(reason, privacy: .public)") + } + guard frame.tracking == .normal else { return refuse(.trackingNotReady, "tracking not normal") } + // A replay has no live surfaces to raycast, so it can't mark the next wall. + let viewPoint = point ?? CGPoint(x: viewSize.width / 2, y: viewSize.height / 2) + guard let hit = liveCapture?.raycastVerticalPlane(from: viewPoint) else { return refuse(.noSurface, "no vertical plane") } + var outward = SIMD3(hit.normal.x, 0, hit.normal.z) + if simd_dot(outward, frame.camera.position - hit.position) < 0 { outward = -outward } + var turned: Result = .failure(.notAWall) + // The new piece's line runs through the hit point facing the hit plane's normal, as the + // meter's does, so its source follows the same rule (`meterLineSource`). + let source = Self.lineSource(of: hit.source) + updateCoverage { map in + turned = Result { () throws(CornerRefusal) in + try map.turnCorner(side == .left ? .left : .right, meeting: hit.position, outward: outward, source: source) + } + } + let corner: WallCorner + switch turned { + case .success(let turn): corner = turn + case .failure(.nearlyParallel): return refuse(.sameWall, "nearly parallel to the current wall") + case .failure(.notAWall): return refuse(.noSurface, "not a wall") + case .failure(.implausible(let s)): return refuse(.notAtCorner, "the walls meet at s=\(s)") + } + spatialRevision += 1 + // The end moves on with the walk; `clearEnd` also publishes the chain for the overlays. + clearEnd(side) + nextWallSide = nil + nextWallRefusal = nil + RuntimeLog.engine.info("corner followed on the \(side.rawValue, privacy: .public) at s=\(corner.s) (\(hit.source == .detectedPlane ? "detected" : "estimated", privacy: .public) plane)") + // Features tapped past the corner were placed on the old wall's line. + reprojectFeatures() + resetGuidanceAfterSkip(camera: frame.camera, time: frame.timestamp) + } + + /// "Open sky or nothing overhead" records the tilt-up view for the export; "A roof edge, + /// porch or stairs" records nothing, so the server treats the stretch as unseen. During an + /// overhead gap request the answer settles the request either way (`settleOverheadGap`). + func answerOverhead(clear: Bool) { + guard state.overheadQuestion else { return } + switch state.phase { + case .wallWalk: + // Something overhead is an answer, not a view: the stretch goes to review unseen. + resolveGuidance(clear ? .met : .skipped) + settleTiltUp(clear: clear) + if let frame = currentFrame { + resetGuidanceAfterSkip(camera: frame.camera, time: frame.timestamp) + } + case .gapRequest: + settleOverheadGap(clear: clear) + default: + return + } + } + + func beginMarking(_ kind: FeatureKind) { + guard state.phase == .wallWalk || state.phase == .markFeatures else { return } + // A mark is a tap into the world frame. The review hides "Add something" while the phone + // has lost its place; this holds if a tap races the change. + guard state.phase != .markFeatures || !state.tracking.hasLostItsPlace else { return } + state.marking = MarkingState(kind: kind, step: 0, refusal: nil) + pendingTaps = [] + } + + func markFeaturePoint(at point: CGPoint?, viewSize: CGSize) { + guard var marking = state.marking, let wall = coverage?.wall, let frame = currentFrame else { return } + guard frame.tracking == .normal else { + marking.refusal = .trackingNotReady + state.marking = marking + return + } + // An AC unit is marked where its front meets the ground: a ray aimed at it meets the + // ground in front of the wall, and the wall's plane only below the floor. + let onGround = marking.kind == .driveway || marking.kind == .fence || marking.kind == .acUnit + let pixel = frame.projection.imagePixel(forViewPoint: point ?? CGPoint(x: viewSize.width / 2, y: viewSize.height / 2), in: viewSize) + let ray = frame.camera.ray(throughPixel: pixel) + if wall.wallPoint(frame.camera.position).out <= 0 { + marking.refusal = .wrongSide + state.marking = marking + return + } + let hit = onGround ? wall.intersectGround(ray) : wall.intersectWall(ray) + guard let hit else { + marking.refusal = .noSurface + state.marking = marking + return + } + // Past 8 m out a ground tap is not about this wall any more. + if onGround, hit.out > 8 || hit.out < 0 { + marking.refusal = .tooFarFromWall + state.marking = marking + return + } + // A fence's feet on two pieces of the wall would be sent as one depth that misses how + // close its line comes to the wall by the corner: refused, and asked for per side. + // So are an AC unit's front corners: no one box from the wall reaches both. + if marking.kind == .fence || marking.kind == .acUnit, let first = pendingTaps.first, !wall.onSamePiece(first, wall.world(hit)) { + marking.refusal = marking.kind == .fence ? .fenceAcrossCorner : .unitAcrossCorner + state.marking = marking + return + } + // The world point, not its wall coordinates: a ground refined before the next tap would + // otherwise move this one's height with it. + pendingTaps.append(wall.world(hit)) + marking.refusal = nil + marking.step += 1 + if marking.step < marking.kind.tapCount { + state.marking = marking + return + } + let marked = feature(marking.kind, taps: pendingTaps, wall: wall) + markTimes[MarkKey.feature(marked.id)] = captureClock + state.features.append(marked) + publishFeaturesPastEnds() + state.marking = nil + pendingTaps = [] + spotMarkPlaced() + } + + private func feature(_ kind: FeatureKind, taps: [SIMD3], wall: WallFrame) -> MarkedFeature { + var marked = MarkedFeature(id: UUID(), kind: kind, span: 0...0, bottom: nil, top: nil, out: nil, points: taps, opens: nil) + Self.project(&marked, onto: wall) + return marked + } + + /// `feature` with its wall coordinates worked out on `wall`. + static func projected(_ feature: MarkedFeature, onto wall: WallFrame) -> MarkedFeature { + var copy = feature + project(©, onto: wall) + return copy + } + + /// Sets a feature's wall coordinates from its tapped world points. Run again whenever the + /// wall frame moves (meter anchor refined, ground measured), since only the world points + /// are what was tapped. + static func project(_ feature: inout MarkedFeature, onto wall: WallFrame) { + let taps = feature.points.map { wall.wallPoint($0) } + let ss = taps.map(\.s) + let span = (ss.min() ?? 0)...(ss.max() ?? 0) + switch feature.kind { + case .door, .window, .gasMeter: + let heights = taps.map(\.height) + feature.span = span + feature.bottom = max(0, heights.min() ?? 0) + feature.top = heights.max() + case .acUnit: + // The front corners on the ground: the unit takes the wall out to the farther one. + feature.span = span + feature.out = taps.map(\.out).max() ?? 0 + case .driveway, .fence: + feature.span = span + // The nearer tap, as the export uses: the narrow end must not be overstated. + feature.out = taps.map(\.out).min() ?? 0 + } + } + + func cancelMarking() { + state.marking = nil + pendingTaps = [] + spotMarkCancelled() + } + + func deleteFeature(_ id: UUID) { + state.features.removeAll { $0.id == id } + } + + func setWindowOpens(_ id: UUID, opens: Bool) { + guard let index = state.features.firstIndex(where: { $0.id == id }) else { return } + state.features[index].opens = opens + } + + func finishWalk() { + guard state.phase == .wallWalk, bothEndsMarked else { return } + if let map = coverage, map.endsTooClose, let left = map.leftEnd, let right = map.rightEnd { + // Ends closer than a battery is wide (`WallFrame.minWallLength`): both sides ended + // without walking, as "Wall ends here" or "Can't get there" at the meter does. + // Nothing between them to scan, and an end can't be moved, so both go and the walk + // goes on; the card says why. + RuntimeLog.engine.info("finish refused: ends at s=\(left) and s=\(right) are closer than \(WallFrame.minWallLength) m") + clearEnd(.left) + clearEnd(.right) + state.wallTooShort = true + if let frame = currentFrame { resetGuidanceAfterSkip(camera: frame.camera, time: frame.timestamp) } + return + } + // Done while a request is still up passes it by. + resolveGuidance(.superseded) + state.marking = nil + nextWallSide = nil + nextWallRefusal = nil + // Leaving with the overhead question unanswered records nothing. + if !tiltUpSettled { settleTiltUp(clear: false) } + go(.markFeatures) + } + + func confirmFeatures() { + guard state.phase == .markFeatures else { return } + runGapCheck() + } + + func skipGap() { + guard state.phase == .gapRequest else { return } + skipCurrentGap() + } + + func cannotAccessArea() { + switch state.phase { + case .gapRequest: + skipCurrentGap() + case .wallWalk: + guard coverage != nil else { return } + let task = ScanEngine.name(state.guidance) + switch state.guidance { + case .aimAtGround, .aimAtWall, .seeBehind, .walk, .markEnd, .tiltUp, .markNextWall: + resolveGuidance(.cannotReach) + default: + return + } + switch state.guidance { + case .aimAtGround(let s): + updateCoverage { $0.markSkipped(.ground, (s - 0.5)...(s + 0.5)) } + case .aimAtWall(let s): + updateCoverage { $0.markSkipped(.wall, (s - 0.5)...(s + 0.5)) } + case .seeBehind(let s): + // Whatever is in the way can't be seen past: its hidden cells go to review. Only + // those: open cells beside them can still be seen and stay asked for. + updateCoverage { map in + for band in seeBehindBands { + for index in ScanEngine.hiddenCells(map, band: band, around: s) { map.markSkipped(band, map.cellRange(index)) } + } + } + case .walk(let side, _), .markEnd(let side): + // The walk can't continue this way: stop the wall where the phone is, as an + // unexplored end (`WalkedEnd`), where the strip's preview showed it. + endWalkCannotGoOn(side) + case .tiltUp: + settleTiltUp(clear: false) + case .markNextWall: + // Not following the corner: the end stays marked, as an unexplored corner. + nextWallSide = nil + nextWallRefusal = nil + default: + return + } + RuntimeLog.engine.info("cannot access area during \(task, privacy: .public)") + if let frame = currentFrame { + resetGuidanceAfterSkip(camera: frame.camera, time: frame.timestamp) + } + default: + return + } + } + + func retryUpload() { + guard state.phase == .uploading else { return } + if case .failed = state.upload { startUpload() } + } + + /// Back to the feature review after a rejected upload. The scan (wall, coverage, keyframes, + /// features) stays; confirming the review runs the gap check and the upload again. + func backToReview() { + guard state.phase == .uploading, case .rejected = state.upload else { return } + state.upload = .idle + go(.markFeatures) + } + + func captureMissing(_ id: String) { + // A stopped camera can't take the view: the answer stays, the request goes to review. + guard mayCapture, state.phase == .result || state.phase == .gapRequest || state.phase == .uploading, + let missing = placement?.missingEvidence, + let index = Int(id.replacingOccurrences(of: "missing-", with: "")), + missing.indices.contains(index) else { return } + let item = missing[index] + guard let map = coverage, let plan = gapPlanner.plan(for: item, leftEnd: map.leftEnd, rightEnd: map.rightEnd, limitEnds: map.limitEnds) else { return } + beginServerGap(item, plan: plan) + } + + func showAR() { + guard state.phase == .result, state.spatialResultAvailable, acceptedAnswerIsCurrent(at: "the AR view") else { return } + go(.resultAR) + } + + func closeAR() { + guard state.phase == .resultAR, acceptedAnswerIsCurrent(at: "the result") else { return } + go(.result) + } + + func startOver() { + releaseFailedSource() + resetAll() + } + + func liveCameraView() -> AnyView { + guard let live = liveCapture else { return AnyView(Color.black) } + return AnyView(LiveCameraView(arView: live.arView)) + } + + // MARK: Helpers + + /// Where a tap lands on the wall plane, in wall coordinates. + func wallHit(_ point: CGPoint?, viewSize: CGSize, frame: SourceFrame, wall: WallFrame) -> WallPoint? { + let pixel = frame.projection.imagePixel(forViewPoint: point ?? CGPoint(x: viewSize.width / 2, y: viewSize.height / 2), in: viewSize) + return nearbyWallHit(frame.camera.ray(throughPixel: pixel), camera: frame.camera, wall: wall) + } + + /// Where `ray` meets the wall, or nil when that is farther along the wall from the camera than + /// the coverage map lets a camera see (`maxDistance`). A ray nearly parallel to the wall meets + /// its line hundreds of meters away; an end placed there, or drawn on the strip, would be + /// nothing the homeowner pointed at. + func nearbyWallHit(_ ray: Ray, camera: CameraFrame, wall: WallFrame) -> WallPoint? { + guard let hit = wall.intersectWall(ray) else { return nil } + let reach = coverage?.config.maxDistance ?? CoverageConfig().maxDistance + return abs(hit.s - wall.wallPoint(camera.position).s) <= reach ? hit : nil + } +} diff --git a/ios/HouseScan/Runtime/ScanEngine+Confirm.swift b/ios/HouseScan/Runtime/ScanEngine+Confirm.swift new file mode 100644 index 000000000..e39be123b --- /dev/null +++ b/ios/HouseScan/Runtime/ScanEngine+Confirm.swift @@ -0,0 +1,463 @@ +import CoreGraphics +import Foundation +import HouseScanKit +import ImageIO +import OSLog + +/// The spot check's state on the engine (`ScanEngine.spotConfirm`). +struct SpotConfirmState { + /// Every answer of this scan, with what each was about. + var confirmations = SpotConfirmations() + /// The last upload's scene.json and the answer to it: what a check asked now is bound to, and + /// the scene the bundle is written again with after an answer that doesn't upload. + var lastScene: Data? + var lastAnswer: Data? + var sceneSHA256: String? + var answerSHA256: String? + /// The check on screen: its area, the photo shown and the exchange it came from. Its answer + /// is filled in when the homeowner gives one. + var pending: SpotConfirmation? + /// The photo on screen, raw as stored, with the image decoded from it: judged again, and its + /// outline's camera corrected again, whenever the wall moves while the question is up + /// (`refreshSpotPhoto`). + var shown: (candidate: SpotPhotoCandidate, image: CGImage)? + /// The unmarked thing the homeowner is marking on the camera, with the check it came from. + var marking: (check: SpotConfirmation, kind: FeatureKind)? + /// The entry in the guidance log for the question on screen. + var request: GuidanceLog.Request? + /// Checks asked in this scan, for their ids. + var asked = 0 + /// Times an accepted answer went stale before it was shown and the scan was sent again + /// (`acceptedAnswerIsCurrent`); back to 0 once an answer is shown as the result. + var staleResends = 0 +} + +/// The spot check: before an answer's spot is shown as the result, the homeowner is shown the +/// kept photo that best sees the footprint and its front clearance (`SpotArea(result:wall:)`, +/// `SpotPhoto.best`) and asked, on that one photo, at most two questions. +/// +/// 1. Is anything in the marked area? "It's clear" backs the camera-only and walked-path claims +/// there (HouseScanKit `CoverageMap`, "Bounded exceptions"). "Something's in the way" withdraws +/// them (`CoverageMap.withdrawClaims`), so the scene reports the area unseen, and uploads again. +/// "A gas meter, AC, window or door" asks which, then opens the camera to mark it with the +/// review's mark flow and uploads again once it is marked; the server checks clearances only +/// from marked objects. When the phone can't mark now (it lost its place, or the camera +/// failed), the area is withdrawn as for "Something's in the way". +/// 2. After "It's clear": what is the ground where the battery would stand? A type goes out as a +/// patch over the footprint and its error margin (`SpotGround`) with an upload; "Not sure" +/// shows the result, and the server reports the surface as not recorded. +/// +/// The obstruction question comes first because either of its other answers changes the scan +/// and can move the spot, which would waste a ground answer about this one. Both are about the +/// same outlined place, so they share the photo and the screen. +/// +/// An answer counts only about what the homeowner was shown. When no kept photo shows the whole +/// area (`SpotPhotoChoice.showsWholeArea`), or the photo can't be decoded, "It's clear" is not +/// offered: the screen says so and the area is withdrawn (`unconfirmed`), and the scan goes to +/// the server again without those claims. Asking for another view of the area would keep more +/// claims, but it needs a capture step of its own; withdrawing needs none and claims nothing +/// unseen. +/// +/// "It's clear" is reused only for the exchange it was given about: the same scene, answer and +/// rules (`SpotConfirmations.settling`). So the upload that sends a ground answer asks the area +/// question once more, about the answer that comes back; the ground answer itself stands for the +/// footprint under the same rules and isn't asked again, and with nothing new to send the result +/// follows. A spot that moved asks both. Answers that left the area out stay. Nothing +/// new goes into scene.json but the ground patch: the answers show as the stretch the scene no +/// longer claims, the marks added, and in the packet's guidance log. +/// +/// The bundled sample is checked like a server's answer, and the screen labels it a sample: the +/// UI tests and the offline demo only ever see the sample, so skipping it would leave the step +/// untested end to end. +extension ScanEngine { + /// How long an answer stays on screen before the result or the upload: 1.2 s, as a closed + /// gap's check mark (`autoAdvanceDelay`), and the autopilot's hold when longer. Not measured. + private var spotAnsweredHold: Double { options.autopilot ? max(1.2, options.autopilotHold) : 1.2 } + + /// Records the upload's scene and the answer to it, which a check asked about this answer + /// is bound to. Call it once the answer has decoded. + func noteExchange(scene: Data, answer: Data) { + spotConfirm.lastScene = scene + spotConfirm.lastAnswer = answer + spotConfirm.sceneSHA256 = PacketFiles.sha256(scene) + spotConfirm.answerSHA256 = PacketFiles.sha256(answer) + } + + /// The ground patches the export sends: the latest ground answer about each footprint. + var spotGroundPatches: [SceneGroundPatch] { + guard let wall = coverage?.wall else { return [] } + return spotConfirm.confirmations.groundPatches(wall: wall, groundGuessError: groundMeasured ? 0 : Self.estimatedGroundError) + } + + /// Shows the answer once the gap loop is done with it: the spot check first when the answer + /// names a spot no earlier check settles, otherwise the result. + func presentAnswer() { + guard acceptedAnswerIsCurrent(at: "the spot check") else { return } + guard let result = state.result, let placement, let wall = coverage?.wall, let area = SpotArea(result: placement, wall: wall), + let sceneSHA256 = spotConfirm.sceneSHA256, let answerSHA256 = spotConfirm.answerSHA256 else { + state.spotCheck = nil + showResult() + return + } + let exchange = SpotExchange(sceneSHA256: sceneSHA256, answerSHA256: answerSHA256, rulesSHA256: placement.policy.rulesSHA256) + spotConfirm.asked += 1 + let id = spotConfirm.asked + if let settled = spotConfirm.confirmations.settling(area, in: exchange) { + // Kept for the result, which says when the area was left out. + var check = Self.spotCheck(id: id, area: area, photo: nil, confirmable: false, isSample: result.isSample) + check.answer = Self.answer(settled.answer) + state.spotCheck = check + RuntimeLog.engine.info("spot check: s=\(area.spot.lowerBound)...\(area.spot.upperBound) was answered before (\(settled.answer.name, privacy: .public)); showing the result") + showResult() + return + } + // Raw cameras with their capture times: each is judged, and drawn, corrected into the frame + // the wall is in now (`SpotPhotoCandidate.camera(correctedBy:)`). + let candidates = store.keyframes.map { + SpotPhotoCandidate(id: $0.id, camera: $0.camera, trackingNormal: $0.tracking == .normal, capturedAt: $0.t) + } + let choice = SpotPhoto.best(candidates, area: area, wall: wall, corrections: poseCorrections) + let candidate = choice.flatMap { choice in candidates.first { $0.id == choice.id } } + let file = choice.flatMap { choice in store.keyframes.first { $0.id == choice.id } }.map { store.directory.appending(path: $0.fileName) } + RuntimeLog.engine.info("spot check \(id): area s=\(area.span.lowerBound)...\(area.span.upperBound) out to \(area.depth) m; photo \(choice?.id ?? "none", privacy: .public) shows \(Int((choice?.footprintInView ?? 0) * 100))% of the spot and \(Int((choice?.areaInView ?? 0) * 100))% of the area") + Task { + let image = await Self.loadPhoto(file) + guard state.phase == .uploading, spotConfirm.asked == id, state.result == result else { return } + // The photo and its outline's camera are fixed together here, after the decode: an + // anchor correction during it moved the wall, so the photo is judged again against + // the wall as it is now. + var photo: SpotCheck.Photo? + var shown: SpotPhotoChoice? + spotConfirm.shown = nil + if let image, let candidate, let wall = coverage?.wall { + shown = SpotPhoto.best([candidate], area: area, wall: wall, corrections: poseCorrections) + let camera = shown?.camera ?? candidate.camera(correctedBy: poseCorrections) + photo = SpotCheck.Photo(image: image, projection: Self.projection(camera)) + spotConfirm.shown = (candidate, image) + } else if choice != nil { + RuntimeLog.engine.error("spot check \(id): photo \(choice?.id ?? "", privacy: .public) could not be decoded; nothing can be confirmed") + } + // Only a photo on screen that shows the whole area can back "It's clear". + let confirmable = shown?.showsWholeArea == true + // The answer is filled in when the homeowner gives one. + spotConfirm.pending = SpotConfirmation(area: area, exchange: exchange, photo: shown, answer: .unconfirmed) + spotConfirm.request = spotCheckRequest(.spotCheck(id: id), question: confirmable ? ScanCopy.spotQuestion : ScanCopy.spotUnconfirmable, + span: area.span, binding: binding(id, area: area, photo: shown)) + state.spotCheck = Self.spotCheck(id: id, area: area, photo: photo, confirmable: confirmable, isSample: result.isSample) + go(.spotConfirm) + } + } + + /// Whether the accepted answer may still be presented as current at `step` (the spot check, + /// the result, the AR view). The geometry can move after the answer was accepted: a ground + /// plane revoked or refined, an anchor correction, while the homeowner answers or looks. When + /// the revision has moved on since the answer's snapshot, the answer is dropped as at the POST + /// (`AnswerFreshness.step`): the scan is sent again from a fresh snapshot, or, when that + /// already happened, the stale-answer failure shows. False when it was dropped. + func acceptedAnswerIsCurrent(at step: String) -> Bool { + guard let sent = lastUploadSnapshot?.revision else { return true } + switch AnswerFreshness.step(changesScan: false, sent: sent, now: spatialRevision, resends: spotConfirm.staleResends) { + case .show, .upload: + return true + case .sendAgain: + spotConfirm.staleResends += 1 + RuntimeLog.engine.info("answer went stale before \(step, privacy: .public) (sent at revision \(sent), now \(self.spatialRevision)): sending again") + dropStaleAnswer() + startUpload() + case .stillChanging: + RuntimeLog.engine.error("answer went stale again before \(step, privacy: .public) (sent at revision \(sent), now \(self.spatialRevision)): not shown") + dropStaleAnswer() + state.upload = UploadFailure.stillChanging + go(.uploading) + } + return false + } + + /// The result once the answer is known to be current. + private func showResult() { + spotConfirm.staleResends = 0 + go(.result) + } + + /// A stale answer and the question about it leave the screen; what the homeowner answered + /// stays recorded, bound to the exchange it was about. + private func dropStaleAnswer() { + state.result = nil + state.spotCheck = nil + spotConfirm.pending = nil + spotConfirm.shown = nil + spotConfirm.request = nil + } + + /// The wall moved while the question is up (an anchor correction, a new ground): the photo's + /// outline is drawn through its camera corrected again, and the photo is judged again. One that + /// no longer shows the whole area can't back "It's clear": the question gives way to leaving + /// the area out. `publishWall` calls it. + func refreshSpotPhoto() { + guard state.phase == .spotConfirm, var check = state.spotCheck, check.answer == nil, let shown = spotConfirm.shown, + var pending = spotConfirm.pending, let wall = coverage?.wall else { return } + let view = SpotPhoto.best([shown.candidate], area: pending.area, wall: wall, corrections: poseCorrections) + let camera = view?.camera ?? shown.candidate.camera(correctedBy: poseCorrections) + check.photo = SpotCheck.Photo(image: shown.image, projection: Self.projection(camera)) + pending.photo = view + spotConfirm.pending = pending + let confirmable = view?.showsWholeArea == true + if confirmable != check.confirmable { + RuntimeLog.engine.info("spot check \(check.id): the wall moved and photo \(shown.candidate.id, privacy: .public) \(confirmable ? "now shows" : "no longer shows", privacy: .public) the whole area") + check.confirmable = confirmable + check.step = .area + } + state.spotCheck = check + } + + private static func projection(_ camera: CameraFrame) -> CameraProjection { + CameraProjection(cameraToWorld: camera.cameraToWorld, intrinsics: camera.intrinsics, imageSize: camera.imageSize) + } + + // MARK: Answers + + /// The first question: is anything in the marked area? + func answerSpotArea(_ answer: SpotAreaAnswer) { + guard state.phase == .spotConfirm, var check = state.spotCheck, check.confirmable, check.step == .area, check.answer == nil else { return } + switch answer { + case .clear: + guard let pending = spotConfirm.pending, let wall = coverage?.wall else { return } + if let known = spotConfirm.confirmations.groundAnswer(for: pending.area, rulesSHA256: pending.exchange.rulesSHA256) { + // Asked again after the upload that sent the ground answer: it still stands. + finish(.clear(ground: known), shown: .clear(Self.groundAnswer(known)), withdraw: false) + return + } + // The ground question, about the footprint, is logged as its own request. + resolveGuidance(.met) + let spot = pending.area.spot + let margin = SpotGround.margin(spot: spot, wall: wall, groundGuessError: groundMeasured ? 0 : Self.estimatedGroundError) + spotConfirm.request = spotCheckRequest(.spotGround(id: check.id), question: ScanCopy.spotGroundQuestion, + span: (spot.lowerBound - margin)...(spot.upperBound + margin), + binding: binding(check.id, area: pending.area, photo: pending.photo)) + check.step = .ground + state.spotCheck = check + noteGuidance() + case .somethingThere: + finish(.somethingThere, shown: .somethingThere, withdraw: true) + case .unmarked: + check.step = .which + state.spotCheck = check + } + } + + /// Which unmarked thing is in the area: mark it on the camera, or leave the area out when the + /// phone can't mark now. Nil goes back to the first question. + func chooseUnmarked(_ kind: FeatureKind?) { + guard state.phase == .spotConfirm, var check = state.spotCheck, check.step == .which, check.answer == nil else { return } + guard let kind, let equipment = Self.equipment(kind) else { + check.step = .area + state.spotCheck = check + return + } + guard !state.tracking.hasLostItsPlace, currentFrame != nil, let pending = spotConfirm.pending else { + RuntimeLog.engine.info("spot check \(check.id): an unmarked \(kind.rawValue, privacy: .public) can't be marked now (tracking \(Self.name(self.state.tracking), privacy: .public)); leaving the area out") + finish(.unmarked(equipment, markedNow: false), shown: .unmarkedCantMark(kind), withdraw: true) + return + } + // The review's mark flow, on the camera. Placing the mark uploads (`spotMarkPlaced`); + // cancelling comes back here (`spotMarkCancelled`). + spotConfirm.marking = (pending, kind) + RuntimeLog.engine.info("spot check \(check.id): marking an unmarked \(kind.rawValue, privacy: .public)") + go(.markFeatures) + beginMarking(kind) + } + + /// The ground question. + func answerSpotGround(_ answer: GroundAnswer) { + guard state.phase == .spotConfirm, let check = state.spotCheck, check.step == .ground, check.answer == nil else { return } + let ground: SpotGroundAnswer = switch answer { + case .type(let type): .type(Self.sceneGroundType(type)) + case .notSure: .notSure + } + finish(.clear(ground: ground), shown: .clear(answer), withdraw: false) + } + + /// No photo shows the whole area: leave it out and check the wall again. + func continueUnconfirmed() { + guard state.phase == .spotConfirm, let check = state.spotCheck, !check.confirmable, check.answer == nil else { return } + finish(.unconfirmed, shown: .unconfirmed, withdraw: true) + } + + /// Records the answer, shows it for a moment, then uploads again, or shows the result when + /// nothing changed ("It's clear", ground not sure). + private func finish(_ answer: SpotConfirmationAnswer, shown: SpotCheckAnswer, withdraw: Bool) { + guard var check = state.spotCheck, var confirmation = spotConfirm.pending else { return } + confirmation.answer = answer + let patchesBefore = spotGroundPatches + do { + try spotConfirm.confirmations.record(confirmation) + } catch { + RuntimeLog.engine.error("spot check \(check.id): \(answer.name, privacy: .public) not recorded without a photo of the whole area") + return + } + spotConfirm.pending = nil + // The scan changes when the answer withdraws the area or sends a ground patch not yet sent. + let uploads = withdraw || spotGroundPatches != patchesBefore + check.answer = shown + check.checksAgain = uploads + state.spotCheck = check + // An answer that withdraws the area sends it to review without the claims. + resolveGuidance(withdraw ? .skipped : .met) + let area = confirmation.area + if withdraw { updateCoverage { $0.withdrawClaims(over: area.span) } } + RuntimeLog.engine.info("spot check \(check.id): \(answer.name, privacy: .public)\(withdraw ? ", claims withdrawn" : "", privacy: .public) over s=\(area.span.lowerBound)...\(area.span.upperBound), photo \(confirmation.photoID ?? "none", privacy: .public), scene \(confirmation.exchange.sceneSHA256.prefix(12), privacy: .public), answer \(confirmation.exchange.answerSHA256.prefix(12), privacy: .public), rules \(confirmation.exchange.rulesSHA256.prefix(12), privacy: .public)") + let id = check.id + Task { + try? await Task.sleep(for: .seconds(spotAnsweredHold)) + guard state.phase == .spotConfirm, state.spotCheck?.id == id else { return } + if uploads { + // The scan with the change (claims withdrawn, or the ground patch) goes again. + startUpload() + return + } + // Nothing to send: the answer shows only if the geometry is still what it answered. + guard acceptedAnswerIsCurrent(at: "the result") else { return } + // The bundle was written at the upload, before this answer: write it again so its + // guidance log holds the answer. + if let scene = spotConfirm.lastScene, let snapshot = lastUploadSnapshot { saveBundle(scene: scene, snapshot: snapshot) } + showResult() + } + } + + // MARK: Marking unmarked equipment + + /// The mark for the spot check's unmarked equipment is placed: record it and upload. + func spotMarkPlaced() { + guard let marking = spotConfirm.marking else { return } + spotConfirm.marking = nil + guard let equipment = Self.equipment(marking.kind) else { return } + var confirmation = marking.check + confirmation.answer = .unmarked(equipment, markedNow: true) + try? spotConfirm.confirmations.record(confirmation) + spotConfirm.pending = nil + RuntimeLog.engine.info("spot check: the unmarked \(marking.kind.rawValue, privacy: .public) is marked; checking the wall again") + startUpload() + } + + /// The spot check's unmarked equipment is being marked on the camera. + var spotConfirmIsMarking: Bool { spotConfirm.marking != nil } + + /// The mark was cancelled: back to the spot check's first question. + func spotMarkCancelled() { + guard let marking = spotConfirm.marking else { return } + spotConfirm.marking = nil + spotConfirm.pending = marking.check + state.spotCheck?.step = .area + go(.spotConfirm) + } + + // MARK: Support + + /// Whether a capture can settle a gap request: not over a stretch whose claims the homeowner + /// withdrew, where a new view would claim the same thing past the same obstruction. Overhead + /// requests are not about that stretch's wall or ground. + func captureCanSettle(_ plan: GapPlan) -> Bool { + if case .overhead = plan.need { return true } + return !(coverage?.hasWithdrawnClaims(overlapping: plan.span) ?? false) + } + + /// Forgets the scan's checks: they name spans along a wall that no longer exists. + func resetSpotChecks() { + spotConfirm = SpotConfirmState() + state.spotCheck = nil + } + + /// The question's guidance-log entry while it is on screen (`guidanceRequest`). + var spotCheckGuidance: GuidanceLog.Request? { spotConfirm.request } + + /// What binds a check, for the guidance log: scene.json and the packet have no field for it. + private func binding(_ id: Int, area: SpotArea, photo: SpotPhotoChoice?) -> String { + let format = { (value: Float) in String(format: "%.2f", value) } + guard let pending = spotConfirm.pending else { return "Spot check \(id)." } + let shown = photo.map { "keyframe \($0.id) shown, \(Int($0.areaInView * 100))% of the area in view" } ?? "no keyframe shown" + return "Spot check \(id): \(shown); spot s \(format(area.spot.lowerBound)) to \(format(area.spot.upperBound)) m, " + + "area s \(format(area.span.lowerBound)) to \(format(area.span.upperBound)) m out to \(format(area.depth)) m; " + + "scene sha256 \(pending.exchange.sceneSHA256), answer sha256 \(pending.exchange.answerSHA256), rules sha256 \(pending.exchange.rulesSHA256)." + } + + /// A gap_band request on the ground over `span`, its message the question and the binding. + private func spotCheckRequest(_ topic: GuidanceLog.Topic, question: Instruction, span: ClosedRange, binding: String) -> GuidanceLog.Request { + GuidanceLog.Request(topic: topic, kind: .gapBand, origin: .phone, message: Self.text(question) + " " + binding, band: .ground, span: span) + } + + private static func spotCheck(id: Int, area: SpotArea, photo: SpotCheck.Photo?, confirmable: Bool, isSample: Bool) -> SpotCheck { + SpotCheck( + id: id, spot: area.spot, spotOut: area.spotOut, spotHeight: area.height, area: area.span, areaDepth: area.depth, + photo: photo, confirmable: confirmable, answer: nil, isSample: isSample) + } + + /// The photo at `url` as a landscape image no wider than 2048 px, decoded off the main actor. + private static func loadPhoto(_ url: URL?) async -> CGImage? { + guard let url else { return nil } + return await Task.detached(priority: .userInitiated) { () -> CGImage? in + guard let source = CGImageSourceCreateWithURL(url as CFURL, nil) else { return nil } + let options: [CFString: Any] = [ + kCGImageSourceCreateThumbnailFromImageAlways: true, + kCGImageSourceThumbnailMaxPixelSize: 2048, + ] + return CGImageSourceCreateThumbnailAtIndex(source, 0, options as CFDictionary) + }.value + } + + private static func groundAnswer(_ answer: SpotGroundAnswer) -> GroundAnswer { + switch answer { + case .type(let type): .type(groundType(type)) + case .notSure: .notSure + } + } + + private static func answer(_ answer: SpotConfirmationAnswer) -> SpotCheckAnswer { + switch answer { + case .clear(let ground): .clear(groundAnswer(ground)) + case .somethingThere: .somethingThere + case .unmarked(let equipment, _): .unmarkedCantMark(featureKind(equipment)) + case .unconfirmed: .unconfirmed + } + } + + static func equipment(_ kind: FeatureKind) -> SpotEquipment? { + switch kind { + case .gasMeter: .gasMeter + case .acUnit: .ac + case .window: .window + case .door: .door + case .driveway, .fence: nil + } + } + + private static func featureKind(_ equipment: SpotEquipment) -> FeatureKind { + switch equipment { + case .gasMeter: .gasMeter + case .ac: .acUnit + case .window: .window + case .door: .door + } + } + + static func sceneGroundType(_ type: GroundType) -> SceneGroundType { + switch type { + case .lawn: .lawn + case .mulch: .mulch + case .gravel: .gravel + case .concrete: .concrete + case .drive: .drive + case .deck: .deck + } + } + + private static func groundType(_ type: SceneGroundType) -> GroundType { + switch type { + case .lawn: .lawn + case .mulch: .mulch + case .gravel: .gravel + case .concrete: .concrete + case .drive: .drive + case .deck: .deck + } + } +} diff --git a/ios/HouseScan/Runtime/ScanEngine+Ends.swift b/ios/HouseScan/Runtime/ScanEngine+Ends.swift new file mode 100644 index 000000000..daf76cf12 --- /dev/null +++ b/ios/HouseScan/Runtime/ScanEngine+Ends.swift @@ -0,0 +1,110 @@ +import Foundation +import HouseScanKit +import OSLog +import simd + +/// Ending the wall during the walk without pointing at the end ("Can't get there", "Wall ends +/// here"), and the preview of where an end would land. +extension ScanEngine { + /// The side ending the wall applies to now, and whether the end lands at the reticle. The + /// walk's own side while it asks to walk or to mark the end; during a request about the wall + /// in front of the homeowner (tilt down, tilt up, step back), the side of the meter the phone + /// is on (`WalkedEnd.side`). Nil when both ends are marked or the walk is doing something else. + private var endOnOffer: (side: WallSide, atReticle: Bool)? { + guard state.phase == .wallWalk, state.marking == nil, state.endQuestion == nil, !state.overheadQuestion, + nextWallSide == nil, let map = coverage else { return nil } + switch state.guidance { + case .walk(let side, _): + return (side, false) + case .markEnd(let side): + return (side, true) + case .aimAtGround, .aimAtWall, .stepBack: + guard let side = WalkedEnd.side(phone: phonePosition, wall: map.wall, leftEnd: map.leftEnd, rightEnd: map.rightEnd) else { return nil } + return (side == .left ? .left : .right, false) + case .findMeter, .aimAtWallForMeter, .holdOnMeter, .walkComplete, .tiltUp, .markNextWall, .gap, .seeBehind: + return nil + } + } + + /// Where ending the walk on `side` puts the end now (`WalkedEnd`): the phone's place along the + /// wall, kept to what was walked on that side. The phone's place doesn't count while it has + /// lost it. + func walkedEnd(_ side: WallSide) -> Float? { + guard let map = coverage else { return nil } + return WalkedEnd.end(side.walk, phone: phonePosition, walked: map.walkedPositions, wall: map.wall) + } + + /// Where the phone is; nil while it has lost its place. + private var phonePosition: SIMD3? { + currentFrame.flatMap { $0.tracking.hasLostItsPlace ? nil : $0.camera.position } + } + + /// The end preview for this moment (`ScanViewState.endPreview`). + func currentEndPreview() -> EndPreview? { + guard let offer = endOnOffer, let map = coverage, let frame = currentFrame else { return nil } + let side: WallSide + let s: Float + if offer.atReticle { + // `markWallEnd` with no point: the middle of the view, which is the sensor image's + // middle whatever the view's size. + guard frame.tracking == .normal, + let hit = nearbyWallHit(frame.camera.ray(throughPixel: frame.camera.imageSize / 2), camera: frame.camera, wall: map.wall) else { return nil } + side = hit.s < 0 ? .left : .right + s = hit.s + } else { + guard let end = walkedEnd(offer.side) else { return nil } + side = offer.side + s = end + } + // Less than one keyframe's spacing short of the farthest view is still the front of the walk. + let walkedPast = map.walkedFarthest(side.walk) - side.walk.sign * s + let leavesOut = walkedPast >= AutoCaptureConfig().spacingMeters ? walkedPast : nil + return EndPreview(side: side, s: s, atReticle: offer.atReticle, leavesOutWalked: leavesOut) + } + + /// Republishes the end preview; called with every guidance update, since the phone moves. + func publishEndPreview() { + let preview = currentEndPreview() + if preview != state.endPreview { state.endPreview = preview } + } + + func endWallHere() { + guard let preview = currentEndPreview(), !preview.atReticle else { return } + // The walk toward that side is met: the homeowner got to its end. + if case .walk(let side, _) = state.guidance, side == preview.side { resolveGuidance(.met) } + logEnd("wall ends here", side: preview.side, at: preview.s) + // Unexplored until the homeowner says something blocks the wall there, as for a marked end. + state.endQuestion = preview.side + setEnd(preview.side, at: preview.s, kind: .unexplored) + } + + /// "Can't get there" while the walk asks to walk `side` or to mark its end: the end goes where + /// the preview showed, as an unexplored end. + func endWalkCannotGoOn(_ side: WallSide) { + guard let s = walkedEnd(side) else { return } + logEnd("can't get there", side: side, at: s) + setEnd(side, at: s, kind: .unexplored) + } + + /// Where the end went and what the old rule, the unbroken covered reach, would have said. + private func logEnd(_ action: String, side: WallSide, at s: Float) { + guard let map = coverage else { return } + let phone = currentFrame.map { map.wall.wallPoint($0.camera.position).s } ?? .nan + let walked = map.walkedFarthest(side.walk) + let covered = GuidancePlanner().reach(side.walk, coverage: map) + RuntimeLog.engine.info("\(action, privacy: .public) on the \(side.rawValue, privacy: .public): end at s=\(s) (phone at s=\(phone), walked \(walked) m, covered reach \(covered) m)") + } + + /// `ScanViewState.featuresPastEnds`, refreshed when the ends move (`publishWall`), the spans + /// follow the wall (`reprojectFeatures`) and a mark is added. + func publishFeaturesPastEnds() { + let left = coverage?.leftEnd + let right = coverage?.rightEnd + let past = Set(state.features.filter { WalkedEnd.liesPastAnEnd($0.span, leftEnd: left, rightEnd: right) }.map(\.id)) + if past != state.featuresPastEnds { state.featuresPastEnds = past } + } +} + +extension WallSide { + var walk: WalkSide { self == .left ? .left : .right } +} diff --git a/ios/HouseScan/Runtime/ScanEngine+Export.swift b/ios/HouseScan/Runtime/ScanEngine+Export.swift new file mode 100644 index 000000000..122fed76e --- /dev/null +++ b/ios/HouseScan/Runtime/ScanEngine+Export.swift @@ -0,0 +1,376 @@ +import Foundation +import HouseScanKit +import OSLog +import simd + +extension ScanEngine { + /// scene.json for the current scan (contract C1), with the LiDAR mesh's measurements on + /// phones that have one. + /// + /// The scene frame is the AR world frame moved down so the ground at the wall is y = 0: the + /// server reads heights (meter.pos y, pose translations) as height above that ground, and + /// scene.json has no field for the ground's height otherwise. + /// + /// Everything comes from `snapshot` (`UploadSnapshot`): nothing live is read, so it + /// describes the one frame the snapshot was taken on, as the mesh measurement and the packet + /// built from the same snapshot do. + func sceneJSON(_ snapshot: UploadSnapshot, mesh: MeshMeasurements = MeshMeasurements()) throws -> Data { + let map = snapshot.map + let wall = map.wall + let drop = SIMD3(0, wall.groundY, 0) + // The corners carry their pieces' sources (`markNextWall`); the meter's piece, its own + // (`meterLineSource`, set on the map in `markMeter`). + let sceneWall = SceneWall( + meter: wall.meter - drop, outward: wall.outward, groundY: 0, leftCorners: wall.leftCorners, rightCorners: wall.rightCorners, + source: wall.source) + + // SceneExport rejects negative heights, a top below a bottom, ground points behind the + // wall and a zero-length driveway edge, and taps can produce each of them once the ground + // or the meter anchor moves after the tap (a window tapped wholly below a guessed ground, + // a fence foot that ends up behind the refined wall line). Such marks are clamped to the + // nearest valid shape here and logged. A driveway or fence with nothing valid left fails + // the export instead (`ExportError.markCollapsed`): dropping it would send the ground + // near it as seen and clear, which can pass its clearance check with the hazard unsent. + // Each mark's wall coordinates, heights among them, from its tapped world points against + // the snapshot's wall and ground, whatever the ground was when it was tapped. + let features: [SceneFeature] = try snapshot.features.map { Self.projected($0, onto: wall) }.map { feature in + let points = feature.points.map { $0 - drop } + switch feature.kind { + case .door, .window, .gasMeter: + let heights = [feature.bottom ?? 0, feature.top ?? 0] + let bottom = max(0, heights.min() ?? 0) + let top = max(bottom, heights.max() ?? 0) + if bottom != feature.bottom || top != feature.top { + RuntimeLog.engine.info("export: \(feature.kind.rawValue, privacy: .public) heights \(feature.bottom ?? .nan)...\(feature.top ?? .nan) clamped to \(bottom)...\(top)") + } + if feature.kind == .gasMeter { + // Two taps on one spot give no size; the homeowner marks it again. + guard feature.span.upperBound - feature.span.lowerBound >= SceneExport.minObjectSize, + top - bottom >= SceneExport.minObjectSize else { throw ExportError.markCollapsed(.gasMeter) } + return .wallObject(kind: .gasMeter, span: feature.span, bottom: bottom, top: top) + } + return .opening(kind: feature.kind == .door ? .door : .window, span: feature.span, bottom: bottom, top: top, + operable: feature.kind == .window ? feature.opens : nil) + case .acUnit: + let front = try Self.groundLine(feature, points, wall: sceneWall) + guard abs(sceneWall.wallCoordinates(of: front[0]).s - sceneWall.wallCoordinates(of: front[1]).s) >= SceneExport.minObjectSize else { + throw ExportError.markCollapsed(.acUnit) + } + return .groundObject(kind: .ac, front: front) + case .fence: + return .fence(foot: try Self.groundLine(feature, points, wall: sceneWall)) + case .driveway: + return .driveway(edge: try Self.groundLine(feature, points, wall: sceneWall)) + } + } + + let keyframes = snapshot.keyframes.map { stored -> SceneKeyframe in + // The photo's raw pose moved by the anchor corrections made after it was taken. + var pose = snapshot.corrections.pose(stored.rawPose, capturedAt: stored.t) + pose.columns.3.y -= wall.groundY + return SceneKeyframe( + id: stored.id, cameraToWorld: pose, intrinsics: stored.camera.intrinsics, + w: Int(stored.camera.imageSize.x), h: Int(stored.camera.imageSize.y), img: stored.fileName + ) + } + + // A guessed ground puts the same error into every height in the scene. scene.json has no + // field for "the ground was estimated", so the error bars say it instead. + let groundError: Float? = snapshot.groundMeasured ? nil : Self.estimatedGroundError + let input = SceneInput( + wall: sceneWall, + baselineS: Self.exportSpan(map), + meterExtraError: groundError, + meterPlane: snapshot.meterPlaneSource, + objectExtraError: groundError, + features: features, + coverage: SceneCoverage( + map, leftEndMarked: snapshot.leftEndIsLimit, rightEndMarked: snapshot.rightEndIsLimit + ), + keyframes: keyframes, + stills: snapshot.stills, + // Written without plus_minus_ft: the server takes its mesh error for both. + meshFacing: mesh.facing, + meshOverheads: mesh.overheads, + // Only what the homeowner said about a checked spot's footprint; before any spot, + // none, and the server reports the surface unknown. + groundPatches: snapshot.groundPatches + ) + return try SceneExport.jsonData(input) + } + + /// How the line of the meter's piece of wall was found, for scene.json's `walls[].source`. + /// + /// Live, `markMeter` puts the wall through the raycast's hit point, facing the normal of the + /// plane it hit. On a detected plane (an ARPlaneAnchor, `.existingPlaneGeometry`) the normal + /// is the anchor's and the hit point lies on the anchor's geometry, so both the line's + /// direction and its distance from the camera are the plane's: `plane`. The tap only picks + /// where along that line the meter is. On an estimated plane there is no anchor: ARKit fits a + /// plane to the feature points around the tapped pixel for that one raycast, which is what an + /// AR tap is, and the schema's `plane` means detected planes, so it is `tap`. Nothing in the + /// capture takes a wall line from the LiDAR mesh (`MeshProbe` only measures in front of and + /// above a line already set), so no piece is `mesh`. + /// + /// A replay's wall is `tap`: a recorded measure-lab wall runs through two tapped ground + /// contacts (its `contacts`), and a wall assumed from the trajectory was never measured at + /// all, which no source describes; replays run only in tests and demos. + var meterLineSource: WallLineSource { + replay == nil ? Self.lineSource(of: meterPlaneSource) : .tap + } + + /// The source of a wall line put through a live vertical-plane raycast's hit, facing the hit + /// plane's normal (`meterLineSource` has the reasons). + static func lineSource(of plane: MeterPlaneSource) -> WallLineSource { + switch plane { + case .detectedPlane: .plane + case .estimatedPlane: .tap + } + } + + /// The stretch of wall the scene describes, meters of s: between the marked ends, or out to + /// what was seen (at least 1 m) on a side without one, always containing the meter. + nonisolated static func exportSpan(_ map: CoverageMap) -> ClosedRange { + let seen = map.seenExtent + let low = min(map.leftEnd ?? min(seen?.lowerBound ?? -1, -1), -0.1) + let high = max(map.rightEnd ?? max(seen?.upperBound ?? 1, 1), 0.1) + return low...high + } + + /// What the LiDAR mesh measured over the exported stretch: the gap from the wall out to + /// whatever faces it, and the clear height under anything overhead, each per stretch of s in + /// meters. Empty without a mesh. + struct MeshMeasurements: Sendable { + var facing: [ObservedSpan] = [] + var overheads: [ObservedSpan] = [] + } + + /// Measures a world-space mesh (meters) against the wall (world meters) over `span`, the + /// exported stretch of s. + nonisolated static func measure(_ mesh: TriangleMesh, wall: WallFrame, over span: ClosedRange) -> MeshMeasurements { + MeshMeasurements(facing: mesh.facingSpans(wall: wall, over: span), overheads: mesh.overheadSpans(wall: wall, over: span)) + } + + /// Error of the chest-height ground guess (camera height minus 1.4 m), meters. Phones held + /// for scanning sit roughly 1.1 to 1.7 m up, so ±0.3 m. A hypothesis; no measured spread exists. + static let estimatedGroundError: Float = 0.3 + + /// A ground point on or behind the wall line moved to 1 mm in front of it, keeping its s. The + /// millimetre keeps float round-off from turning an on-the-line point into a negative depth. + private static func inFront(of wall: SceneWall, _ point: SIMD3) -> SIMD3 { + let c = wall.wallCoordinates(of: point) + return c.out >= 0.001 ? point : wall.world(s: c.s, height: c.height, out: 0.001) + } + + /// The two ground points of a driveway edge, a fence foot or an AC unit's front, each moved in + /// front of the wall. + /// Throws `markCollapsed` when they are not two points 1 cm apart in plan: a wall line refined + /// past both taps puts them on one point. SceneExport's own degenerate-edge tolerance is + /// 1e-3 ft; 1 cm keeps well clear of it. + private static func groundLine(_ feature: MarkedFeature, _ points: [SIMD3], wall: SceneWall) throws(ExportError) -> [SIMD3] { + let line = points.map { inFront(of: wall, $0) } + guard line.count == 2, simd_distance(SIMD2(line[0].x, line[0].z), SIMD2(line[1].x, line[1].z)) > 0.01 else { + RuntimeLog.engine.error("export: \(feature.kind.rawValue, privacy: .public) \(feature.id.uuidString, privacy: .public) has \(points.count) taps that don't make a line; asking for it to be marked again") + throw .markCollapsed(feature.kind) + } + return line + } + + enum ExportError: Error, CustomStringConvertible { + case noWall + /// A driveway or fence whose taps no longer make a line. The homeowner has to mark it + /// again; `UploadFailure.packaging(_:)` says so. + case markCollapsed(FeatureKind) + + var description: String { + switch self { + case .noWall: "There is no wall to export yet." + case .markCollapsed(let kind): "The \(kind.rawValue) mark's taps don't make a line." + } + } + } + + /// The server's result in the terms the screens use (meters, wall coordinates). + func presentation(of result: PlacementResult, isSample: Bool) -> ResultPresentation { + let sceneWall = coverage.map { + SceneWall(meter: $0.wall.meter, outward: $0.wall.outward, groundY: $0.wall.groundY, leftCorners: $0.wall.leftCorners, rightCorners: $0.wall.rightCorners) + } + // A view is capturable only when a gap request can be built from it: a band item needs + // its span (and a facing item its out_ft, which a walk can reach), a past_end item its + // side. Otherwise the button would do nothing. A request the homeowner already skipped + // or answered with something overhead stays with the installer, and so does one over a + // stretch the homeowner said something stands on (`captureCanSettle`). + let plans = result.missingEvidence.map { item in + gapPlanner.plan(for: item, leftEnd: coverage?.leftEnd, rightEnd: coverage?.rightEnd, limitEnds: coverage?.limitEnds ?? []) + } + let settles = plans.map { plan in plan.map { !skippedGaps.contains($0) && captureCanSettle($0) } ?? false } + return Self.presentation(of: result, isSample: isSample, wall: sceneWall) { item in + result.missingEvidence.firstIndex(of: item).map { settles[$0] } ?? false + } + } + + /// `presentation(of:isSample:)` without an engine, for the UI demo: `wall` is the scan's wall + /// in world meters (nil draws no route off a sample's frame), `capturable` says whether a + /// requested view can be taken now. + static func presentation( + of result: PlacementResult, isSample: Bool, wall sceneWall: SceneWall?, + capturable: (PlacementMissingEvidence) -> Bool + ) -> ResultPresentation { + let meters: (Double) -> Float = { Float($0 * 0.3048) } + + let spot = result.spot.map(Self.batterySpot) + let nearestSpot = result.nearestConsidered.map(Self.batterySpot) + + var route: [SIMD2] = [] + if let cable = result.route { + let height = meters(cable.heightFt) + // Each point as an offset from the meter: the result's meter is the spot's centre + // minus its offset from the meter. The answer to this scan is in the scene's plan + // frame (the AR world's x and z in feet, scene.json's frame; result.schema.json's + // meter_offset_ft is "in the scene frame's axes"), so the offsets are put on the + // current wall chain by `chainS`, which follows the route round corners and adds a + // vertex at each so the drawn line bends there. Offsets rather than the points + // themselves keep the route on the meter's anchor as it moves, like the spot's s. + // The bundled sample is in a frame of its own, unrelated to this wall, so its route + // is measured along the sample spot's own direction, as before. + if isSample { + if let placed = result.spot { + let meterPlan = placed.center - placed.meterOffsetFt + route = cable.polyline.map { point in + let d = point - meterPlan + return SIMD2(meters(d.x * placed.along.x + d.y * placed.along.y), height) + } + } + } else if let sceneWall { + // A route comes with a spot (result.schema.json); without one, the meter as it is now. + let meterNow = SIMD2(Double(sceneWall.meter.x), Double(sceneWall.meter.z)) * SceneUnits.feetPerMeter + let meterPlan = result.spot.map { $0.center - $0.meterOffsetFt } ?? meterNow + route = sceneWall.chainS(ofPlanOffsetsFeet: cable.polyline.map { $0 - meterPlan }).map { SIMD2($0, height) } + } + } + + let checks = result.checks.map { check in + CheckRow( + id: check.id, title: check.label, outcome: Self.outcome(check.outcome), reason: check.reason, + needsPerson: check.needsPerson, + measured: check.measuredFt.map(meters), threshold: check.thresholdFt.map(meters), plusMinus: check.plusMinusFt.map(meters), + comparison: check.comparison.map(Self.comparison), + settledBy: result.evidenceIndex(settling: check.id).map(Self.missingID) + ) + } + + // A run's start_ft is a range of battery LEFT edges, so the wall it describes reaches one + // battery width past its last start. The spot carries the size, or after a reject the + // nearest spot, which the server sized the same way. With neither, the zone is drawn over + // the starts alone at 0.3 m deep, which understates it. + let sized = spot ?? nearestSpot + let depth = sized?.depth ?? 0.3 + let width = sized.map { $0.span.upperBound - $0.span.lowerBound } ?? 0 + let clearances = result.sweep.enumerated().map { index, run in + let first = meters(min(run.startFt.x, run.startFt.y)) + let last = meters(max(run.startFt.x, run.startFt.y)) + return ClearanceZone( + id: "sweep-\(index)", + label: (run.failing + run.unsure).joined(separator: ", "), + outcome: Self.outcome(run.outcome), + span: first...(last + width), + depth: depth + ) + } + + let missing = result.missingEvidence.enumerated().map { index, item in + MissingEvidence( + id: Self.missingID(index), text: item.message, + capturable: capturable(item), + checkIDs: item.checks ?? [] + ) + } + + var unseen: WallSide? + if let pastEnd = result.missingEvidence.first(where: { $0.kind == .pastEnd })?.side { + unseen = pastEnd == .left ? .left : .right + } else if result.ends.left.kind == .unexplored, result.ends.left.beyondReach != true { + // An end beyond cable reach can't hold the battery whatever lies past it. + unseen = .left + } else if result.ends.right.kind == .unexplored, result.ends.right.beyondReach != true { + unseen = .right + } + + return ResultPresentation( + decision: Self.decision(result.decision), + // The solver ends the summary with the policy notice; the screen shows it apart. + summary: result.summaryWithoutNotice, + policyApproved: result.policy.autoApprove, + rulesNotice: result.policy.notice.flatMap { $0.isEmpty ? nil : $0 }, + rulesHash: result.policy.rulesShortHash, + spot: spot, + nearestSpot: nearestSpot, + nearestFailingCheck: result.nearestFailure?.id, + cableRoute: route, + cableLength: result.route.map { meters($0.lengthFt) }, + checks: checks, + clearances: clearances, + missing: missing, + unseenSide: unseen, + isSample: isSample + ) + } + + static func missingID(_ index: Int) -> String { + "missing-\(index)" + } + + /// A server spot in wall terms: its stretch of s, size and gap to the wall, meters. + /// + /// Placed by its `span_ft`, its stretch in s along the wall chain, which the screens turn into + /// world points piece by piece (`WallGeometry.world(s:)`), so a spot round a corner lands on + /// the right piece and still moves with the meter's anchor. The meter offset split along the + /// spot's own `along` gave s only on the meter's piece: past a corner it measured along the + /// other piece's direction and drew the spot on the meter's wall. Both s and the depth below + /// are frame-free, so the bundled sample (in its own frame) still means the same on any wall. + /// + /// The size is the server's (`span_ft`, `depth_ft`, `height_ft`), never a battery size of the + /// app's own, for `nearest_considered` as for `spot`. There is no fallback for a missing one: + /// decoding requires every dimension (`PlacementSpot.init(from:)`), so an answer without one + /// fails with its path, such as `nearest_considered.depth_ft`, and the upload screen says the + /// answer couldn't be read (`ResultReadingTests.aNearestSpotWithoutADimensionIsRefused`). + static func batterySpot(_ placed: PlacementSpot) -> BatterySpot { + let meters: (Double) -> Float = { Float($0 * 0.3048) } + let low = meters(min(placed.spanFt.x, placed.spanFt.y)) + let high = meters(max(placed.spanFt.x, placed.spanFt.y)) + let depth = meters(placed.depthFt) + // How far the footprint's centre stands out from its back edge (back-left and back-right + // corners come first), along the spot's own outward: the gap to the wall is what exceeds + // half the depth. The server puts the back on the wall line. Decoding refuses a footprint + // without exactly four corners. + let d = placed.center - (placed.footprint[0] + placed.footprint[1]) / 2 + let centerOut = meters(d.x * placed.outward.x + d.y * placed.outward.y) + return BatterySpot( + span: low...high, + depth: depth, height: meters(placed.heightFt), + offsetFromWall: max(0, centerOut - depth / 2) + ) + } + + static func outcome(_ outcome: PlacementOutcome) -> CheckOutcome { + switch outcome { + case .pass: .pass + case .fail: .fail + case .unsure: .unsure + } + } + + static func comparison(_ comparison: PlacementComparison) -> RuleComparison { + switch comparison { + case .atLeast: .atLeast + case .atMost: .atMost + } + } + + static func decision(_ decision: PlacementDecision) -> ResultPresentation.Decision { + switch decision { + case .pass: .pass + case .manualReview: .manualReview + case .reject: .reject + } + } +} diff --git a/ios/HouseScan/Runtime/ScanEngine+GuidanceLog.swift b/ios/HouseScan/Runtime/ScanEngine+GuidanceLog.swift new file mode 100644 index 000000000..0f689f475 --- /dev/null +++ b/ios/HouseScan/Runtime/ScanEngine+GuidanceLog.swift @@ -0,0 +1,131 @@ +import Foundation +import HouseScanKit + +/// How the app's steps map onto the packet's guidance kinds (`GuidanceLog`), and how a request +/// that leaves the screen without an action settling it is judged. +/// +/// | App step | Packet kind | band, span | +/// | --- | --- | --- | +/// | hold on the meter (close-up) | closeup | | +/// | walk to a side; walk round a corner and mark the next wall | walk | | +/// | mark the end on a side | mark_end | | +/// | tilt down to the ground at s | tilt_to_ground | ground, s ± 0.3 m | +/// | tilt up to show more wall at s | gap_band | wall, s ± 0.3 m | +/// | step back | step_back | | +/// | tilt up above the meter's stretch | gap_band | overhead, its stretch | +/// | something in front of the wall at s (LiDAR) | gap_band | its hidden band, s ± 1 m | +/// | a gap request, phone's or server's | gap_band | wall, ground, overhead, or facing for a walk-out | +/// | a server past_end request | gap_past_end | its span | +/// | the spot check before the result | gap_band | ground, the footprint and its front clearance | +/// | its ground question | gap_band | ground, the footprint and its margin | +/// +/// Finding and marking the meter, "that's the whole wall" and the questions (what is at an end, +/// what is overhead) are not requests of any packet kind and are not logged. Coaching (slow +/// down, hold steady) is not logged either: it pauses a request rather than replacing it. +extension ScanEngine { + /// The request on screen now, or nil. + func guidanceRequest() -> GuidanceLog.Request? { + switch state.phase { + case .meterCloseUp: + guard state.guidance == .holdOnMeter else { return nil } + return request(.closeUp, .closeup, ScanCopy.guidance(.holdOnMeter)) + case .wallWalk: + return walkRequest(state.guidance) + case .gapRequest: + guard let gap = state.gap, let plan = gapPlan else { return nil } + let band: GuidanceLog.Band = switch plan.need { + case .overhead: .overhead + case .walkOut: .facing + case .groundOut: .ground + case .cells, .wallUp: plan.band == .ground ? .ground : .wall + } + return GuidanceLog.Request( + topic: .gap(id: gap.id), kind: pastEndSide == nil ? .gapBand : .gapPastEnd, origin: gap.origin == .server ? .server : .phone, + message: Self.text(ScanCopy.gap(gap)), band: pastEndSide == nil ? band : nil, span: plan.span + ) + case .spotConfirm: + return spotCheckGuidance + case .onboarding, .findMeter, .markFeatures, .uploading, .result, .resultAR, .unsupported: + return nil + } + } + + private func walkRequest(_ step: GuidanceStep) -> GuidanceLog.Request? { + let copy = ScanCopy.guidance(step) + let cell = { (s: Float) in self.coverage?.cellIndex(forS: s) ?? 0 } + switch step { + case .walk(let side, _): + return request(.walk(side), .walk, copy) + case .markNextWall(let side, _): + return request(.nextWall(side), .walk, copy) + case .markEnd(let side): + return request(.markEnd(side), .markEnd, copy) + case .aimAtGround(let s): + // The planner's own window for an aim task (`GuidancePlanner.isSatisfied`). + return request(.aimAtGround(cell: cell(s)), .tiltToGround, copy, band: .ground, span: (s - 0.3)...(s + 0.3)) + case .aimAtWall(let s): + return request(.aimAtWall(cell: cell(s)), .gapBand, copy, band: .wall, span: (s - 0.3)...(s + 0.3)) + case .stepBack: + return request(.stepBack, .stepBack, copy) + case .tiltUp(let span): + return request(.tiltUp, .gapBand, copy, band: .overhead, span: span) + case .seeBehind(let s): + let band: GuidanceLog.Band = seeBehindBands.contains(.wall) ? .wall : .ground + return request(.seeBehind(cell: cell(s)), .gapBand, copy, band: band, span: (s - Self.seeBehindReach)...(s + Self.seeBehindReach)) + case .findMeter, .aimAtWallForMeter, .holdOnMeter, .walkComplete, .gap: + return nil + } + } + + private func request( + _ topic: GuidanceLog.Topic, _ kind: GuidanceLog.Kind, _ copy: Instruction, band: GuidanceLog.Band? = nil, span: ClosedRange? = nil + ) -> GuidanceLog.Request { + GuidanceLog.Request(topic: topic, kind: kind, origin: .phone, message: Self.text(copy), band: band, span: span) + } + + /// The words on the card: the title, then the detail. + static func text(_ copy: Instruction) -> String { + guard let detail = copy.detail else { return copy.title } + let ends = copy.title.last.map { ".?!".contains($0) } ?? false + return copy.title + (ends ? " " : ". ") + detail + } + + /// The outcome of a request that left the screen without an action closing it (`resolve` + /// covers "I can't get there", answered questions and closed gaps). Met when what it asked for + /// is now there, superseded when another request took its place first. + func closingOutcome(_ old: GuidanceLog.Request, next: GuidanceLog.Request?) -> GuidanceLog.Outcome { + let s = old.span.map { ($0.lowerBound + $0.upperBound) / 2 } + switch old.topic { + case .closeUp: + switch state.closeUp { + case .captured: return .met + case .skipped: return .skipped + case .aiming: return .superseded + } + case .walk(let side): + // Walked as far as that side goes: the planner asks for its end, or the walk is done. + return next?.topic == .markEnd(side) || state.guidance == .walkComplete ? .met : .superseded + case .nextWall: + // Marking the next wall clears the question; "I can't get there" and Done resolve it + // before it leaves. + return nextWallSide == nil ? .met : .superseded + case .markEnd(let side): + return wallEndKinds[side] != nil ? .met : .superseded + case .stepBack: + guard let camera = currentFrame?.camera, let wall = coverage?.wall else { return .superseded } + return wall.wallPoint(camera.position).out > GuidancePlanner().config.tooClose ? .met : .superseded + case .aimAtGround, .aimAtWall: + guard let s, let map = coverage else { return .superseded } + let band: SurfaceBand = old.band == .ground ? .ground : .wall + return map.level(band, map.cellIndex(forS: s)) == .covered ? .met : .superseded + case .seeBehind: + guard let s, let map = coverage else { return .superseded } + let band: SurfaceBand = old.band == .wall ? .wall : .ground + return Self.hiddenCells(map, band: band, around: s).isEmpty ? .met : .superseded + case .tiltUp, .gap, .spotCheck, .spotGround: + // These close through their answers (`resolveGuidance`); leaving any other way means + // something else took over. + return .superseded + } + } +} diff --git a/ios/HouseScan/Runtime/ScanEngine+Packet.swift b/ios/HouseScan/Runtime/ScanEngine+Packet.swift new file mode 100644 index 000000000..9a8a490e8 --- /dev/null +++ b/ios/HouseScan/Runtime/ScanEngine+Packet.swift @@ -0,0 +1,488 @@ +import Foundation +import HouseScanKit +import OSLog +import simd +import UIKit + +/// The capture packet (packet/README.md on t3/packet, version 1.1): what Share scan hands over. +/// Everything in it is in meters, seconds of device uptime and the meter frame, except +/// scene.json, which travels inside unchanged. Nothing uploads it: photos leave the phone only +/// when the homeowner shares the scan. +extension ScanEngine { + /// Everything the packet needs from the main actor, read in one turn so it describes one + /// moment of the scan. The files are written from it off the main actor (`writePacket`). + struct PacketInputs: Sendable { + var folder: URL + var zip: URL + var storeDirectory: URL + var recorder: CaptureRecorder + /// A replay's own frames, which stand in for the live trajectory: nil live. + var replayTrajectory: [ReplayPose]? + /// A replay's frames that recorded depth, the source of its depth frames: nil live, where + /// the recorder holds them. + var replayDepth: ReplayDepth? + var producer: PacketManifest.Producer + var device: PacketManifest.Device + var meterFrame: MeterFrame + /// The meter anchor's corrections: each photo, trajectory row and depth frame is moved by + /// those made after its time before it goes into the meter frame. The raw poses stay as + /// they were recorded. + var corrections: PoseCorrections + /// The latest frame time the engine had seen when these inputs were taken. Live stream + /// rows after it are left out: the recorder goes on taking rows while the packet is + /// queued, and a row after a correction that isn't in `corrections` would be written in + /// the wrong frame. The capture window ends at the last row kept, so depth frames and + /// motion rows stop there too. Nil keeps every row (a replay has no corrections). + var streamCutoff: Double? + var groundWorldY: Float + /// Keyframes and stills; `writePacket` orders them by time. + var photos: [StoredKeyframe] + var mesh: LiveCapture.MeshSnapshot? + var planes: [PacketPlane] + /// `photoIDs` hold store ids ("meter_close"); `writePacket` swaps in packet ids. + var marks: [PacketMark] + var guidance: [PacketGuidanceEntry] + var scene: Data + /// The camera's frame rate live; nil on a replay, whose rate comes from its frames. + var trajectoryRate: Double? + var motionStreams: Set + } + + struct ReplayPose: Sendable { + var t: Double + var normal: Bool + var cameraToWorld: simd_float4x4 + } + + struct ReplayDepth: Sendable { + var folder: URL + var frames: [ReplayFrame] + } + + /// Everything spatial comes from `snapshot` (the wall, the marks, the photos, the mesh, the + /// planes, the stream cutoff); only the logs (guidance, mark times) are read here. Nil when + /// the wall has no outward to build the meter frame from. + func packetInputs(scene: Data, snapshot: UploadSnapshot) -> PacketInputs? { + let map = snapshot.map + let wall = map.wall + // The wall in world meters, as scene.json's wall type describes it, for the marks. + let sceneWall = SceneWall(meter: wall.meter, outward: wall.outward, groundY: wall.groundY, leftCorners: wall.leftCorners, rightCorners: wall.rightCorners) + guard let frame = MeterFrame(meter: wall.meter, outward: wall.outward) else { return nil } + let settings = liveCapture?.settings + let info = Bundle.main.infoDictionary ?? [:] + let version = [info["CFBundleShortVersionString"], info["CFBundleVersion"]].compactMap { $0 as? String } + return PacketInputs( + folder: store.packetFolder, + zip: store.bundleURL, + storeDirectory: store.directory, + recorder: recorder, + replayTrajectory: replay.map { player in + player.frames.map { ReplayPose(t: $0.timestamp, normal: $0.trackingNormal, cameraToWorld: $0.cameraToWorld) } + }, + replayDepth: replay.map { ReplayDepth(folder: $0.folder, frames: $0.frames.filter { $0.depth != nil }) }, + producer: PacketManifest.Producer( + kind: .app, name: info["CFBundleName"] as? String ?? "HouseScan", + version: version.count == 2 ? "\(version[0]) (\(version[1]))" : version.first ?? "unknown", + // The "Stamp the git commit" build phase writes it (ios/project.yml). + commit: (info["HouseScanGitCommit"] as? String).flatMap { $0 == "unknown" ? nil : $0 } + ), + device: PacketManifest.Device( + model: Self.hardwareModel(), iosVersion: UIDevice.current.systemVersion, lidar: LiveCapture.supportsDepth, + // A replay runs no ARKit session, so none of them ran. + sceneDepthEnabled: settings?.sceneDepth ?? false, meshEnabled: settings?.mesh ?? false, + meshClassificationEnabled: settings?.meshClassification ?? false + ), + meterFrame: frame, + corrections: snapshot.corrections, + streamCutoff: replay == nil ? snapshot.frameTime : nil, + groundWorldY: wall.groundY, + photos: snapshot.keyframes + snapshot.stillFrames.keys.sorted().compactMap { snapshot.stillFrames[$0] }, + mesh: snapshot.mesh, + planes: snapshot.planes.map { plane in + PacketPlane( + id: plane.id, alignment: plane.vertical ? .vertical : .horizontal, classification: plane.classification, + anchorToMeter: frame.pose(plane.anchorToWorld), center: plane.center, rotationOnYAxis: plane.rotationOnYAxis, + extent: plane.extent, boundaryVertices: plane.boundary + ) + }, + marks: packetMarks(snapshot, wall: sceneWall, frame: frame), + guidance: guidanceLog.entries.map { Self.packetEntry($0, wall: sceneWall, frame: frame) }, + scene: scene, + trajectoryRate: settings.map { Double($0.framesPerSecond) }, + motionStreams: motionRunsLive ? motionAvailable : [] + ) + } + + /// `utsname.machine`, such as "iPhone16,1": the hardware, never the phone's name. "arm64" in + /// the Simulator. + nonisolated static func hardwareModel() -> String { + var system = utsname() + uname(&system) + return withUnsafeBytes(of: system.machine) { raw in + String(decoding: raw.prefix { $0 != 0 }, as: UTF8.self) + } + } + + // MARK: Marks + + /// The meter, the marked wall ends and every marked feature, in the meter frame. + private func packetMarks(_ snapshot: UploadSnapshot, wall: SceneWall, frame: MeterFrame) -> [PacketMark] { + let map = snapshot.map + var marks = [PacketMark.meter( + id: "meter", t: markTimes[MarkKey.meter], photoIDs: snapshot.stillFrames["meter_close"] == nil ? nil : ["meter_close"] + )] + for (side, s) in [(WallSide.left, map.leftEnd), (.right, map.rightEnd)] { + guard let s else { continue } + marks.append(.wallEnd( + id: "wall_end_\(side.rawValue)", side: side == .left ? .left : .right, + endKind: (side == .left ? snapshot.leftEndIsLimit : snapshot.rightEndIsLimit) ? .limit : .unexplored, s: s, wall: wall, frame: frame, t: markTimes[MarkKey.end(side)] + )) + } + for feature in snapshot.features.map({ Self.projected($0, onto: map.wall) }) { + let id = feature.id.uuidString.lowercased() + let t = markTimes[MarkKey.feature(feature.id)] + let points = feature.points.map { frame.point($0) } + switch feature.kind { + case .door, .window: + let bottom = feature.bottom ?? 0 + marks.append(.opening( + feature.kind == .door ? .door : .window, id: id, span: feature.span, bottom: bottom, top: feature.top ?? bottom, + operable: feature.kind == .window ? feature.opens : nil, wall: wall, frame: frame, t: t + )) + case .gasMeter, .acUnit: + // The manifest gives these one point: the middle of the two corners. + guard points.count == 2 else { + RuntimeLog.engine.error("packet: \(feature.kind.rawValue, privacy: .public) mark has \(points.count) taps, not 2; left out") + continue + } + marks.append(.pointObject(feature.kind == .gasMeter ? .gasMeter : .ac, id: id, point: (points[0] + points[1]) / 2, t: t)) + case .driveway, .fence: + guard points.count == 2 else { + RuntimeLog.engine.error("packet: \(feature.kind.rawValue, privacy: .public) mark has \(points.count) taps, not 2; left out") + continue + } + marks.append(feature.kind == .fence + ? .fence(id: id, from: points[0], to: points[1], t: t) + : .driveEdge(id: id, from: points[0], to: points[1], t: t)) + } + } + return marks + } + + /// Keys of `markTimes`. + enum MarkKey { + static let meter = "meter" + static func end(_ side: WallSide) -> String { "end.\(side.rawValue)" } + static func feature(_ id: UUID) -> String { id.uuidString } + } + + // MARK: Guidance + + /// A logged request with its span moved from s to meter-frame x. + private static func packetEntry(_ entry: GuidanceLog.Entry, wall: SceneWall, frame: MeterFrame) -> PacketGuidanceEntry { + let span = entry.request.span.map { span -> ClosedRange in + let low = frame.point(on: wall, s: span.lowerBound, height: 0, out: 0).x + let high = frame.point(on: wall, s: span.upperBound, height: 0, out: 0).x + return min(low, high)...max(low, high) + } + return PacketGuidanceEntry( + id: entry.id, kind: entry.request.kind, origin: entry.request.origin, message: entry.request.message, + band: entry.request.band, span: span, tShown: entry.shown, tResolved: entry.resolved, outcome: entry.outcome + ) + } + + // MARK: Writing + + /// Writes the packet folder, zips it to `inputs.zip` and removes the folder. Returns the zip + /// and a one-line summary for the log. Off the main actor: it copies every photo and hashes + /// every file. + /// + /// A photo, a depth map, a trajectory row, a plane or a section the writer refuses is left + /// out and logged, so one bad input doesn't cost the homeowner the whole packet; the writer's + /// checks are the validator's, so what is written validates. + nonisolated static func writePacket(_ inputs: PacketInputs) throws -> (url: URL, summary: String) { + let frame = inputs.meterFrame + let trajectory = trajectoryRows(inputs) + guard let started = trajectory.rows.first?.t, let ended = trajectory.rows.last?.t, ended > started else { + throw PacketBuildError.noTrajectory + } + + try? FileManager.default.removeItem(at: inputs.folder) + var writer = try PacketWriter(folder: inputs.folder, session: PacketSessionInfo( + id: trajectory.sessionID, producer: inputs.producer, device: inputs.device, startedAt: trajectory.startedAt, + startedAtUptime: started, meterFrame: frame, groundWorldY: inputs.groundWorldY + )) + + // Photos in time order, one per source frame, inside the capture. + var packetID: [String: String] = [:] + var last: (t: Double, id: String)? + var added = 0 + // Times of photos written with depth: a depth frame at one of them would repeat it. + var photoDepthTimes: Set = [] + for stored in inputs.photos.sorted(by: { $0.t < $1.t }) { + if let last, last.t == stored.t { + // The same frame kept twice (a close-up that was also a walk frame, or a view kept + // again for the overhead answer): one photo. + packetID[stored.id] = last.id + continue + } + guard (started...ended).contains(stored.t), let sharpness = stored.sharpness else { + RuntimeLog.engine.error("packet: \(stored.id, privacy: .public) at t=\(stored.t) left out: outside the capture \(started)...\(ended) or no sharpness") + continue + } + let id = PacketPhoto.id(number: added + 1) + do { + var depth = stored.depth.flatMap { KeyframeStore.loadDepth($0, in: inputs.storeDirectory) }.flatMap(Self.measured) + do { + try writer.addPhoto(Self.photo(stored, id: id, sharpness: sharpness, depth: depth, inputs: inputs)) + } catch PacketError.invalidDepth(_, let reason) { + // The photo is worth more than its depth. + RuntimeLog.engine.error("packet: \(stored.id, privacy: .public) depth left out: \(reason, privacy: .public)") + depth = nil + try writer.addPhoto(Self.photo(stored, id: id, sharpness: sharpness, depth: nil, inputs: inputs)) + } + packetID[stored.id] = id + last = (stored.t, id) + added += 1 + if depth != nil { photoDepthTimes.insert(stored.t) } + } catch { + RuntimeLog.engine.error("packet: \(stored.id, privacy: .public) left out: \(String(describing: error), privacy: .public)") + } + } + + var skippedRows = 0 + try writer.setNominalRate(trajectory.rate, for: .trajectory) + for row in trajectory.rows { + do { try writer.appendTrajectory(t: row.t, tracking: row.tracking, pose: frame.pose(inputs.corrections.pose(row.cameraToWorld, capturedAt: row.t))) } catch { skippedRows += 1 } + } + skippedRows += writeMotion(inputs, window: started...ended, into: &writer) + if skippedRows > 0 { RuntimeLog.engine.error("packet: \(skippedRows) stream rows refused and left out") } + + let depthFrames = writeDepthFrames(inputs, window: started...ended, skipping: photoDepthTimes, into: &writer) + + if let mesh = inputs.mesh { + do { try writer.setMesh(frame.mesh(mesh.mesh), classification: mesh.classification) } catch { Self.logLeftOut("mesh", error) } + } + var planes = 0 + for plane in inputs.planes { + do { + try writer.addPlane(plane) + planes += 1 + } catch let error where plane.boundary != nil { + // A boundary outside its extent means the two disagree; the extent still holds. + Self.logLeftOut("plane \(plane.id) boundary", error) + var bare = plane + bare.boundary = nil + do { + try writer.addPlane(bare) + planes += 1 + } catch { + Self.logLeftOut("plane \(plane.id)", error) + } + } catch { + Self.logLeftOut("plane \(plane.id)", error) + } + } + let clamp = { (t: Double) in min(max(t, started), ended) } + do { + try writer.setMarks(inputs.marks.map { mark in + var mark = mark + let ids = (mark.photoIDs ?? []).compactMap { packetID[$0] } + mark.photoIDs = ids.isEmpty ? nil : ids + mark.t = mark.t.map(clamp) + return mark + }) + } catch { Self.logLeftOut("marks", error) } + do { + try writer.setGuidance(inputs.guidance.map { entry in + var entry = entry + entry.tShown = clamp(entry.tShown) + entry.tResolved = entry.tResolved.map { max(clamp($0), entry.tShown) } + return entry + }) + } catch { Self.logLeftOut("guidance", error) } + try writer.setScene(inputs.scene) + let folder = try writer.finish() + try KeyframeStore.zipPacket(folder, to: inputs.zip) + let summary = "\(added) photos (\(photoDepthTimes.count) with depth), \(depthFrames) depth frames, " + + "\(trajectory.rows.count) trajectory rows over \(String(format: "%.1f", ended - started)) s, " + + "\(inputs.motionStreams.count) motion streams, \(inputs.mesh == nil ? "no mesh" : "mesh"), \(planes) planes, " + + "\(inputs.marks.count) marks, \(inputs.guidance.count) guidance entries" + return (inputs.zip, summary) + } + + private struct Trajectory { + var sessionID: String + var startedAt: Date? + /// Nominal rate, Hz. + var rate: Double + var rows: [(t: Double, tracking: PacketTracking, cameraToWorld: simd_float4x4)] + } + + /// The camera path, world frame: every recorded ARFrame live, or one row per replay frame. + /// Its first and last times are the capture's window. + private nonisolated static func trajectoryRows(_ inputs: PacketInputs) -> Trajectory { + if let poses = inputs.replayTrajectory { + var rows: [(t: Double, tracking: PacketTracking, cameraToWorld: simd_float4x4)] = [] + for pose in poses.sorted(by: { $0.t < $1.t }) where pose.t > (rows.last?.t ?? -.infinity) { + rows.append((pose.t, pose.normal ? .normal : .limited(nil), pose.cameraToWorld)) + } + // The recording's own rate. It has no wall-clock start. + let span = (rows.last?.t ?? 0) - (rows.first?.t ?? 0) + let rate = rows.count > 1 && span > 0 ? Double(rows.count - 1) / span : 1 + return Trajectory(sessionID: UUID().uuidString, startedAt: nil, rate: rate, rows: rows) + } + let snapshot = inputs.recorder.flush() + let cutoff = inputs.streamCutoff ?? .infinity + let recorded: [[Double]] = inputs.recorder.rows(.trajectory).filter { $0[0] <= cutoff } + let rows = recorded.map { row in + let pose = simd_float4x4( + SIMD4(Float(row[3]), Float(row[4]), Float(row[5]), Float(row[6])), + SIMD4(Float(row[7]), Float(row[8]), Float(row[9]), Float(row[10])), + SIMD4(Float(row[11]), Float(row[12]), Float(row[13]), Float(row[14])), + SIMD4(Float(row[15]), Float(row[16]), Float(row[17]), Float(row[18])) + ) + return (t: row[0], tracking: TrackingCode(state: Int(row[1]), reason: Int(row[2])).packetTracking, cameraToWorld: pose) + } + return Trajectory(sessionID: snapshot.sessionID, startedAt: snapshot.startedAt, rate: inputs.trajectoryRate ?? 60, rows: rows) + } + + private nonisolated static func photo(_ stored: StoredKeyframe, id: String, sharpness: Double, depth: DepthPacket?, inputs: PacketInputs) -> PacketPhoto { + // Intrinsics belong to the stored image; a JPEG of another size than the camera reported + // scales them with it. + let scale = SIMD2(Float(stored.width), Float(stored.height)) / stored.camera.imageSize + let k = stored.camera.intrinsics + return PacketPhoto( + id: id, jpeg: inputs.storeDirectory.appending(path: "\(stored.id).jpg"), width: stored.width, height: stored.height, + t: stored.t, pose: inputs.meterFrame.pose(inputs.corrections.pose(stored.rawPose, capturedAt: stored.t)), + intrinsics: SIMD4(k.x * scale.x, k.y * scale.y, k.z * scale.x, k.w * scale.y), + tracking: TrackingCode(stored.tracking).packetTracking, + exposure: stored.exposure.map { PacketExposure(durationS: $0.durationS, iso: $0.iso, offsetEV: $0.offsetEV) }, + // ARKit's frames come from the back wide camera; a replay says nothing about its lens. + lens: stored.exposure.map { PacketLens(focalLengthMM: $0.focalLengthMM, fNumber: $0.fNumber, camera: "wide") }, + sharpness: sharpness, depth: depth + ) + } + + /// Nil for a depth map with under 1% of pixels measured, which the validator calls empty: the + /// photo goes in without it. + private nonisolated static func measured(_ depth: DepthPacket) -> DepthPacket? { + let count = depth.meters.reduce(0) { $0 + ($1.isFinite && $1 > 0 ? 1 : 0) } + return Double(count) >= 0.01 * Double(depth.meters.count) ? depth : nil + } + + /// Depth recorded between photos, inside the capture: the live recorder's LiDAR frames, or a + /// replay's recorded depth under the same `DepthFrameBudget` and the same rule of normal + /// tracking. A frame at the time of a photo written with depth would repeat it and is left + /// out. Each map is read, written and dropped in turn, so memory holds one at a time. Returns + /// how many were written. + private nonisolated static func writeDepthFrames( + _ inputs: PacketInputs, window: ClosedRange, skipping photoDepthTimes: Set, into writer: inout PacketWriter + ) -> Int { + var written = 0 + var refused: [String] = [] + func add(t: Double, tracking: PacketTracking, cameraToWorld: simd_float4x4, intrinsics: SIMD4, depth: DepthPacket?) { + guard let depth else { + refused.append("t=\(t): its files are missing or the wrong size") + return + } + do { + try writer.addDepthFrame(PacketDepthFrame( + id: PacketDepthFrame.id(number: written + 1), t: t, pose: inputs.meterFrame.pose(inputs.corrections.pose(cameraToWorld, capturedAt: t)), intrinsics: intrinsics, + tracking: tracking, depth: depth + )) + written += 1 + } catch { + refused.append(String(describing: error)) + } + } + if let replay = inputs.replayDepth { + var budget = DepthFrameBudget() + for frame in replay.frames where frame.trackingNormal && window.contains(frame.timestamp) && budget.admit(t: frame.timestamp) { + guard !photoDepthTimes.contains(frame.timestamp), let file = frame.depth else { continue } + let intrinsics = DepthImage.intrinsics( + scaling: frame.intrinsics, from: SIMD2(Float(frame.width), Float(frame.height)), toWidth: file.width, height: file.height + ) + add(t: frame.timestamp, tracking: .normal, cameraToWorld: frame.cameraToWorld, intrinsics: intrinsics, depth: replayDepth(file, in: replay.folder)) + } + } else { + for frame in inputs.recorder.depthFrames() where window.contains(frame.t) && !photoDepthTimes.contains(frame.t) { + add( + t: frame.t, tracking: frame.tracking.packetTracking, cameraToWorld: frame.cameraToWorld, intrinsics: frame.intrinsics, + depth: inputs.recorder.loadDepth(frame) + ) + } + } + if let first = refused.first { + RuntimeLog.engine.error("packet: \(refused.count) depth frames left out; the first: \(first, privacy: .public)") + } + return written + } + + /// A replay frame's recorded depth as the packet takes it: meters and, when recorded, + /// confidence, with no source, since the recording says neither which ARKit depth it saved + /// nor whether it was rendered (the synthetic fixtures' is). Nil when the files are missing + /// or the wrong size. + private nonisolated static func replayDepth(_ file: ReplayDepthFile, in folder: URL) -> DepthPacket? { + let count = file.width * file.height + guard let map = try? Data(contentsOf: folder.appending(path: file.file)), map.count == count * 4 else { return nil } + var confidence: [UInt8]? + if let path = file.confidenceFile { + guard let data = try? Data(contentsOf: folder.appending(path: path)), data.count == count else { return nil } + confidence = [UInt8](data) + } + return DepthPacket(meters: PacketFiles.floats(littleEndian: map), width: file.width, height: file.height, confidence: confidence, source: nil) + } + + /// Core Motion's streams, rows inside the capture only. Returns how many rows were refused. + private nonisolated static func writeMotion(_ inputs: PacketInputs, window: ClosedRange, into writer: inout PacketWriter) -> Int { + var refused = 0 + for stream in CaptureRecorder.Stream.allCases where stream != .trajectory && inputs.motionStreams.contains(stream) { + let rows = inputs.recorder.rows(stream).filter { window.contains($0[0]) } + guard !rows.isEmpty else { continue } + let packetStream: PacketStream = switch stream { + case .trajectory: .trajectory + case .accelerometer: .accelerometer + case .gyroscope: .gyroscope + case .magnetometer: .magnetometer + case .deviceMotion: .deviceMotion + case .barometer: .barometer + } + // CMAltimeter sets its own rate (about 1 Hz), so the barometer claims none. + if stream != .barometer { try? writer.setNominalRate(MotionSource.rate, for: packetStream) } + for r in rows { + do { + switch stream { + case .accelerometer: try writer.appendAccelerometer(t: r[0], g: SIMD3(r[1], r[2], r[3])) + case .gyroscope: try writer.appendGyroscope(t: r[0], radiansPerSecond: SIMD3(r[1], r[2], r[3])) + case .magnetometer: try writer.appendMagnetometer(t: r[0], microtesla: SIMD3(r[1], r[2], r[3])) + case .deviceMotion: + try writer.appendDeviceMotion(DeviceMotionSample( + t: r[0], attitude: SIMD4(r[1], r[2], r[3], r[4]), gravity: SIMD3(r[5], r[6], r[7]), + userAcceleration: SIMD3(r[8], r[9], r[10]), rotationRate: SIMD3(r[11], r[12], r[13]), headingDegrees: r[14] + )) + case .barometer: try writer.appendBarometer(t: r[0], pressureKPa: r[1], relativeAltitudeM: r[2]) + case .trajectory: break + } + } catch { + refused += 1 + } + } + } + return refused + } + + private nonisolated static func logLeftOut(_ section: String, _ error: any Error) { + RuntimeLog.engine.error("packet: \(section, privacy: .public) left out: \(String(describing: error), privacy: .public)") + } + + enum PacketBuildError: Error, CustomStringConvertible { + case noTrajectory + + var description: String { + switch self { + case .noTrajectory: "no trajectory was recorded, so the capture has no time span" + } + } + } +} diff --git a/ios/HouseScan/Runtime/ScanEngine+Snapshot.swift b/ios/HouseScan/Runtime/ScanEngine+Snapshot.swift new file mode 100644 index 000000000..c5829d178 --- /dev/null +++ b/ios/HouseScan/Runtime/ScanEngine+Snapshot.swift @@ -0,0 +1,101 @@ +import Foundation +import HouseScanKit +import OSLog +import simd + +/// Everything an upload sends, frozen on the main actor at one frame: scene.json, the mesh's +/// measurements and the packet are all built from it and read nothing live afterwards. +/// +/// It is taken on the last frame the engine ingested (live, the one that also carried its own +/// mesh and planes, `LiveCapture.requestSpatialCapture`), after that frame's anchor correction +/// and ground update. So the wall, the ground, the coverage, the marks and the mesh describe one +/// world frame. Raw poses are kept as ARKit reported them (`StoredKeyframe.rawPose`); the +/// corrections to apply for scene.json and the 1.1 packet travel with them (`corrections`). +struct UploadSnapshot: Sendable { + /// `ScanEngine.spatialRevision` when taken: an answer to a scan whose geometry has changed + /// since is stale (`ScanEngine.upload`). + var revision: Int + /// The time of the last frame ingested when it was taken, whose corrections it includes; + /// nil before any frame. + var frameTime: Double? + var map: CoverageMap + var groundMeasured: Bool + /// The marks with their tapped world points; their wall coordinates are worked out against + /// `map.wall` when exported. + var features: [MarkedFeature] + /// Raw, as stored; `corrections` moves each for the export. + var keyframes: [StoredKeyframe] + var stills: [String: String] + var stillFrames: [String: StoredKeyframe] + var corrections: PoseCorrections + var meterPlaneSource: MeterPlaneSource + var leftEndIsLimit: Bool + var rightEndIsLimit: Bool + /// The ground at checked spots' footprints, against this snapshot's wall and ground. + var groundPatches: [SceneGroundPatch] + /// The frame's own mesh and planes, when the frame carried them; nil when it didn't (no + /// LiDAR, a replay, or no matching frame arrived in time), and the scan goes without. + var mesh: LiveCapture.MeshSnapshot? + var planes: [LiveCapture.PlaneSnapshot] +} + +extension ScanEngine { + /// How long an upload waits for a frame carrying its mesh and planes. Frames are sampled + /// about ten times a second; 2 s allows for a stall. A guess, not measured. + static let spatialFrameWait: Duration = .seconds(2) + + /// The snapshot as of the last frame ingested, with `spatial` only when it came with that + /// frame (same time). + func makeUploadSnapshot(spatial: FrameSpatialCapture?) -> UploadSnapshot? { + guard let map = coverage else { return nil } + let frameTime = captureClock + let matched = spatial.flatMap { $0.time == frameTime ? $0 : nil } + if spatial != nil, matched == nil { + RuntimeLog.engine.error("snapshot: the mesh came from another frame; the scan goes without it") + } + return UploadSnapshot( + revision: spatialRevision, frameTime: frameTime, map: map, groundMeasured: groundMeasured, + features: state.features, keyframes: store.keyframes, stills: store.stills, stillFrames: store.stillFrames, + corrections: poseCorrections, meterPlaneSource: meterPlaneSource, + leftEndIsLimit: wallEndKinds[.left] == .limit, rightEndIsLimit: wallEndKinds[.right] == .limit, + groundPatches: spotGroundPatches, mesh: matched?.mesh, planes: matched?.planes ?? []) + } + + /// Takes the upload's snapshot. Live, on the next frame that carries its own mesh and planes + /// (taken in `ingest`, right after that frame's corrections); if none arrives within + /// `spatialFrameWait`, or the source is gone, at the current frame without mesh or planes. + /// On a replay, at once, without. + func takeUploadSnapshot() async -> UploadSnapshot? { + guard let live = liveCapture, mayCapture else { return makeUploadSnapshot(spatial: nil) } + // A request still waiting belongs to an upload that was replaced: it gets nothing. + cancelPendingSnapshot() + snapshotRequest += 1 + let request = snapshotRequest + return await withCheckedContinuation { (continuation: CheckedContinuation) in + pendingSnapshot = continuation + live.requestSpatialCapture() + Task { + try? await Task.sleep(for: Self.spatialFrameWait) + guard snapshotRequest == request, let waiting = pendingSnapshot else { return } + pendingSnapshot = nil + RuntimeLog.engine.error("snapshot: no frame with its mesh arrived; the scan goes without the mesh") + waiting.resume(returning: makeUploadSnapshot(spatial: nil)) + } + } + } + + /// Called by `ingest` after a frame's spatial update: a frame carrying its mesh completes a + /// pending snapshot, taken on that frame. + func completeUploadSnapshot(with frame: SourceFrame) { + guard let spatial = frame.spatial, let waiting = pendingSnapshot else { return } + pendingSnapshot = nil + waiting.resume(returning: makeUploadSnapshot(spatial: spatial)) + } + + /// Ends a waiting snapshot request with nothing (a reset, a replaced upload). + func cancelPendingSnapshot() { + guard let waiting = pendingSnapshot else { return } + pendingSnapshot = nil + waiting.resume(returning: nil) + } +} diff --git a/ios/HouseScan/Runtime/ScanEngine+Stamp.swift b/ios/HouseScan/Runtime/ScanEngine+Stamp.swift new file mode 100644 index 000000000..8bff50086 --- /dev/null +++ b/ios/HouseScan/Runtime/ScanEngine+Stamp.swift @@ -0,0 +1,27 @@ +import Foundation +import HouseScanKit +import OSLog + +extension ScanEngine { + /// Writes `scan-stamp.json` into the scan's folder: the app version, build and commit, the + /// server the scan goes to, and the latest answer's rules (`ScanStamp`). Written when the + /// scan is packaged and again when an answer arrives, so a scan pulled off the phone says what + /// built and judged it. A failed write is logged; the scan goes on without it. + func writeScanStamp(answer: PlacementResult?) { + let info = Bundle.main.infoDictionary ?? [:] + let stamp = ScanStamp( + app: .init( + version: info["CFBundleShortVersionString"] as? String ?? "unknown", + build: info["CFBundleVersion"] as? String ?? "unknown", + // Written into the built Info.plist by the "Stamp the git commit" build phase. + commit: info["HouseScanGitCommit"] as? String ?? "unknown"), + server: .init(url: (resultClient as? HTTPResultClient)?.serverURL.absoluteString), + answer: answer.map { ScanStamp.Answer($0, sample: resultClient.isSample) }) + let file = store.directory.appending(path: ScanStamp.fileName) + do { + try stamp.jsonData().write(to: file, options: .atomic) + } catch { + RuntimeLog.engine.error("scan stamp not written: \(String(describing: error), privacy: .public)") + } + } +} diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift new file mode 100644 index 000000000..60013075d --- /dev/null +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -0,0 +1,1854 @@ +import ARKit +import CoreGraphics +import Foundation +import HouseScanKit +import OSLog +import simd +import SwiftUI + +/// The capture engine: the only writer of `ScanViewState`, and the implementation of the +/// homeowner's intents. Frames arrive from `LiveCapture` (ARKit) or `ReplayPlayer` (a recorded +/// session) and take the same path through auto-capture, coverage and guidance. +@MainActor +final class ScanEngine { + let state = ScanViewState() + let options: LaunchOptions + + // Sources + private(set) var replay: ReplayPlayer? + /// Which kind of plane the meter tap hit; an estimated plane widens the meter's error in the + /// export. A replay's wall comes from the recording, so it counts as detected. + var meterPlaneSource: MeterPlaneSource = .detectedPlane + private var live: LiveCapture? + + // Capture logic (HouseScanKit) + private(set) var coverage: CoverageMap? + private var autoCapture = AutoCapture() + private var closeUpGate = CloseUpGate() + private var planner = GuidancePlanner() + let gapPlanner = GapPlanner() + + // Stored evidence + private(set) var store: KeyframeStore + private var keptSourceIDs: Set = [] + /// Debounces coaching that comes from the capture gate (see `gateCoaching`). + private var gateProblem: (coaching: Coaching, since: Double)? + private var gateClearSince: Double? + private var closeUpPending = false + /// Why the last close-up has to be retaken (from the meter-number reader, or "None of + /// these"), and when that was said, in screen seconds. Shown until the next shot fires. + private var closeUpRetake: (problem: CloseUpProblem, since: Double)? + /// The reader's answer for the close-up on screen, for the advice after "None of these". + private var meterReadout: MeterReadout? + /// The close-up view coverage may take when the close-up step ends (`observeCloseUpView`): + /// that of the photo on disk, once the reader's image checks passed it. + var closeUpCredit = CloseUpCredit() + /// Keyframe writes still in flight, by the `generation` they started in; the bundle waits + /// for its own generation's. Keyed so a write finishing after a reset can't count against + /// the new scan (a plain counter went negative when `resetAll` zeroed it mid-write). + private var pendingSaves: [Int: Int] = [:] + /// Bumped whenever the world frame or the whole scan is thrown away, so work that finishes + /// afterwards (a keyframe write, an upload) can tell it belongs to a scan that no longer exists. + private var generation = 0 + + // Wall geometry inputs + private var meterAnchorID: UUID? + /// The meter anchor's pose the wall and everything captured agree with, and the corrections + /// still to apply to them as ARKit refines it (`refreshMeterFromAnchor`). Nil on a replay. + private var meterTracking: MeterAnchorTracking? + /// Whether a frame source may start, and what a failure and Start over do to it. + private var sourceState = CaptureSourceState() + /// Counts changes to the captured geometry: every anchor correction, ground refinement or + /// revocation, new wall and followed corner. An upload's snapshot records it; an answer to a + /// scan whose count has moved on since is stale (`upload`). + var spatialRevision = 0 + /// An upload waiting for a frame with its mesh (`takeUploadSnapshot`), and its number. + var pendingSnapshot: CheckedContinuation? + var snapshotRequest = 0 + /// The snapshot the last upload sent, which the spot check's repackaging writes again. + private(set) var lastUploadSnapshot: UploadSnapshot? + /// Detected horizontal planes, with their classes and outlines. + private var groundPlanes: [GroundPlaneEvidence] = [] + private var lastFrame: SourceFrame? + /// Whether `WallFrame.groundY` comes from a detected plane (or a recording's wall taps) rather + /// than the chest-height guess, and the guess's error. The export widens position errors + /// while it is a guess. + private var groundEvidence = GroundEvidence(measured: false, guessError: ScanEngine.estimatedGroundError) + private(set) var groundMeasured: Bool { + get { groundEvidence.measured } + set { groundEvidence = GroundEvidence(measured: newValue, guessError: Self.estimatedGroundError) } + } + private var endKinds: [WallSide: EndKind] = [:] + /// The side whose end turns a corner the walk is to follow, while it waits for the next wall + /// to be marked (`GuidanceStep.markNextWall`), and why the last mark was refused. + var nextWallSide: WallSide? + var nextWallRefusal: NextWallRefusal? + + // Gap loop + private(set) var gapPlan: GapPlan? + private var gapCounter = 0 + /// Keyframes stored when the current gap request began: a request is closed only by new views. + private var keyframesAtGapStart = 0 + /// Tilt-up views kept when the current gap request began: an overhead request is closed by a + /// new one (`keepOverheadView`); counting keyframes would not do, since the walk keeps them too. + private var overheadViewsAtGapStart = 0 + /// Requests the homeowner skipped or answered with something overhead: they go to installer + /// review, and the result doesn't offer them as captures again. + private(set) var skippedGaps: [GapPlan] = [] + /// The side of a server past_end request being captured: that end was cleared, and marking + /// it again settles the request (see `markWallEnd`). + var pastEndSide: WallSide? + /// The end the past_end request cleared: where it was, its kind and when it was marked, put + /// back or moved on when the request ends without it marked again (`settleClearedEnd`). + private var clearedEnd: (s: Float, kind: EndKind, t: Double?)? + /// Server requests raised without a tap since the review was confirmed (`nextAutomaticGap`), + /// oldest first. Each is raised once; the result still offers it as a capture. + private var automaticGaps: [GapPlan] = [] + /// Set when the homeowner says they can't get to a server request's view: after the upload + /// that follows, the result shows instead of the next request. + private var automaticGapsStopped = false + /// At most this many server requests are raised without a tap per confirmed review, so a + /// server that keeps finding new gaps can't hold the homeowner in the loop. A guess, not + /// measured: an answer usually lists one to three capturable items, and each round is a + /// capture and an upload. + static let maxAutomaticGaps = 5 + /// How long the upload screen shows "One more view to finish" before a request raised from + /// the answer opens the camera: 1.5 s, about the time to read four words and see the new + /// step appear, and what the UI lane asked for. Not measured with homeowners. + static let followUpHold: Double = 1.5 + + // LiDAR + /// The bands the see-behind step is about, while `state.guidance` is `.seeBehind`: those with + /// hidden cells near its s (`hiddenCells(_:band:around:)`). + private(set) var seeBehindBands: [SurfaceBand] = [] + + // Tilt-up step and overhead requests + /// Set once the tilt-up step is answered or skipped: the walk asks it once per scan. + var tiltUpSettled = false + /// The tilted-up view the overhead question is about, with the walked-path segment it was + /// captured in. "Open sky or nothing overhead" keeps it as a keyframe and in the coverage map + /// (`keepOverheadView`), which the export sends as the overhead band; "A roof edge, porch or + /// stairs" keeps nothing, so the server treats that stretch as unseen. + private var pendingOverhead: (frame: SourceFrame, segment: Int?)? + + // Tracking recovery + private var relocalizingSince: Double? + + // Upload + let resultClient: any ResultClient + private var uploadTask: Task? + private(set) var placement: PlacementResult? + /// The latest scan-bundle write (`saveBundle`), and a count of writes started, so only the + /// latest one offers its bundle. + private var bundleTask: Task? + private var bundleSerial = 0 + + // Packet + /// The packet's sensor streams for the current world frame. + private(set) var recorder: CaptureRecorder + private let motion = MotionSource() + /// Every request the homeowner was shown, for the packet. + var guidanceLog = GuidanceLog() + /// The spot check (`ScanEngine+Confirm.swift`). + var spotConfirm = SpotConfirmState() + /// When each mark was made, on the capture clock (`MarkKey`). + var markTimes: [String: Double] = [:] + /// The packet's clock for guidance and marks: the latest frame's time, ARFrame.timestamp + /// live. A replay plays parts of its recording more than once, so its clock is the latest + /// frame time seen and never runs back. Nil until the first frame. + private(set) var captureClock: Double? + + private var lastGuidanceLog = "" + private var lastGateLog = "" + /// Taps of the feature being marked, in wall coordinates. + /// Tapped world points of the feature being marked. + var pendingTaps: [SIMD3] = [] + + enum EndKind { + /// The homeowner marked the end and said something blocks the wall there (a fence, gate + /// or property line). + case limit + /// The wall may go on past this end: it turns a corner, the walk stopped there ("I can't + /// get there"), or the homeowner has not said what is there yet. + case unexplored + } + + init(options: LaunchOptions) { + self.options = options + store = KeyframeStore() + recorder = Self.makeRecorder(store) + if let url = options.serverURL, !options.sampleResult { + resultClient = HTTPResultClient(serverURL: url) + } else { + resultClient = SampleResultClient(pace: options.autopilot ? options.autopilotHold : 1.2) + } + state.isAutopilot = options.autopilot + state.isReplay = options.replayFolder != nil + state.usesSampleResult = resultClient.isSample + } + + // MARK: Start + + func start() { + RuntimeLog.state.info("STATE=\(self.state.phase.rawValue, privacy: .public)") + if let folder = options.replayFolder { + Task { await loadReplay(folder) } + } else if !ARWorldTrackingConfiguration.isSupported { + fail(.arUnsupported) + } + } + + /// Every failure the homeowner has to act on (no AR, camera denied, a failed session, an + /// unreadable replay) ends on the failure screen, which reads `state.failure` for its words. + /// Setting the failure alone left the flow on whatever screen was up. + private func fail(_ failure: ScanFailure) { + _ = sourceState.sourceFailed(failure == .arUnsupported ? .unsupported : .recoverable, afterCapture: false) + state.failure = failure + go(.unsupported) + } + + private func loadReplay(_ folder: URL) async { + do { + let loaded = try await Task.detached(priority: .userInitiated) { try ReplayPlayer.load(folder: folder) }.value + let source = sourceState.sourceStarted() + let player = ReplayPlayer(folder: folder, loaded: loaded) { [weak self] frame in self?.ingest(frame, from: source) } + replay = player + state.spatialResultAvailable = true + let withDepth = player.frames.filter { $0.depth != nil }.count + state.depthAvailable = withDepth > 0 + player.show(index: 0) + RuntimeLog.engine.info("replay \(player.session.id, privacy: .public): \(player.frames.count) frames, \(withDepth) with depth, wall \(player.wallDescription, privacy: .public)") + } catch { + RuntimeLog.engine.error("replay unreadable: \(String(describing: error), privacy: .public)") + fail(.replayUnreadable(String(describing: error))) + } + } + + // MARK: Phases + + func go(_ phase: ScanPhase) { + guard state.phase != phase else { return } + if state.phase == .wallWalk || state.phase == .gapRequest { + breakWalkedPath(because: "the walk paused (\(state.phase.rawValue) -> \(phase.rawValue))") + } + if state.phase == .resultAR { hideResultInCamera() } + state.phase = phase + RuntimeLog.state.info("STATE=\(phase.rawValue, privacy: .public)") + switch phase { + case .findMeter: + state.guidance = .findMeter + startSourceIfNeeded() + live?.setMode(.idle) + case .meterCloseUp: + state.guidance = .holdOnMeter + state.closeUp = .aiming(hold: 0, problem: nil) + closeUpGate = CloseUpGate() + closeUpPending = false + closeUpRetake = nil + meterReadout = nil + closeUpCredit = CloseUpCredit() + state.meterNumber = nil + state.closeUpFailedAttempts = 0 + live?.setMode(.closeUp) + if let replay { replay.play(range: 0.. CaptureRecorder { + CaptureRecorder(directory: store.directory.appending(path: "streams-raw", directoryHint: .isDirectory)) + } + + /// Streams record while a capture is under way, from the meter search through the upload that + /// sends it. They stop on the result, and when an upload fails or is refused, since the phone + /// may then sit idle for minutes; a request raised from the result, going back to the review + /// or sending again starts them again. Depth frames record only where the camera is meant to + /// be on the wall (the close-up, the walk, a gap request), so the meter search and the review + /// don't spend `DepthFrameBudget`. Core Motion runs only with the live camera: a replay's + /// frames were recorded by another phone at another time. + private func updateRecording() { + let capturing = switch state.phase { + case .findMeter, .meterCloseUp, .wallWalk, .markFeatures, .gapRequest: true + case .uploading: + switch state.upload { + case .failed, .rejected: false + case .idle, .packaging, .uploading, .analyzing, .done: true + } + case .onboarding, .spotConfirm, .result, .resultAR, .unsupported: false + } + let depthFrames = switch state.phase { + case .meterCloseUp, .wallWalk, .gapRequest: true + case .onboarding, .findMeter, .markFeatures, .uploading, .spotConfirm, .result, .resultAR, .unsupported: false + } + recorder.setRecording(capturing, depthFrames: depthFrames) + guard live != nil else { return } + if capturing { motion.start(into: recorder) } else { motion.stop() } + } + + var motionRunsLive: Bool { live != nil } + var motionAvailable: Set { motion.available } + + // MARK: Frames + + /// A frame from the source started as `source`. One from a source that failed or was + /// replaced is dropped: a frame it queued before failing would set tracking back to normal. + func ingest(_ frame: SourceFrame, from source: Int) { + guard sourceState.accepts(source) else { return } + captureClock = max(captureClock ?? frame.timestamp, frame.timestamp) + if !frame.isPoseOnly { lastFrame = frame } + if let still = frame.still { state.feed = .still(still) } + state.projection = frame.projection + if state.tracking != frame.tracking { + RuntimeLog.capture.info("tracking \(Self.name(self.state.tracking), privacy: .public) -> \(Self.name(frame.tracking), privacy: .public)") + if state.tracking == .normal { breakWalkedPath(because: "tracking left normal") } + state.tracking = frame.tracking + live?.setResultVisible(frame.tracking == .normal) + } + if let planes = frame.groundPlanes { groundPlanes = planes } + applySpatialUpdate(frame) + // After the frame's corrections, so a snapshot taken on it agrees with its mesh. + completeUploadSnapshot(with: frame) + guard !frame.isPoseOnly else { return } + trackRelocalization(frame) + guard !frame.isReview else { + refreshCues(camera: frame.camera) + return + } + + switch state.phase { + case .meterCloseUp: + closeUp(frame) + case .wallWalk, .gapRequest: + walk(frame) + case .findMeter: + state.coaching = coaching(for: frame.tracking, skip: nil) + default: + break + } + } + + /// ARKit's correction to the meter's anchor, then the ground from the planes this frame + /// reports, in that order (`SpatialUpdate`): the correction moves everything captured (the + /// wall and corners, the kept cameras, and here the tapped marks) as one body into the frame + /// ARKit reports now, and the ground is then read in that same frame. A plane that stops + /// supporting the ground (reclassified, removed) puts it back to a guess. + private func applySpatialUpdate(_ frame: SourceFrame) { + guard var map = coverage else { return } + let before = (y: map.wall.groundY, measured: groundEvidence.measured) + let outcome = SpatialUpdate.apply( + anchor: frame.meterAnchor, planes: frame.groundPlanes, time: frame.timestamp, + map: &map, tracking: &meterTracking, ground: &groundEvidence) + guard outcome.changed else { return } + spatialRevision += 1 + coverage = map + if let correction = outcome.correction { + var features = state.features + for index in features.indices { features[index].points = features[index].points.map(correction.point) } + if features != state.features { state.features = features } + // The first tap of a mark still being made was in the old frame too. + pendingTaps = pendingTaps.map(correction.point) + } + if outcome.groundChanged { + RuntimeLog.engine.info("ground \(self.groundEvidence.measured ? "measured" : "a guess again", privacy: .public) at y=\(map.wall.groundY) (was \(before.y), \(before.measured ? "measured" : "estimated", privacy: .public))") + } + publishWall() + publishCoverage() + reprojectFeatures() + } + + /// A raw pose ARKit reported at `time`, in the frame the wall agrees with now. The same raw + /// pose without an anchor (a replay). + func correctedPose(_ raw: simd_float4x4, capturedAt time: Double) -> simd_float4x4 { + meterTracking?.correctedPose(raw, capturedAt: time) ?? raw + } + + /// The anchor corrections, for the packet writer off the main actor. + var poseCorrections: PoseCorrections { PoseCorrections(meterTracking) } + + /// A frame's camera in the frame the wall agrees with now: a frame kept before a correction + /// and stored after it (its photo saving, the close-up being read, the overhead question + /// waiting) was captured in the old frame. + func correctedCamera(_ frame: SourceFrame) -> CameraFrame { + meterTracking?.correctedCamera(frame.camera, capturedAt: frame.timestamp) ?? frame.camera + } + + /// Door and window heights, spans and fence distances follow the wall frame; the tapped world + /// points stay put. + func reprojectFeatures() { + guard let wall = coverage?.wall, !state.features.isEmpty else { return } + var features = state.features + for index in features.indices { Self.project(&features[index], onto: wall) } + if features != state.features { state.features = features } + publishFeaturesPastEnds() + } + + private func closeUp(_ frame: SourceFrame) { + guard let wall = coverage?.wall else { return } + if case .captured = state.closeUp { return } + if case .skipped = state.closeUp { return } + state.coaching = coaching(for: frame.tracking, skip: nil) + // After a retake request the shutter waits long enough for the reason to be read (and, + // for "move closer", acted on) before the hold can start again. + if let retake = closeUpRetake, screenTime - retake.since < Self.retakeNotice { + state.closeUp = .aiming(hold: 0, problem: retake.problem) + return + } + let sample = FrameSample(timestamp: frame.timestamp, camera: frame.camera, tracking: frame.captureTracking, quality: frame.quality) + let status = closeUpGate.evaluate(sample, meter: wall.meter) + state.closeUpFailedAttempts = status.failedAttempts + if let issue = status.issue { + logGate("close-up held back: \(issue)") + } else if status.fire || closeUpPending, !frame.jpeg.isAvailable { + logGate("close-up waiting: no photo on this frame") + } else if !status.fire, !closeUpPending { + logGate("close-up holding") + } + if status.fire || closeUpPending, status.issue == nil, frame.jpeg.isAvailable { + logGate("close-up taken from \(frame.id)", always: true) + closeUpPending = false + closeUpRetake = nil + captureCloseUp(frame) + } else { + // A hold that finished on a frame without a photo waits for the next photo, but only + // while the gates keep passing; any problem restarts the hold. + closeUpPending = status.issue == nil && (closeUpPending || status.fire) + state.closeUp = .aiming(hold: status.hold, problem: status.issue.map(Self.problem) ?? closeUpRetake?.problem) + } + } + + /// Seconds a retake reason stays up before the next close-up can be taken. A guess to try + /// on a phone, not measured: long enough to read one short line. + private static let retakeNotice: Double = 2 + + private var screenTime: Double { Double(DispatchTime.now().uptimeNanoseconds) / 1_000_000_000 } + + /// Back to aiming because the close-up photo was not usable (not saved, no number read, or + /// "None of these"). Counts as a failed attempt, so "Can't get a clear shot" appears from + /// the second one on. + func retakeCloseUp(_ problem: CloseUpProblem) { + guard state.phase == .meterCloseUp else { return } + RuntimeLog.engine.info("close-up retake: \(String(describing: problem), privacy: .public)") + closeUpGate.photoRejected() + state.closeUpFailedAttempts = closeUpGate.failedAttempts + closeUpPending = false + closeUpRetake = (problem, screenTime) + meterReadout = nil + state.meterNumber = nil + state.closeUp = .aiming(hold: 0, problem: problem) + } + + /// The meter number was confirmed: on to the walk once the confirmation has been seen. + func finishCloseUp() { + let scan = generation + Task { + try? await Task.sleep(for: .seconds(autoAdvanceDelay)) + await waitForGate(.meterCloseUp) + if scan == generation, state.phase == .meterCloseUp { go(.wallWalk) } + } + } + + /// The reader's answer for the close-up on screen, or nil while none is showing. + var currentMeterReadout: MeterReadout? { meterReadout } + + private func captureCloseUp(_ frame: SourceFrame) { + state.closeUp = .captured(nil) + // This shot's save replaces the photo on disk: the last one's view no longer counts. + closeUpCredit.shotStarted() + let view = frame.tracking == .normal ? CloseUpView(camera: frame.camera, depth: frame.depth, time: frame.timestamp) : nil + let scan = generation + let store = store + Task { + // The task can start after a reset or after the flow left the close-up. + guard scan == generation, state.phase == .meterCloseUp else { return } + let saved = await store.saveStill(frame, name: "meter_close.jpg") + guard scan == generation, state.phase == .meterCloseUp else { return } + if !saved { + retakeCloseUp(.blurry) + return + } + let thumbnail = await store.thumbnail(ofStill: "meter_close.jpg") + guard scan == generation, state.phase == .meterCloseUp else { return } + state.closeUp = .captured(thumbnail) + state.captureCount += 1 + state.lastCapture = CaptureEvent(id: state.captureCount, kind: .closeUp, thumbnail: thumbnail) + await readMeterNumber(scan: scan, view: view) + } + } + + /// Reads the meter number from the saved close-up, off the main actor, then offers the + /// candidates for the homeowner to pick from (never filling one in) or asks for a retake. + /// `view` is the shot's, which coverage may take once the reader has passed its photo. + private func readMeterNumber(scan: Int, view: CloseUpView?) async { + state.meterNumber = .reading + let reader = MeterNumberReaders.make() + let photo = store.directory.appending(path: "meter_close.jpg") + let readout = await Task.detached(priority: .userInitiated) { () -> MeterReadout? in + guard let jpeg = try? Data(contentsOf: photo) else { return nil } + return await reader.read(jpeg: jpeg) + }.value + guard scan == generation, state.phase == .meterCloseUp, state.meterNumber == .reading else { return } + closeUpCredit.photoChecked(view, passed: readout?.photoPassedChecks == true) + guard let readout else { + RuntimeLog.engine.error("close-up photo could not be read back for the meter number") + retakeCloseUp(.noNumber) + return + } + guard !readout.candidates.isEmpty else { + retakeCloseUp(readout.retake ?? .noNumber) + return + } + RuntimeLog.engine.info("meter number: \(readout.candidates.count) candidates to choose from") + meterReadout = readout + state.meterBrand = readout.brand + state.meterNumber = .choose(readout.candidates) + } + + private func walk(_ frame: SourceFrame) { + guard let map = coverage else { return } + let sample = FrameSample(timestamp: frame.timestamp, camera: frame.camera, tracking: frame.captureTracking, quality: frame.quality) + let decision = autoCapture.evaluate(sample, newlySeenCells: map.newlySeenCount(from: frame.camera)) + // Past an end it can't see back from, a photo adds nothing: it is refused and the screen says so. + let pastEnd = map.unexploredEndPassed(by: frame.camera) + var skip: CaptureDecision.SkipReason? + switch decision { + case .skip(let reason): + skip = reason + logGate("skipped: \(reason)") + case .keep where pastEnd != nil: + logGate("refused: past the \(pastEnd?.rawValue ?? "") end, nothing between the ends in view") + // The gate judges sharpness and exposure from this frame's own quality when it has one, + // else from the last measured frame's. A kept photo must have been judged itself. + case .keep where frame.quality == nil: + logGate("refused: no quality measured on this frame") + case .keep where !frame.jpeg.isAvailable: + logGate("refused: no photo on this frame") + case .keep where keptSourceIDs.contains(frame.id): + logGate("refused: frame already kept") + case .keep(let reason): + logGate("kept \(frame.id) (\(reason))", always: true) + autoCapture.didKeep(sample) + keptSourceIDs.insert(frame.id) + keep(frame) + } + state.coaching = walkCoaching(tracking: frame.tracking, skip: skip, pastEnd: pastEnd != nil, time: frame.timestamp) + afterCoverageChange(camera: frame.camera, time: frame.timestamp) + askOverheadIfTiltedUp(frame) + } + + /// Tracking problems show at once. A problem the capture gate reports (moving, blurry, too + /// dark) shows only once it has lasted `showAfter` seconds and clears after `clearAfter` + /// seconds of frames without it: the gate judges every frame, and one fast frame at 30 fps + /// would otherwise flash a prompt for a single frame. During the walk moving and blurry read + /// as "Slow down", never "Hold steady", which would tell a walking homeowner to stop. Both + /// durations are guesses to try on a phone, not measured. Standing past an end the phone + /// can't see back from (`pastEnd`) is debounced the same way and comes before the gate's + /// reasons: no photo is kept there whatever the gate says. + private func walkCoaching(tracking: TrackingQuality, skip: CaptureDecision.SkipReason?, pastEnd: Bool, time: Double) -> Coaching? { + let showAfter = 0.7 + let clearAfter = 0.5 + guard tracking == .normal else { + gateProblem = nil + gateClearSince = nil + return coaching(for: tracking, skip: nil) + } + var candidate: Coaching? = switch skip { + case .moving?, .blurry?: .slowDown + case .tooDark?: .tooDark + default: nil + } + if pastEnd { candidate = .pastWallEnd } + if let problem = gateProblem, time < problem.since { gateProblem = nil } // replay restarted + if let candidate { + gateClearSince = nil + if gateProblem?.coaching != candidate { + gateProblem = (candidate, time) + } + } else if let problem = gateProblem { + let clearSince = gateClearSince ?? time + gateClearSince = clearSince + if time - clearSince >= clearAfter || time - problem.since < showAfter { + gateProblem = nil + gateClearSince = nil + } + } + guard let problem = gateProblem, time - problem.since >= showAfter else { return nil } + return problem.coaching + } + + private func afterCoverageChange(camera: CameraFrame?, time: Double) { + publishCoverage() + if state.phase == .wallWalk, let camera { + updateGuidance(camera: camera, time: time) + } else if state.phase == .gapRequest { + updateGap(camera: camera) + } + } + + /// Stores a kept frame; coverage and the capture count move only once its photo is on disk, + /// so the strip never claims a view the bundle lacks. The photo, pose and tracking all come + /// from the one `SourceFrame`, so what is credited is the pose of the stored image. With + /// `overhead`, the stored view is also kept as a clear overhead view. `segment` is the + /// walked-path segment the frame was captured in, when it was captured before now. + private func keep(_ frame: SourceFrame, overhead: Bool = false, capturedIn segment: Int? = nil) { + let kind: CaptureEvent.Kind = state.phase == .gapRequest ? .gap : .walk + let index = store.nextKeyframeIndex() + let scan = generation + let store = store + // The walked-path segment the frame was captured in: a break while its photo stores must + // not join it to frames captured after the break. + let segment = segment ?? coverage?.pathSegment + pendingSaves[scan, default: 0] += 1 + Task { + // Every exit drains this generation's count, so the upload never waits on a write + // that was refused or failed. + defer { + let left = (pendingSaves[scan] ?? 1) - 1 + pendingSaves[scan] = left > 0 ? left : nil + } + // A frame queued before a reset must not be written into the new scan's store. + guard scan == generation else { return } + let saved = await store.saveKeyframe(frame, index: index) + guard scan == generation else { return } + guard saved.stored else { + // Not stored, so not kept: a later pass over the same replay frame may keep it. + keptSourceIDs.remove(frame.id) + if overhead { RuntimeLog.engine.error("overhead: the view asked about was not stored; nothing recorded") } + return + } + // Coverage only moves on kept frames with normal tracking (checklist R3). + // The frame's own time lets the walked path join only poses kept close together in time. + // With LiDAR depth, a cell counts only where depth confirms the camera saw it. + // Corrected for anchor moves made while the photo was saving. + let delta = coverage?.observe(correctedCamera(frame), trackingNormal: frame.tracking == .normal, time: frame.timestamp, depth: frame.depth, segment: segment) + RuntimeLog.capture.info("stored \(frame.id, privacy: .public) as keyframe \(index)\(frame.depth == nil ? "" : " with depth", privacy: .public): \(delta?.newlySeen ?? 0) cells newly seen, \(delta?.newlyCovered ?? 0) newly covered, \(delta?.newlyHidden ?? 0) newly hidden") + if overhead { recordOverhead(frame) } + state.captureCount += 1 + state.lastCapture = CaptureEvent(id: state.captureCount, kind: kind, thumbnail: saved.thumbnail) + afterCoverageChange(camera: lastFrame?.camera, time: lastFrame?.timestamp ?? frame.timestamp) + } + } + + // MARK: Guidance + + /// The planner's dwell runs on the screen's clock, not on `time` (the frame's timestamp). + /// "Don't change the instruction for 3 s" is about what the person looking at the screen + /// reads. Live, the two clocks agree; a replay at 3x speed on frame time changed the card + /// every second or so, which the verify lane's wall-clock check (I6) and the accessibility + /// audit both caught. + private func updateGuidance(camera: CameraFrame, time _: Double) { + // A question is on screen: the next step waits for its answer. + guard let map = coverage, state.endQuestion == nil, !state.overheadQuestion else { return } + if let side = nextWallSide { + state.guidance = .markNextWall(side: side, refusal: nextWallRefusal) + state.target = nil + state.path = [] + logGuidance() + return + } + if let span = tiltUpSpanIfDue(map) { + state.guidance = .tiltUp(span: span) + state.target = map.wall.world(s: (span.lowerBound + span.upperBound) / 2, height: Self.tiltUpHeight(map)) + state.path = [] + logGuidance() + return + } + let screenTime = Double(DispatchTime.now().uptimeNanoseconds) / 1_000_000_000 + let output = planner.update(coverage: map, camera: camera, time: screenTime) + if let hidden = hiddenBlock(output.task, map, camera: camera) { + seeBehindBands = hidden.bands + state.guidance = .seeBehind(s: hidden.s) + state.target = hidden.bands.contains(.wall) + ? map.wall.world(s: hidden.s, height: 1) + : map.wall.world(s: hidden.s, height: 0, out: map.config.groundBandDepth / 2) + // Where to stand for another angle depends on what is in the way, which the map + // doesn't know; no path. + state.path = [] + } else { + seeBehindBands = [] + state.guidance = Self.step(output.task) + state.target = output.target + state.path = output.path + } + logGuidance() + } + + // MARK: See-behind step (LiDAR) + + /// How far past a hidden stretch the camera must be before the walk asks to look behind it + /// instead of walking on: 1 m, about two strides, so the stretch has had a chance to show from + /// the angles the walk passes through. A guess to try on a phone, not measured. + static let seeBehindPassed: Float = 1 + + /// LiDAR phones: the stretch the walk's task is waiting on, when what keeps it open is hidden + /// cells (the camera looked, and depth showed something nearer) rather than unseen ones. + /// Walking up to it again gives the same view; another angle, or stepping round, can show it. + /// + /// An aim task's stretch counts as hidden when at least half of its cells still open (neither + /// covered nor skipped) within 0.3 m of it are hidden: the planner's own window for the task + /// (`GuidancePlanner.isSatisfied`), and "half" is a guess. A walk counts as blocked when the + /// first open cell on its side, where the planner's reach stops, is hidden and the camera is + /// `seeBehindPassed` beyond it: without this, a bush hiding the whole band leaves the walk + /// saying "walk on" while its reach never moves. + /// + /// The planner raises `.seeBehind` itself when hidden cells lie near the camera; the step is + /// then about the bands holding hidden cells near its s. Nil when there are none, which leaves + /// no cell for "Can't see past it" to settle. + private func hiddenBlock(_ task: GuidanceTask, _ map: CoverageMap, camera: CameraFrame) -> (s: Float, bands: [SurfaceBand])? { + func open(_ band: SurfaceBand, _ index: Int) -> Bool { + let level = map.level(band, index) + return level != .covered && level != .skipped + } + func mostlyHidden(_ band: SurfaceBand, around s: Float) -> Bool { + let cells = map.indices(overlapping: (s - 0.3)...(s + 0.3)).filter { map.isWithinEnds($0) && open(band, $0) } + let hidden = cells.filter { map.level(band, $0) == .hidden }.count + return hidden > 0 && hidden * 2 >= cells.count + } + switch task { + case .seeBehind(let s): + let bands = SurfaceBand.allCases.filter { !Self.hiddenCells(map, band: $0, around: s).isEmpty } + return bands.isEmpty ? nil : (s, bands) + case .aimAtGround(let s): + return mostlyHidden(.ground, around: s) ? (s, [.ground]) : nil + case .aimAtWall(let s): + return mostlyHidden(.wall, around: s) ? (s, [.wall]) : nil + case .walk(let side): + let reach = planner.reach(side, coverage: map) + let index = map.cellIndex(forS: side.sign * (reach + map.config.cellWidth / 2)) + let bands = SurfaceBand.allCases.filter { map.level($0, index) == .hidden } + let beyond = side.sign * map.wall.wallPoint(camera.position).s - reach + guard !bands.isEmpty, beyond >= Self.seeBehindPassed else { return nil } + let cell = map.cellRange(index) + return ((cell.lowerBound + cell.upperBound) / 2, bands) + case .markEnd, .stepBack, .complete: + return nil + } + } + + /// How far either side of a see-behind step's s its hidden cells are looked for: 1 m, the + /// half-width of the window around the camera in which the planner looks for them + /// (`GuidancePlanner.preferredTask`). + static let seeBehindReach: Float = 1 + + /// The hidden cells of `band` within `seeBehindReach` of `s`: what the see-behind step asks + /// to see past, and what "Can't see past it" hands to review. + static func hiddenCells(_ map: CoverageMap, band: SurfaceBand, around s: Float) -> [Int] { + map.indices(overlapping: (s - seeBehindReach)...(s + seeBehindReach)).filter { map.level(band, $0) == .hidden } + } + + /// A frame shown for review (not captured) keeps the current task but re-aims its target and + /// path from the camera now on screen; otherwise the arrow points from where the walk last was. + private func refreshCues(camera: CameraFrame) { + guard state.phase == .wallWalk, state.endQuestion == nil, !state.overheadQuestion, nextWallSide == nil, + let map = coverage, let task = planner.current else { return } + if case .tiltUp = state.guidance { return } + if case .seeBehind = state.guidance { return } + let output = planner.cues(for: task, coverage: map, camera: camera) + state.target = output.target + state.path = output.path + } + + /// "I can't get there" or an answered end question settled the current task: choose the next + /// one now. + func resetGuidanceAfterSkip(camera: CameraFrame, time: Double) { + planner.reset() + updateGuidance(camera: camera, time: time) + } + + // MARK: Tilt-up step + + /// Half the stretch the tilt-up step asks about: 1.5 m (about 5 ft) each side of the meter, + /// inside the marked ends. The spots the server tries first sit beside the meter, where the + /// cable run is shortest, and a 2.58 ft battery fits on either side within it. 3 m is also + /// about what one upright view takes in from 3 m out. A judgment call, not measured: a spot + /// farther along gets its own `.overhead` request from the server. + static let tiltUpReach: Float = 1.5 + + /// The stretch to ask about, once both ends are marked and answered and the step hasn't been + /// answered or skipped; nil otherwise, and when the marked ends leave no stretch. + private func tiltUpSpanIfDue(_ map: CoverageMap) -> ClosedRange? { + guard !tiltUpSettled, let left = map.leftEnd, let right = map.rightEnd else { return nil } + let low = max(left, -Self.tiltUpReach) + let high = min(right, Self.tiltUpReach) + return low < high ? low...high : nil + } + + /// How far above the top wall row (`CoverageConfig.wallCaptureHeight`, 7.5 ft) a view must + /// reach to count as tilted up: 0.7 m, so about 9.8 ft above the ground, past a one-storey + /// eave, where the view shows whether one is there. It is a height, not a pitch: a + /// level view from 2 m out reaches about 8.5 ft and does not count, one from 2.6 m out + /// reaches about 10 ft and does, and it shows what is overhead as well as a tilted one. A + /// guess to try on a phone, not measured. + static let tiltUpAbove: Float = 0.7 + + /// The height the tilt-up step and an overhead request aim at, meters above the ground. + static func tiltUpHeight(_ map: CoverageMap) -> Float { map.config.wallCaptureHeight + tiltUpAbove } + + /// The stretches a view shows at least `tiltUpHeight` up the wall (`CoverageMap.overheadReach`): + /// empty unless the camera is tilted up at the wall. + static func tiltedUp(_ camera: CameraFrame, _ map: CoverageMap) -> [ClosedRange] { + let height = tiltUpHeight(map) + return map.overheadReach(from: camera).filter { $0.out >= height }.map(\.span) + } + + /// Raises the overhead question when a frame with normal tracking is tilted up where the scan + /// needs it: over any of the tilt-up step's stretch during the walk, or, for an overhead gap + /// request, over enough of the requested span that recording it settles the request + /// (`GapPlanner.overheadViewSettles`), so "nothing overhead" always closes it. Only the + /// homeowner can say whether what is above is open sky or an eave. Any frame with a photo + /// counts, not only frames auto-capture kept: "nothing overhead" stores that photo as a + /// keyframe, and the view counts only once it is stored (`keepOverheadView`). + private func askOverheadIfTiltedUp(_ frame: SourceFrame) { + guard !state.overheadQuestion, frame.tracking == .normal, frame.jpeg.isAvailable, let map = coverage else { return } + let wanted: ClosedRange + switch state.phase { + case .wallWalk: + guard case .tiltUp(let span) = state.guidance else { return } + wanted = span + case .gapRequest: + guard let plan = gapPlan, case .overhead = plan.need, state.gap?.isSatisfied == false else { return } + wanted = plan.span + default: + return + } + let seen = Self.tiltedUp(frame.camera, map) + guard seen.contains(where: { $0.overlaps(wanted) }) else { return } + if state.phase == .gapRequest, let plan = gapPlan, !gapPlanner.overheadViewSettles(plan, map, camera: frame.camera) { return } + // The segment now: the answer can come after a tracking break. + pendingOverhead = (frame, coverage?.pathSegment) + state.overheadQuestion = true + RuntimeLog.engine.info("tilt-up view over s \(seen.first?.lowerBound ?? 0)...\(seen.last?.upperBound ?? 0): asking what is overhead") + } + + /// Ends the tilt-up step. `clear` is the homeowner's "Open sky or nothing overhead", which + /// keeps the view the question was about; false for "A roof edge, porch or stairs", "Can't + /// get there" or leaving the walk, which keep nothing. + func settleTiltUp(clear: Bool) { + let pending = pendingOverhead + pendingOverhead = nil + state.overheadQuestion = false + tiltUpSettled = true + // After settling, so the guidance recomputed once it is stored moves past the tilt-up step. + let storing = clear && pending.map { keepOverheadView($0.frame, capturedIn: $0.segment) } == true + RuntimeLog.engine.info("tilt-up step settled: \(storing ? "storing the clear overhead view" : "nothing recorded", privacy: .public)") + } + + /// The answer to the overhead question during an overhead gap request. "Nothing overhead" + /// keeps the view, which was checked to settle the request when the question was raised, so + /// the request closes through `updateGap`. Something overhead means no view can settle it: + /// the request goes to installer review and the gap loop moves on to the upload. + func settleOverheadGap(clear: Bool) { + let pending = pendingOverhead + pendingOverhead = nil + state.overheadQuestion = false + guard clear else { + skipCurrentGap(because: "something is overhead", refused: false) + return + } + if pending.map({ keepOverheadView($0.frame, capturedIn: $0.segment) }) != true { + RuntimeLog.engine.error("overhead answer: the view asked about could not be kept") + } + } + + /// The replay's closing run of tilted-up frames (`tiltedUp`): the walk leaves them out, and + /// the tilt-up step and overhead gap requests play them (`playReplayTiltUp`). Empty at the end + /// when the recording has none. + static func tiltUpFrames(in replay: ReplayPlayer, map: CoverageMap) -> Range { + var start = replay.frames.count + while start > 0, !tiltedUp(replay.camera(at: start - 1), map).isEmpty { start -= 1 } + return start.. Bool { + guard let replay, let map = coverage else { return false } + let frames = Self.tiltUpFrames(in: replay, map: map) + guard !frames.isEmpty else { return false } + replay.play(range: frames, speed: replaySpeed) + return true + } + + private func updateGap(camera: CameraFrame?) { + guard let map = coverage, let plan = gapPlan, var request = state.gap else { return } + request.progress = gapPlanner.progress(of: plan, map) + let fresh = if case .overhead = plan.need { + map.overheadCameras.count > overheadViewsAtGapStart + } else { + store.keyframes.count > keyframesAtGapStart + } + let satisfied = gapPlanner.isSatisfied(plan, map) && fresh + state.guidance = .gap + let center = (plan.span.lowerBound + plan.span.upperBound) / 2 + let cue = gapCue(plan, map, center: center) + state.target = cue.target + if let camera { + let from = map.wall.wallPoint(camera.position).s + // Every request but an overhead one needs its whole span seen or walked, and the + // server's can run 20 ft or more (ground out to a pool's clearance), so the line runs + // on to the span's far end. One tilted-up view from the middle covers an overhead one. + let farEnd = abs(plan.span.lowerBound - from) > abs(plan.span.upperBound - from) ? plan.span.lowerBound : plan.span.upperBound + let to = if case .overhead = plan.need { center } else { farEnd } + state.path = [from, to].map { map.wall.world(s: $0, height: 0, out: cue.standOut) } + } + logGuidance() + if satisfied, !request.isSatisfied { + resolveGuidance(.met) + request.isSatisfied = true + request.progress = max(request.progress, gapPlanner.config.satisfiedFraction) + state.gap = request + RuntimeLog.engine.info("gap \(request.id) satisfied") + let id = request.id + Task { + try? await Task.sleep(for: .seconds(autoAdvanceDelay)) + await waitForGate(.gapRequest) + // Only if this same request is still showing (not skipped or replaced meanwhile). + guard state.phase == .gapRequest, state.gap?.id == id else { return } + afterGapResolved() + } + } else if !request.isSatisfied { + state.gap = request + } + } + + /// Where a gap request points the camera, and how far out from the wall to walk for it. + private func gapCue(_ plan: GapPlan, _ map: CoverageMap, center: Float) -> (target: SIMD3, standOut: Float) { + let standOff = planner.config.standOff + switch plan.need { + case .cells: + let target = plan.band == .ground + ? map.wall.world(s: center, height: 0, out: map.config.groundBandDepth / 2) + : map.wall.world(s: center, height: 1.2) + return (target, standOff) + case .groundOut(let out): + // 1 m beyond the requested depth: a phone at chest height (about 1.4 m) tilted down + // there sees the ground from about 2 m nearer the wall out to that depth, within the + // 65 degree view limit. Geometry only; not tried on a device. + return (map.wall.world(s: center, height: 0, out: out), max(standOff, out + 1)) + case .walkOut(let out): + // The walk has to pass `out` plus the wall's position error at the span's end where it + // is larger (farther from the meter, or on a piece with a larger default); 0.3 m more + // leaves room for drifting toward the wall. The 0.3 m is a guess. + let error = max(map.positionError(atS: plan.span.lowerBound), map.positionError(atS: plan.span.upperBound)) + return (map.wall.world(s: center, height: 1.2), max(standOff, out + error + 0.3)) + case .overhead(let height): + // Aim where a tilted-up view reaches, or at the height asked for when that is higher. + let aim = max(Self.tiltUpHeight(map), height ?? 0) + return (map.wall.world(s: center, height: aim), standOff) + case .wallUp(let height): + // At the height the view must pass, walked along the whole span like a cell request: + // each stretch needs it from two places. + return (map.wall.world(s: center, height: height), standOff) + } + } + + private func logGuidance() { + noteGuidance() + let name = Self.name(state.guidance) + guard name != lastGuidanceLog else { return } + lastGuidanceLog = name + RuntimeLog.guidance.info("GUIDANCE=\(name, privacy: .public)") + } + + /// Logs a capture-gate decision. The gate judges about ten frames a second, so a reason is + /// logged when it differs from the last one logged; `always` logs every time (a kept frame, + /// a shot taken). `decision` holds only enum reasons and frame ids, never image content. + private func logGate(_ decision: String, always: Bool = false) { + guard always || decision != lastGateLog else { return } + lastGateLog = decision + RuntimeLog.capture.info("gate \(decision, privacy: .public)") + } + + /// The homeowner's path is not continuous across this point (tracking left normal, the + /// session was interrupted, or the walk paused), so walked-path evidence must not join the + /// poses on either side. + private func breakWalkedPath(because reason: String) { + guard coverage != nil else { return } + coverage?.breakWalkedPath() + RuntimeLog.capture.info("walked path broken: \(reason, privacy: .public)") + } + + // MARK: Tracking recovery + + private func trackRelocalization(_ frame: SourceFrame) { + guard live != nil else { return } + guard case .limited(.relocalizing) = frame.tracking else { + if let since = relocalizingSince { + RuntimeLog.capture.info("relocalization ended after \(frame.timestamp - since, format: .fixed(precision: 1)) s: tracking \(Self.name(frame.tracking), privacy: .public)") + } + relocalizingSince = nil + return + } + if relocalizingSince == nil { + RuntimeLog.capture.info("relocalization started (phase \(self.state.phase.rawValue, privacy: .public))") + } + let since = relocalizingSince ?? frame.timestamp + relocalizingSince = since + // After 20 s ARKit is unlikely to relocalize; the old world frame is gone (checklist R5). + guard frame.timestamp - since > 20 else { return } + switch state.phase { + case .findMeter, .meterCloseUp, .wallWalk: + // The walk still needs the world frame: start again from the meter. + RuntimeLog.capture.info("relocalization timed out after 20 s: resetting to the meter") + resetSpatialState(reason: "relocalization timed out") + case .gapRequest: + // The scan so far is whole; only this request needs the lost frame. It goes to installer + // review as if the homeowner couldn't get there, and the upload that follows keeps the + // scan (and, for a request raised from the result, replaces the result with the new + // answer). The clock restarts, so a later request gets its own 20 s. + relocalizingSince = nil + // A request already met is on its way to the upload (`afterGapResolved`). + guard state.gap?.isSatisfied != true else { return } + RuntimeLog.capture.info("relocalization timed out after 20 s: leaving the gap for installer review") + skipCurrentGap(because: "the phone lost its place for 20 s") + case .markFeatures: + // The review needs no live frame: "Looks complete" uploads the scan as it is, and + // "Add something", which taps into the world frame, waits for tracking to return + // (`beginMarking`). Nothing is thrown away. + break + case .uploading, .spotConfirm, .result, .resultAR, .onboarding, .unsupported: + // The bundle is already packed and the server's answer does not depend on the live + // world frame, so the scan and the result stay. The AR result hides its overlay while + // tracking is not normal and shows it again if ARKit does relocalize. + relocalizingSince = nil + } + } + + private func handle(_ event: LiveEvent, from source: Int) { + // An event from a source that failed or was replaced says nothing about the running one. + guard sourceState.accepts(source) else { return } + switch event { + case .interrupted(let lastFrameTime): + // The phase, captures and strip stay as they are; ARKit relocalizes into the same + // world frame when the session resumes (checklist R4). The break is placed after the + // last frame the session delivered, which may still be on its way here. + RuntimeLog.capture.info("session interrupted") + if let lastFrameTime { coverage?.breakWalkedPath(at: lastFrameTime) } + breakWalkedPath(because: "session interrupted") + state.coaching = .relocalizing + case .interruptionEnded: + RuntimeLog.capture.info("session interruption ended") + state.coaching = .relocalizing + case .cameraDenied: + fail(.cameraDenied) + case .failed(let message): + // Once the scan is sent, the upload and its result no longer need the camera: keep + // them on screen. So does the spot check, which shows a stored photo; with tracking + // gone, unmarked equipment can't be marked there and its area is left out. Only the AR + // view needs the camera, and it already hides the battery while it isn't tracking. + switch state.phase { + case .uploading, .spotConfirm, .result, .resultAR: + RuntimeLog.engine.error("camera session failed after capture: \(message, privacy: .public)") + _ = sourceState.sourceFailed(.recoverable, afterCapture: true) + loseSpatialResult() + case .markFeatures where spotConfirmIsMarking: + // Marking unmarked equipment from the spot check: back to its photo, where the + // thing can't be marked now and its area is left out. + RuntimeLog.engine.error("camera session failed while marking for the spot check: \(message, privacy: .public)") + _ = sourceState.sourceFailed(.recoverable, afterCapture: true) + loseSpatialResult() + cancelMarking() + default: + // The screen shows plain words, so the camera's own error is only recorded here. + RuntimeLog.engine.error("camera session failed: \(message, privacy: .public)") + fail(.sessionFailed(message)) + } + } + } + + /// Forgets everything tied to the old world frame and asks for the meter again, the spot + /// check's answers with it: even the ground answer is about a footprint in that frame. The + /// close-up's own state (gate, readout, view) is reset when the close-up starts again + /// (`go(.meterCloseUp)`). + func resetSpatialState(reason: String) { + RuntimeLog.engine.info("spatial reset: \(reason, privacy: .public)") + generation += 1 + spatialRevision += 1 + cancelPendingSnapshot() + lastUploadSnapshot = nil + // A new world frame is a new packet session: what was recorded is in the old frame. + recorder.restart() + resetPacketLog() + relocalizingSince = nil + groundPlanes = [] + groundMeasured = false + lastFrame = nil + // A fresh map: the old world frame is gone, so its anchors and planes are meaningless. + live?.restart() + coverage = nil + meterAnchorID.map { live?.removeAnchor($0) } + meterAnchorID = nil + meterPlaneSource = .detectedPlane + state.wall = nil + state.coverage = .empty + state.target = nil + state.path = [] + state.features = [] + // A mark half placed holds taps in the old frame's wall coordinates. + state.marking = nil + pendingTaps = [] + state.gap = nil + gapPlan = nil + pastEndSide = nil + clearedEnd = nil + // Skipped requests name spans along the old wall. + skippedGaps = [] + automaticGaps = [] + automaticGapsStopped = false + seeBehindBands = [] + endKinds = [:] + state.endQuestion = nil + state.wallTooShort = false + nextWallSide = nil + nextWallRefusal = nil + resetTiltUp() + resetSpotChecks() + // An answer describes a scan that no longer exists; the next upload brings a new one. + uploadTask?.cancel() + placement = nil + state.result = nil + state.upload = .idle + // The keyframes and stills, the meter close-up included, were taken in the old frame. + store.discardKeyframes() + // A bundle packed before this holds keyframes of the world frame just discarded. + state.shareableScan = nil + keptSourceIDs = [] + state.captureCount = 0 + state.lastCapture = nil + gateProblem = nil + gateClearSince = nil + autoCapture.reset() + planner.reset() + go(.findMeter) + } + + // MARK: Wall + + /// Sets the wall from a meter point and the wall's outward normal, and starts coverage. + func setWall(meter: SIMD3, outward: SIMD3, groundY: Float, groundMeasured: Bool) -> Bool { + guard let frame = WallFrame(meter: meter, outward: outward, groundY: groundY) else { return false } + spatialRevision += 1 + coverage = CoverageMap(wall: frame) + coverage?.heightError = groundMeasured ? 0 : Self.estimatedGroundError + self.groundMeasured = groundMeasured + endKinds = [:] + state.endQuestion = nil + state.wallTooShort = false + nextWallSide = nil + nextWallRefusal = nil + resetTiltUp() + publishWall() + publishCoverage() + return true + } + + func publishWall() { + guard let map = coverage else { state.wall = nil; return } + let wall = map.wall + state.wall = WallGeometry( + meter: wall.meter, along: wall.along, outward: wall.outward, groundY: wall.groundY, + leftEnd: map.leftEnd, rightEnd: map.rightEnd, + cornerSegments: wall.segments.indices.filter { $0 != wall.meterSegmentIndex }.map { index in + let piece = wall.segments[index] + return WallGeometry.Segment(span: piece.span, along: piece.along, outward: piece.outward, anchor: piece.anchor, anchorS: piece.anchorS) + } + ) + if state.phase == .resultAR { showResultInCamera(rising: false) } + publishFeaturesPastEnds() + refreshSpotPhoto() + } + + /// "See it on your wall" on the live camera: the result goes into the AR scene on the meter's + /// anchor, where people and objects in front of it hide it. A replay, or a meter without an + /// anchor, leaves it to the screen's overlay. + private func showResultInCamera(rising: Bool) { + guard let live, let wall = state.wall, let result = state.result, let pose = meterTracking?.pose, + state.spatialResultAvailable else { return } + let model = ResultARModel.build(wall: wall, result: result) + state.resultInCamera = live.showResult(model, builtFor: pose) + guard state.resultInCamera else { return } + live.setResultVisible(state.tracking == .normal) + if rising { ResultARModel.rise(model) } + } + + private func hideResultInCamera() { + live?.hideResult() + state.resultInCamera = false + } + + func publishCoverage() { + guard let map = coverage else { return } + guard map.revision != state.coverage.revision || state.coverage.wall.isEmpty else { return } + let range = map.visibleRange + let indices = map.indices(overlapping: range) + state.coverage = CoverageStrip( + cellWidth: map.config.cellWidth, + firstCellS: map.cellRange(indices.lowerBound).lowerBound, + wall: indices.map { Self.cell(map.level(.wall, $0)) }, + ground: indices.map { Self.cell(map.level(.ground, $0)) }, + // The band the walk asks for; heights above it are still reported when seen. + wallBandHeight: map.config.wallWalkHeight, + groundBandDepth: map.config.groundBandDepth, + visibleRange: range, + revision: map.revision + ) + } + + func setEnd(_ side: WallSide, at s: Float, kind: EndKind) { + guard var map = coverage else { return } + markTimes[MarkKey.end(side)] = captureClock + map.setEnd(side == .left ? .left : .right, at: s) + // Ground past a limit end still counts toward clearances (server contract, "Ends and + // corners"); past an unexplored end it doesn't. + map.setEndIsLimit(side == .left ? .left : .right, kind == .limit) + coverage = map + endKinds[side] = kind + state.wallTooShort = false + publishWall() + publishCoverage() + RuntimeLog.engine.info("end \(side.rawValue, privacy: .public) at s=\(s) (\(kind == .limit ? "limit" : "unexplored", privacy: .public))") + if let camera = lastFrame?.camera, state.phase == .wallWalk { + updateGuidance(camera: camera, time: lastFrame?.timestamp ?? 0) + } + } + + /// What the homeowner said is at an already marked end. + func setEndKind(_ side: WallSide, _ kind: EndKind) { + endKinds[side] = kind + updateCoverage { $0.setEndIsLimit(side == .left ? .left : .right, kind == .limit) } + RuntimeLog.engine.info("end \(side.rawValue, privacy: .public) is \(kind == .limit ? "limit" : "unexplored", privacy: .public)") + } + + func clearEnd(_ side: WallSide) { + markTimes[MarkKey.end(side)] = nil + updateCoverage { $0.clearEnd(side == .left ? .left : .right) } + endKinds[side] = nil + if state.endQuestion == side { state.endQuestion = nil } + publishWall() + } + + var bothEndsMarked: Bool { coverage?.leftEnd != nil && coverage?.rightEnd != nil } + + // MARK: Gap loop + + /// After the review: ask for the planner's gap, or upload. Confirming the review starts a new + /// pass of server requests raised without a tap. + func runGapCheck() { + automaticGaps = [] + automaticGapsStopped = false + // A request needs the camera in the scan's world frame; while the phone has lost its + // place, the scan goes as it is and the server's answer lists what is still unseen. + guard !state.tracking.hasLostItsPlace else { + RuntimeLog.engine.info("gap check skipped: the phone has lost its place; uploading the scan as it is") + startUpload() + return + } + guard let map = coverage, let plan = gapPlanner.plan(map), !skippedGaps.contains(plan) else { + startUpload() + return + } + beginGap(plan, origin: .phone, reason: plan.band == .ground ? .groundNearCandidate : .wallAboveCandidate) + } + + func beginGap(_ plan: GapPlan, origin: GapRequest.Origin, reason: GapRequest.Reason) { + // A request with a reach says what to do in its own terms, whatever the caller passed. + let reason: GapRequest.Reason = switch plan.need { + case .cells: reason + case .groundOut(let out): .groundOut(out: out) + case .walkOut(let out): .walkOut(out: out) + case .overhead: .overhead + // The server's own words name the height ("seen at least 6 ft 6 in up the wall"). + case .wallUp: reason + } + gapCounter += 1 + gapPlan = plan + keyframesAtGapStart = store.keyframes.count + overheadViewsAtGapStart = coverage?.overheadCameras.count ?? 0 + let progress = coverage.map { gapPlanner.progress(of: plan, $0) } ?? 0 + state.gap = GapRequest(id: gapCounter, origin: origin, reason: reason, band: plan.band == .ground ? .ground : .wall, span: plan.span, progress: progress, isSatisfied: false) + state.guidance = .gap + go(.gapRequest) + updateGap(camera: lastFrame?.camera) + } + + /// A closed or skipped gap goes straight to the upload, which re-runs the server's checks + /// with the new evidence (the closed loop: gap, instruction, capture, updated result). The + /// answer then leads to the next capturable request or to the result (`upload`). + private func afterGapResolved() { + settleClearedEnd() + gapPlan = nil + pastEndSide = nil + pendingOverhead = nil + state.overheadQuestion = false + state.gap = nil + startUpload() + } + + /// The past_end request's end was marked again and its question answered: the request is + /// settled, so the scan goes to the upload like a closed gap. + func settlePastEnd() { + guard state.phase == .gapRequest, var request = state.gap, !request.isSatisfied else { return } + resolveGuidance(.met) + request.isSatisfied = true + request.progress = 1 + state.gap = request + RuntimeLog.engine.info("gap \(request.id) settled by marking the end again") + let id = request.id + Task { + try? await Task.sleep(for: .seconds(autoAdvanceDelay)) + await waitForGate(.gapRequest) + guard state.phase == .gapRequest, state.gap?.id == id else { return } + afterGapResolved() + } + } + + /// Keeps the tilted-up view the overhead question was about as overhead evidence. Call it + /// only once the homeowner answered that nothing is overhead (`answerOverhead(clear: true)`): + /// the camera can't tell open sky from an eave. The view's photo is stored as a keyframe + /// first, and it counts as overhead evidence only once stored (`recordOverhead`), like every + /// other view. Returns false when it can't be kept: no photo, tracking not normal, or the + /// view doesn't show the wall from the top of the sampled rows (7.5 ft, `wallCaptureHeight`) upward. + /// `segment` is the walked-path segment the view was captured in. + func keepOverheadView(_ frame: SourceFrame, capturedIn segment: Int?) -> Bool { + guard let map = coverage, frame.jpeg.isAvailable, frame.tracking == .normal, + !map.overheadReach(from: correctedCamera(frame)).isEmpty else { return false } + keep(frame, overhead: true, capturedIn: segment) + return true + } + + /// Records a stored tilted-up view in the coverage map; the export then sends each stretch it + /// reached as an overhead entry with the height seen. + private func recordOverhead(_ frame: SourceFrame) { + guard var map = coverage else { return } + let reach = map.recordOverhead(correctedCamera(frame), trackingNormal: frame.tracking == .normal) + guard !reach.isEmpty else { + RuntimeLog.engine.error("overhead: the stored view no longer reaches above the wall band; nothing recorded") + return + } + coverage = map + RuntimeLog.engine.info("overhead: kept a view reaching \(reach.map(\.out).min() ?? 0) m over s=\(reach.first?.span.lowerBound ?? 0)...\(reach.last?.span.upperBound ?? 0)") + } + + /// Ends the current request without the view it asked for ("I can't get there", or something + /// overhead): recorded for installer review, then on to the upload. "I can't get there" on a + /// server request also ends the requests raised without a tap: after that upload the result + /// shows. Something overhead is an answer, not a refusal, so the loop goes on. + func skipCurrentGap(because reason: String = "the homeowner can't get there", refused: Bool = true) { + guard let plan = gapPlan else { return } + // Something overhead is an answer: the request goes to review without its view. + resolveGuidance(refused ? .cannotReach : .skipped) + if refused, state.gap?.origin == .server { automaticGapsStopped = true } + // Only a cell request marks cells: skipping a deeper, walked or overhead view says + // nothing about the band the strip draws. + if plan.need == .cells { coverage?.markSkipped(plan.band, plan.span) } + skippedGaps.append(plan) + publishCoverage() + RuntimeLog.engine.info("gap \(self.gapCounter) left for installer review: \(reason, privacy: .public)") + afterGapResolved() + } + + // MARK: Upload + + func startUpload() { + go(.uploading) + uploadTask?.cancel() + uploadTask = Task { await upload() } + } + + private func upload() async { + let scan = generation + state.upload = .packaging + // Sending again from a failed upload stays on this phase, so `go` doesn't restart them. + updateRecording() + // Keyframe writes still in flight belong in the scene's keyframe list. + for _ in 0..<200 where (pendingSaves[scan] ?? 0) > 0 { + try? await Task.sleep(for: .milliseconds(50)) + } + guard scan == generation else { return } + var resends = 0 + while true { + // Everything this attempt sends is built from one snapshot of one frame + // (`UploadSnapshot`): scene.json, the mesh's measurements and the packet. + guard let snapshot = await takeUploadSnapshot() else { + guard scan == generation, !Task.isCancelled else { return } + RuntimeLog.engine.error("scene.json export failed: no wall") + state.upload = UploadFailure.packaging(ExportError.noWall) + updateRecording() + return + } + guard scan == generation, !Task.isCancelled else { return } + // LiDAR phones: the frame's mesh, measured for what faces the wall and what is + // overhead against the snapshot's wall, off the main actor since ray casts over a + // whole mesh take a while. Nothing the measurement or the export reads can move + // meanwhile: both read the snapshot. + var measured = MeshMeasurements() + if let mesh = snapshot.mesh?.mesh { + let wall = snapshot.map.wall + let span = Self.exportSpan(snapshot.map) + measured = await Task.detached(priority: .userInitiated) { Self.measure(mesh, wall: wall, over: span) }.value + guard scan == generation, !Task.isCancelled else { return } + RuntimeLog.engine.info("mesh: \(mesh.vertices.count) vertices, \(mesh.indices.count / 3) triangles; \(measured.facing.count) facing and \(measured.overheads.count) overhead measurements") + } + let scene: Data + do { + scene = try sceneJSON(snapshot, mesh: measured) + } catch { + RuntimeLog.engine.error("scene.json export failed: \(String(describing: error), privacy: .public)") + state.upload = UploadFailure.packaging(error) + updateRecording() + return + } + writeScanStamp(answer: placement) + saveBundle(scene: scene, snapshot: snapshot) + guard !Task.isCancelled else { return } + state.upload = .uploading(fraction: 0) + let result: PlacementResult + let data: Data + do { + data = try await resultClient.submit(scene: scene) { [weak self] fraction in + Task { @MainActor in + guard let self, scan == self.generation, case .uploading = self.state.upload else { return } + self.state.upload = fraction >= 1 ? .analyzing : .uploading(fraction: fraction) + } + } + guard scan == generation else { return } + state.upload = .analyzing + result = try PlacementResult.decode(data) + } catch is CancellationError { + return + } catch { + guard scan == generation else { return } + RuntimeLog.engine.error("upload failed: \(String(describing: error), privacy: .public)") + state.upload = UploadFailure.state(for: error) + updateRecording() + return + } + // An answer about geometry the phone has since corrected would be drawn on the + // wrong wall (`AnswerFreshness`). + switch AnswerFreshness.of(sent: snapshot.revision, now: spatialRevision, resends: resends) { + case .current: + break + case .sendAgain: + resends += 1 + RuntimeLog.engine.info("answer is stale (sent at revision \(snapshot.revision), now \(self.spatialRevision)): sending again") + state.upload = .packaging + continue + case .stillChanging: + RuntimeLog.engine.error("answer is stale again (sent at revision \(snapshot.revision), now \(self.spatialRevision)): not shown") + state.upload = UploadFailure.stillChanging + updateRecording() + return + } + placement = result + lastUploadSnapshot = snapshot + noteExchange(scene: scene, answer: data) + break + } + guard let result = placement else { return } + writeScanStamp(answer: result) + state.result = presentation(of: result, isSample: resultClient.isSample) + state.upload = .done + await waitForGate(.uploading) + guard scan == generation else { return } + if let next = nextAutomaticGap(result) { + // The upload screen says "One more view to finish" once the answer is in + // (`state.result` set, upload `.done`, both kept through the request): long + // enough to read before the camera takes over. + guard (try? await Task.sleep(for: .seconds(Self.followUpHold))) != nil, + scan == generation, state.phase == .uploading else { return } + automaticGaps.append(next.plan) + RuntimeLog.engine.info("answer lists capturable evidence: asking for it (\(self.automaticGaps.count) of at most \(Self.maxAutomaticGaps))") + beginServerGap(next.item, plan: next.plan) + return + } + // The result appears only after the server answered (checklist R6), and after the + // homeowner checked its spot (`presentAnswer`). + presentAnswer() + } + + /// The first item of the answer's missing evidence a capture can settle (the result's + /// "capturable"), not skipped and not yet raised in this pass; nil once the homeowner said + /// they can't get to one, or after `maxAutomaticGaps` requests. + private func nextAutomaticGap(_ result: PlacementResult) -> (item: PlacementMissingEvidence, plan: GapPlan)? { + // A request raised while the phone has lost its place could only time out: show the result. + guard !automaticGapsStopped, automaticGaps.count < Self.maxAutomaticGaps, !state.tracking.hasLostItsPlace, + sourceState.mayCapture, let map = coverage else { return nil } + for item in result.missingEvidence { + guard let plan = gapPlanner.plan(for: item, leftEnd: map.leftEnd, rightEnd: map.rightEnd, limitEnds: map.limitEnds), + !skippedGaps.contains(plan), !automaticGaps.contains(plan), captureCanSettle(plan) else { continue } + return (item, plan) + } + return nil + } + + /// Starts the capture for an item of the server's missing evidence, whether tapped on the + /// result or raised after an upload. + func beginServerGap(_ item: PlacementMissingEvidence, plan: GapPlan) { + var pastEnd: WallSide? + if item.kind == .pastEnd, let side = item.side { + // The walk has to go past the end it stopped at; that end is no longer a limit. It + // stays cleared until the homeowner marks it again (markWallEnd) or the request ends + // (settleClearedEnd). + let wallSide: WallSide = side == .left ? .left : .right + pastEnd = wallSide + let old = wallSide == .left ? coverage?.leftEnd : coverage?.rightEnd + clearedEnd = old.map { (s: $0, kind: endKinds[wallSide] ?? EndKind.unexplored, t: markTimes[MarkKey.end(wallSide)]) } + clearEnd(wallSide) + } + // Set first, so the guidance log records the request as a past-end one. + pastEndSide = pastEnd + beginGap(plan, origin: .server, reason: .server(detail: item.message)) + } + + /// A past_end request ending without its end marked again (the gap screen offers only "I + /// can't get there") must not leave that side without an end: the export would run the wall + /// out to whatever the fog saw, and the next past_end request would be planned from the + /// meter (issue #35). Met, the end moves on past the ground the request showed, still + /// unexplored (`GapPlanner.endAfterPastEnd`); skipped, the end it cleared comes back with its + /// kind and mark time. + private func settleClearedEnd() { + guard let side = pastEndSide, let old = clearedEnd, let plan = gapPlan else { return } + clearedEnd = nil + guard (side == .left ? coverage?.leftEnd : coverage?.rightEnd) == nil else { return } + if state.gap?.isSatisfied == true { + setEnd(side, at: gapPlanner.endAfterPastEnd(plan, side: side == .left ? .left : .right, clearedAt: old.s), kind: .unexplored) + } else { + setEnd(side, at: old.s, kind: old.kind) + markTimes[MarkKey.end(side)] = old.t + } + } + + /// Writes the capture packet (`ScanEngine+Packet.swift`) with this scene.json inside, zipped + /// into the scan folder's `scan.zip`: the bundle "Share scan" offers (`state.shareableScan`) + /// whatever the upload then does: it fails, is refused or answers. Nothing uploads the + /// packet, and the upload never waits for it or fails because of it. The zip is rewritten in + /// place, so it is not offered while a write is under way, and writes run one after another: + /// a retry's write waits for the last one, and a write already superseded is skipped. + func saveBundle(scene: Data, snapshot: UploadSnapshot) { + state.shareableScan = nil + bundleSerial += 1 + let serial = bundleSerial + let scan = generation + let previous = bundleTask + guard let inputs = packetInputs(scene: scene, snapshot: snapshot) else { + RuntimeLog.engine.error("scan bundle not written: no wall") + return + } + bundleTask = Task { + await previous?.value + guard scan == generation, serial == bundleSerial else { return } + do { + let written = try await Task.detached(priority: .userInitiated) { try Self.writePacket(inputs) }.value + RuntimeLog.engine.info("bundle \(written.url.path, privacy: .public): packet \(PacketManifest.version, privacy: .public) with \(written.summary, privacy: .public) (kept on the phone)") + guard scan == generation, serial == bundleSerial else { return } + state.shareableScan = written.url + } catch { + RuntimeLog.engine.error("scan bundle not written: \(String(describing: error), privacy: .public)") + } + } + } + + // MARK: Result for AR + + /// The replay frame that sees the chosen spot best, for the AR result on a replay. + private func bestFrameForResult() -> Int? { + guard let replay, let wall = coverage?.wall else { return nil } + let spot = state.result?.spot + let s = spot.map { ($0.span.lowerBound + $0.span.upperBound) / 2 } ?? 0 + let point = wall.world(s: s, height: 0.5, out: 0.3) + return replay.bestFrame(showing: point) + } + + // MARK: Start over + + func resetAll() { + generation += 1 + spatialRevision += 1 + uploadTask?.cancel() + cancelPendingSnapshot() + lastUploadSnapshot = nil + replay?.stop() + coverage = nil + meterAnchorID.map { live?.removeAnchor($0) } + meterAnchorID = nil + meterPlaneSource = .detectedPlane + store = KeyframeStore() + recorder = Self.makeRecorder(store) + live?.setRecorder(recorder) + motion.stop() + resetPacketLog() + keptSourceIDs = [] + autoCapture.reset() + planner.reset() + closeUpGate = CloseUpGate() + gapPlan = nil + skippedGaps = [] + pastEndSide = nil + clearedEnd = nil + automaticGaps = [] + automaticGapsStopped = false + seeBehindBands = [] + endKinds = [:] + state.endQuestion = nil + state.wallTooShort = false + nextWallSide = nil + nextWallRefusal = nil + resetTiltUp() + resetSpotChecks() + placement = nil + // The bundle belongs to the scan being thrown away; `generation` stops a write in flight + // from offering it again. + state.shareableScan = nil + state.wall = nil + state.coverage = .empty + state.features = [] + state.marking = nil + state.gap = nil + state.result = nil + state.upload = .idle + state.captureCount = 0 + state.lastCapture = nil + state.target = nil + state.path = [] + state.coaching = nil + state.guidance = .findMeter + state.closeUp = .aiming(hold: 0, problem: nil) + state.closeUpFailedAttempts = 0 + state.meterNumber = nil + closeUpRetake = nil + meterReadout = nil + go(.onboarding) + replay?.show(index: 0) + } + + /// The overhead views belong to the world frame and the scan they were seen in. + private func resetTiltUp() { + tiltUpSettled = false + pendingOverhead = nil + state.overheadQuestion = false + } + + // MARK: Internal accessors for marking and export + + var currentFrame: SourceFrame? { lastFrame } + var liveCapture: LiveCapture? { live } + var wallEndKinds: [WallSide: EndKind] { endKinds } + + /// The meter's anchor, and the pose the wall set with it agrees with. + func setMeterAnchor(_ id: UUID?, pose: simd_float4x4?) { + meterAnchorID = id + meterTracking = pose.map(MeterAnchorTracking.init) + } + + var detectedGroundPlanes: [GroundPlaneEvidence] { groundPlanes } + + /// The camera failed after the scan was sent: no frame will come to say tracking was lost, so + /// the last one's "normal" would keep the AR result up and offered. The answer stays; the + /// spatial result goes. + private func loseSpatialResult() { + state.tracking = .notAvailable + state.spatialResultAvailable = false + hideResultInCamera() + if state.phase == .resultAR { go(.result) } + } + + /// Whether a capture (a server request's view) can start: only with a running source. + var mayCapture: Bool { sourceState.mayCapture } + + /// Start over after a failure: the failed source is let go and the failure cleared, so the + /// next scan starts a new session (`startSourceIfNeeded`) or reads the replay again. A + /// device that can't run world tracking stays failed. + func releaseFailedSource() { + let discard = sourceState.startOver() + if sourceState.failure == nil { state.failure = nil } + guard discard else { return } + live?.pause() + live = nil + state.feed = .none + // What the retired source saw is in its own world frame; the next source starts another. + groundPlanes = [] + groundMeasured = false + lastFrame = nil + meterTracking = nil + meterAnchorID = nil + state.projection = nil + state.tracking = .notAvailable + if replay == nil, let folder = options.replayFolder { Task { await loadReplay(folder) } } + } + + func updateCoverage(_ body: (inout CoverageMap) -> Void) { + guard var map = coverage else { return } + body(&map) + coverage = map + publishCoverage() + } + + // MARK: Packet log + + /// Forgets the guidance log, the mark times and the clock: they belong to the packet session + /// being thrown away. + private func resetPacketLog() { + guidanceLog = GuidanceLog() + markTimes = [:] + captureClock = nil + } + + /// Records what request is on screen now in the guidance log. + func noteGuidance() { + publishEndPreview() + guard let t = captureClock else { return } + guidanceLog.show(guidanceRequest(), at: t) { old, next in closingOutcome(old, next: next) } + } + + /// Closes the open request with an outcome an action settled. + func resolveGuidance(_ outcome: GuidanceLog.Outcome) { + guard let t = captureClock else { return } + guidanceLog.resolve(outcome, at: t) + } +} + +// MARK: - Mapping to contract values + +extension ScanEngine { + static func cell(_ level: CoverageLevel) -> CellState { + switch level { + case .unseen: .unseen + case .seen: .seen + case .covered: .covered + case .skipped: .skipped + case .hidden: .hidden + } + } + + static func step(_ task: GuidanceTask) -> GuidanceStep { + switch task { + case .walk(let side): .walk(side: side == .left ? .left : .right, remaining: nil) + case .markEnd(let side): .markEnd(side: side == .left ? .left : .right) + case .aimAtGround(let s): .aimAtGround(s: s) + case .aimAtWall(let s): .aimAtWall(s: s) + case .stepBack: .stepBack + case .seeBehind(let s): .seeBehind(s: s) + case .complete: .walkComplete + } + } + + static func name(_ tracking: TrackingQuality) -> String { + switch tracking { + case .notAvailable: "notAvailable" + case .normal: "normal" + case .limited(let reason): "limited.\(reason)" + } + } + + static func name(_ step: GuidanceStep) -> String { + switch step { + case .findMeter: "findMeter" + case .aimAtWallForMeter: "aimAtWallForMeter" + case .holdOnMeter: "holdOnMeter" + case .walk(let side, _): "walk.\(side.rawValue)" + case .markEnd(let side): "markEnd.\(side.rawValue)" + case .aimAtGround: "aimAtGround" + case .aimAtWall: "aimAtWall" + case .stepBack: "stepBack" + case .walkComplete: "walkComplete" + case .tiltUp: "tiltUp" + case .markNextWall(let side, _): "markNextWall.\(side.rawValue)" + case .gap: "gap" + case .seeBehind: "seeBehind" + } + } + + static func problem(_ issue: CloseUpIssue) -> CloseUpProblem { + switch issue { + case .blurry: .blurry + case .tooDark: .tooDark + case .tooBright: .tooBright + case .notCentered: .meterNotCentered + case .tooFar: .tooFar + case .tracking: .tracking + } + } + + /// Tracking problems first (they block everything), then the capture gate's reason. + func coaching(for tracking: TrackingQuality, skip: CaptureDecision.SkipReason?) -> Coaching? { + switch tracking { + case .notAvailable: return .trackingLost + case .limited(.initializing): return .initializing + case .limited(.excessiveMotion): return .slowDown + case .limited(.insufficientFeatures): return .needsTexture + case .limited(.relocalizing): return .relocalizing + case .limited(.unknown): return .trackingLost + case .normal: + switch skip { + case .moving?, .blurry?: return .holdSteady + case .tooDark?: return .tooDark + default: return nil + } + } + } +} diff --git a/ios/HouseScan/Runtime/SourceFrame.swift b/ios/HouseScan/Runtime/SourceFrame.swift new file mode 100644 index 000000000..257b4128d --- /dev/null +++ b/ios/HouseScan/Runtime/SourceFrame.swift @@ -0,0 +1,154 @@ +import CoreGraphics +import Foundation +import HouseScanKit +import ImageIO +import simd + +/// One camera frame from the live session or a replay, in the form the engine consumes. +struct SourceFrame: Sendable { + /// Replay keyframe id, or "live-". + var id: String + /// Seconds on the frame clock (ARFrame.timestamp, or the replay's timestamps). + var timestamp: Double + var camera: CameraFrame + var tracking: TrackingQuality + /// Measured on some frames only; auto-capture reuses the last measurement in between. + var quality: FrameQuality? + /// Where the full-resolution JPEG comes from if the frame is kept. + var jpeg: JPEGPayload + /// The replay image, drawn as the feed. Nil for live frames. + var still: CGImage? + /// World transform of the meter's ARAnchor in this frame, when one exists. + var meterAnchor: simd_float4x4? + /// Detected horizontal planes, with their classes and outlines, on a frame that reports them: + /// empty when ARKit has found none (or removed the ones it had). Nil on a frame that doesn't + /// report planes (a pose-only frame, a replay), which says nothing about the ground. + var groundPlanes: [GroundPlaneEvidence]? = nil + /// LiDAR depth copied with the photo of a frame that could be kept; nil without LiDAR, and on + /// frames without a photo. Coverage reads it; a replay's recorded depth plays in here too. + var depth: DepthImage? + /// The same ARKit depth map as float32 meters with its confidence, for the packet. Live frames + /// with a photo only: a replay's depth is not ARKit's own, so it never goes into the packet. + var sensorDepth: DepthPacket? + /// Camera settings of a live frame with a photo; nil on a replay. + var exposure: PhotoExposure? + /// Live frames only, when the engine asked (`LiveCapture.requestSpatialCapture`): this frame's + /// own mesh and planes, for the upload's snapshot. + var spatial: FrameSpatialCapture? + /// Shown for review or tapping only; never offered to auto-capture. + var isReview = false + /// Carries only pose and tracking, so overlays follow the camera between sampled frames. + var isPoseOnly = false + + var projection: CameraProjection { + CameraProjection(cameraToWorld: camera.cameraToWorld, intrinsics: camera.intrinsics, imageSize: camera.imageSize) + } + + var captureTracking: TrackingStatus { + switch tracking { + case .normal: .normal + case .limited: .limited + case .notAvailable: .notAvailable + } + } +} + +/// The camera settings a photo was taken with: `ARCamera.exposureDuration` and +/// `exposureOffset`, and ISO, focal length and f-number from `ARFrame.exifData` when it has them. +struct PhotoExposure: Sendable, Equatable { + var durationS: Double? + var offsetEV: Double? + var iso: Double? + var focalLengthMM: Double? + var fNumber: Double? +} + +enum JPEGPayload: Sendable { + /// A JPEG already on disk (replay keyframes). + case file(URL) + /// A JPEG encoded on LiveCapture's encode queue, off the AR delegate queue. + case data(Data) + /// Not encoded; the frame can't be kept. + case none + + var isAvailable: Bool { + if case .none = self { return false } + return true + } +} + +/// Image helpers that run off the main actor. +enum ImageWork { + /// Decodes a JPEG and measures its quality from a quarter-size grayscale copy. + static func decode(_ url: URL) -> (CGImage, FrameQuality)? { + guard let source = CGImageSourceCreateWithURL(url as CFURL, nil), + let image = CGImageSourceCreateImageAtIndex(source, 0, nil) else { return nil } + return (image, quality(of: image)) + } + + static func quality(of image: CGImage) -> FrameQuality { + let width = max(8, image.width / 4) + let height = max(8, image.height / 4) + var pixels = [UInt8](repeating: 0, count: width * height) + pixels.withUnsafeMutableBytes { buffer in + guard let context = CGContext( + data: buffer.baseAddress, width: width, height: height, bitsPerComponent: 8, + bytesPerRow: width, space: CGColorSpaceCreateDeviceGray(), bitmapInfo: CGImageAlphaInfo.none.rawValue + ) else { return } + context.interpolationQuality = .medium + context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height)) + } + return FrameQuality(LumaImage(width: width, height: height, pixels: pixels)) + } + + /// The JPEG's luma as Pillow's `convert("L")` computes it (`PacketSharpness.luma`), row-major, + /// at the JPEG's full size: the input of the packet's sharpness. + /// The image is drawn into an RGB context in its own color space, so no color conversion + /// changes the decoded values. Nil when it does not decode. + static func luma(jpeg: Data) -> (pixels: [UInt8], width: Int, height: Int)? { + guard let source = CGImageSourceCreateWithData(jpeg as CFData, nil), + let image = CGImageSourceCreateImageAtIndex(source, 0, nil) else { return nil } + let width = image.width + let height = image.height + let space = image.colorSpace.flatMap { $0.model == .rgb ? $0 : nil } ?? CGColorSpace(name: CGColorSpace.sRGB) ?? CGColorSpaceCreateDeviceRGB() + var rgbx = [UInt8](repeating: 0, count: width * height * 4) + let drawn = rgbx.withUnsafeMutableBytes { buffer -> Bool in + guard let context = CGContext( + data: buffer.baseAddress, width: width, height: height, bitsPerComponent: 8, + bytesPerRow: width * 4, space: space, bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue + ) else { return false } + context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height)) + return true + } + guard drawn else { return nil } + let luma = rgbx.withUnsafeBytes { PacketSharpness.luma(rgbx: $0, width: width, height: height, bytesPerRow: width * 4) } + return (luma, width, height) + } + + /// A small upright thumbnail: the landscape sensor image rotated 90° clockwise, as the + /// screen shows it. + static func uprightThumbnail(jpeg: Data, maxPixels: Int = 240) -> CGImage? { + guard let source = CGImageSourceCreateWithData(jpeg as CFData, nil) else { return nil } + let options: [CFString: Any] = [ + kCGImageSourceCreateThumbnailFromImageAlways: true, + kCGImageSourceThumbnailMaxPixelSize: maxPixels, + ] + guard let thumb = CGImageSourceCreateThumbnailAtIndex(source, 0, options as CFDictionary) else { return nil } + return rotatedClockwise(thumb) + } + + static func rotatedClockwise(_ image: CGImage) -> CGImage? { + let width = image.height + let height = image.width + guard let context = CGContext( + data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 0, + space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) else { return nil } + // Core Graphics has y up: rotating the content by -90° about the origin and shifting it up + // by the new height turns the image clockwise as seen on screen. + context.translateBy(x: 0, y: CGFloat(height)) + context.rotate(by: -.pi / 2) + context.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height)) + return context.makeImage() + } +} diff --git a/ios/HouseScan/SessionDelegate.swift b/ios/HouseScan/SessionDelegate.swift deleted file mode 100644 index 00ca7866e..000000000 --- a/ios/HouseScan/SessionDelegate.swift +++ /dev/null @@ -1,46 +0,0 @@ -import ARKit - -/// Receives ARSession callbacks and forwards them to `CaptureSessionModel`. -/// -/// ARKit calls these methods on the session's `delegateQueue`, which `ARCaptureView` sets to a -/// private serial queue so per-frame work added later stays off the main thread. Each callback -/// copies what it needs into Sendable values on that queue, then hops to the main actor with -/// `Task { @MainActor in ... }`. The model is main-actor isolated and therefore Sendable, so -/// no `@unchecked Sendable` or `nonisolated(unsafe)` is needed. ARKit's own objects stay on the -/// delegate queue. -final class SessionDelegate: NSObject, ARSessionDelegate { - private let model: CaptureSessionModel - - init(model: CaptureSessionModel) { - self.model = model - } - - func session(_ session: ARSession, cameraDidChangeTrackingState camera: ARCamera) { - let state = TrackingState(camera.trackingState) - Task { @MainActor [model] in model.trackingChanged(to: state) } - } - - func session(_ session: ARSession, didAdd anchors: [ARAnchor]) { - let planes = anchors.compactMap(DetectedPlane.init) - guard !planes.isEmpty else { return } - Task { @MainActor [model] in model.planesAdded(planes) } - } - - func session(_ session: ARSession, didRemove anchors: [ARAnchor]) { - let planes = anchors.compactMap(DetectedPlane.init) - guard !planes.isEmpty else { return } - Task { @MainActor [model] in model.planesRemoved(planes) } - } - - func sessionWasInterrupted(_ session: ARSession) { - Task { @MainActor [model] in model.interruptionChanged(isInterrupted: true) } - } - - func sessionInterruptionEnded(_ session: ARSession) { - Task { @MainActor [model] in model.interruptionChanged(isInterrupted: false) } - } - - func session(_ session: ARSession, didFailWithError error: any Error) { - Task { @MainActor [model] in model.sessionFailed(error) } - } -} diff --git a/ios/HouseScan/UI/Camera/CameraBackdrop.swift b/ios/HouseScan/UI/Camera/CameraBackdrop.swift new file mode 100644 index 000000000..25adbfc2f --- /dev/null +++ b/ios/HouseScan/UI/Camera/CameraBackdrop.swift @@ -0,0 +1,34 @@ +import SwiftUI + +/// What sits behind every overlay: the live camera, a replayed still, or a dark placeholder. +/// +/// A still is the unrotated landscape sensor image. It is shown rotated 90° clockwise and +/// scaled to fill, centered and cropped, which is exactly what `CameraProjection.viewPoint` +/// assumes, so overlays drawn from the projection line up with the picture. +struct CameraBackdrop: View { + var feed: CameraFeed + var actions: any ScanActions + + var body: some View { + GeometryReader { proxy in + switch feed { + case .live: + actions.liveCameraView() + .frame(width: proxy.size.width, height: proxy.size.height) + case .still(let image): + Image(decorative: image, scale: 1, orientation: .right) + .resizable() + .aspectRatio(contentMode: .fill) + .frame(width: proxy.size.width, height: proxy.size.height) + .clipped() + case .none: + LinearGradient( + colors: [Color(white: 0.16), Color(white: 0.05)], + startPoint: .top, endPoint: .bottom + ) + } + } + .ignoresSafeArea() + .accessibilityHidden(true) + } +} diff --git a/ios/HouseScan/UI/Camera/CameraOverlays.swift b/ios/HouseScan/UI/Camera/CameraOverlays.swift new file mode 100644 index 000000000..da1916cee --- /dev/null +++ b/ios/HouseScan/UI/Camera/CameraOverlays.swift @@ -0,0 +1,37 @@ +import SwiftUI + +/// The overlays every walking screen draws over the camera, in paint order. +struct CameraOverlays: View { + let state: ScanViewState + var highlight: GapRequest? + + var body: some View { + ZStack { + overlays + } + .animation(.easeOut(duration: 0.2), value: state.tracking == .normal) + } + + @ViewBuilder + private var overlays: some View { + // Hidden while tracking isn't normal: drawn from a pose the phone doesn't trust, the haze, + // path and pins would sit in the wrong place (checklist T3). They fade back on recovery. + if state.tracking == .normal, let projection = state.projection, let wall = state.wall { + ZStack { + FogOverlay(coverage: state.coverage, wall: wall, projection: projection, highlight: highlight) + .ignoresSafeArea() + WallMarksOverlay( + projection: projection, + wall: wall, + features: state.features, + wallBandHeight: state.coverage.wallBandHeight + ) + if state.marking == nil { + // While marking, the reticle is the only aim; the path and ring would compete. + WayfindingOverlay(projection: projection, wall: wall, path: state.path, target: state.target) + .transition(.opacity) + } + } + } + } +} diff --git a/ios/HouseScan/UI/Camera/FogOverlay.swift b/ios/HouseScan/UI/Camera/FogOverlay.swift new file mode 100644 index 000000000..6e2ac6800 --- /dev/null +++ b/ios/HouseScan/UI/Camera/FogOverlay.swift @@ -0,0 +1,334 @@ +import SwiftUI +import simd + +/// The haze over the parts of the wall and ground the phone hasn't seen yet. +/// +/// Unseen cells carry a soft frosted haze, seen cells a thinner one, covered cells none. When a +/// cell's state changes (a new `coverage.revision`), its haze fades and drifts upward over +/// `Motion.fogLift`, like mist lifting: the signature moment that tells the homeowner "the phone +/// saw that" without a word. With Reduce Motion the haze only fades. The haze is guidance, never +/// proof that space is clear. +/// +/// A gap request replaces the fog on its cells with an amber highlight, so the camera itself +/// shows where to aim. +/// +/// On a phone with depth, a stretch the camera looked at through something nearer (a bush, a +/// bin) is hidden: no frost, since the phone did look, but a soft dark veil with a dashed violet +/// outline round the whole stretch. It fades in without rising: the rising drift belongs to +/// "the phone saw that", which a hidden stretch is not. +struct FogOverlay: View { + var coverage: CoverageStrip + var wall: WallGeometry + var projection: CameraProjection + var highlight: GapRequest? + + @State private var memory = FogMemory() + + @State private var liftDeadline = Date.distantPast + @Environment(\.accessibilityReduceMotion) private var reduceMotion + + var body: some View { + TimelineView(.animation(paused: liftDeadline < .now)) { timeline in + let frame = FogFrame( + coverage: coverage, wall: wall, projection: projection, highlight: highlight, + lifts: memory.lifts, now: timeline.date, drift: reduceMotion ? 0 : 10 + ) + ZStack { + // The frost itself: a material blurs whatever the camera shows, so the haze + // reads on a white wall in sun as well as on dark brick. The canvas only + // decides where, and how thick. + Rectangle() + .fill(.thinMaterial) + .environment(\.colorScheme, .light) + .mask { + Canvas(rendersAsynchronously: false) { context, size in + frame.drawHaze(in: &context, size: size, color: .black) + } + } + Canvas(rendersAsynchronously: false) { context, size in + frame.drawHaze(in: &context, size: size, color: Color(white: 0.98).opacity(0.35)) + frame.drawMarks(in: &context, size: size) + } + } + } + .allowsHitTesting(false) + .accessibilityHidden(true) + .onAppear { memory.seed(coverage) } + .onChange(of: coverage.revision) { _, _ in + if memory.update(to: coverage, at: .now, highlight: highlight) { + liftDeadline = Date.now.addingTimeInterval(Motion.fogLift) + } + } + .task(id: liftDeadline) { + // Ends the timeline once the last lift has finished, so a still replay frame + // stops redrawing. + let remaining = liftDeadline.timeIntervalSinceNow + guard remaining > 0 else { return } + // A newer revision replaces this task; it must not pause the timeline for the lifts + // that revision started, so cancellation ends it here. + do { try await Task.sleep(for: .seconds(remaining + 0.05)) } catch { return } + memory.prune(at: .now) + liftDeadline = .distantPast + } + } + + // MARK: Drawing + + /// Haze strength per state. Hypothesis, picked by eye on the demo wall and the synthetic + /// replay: unseen must read as "not done" over a bright wall; seen must be visibly lighter + /// than unseen but still clearly not clear. Full strength (with the regular material and a + /// 50% wash) whited out the wall the homeowner is walking toward, so unseen stays below it + /// and the wall's shapes show through. Tune on device in sun. + nonisolated static func haze(_ state: CellState) -> Double { + switch state { + case .unseen: 0.75 + case .seen: 0.35 + // Hidden is drawn by `veil`: frost would say "not looked at yet", and it was. + case .covered, .skipped, .hidden: 0 + } + } + + /// Strength of the dark veil and outline per state: only a hidden stretch has one. + nonisolated static func veil(_ state: CellState) -> Double { + state == .hidden ? 1 : 0 + } + +} + +/// One frame of fog: which cells are hazy, lifting, veiled, skipped or requested, at a moment in time. +private struct FogFrame { + var coverage: CoverageStrip + var wall: WallGeometry + var projection: CameraProjection + var highlight: GapRequest? + var lifts: [FogMemory.Key: FogMemory.Lift] + var now: Date + /// Points a lifting cell rises; zero with Reduce Motion, which keeps only the fade. + var drift: CGFloat + + private struct Cell { + var quad: Path + var level: Double + var lift: CGFloat + } + + private struct Layers { + var haze: [Cell] = [] + var skipped: [Path] = [] + var amber: [Path] = [] + /// Per-cell veil fills, blurred together so a stretch reads as one shadow. + var veil: [(quad: Path, level: Double)] = [] + /// One outline per run of hidden cells in a band, so the dash runs on unbroken. + var outlines: [(path: Path, level: Double)] = [] + } + + private func layers(size: CGSize) -> Layers { + let geometry = WallProjection(projection: projection, wall: wall, size: size) + let visible = coverage.visibleRange + var layers = Layers() + guard visible.upperBound > visible.lowerBound else { return layers } + for band in CoverageBand.allCases { + let states = coverage.cells(band) + // The hidden run being collected: its clipped s edges and its strongest veil. + var run: (edges: [Float], level: Double)? + func closeRun() { + if let edges = run?.edges, let level = run?.level, let outline = outline(band, edges, geometry) { + layers.outlines.append((outline, level)) + } + run = nil + } + for index in states.indices { + let range = coverage.cellRange(index) + guard FogMemory.interiorsOverlap(range, visible) else { closeRun(); continue } + let clipped = max(range.lowerBound, visible.lowerBound)...min(range.upperBound, visible.upperBound) + guard let quad = quad(band, clipped, geometry) else { closeRun(); continue } + let state = states[index] + if let gap = highlight, gap.band == band, FogMemory.interiorsOverlap(gap.span, range), state != .covered { + layers.amber.append(quad) + closeRun() + continue + } + if state == .skipped { layers.skipped.append(quad) } + let hazeTarget = FogOverlay.haze(state) + let veilTarget = FogOverlay.veil(state) + var haze = hazeTarget, veil = veilTarget, rise: CGFloat = 0 + if let lift = lifts[FogMemory.Key(band: band, index: index)], let progress = lift.progress(at: now), progress < 1 { + let eased = 1 - pow(1 - progress, 3) + haze = lift.from + (hazeTarget - lift.from) * eased + veil = lift.fromVeil + (veilTarget - lift.fromVeil) * eased + rise = lift.drifts ? CGFloat(eased) : 0 + } + if haze > 0 { layers.haze.append(Cell(quad: quad, level: haze, lift: rise)) } + if veil > 0 { layers.veil.append((quad, veil)) } + if state == .hidden { + var current = run ?? ([clipped.lowerBound], 0) + current.edges.append(clipped.upperBound) + current.level = max(current.level, veil) + run = current + } else { + closeRun() + } + } + closeRun() + } + return layers + } + + /// Haze cells in one blurred layer, so neighbours merge into one bank of fog; lifting cells + /// rise a few points as they thin out. + func drawHaze(in context: inout GraphicsContext, size: CGSize, color: Color) { + let haze = layers(size: size).haze + context.drawLayer { layer in + layer.addFilter(.blur(radius: 9)) + for cell in haze { + var shifted = layer + shifted.translateBy(x: 0, y: -drift * cell.lift) + shifted.fill(cell.quad, with: .color(color.opacity(cell.level))) + } + } + } + + /// Skipped cells get a slate hatch, so "can't get there" never reads as clear or as fog; + /// hidden stretches get a dark veil and a dashed violet outline; requested cells glow amber. + func drawMarks(in context: inout GraphicsContext, size: CGSize) { + let layers = layers(size: size) + if !layers.veil.isEmpty { + context.drawLayer { layer in + layer.addFilter(.blur(radius: 4)) + for cell in layers.veil { + layer.fill(cell.quad, with: .color(.black.opacity(0.42 * cell.level))) + } + } + } + // The map's dash, lengthened for the larger shapes on the camera. + let dash = StrokeStyle(lineWidth: 2, lineCap: .round, lineJoin: .round, dash: Palette.hiddenDash.map { $0 * 1.6 }) + for outline in layers.outlines { + // A dark halo under the dash keeps it legible on a sunlit wall. + context.stroke(outline.path, with: .color(.black.opacity(0.35 * outline.level)), style: StrokeStyle(lineWidth: 4, lineJoin: .round)) + context.stroke(outline.path, with: .color(Palette.hidden.opacity(outline.level)), style: dash) + } + for quad in layers.skipped { + context.fill(quad, with: .color(Palette.skipped.opacity(0.3))) + var hatched = context + hatched.clip(to: quad) + let bounds = quad.boundingRect + var lines = Path() + var x = bounds.minX - bounds.height + while x < bounds.maxX { + lines.move(to: CGPoint(x: x, y: bounds.maxY)) + lines.addLine(to: CGPoint(x: x + bounds.height, y: bounds.minY)) + x += 10 + } + hatched.stroke(lines, with: .color(.white.opacity(0.55)), lineWidth: 1.5) + } + for quad in layers.amber { + context.fill(quad, with: .color(Palette.caution.opacity(0.34))) + context.stroke(quad, with: .color(Palette.caution.opacity(0.9)), lineWidth: 1.5) + } + } + + /// The outline of a run of cells whose s edges are `edges`, through every cell edge so it + /// follows the wall round a corner. Nil when any point is behind the camera. + private func outline(_ band: CoverageBand, _ edges: [Float], _ geometry: WallProjection) -> Path? { + let corners: [SIMD3] = switch band { + case .wall: + edges.map { wall.world(s: $0, height: coverage.wallBandHeight) } + edges.reversed().map { wall.world(s: $0, height: 0) } + case .ground: + edges.map { wall.world(s: $0, height: 0, out: coverage.groundBandDepth) } + edges.reversed().map { wall.world(s: $0, height: 0) } + } + return geometry.polygon(corners) + } + + private func quad(_ band: CoverageBand, _ range: ClosedRange, _ geometry: WallProjection) -> Path? { + switch band { + case .wall: + geometry.wallQuad(s: range, height: 0...coverage.wallBandHeight) + case .ground: + geometry.groundQuad(s: range, out: 0...coverage.groundBandDepth) + } + } +} + +/// Remembers the last cell states so a new revision can be diffed into lifts. +@MainActor +final class FogMemory { + struct Key: Hashable { + var band: CoverageBand + var index: Int + } + + struct Lift { + /// Haze and veil strength when the change began. + var from: Double + var fromVeil: Double + var start: Date + /// Rises as it thins: only fog lifting off something the phone saw. + var drifts: Bool + + func progress(at now: Date) -> Double? { + let t = now.timeIntervalSince(start) / Motion.fogLift + return t < 0 ? 0 : t + } + } + + private var firstCellS: Float = 0 + private var states: [Key: CellState] = [:] + private(set) var lifts: [Key: Lift] = [:] + + func seed(_ coverage: CoverageStrip) { + firstCellS = coverage.firstCellS + states = Self.snapshot(coverage) + } + + /// Records a lift for every cell whose haze went down or whose veil changed. Returns true + /// when any lift started. + /// Cells inside `highlight` are drawn amber, not fogged, so they get no lift. + func update(to coverage: CoverageStrip, at now: Date, highlight: GapRequest?) -> Bool { + // The strip can grow to the left, which shifts every index; realign by s. + let shift = Int(((firstCellS - coverage.firstCellS) / coverage.cellWidth).rounded()) + let next = Self.snapshot(coverage) + if shift != 0 { + lifts = Dictionary(uniqueKeysWithValues: lifts.map { (Key(band: $0.key.band, index: $0.key.index + shift), $0.value) }) + } + var started = false + for (key, state) in next { + let oldKey = Key(band: key.band, index: key.index - shift) + let old = states[oldKey] ?? .unseen + if let highlight, highlight.band == key.band, + Self.interiorsOverlap(highlight.span, coverage.cellRange(key.index)) { + continue + } + let thinned = FogOverlay.haze(state) < FogOverlay.haze(old) + if thinned || FogOverlay.veil(state) != FogOverlay.veil(old) { + lifts[key] = Lift( + from: FogOverlay.haze(old), fromVeil: FogOverlay.veil(old), start: now, + drifts: thinned && state != .hidden + ) + started = true + } + } + states = next + firstCellS = coverage.firstCellS + return started + } + + /// True when the ranges share more than an endpoint. `ClosedRange.overlaps` is also true + /// for ranges that only touch, which would tint the cells either side of a gap. + nonisolated static func interiorsOverlap(_ a: ClosedRange, _ b: ClosedRange) -> Bool { + a.lowerBound < b.upperBound && b.lowerBound < a.upperBound + } + + func prune(at now: Date) { + lifts = lifts.filter { ($0.value.progress(at: now) ?? 1) < 1 } + } + + private static func snapshot(_ coverage: CoverageStrip) -> [Key: CellState] { + var result: [Key: CellState] = [:] + for band in CoverageBand.allCases { + for (index, state) in coverage.cells(band).enumerated() { + result[Key(band: band, index: index)] = state + } + } + return result + } +} diff --git a/ios/HouseScan/UI/Camera/WallMarksOverlay.swift b/ios/HouseScan/UI/Camera/WallMarksOverlay.swift new file mode 100644 index 000000000..00f4ccd49 --- /dev/null +++ b/ios/HouseScan/UI/Camera/WallMarksOverlay.swift @@ -0,0 +1,71 @@ +import SwiftUI + +/// What the homeowner has marked, pinned onto the camera image: the meter, the wall ends and +/// the marked features (openings as outlines, everything else as pins or lines). +struct WallMarksOverlay: View { + var projection: CameraProjection + var wall: WallGeometry + var features: [MarkedFeature] + var wallBandHeight: Float + + var body: some View { + Canvas { context, size in + let geometry = WallProjection(projection: projection, wall: wall, size: size) + drawEnds(in: &context, geometry) + for feature in features { + drawFeature(feature, in: &context, geometry) + } + drawMeter(in: &context, geometry) + } + .ignoresSafeArea() + .allowsHitTesting(false) + .accessibilityHidden(true) + } + + private func drawMeter(in context: inout GraphicsContext, _ geometry: WallProjection) { + guard let point = geometry.point(wall.meter) else { return } + let outer = CGRect(x: point.x - 13, y: point.y - 13, width: 26, height: 26) + context.fill(Path(ellipseIn: outer), with: .color(.white)) + context.fill(Path(ellipseIn: outer.insetBy(dx: 4, dy: 4)), with: .color(Palette.signal)) + context.stroke(Path(ellipseIn: outer.insetBy(dx: -5, dy: -5)), with: .color(.white.opacity(0.6)), lineWidth: 2) + } + + private func drawEnds(in context: inout GraphicsContext, _ geometry: WallProjection) { + for end in [wall.leftEnd, wall.rightEnd].compactMap(\.self) { + guard let bottom = geometry.point(s: end, height: 0), let top = geometry.point(s: end, height: wallBandHeight) else { continue } + var line = Path() + line.move(to: bottom) + line.addLine(to: top) + context.stroke(line, with: .color(.black.opacity(0.35)), style: StrokeStyle(lineWidth: 7, lineCap: .round)) + context.stroke(line, with: .color(.white), style: StrokeStyle(lineWidth: 4, lineCap: .round)) + let cap = CGRect(x: bottom.x - 7, y: bottom.y - 7, width: 14, height: 14) + context.fill(Path(ellipseIn: cap), with: .color(.white)) + } + } + + private func drawFeature(_ feature: MarkedFeature, in context: inout GraphicsContext, _ geometry: WallProjection) { + let color = Color.white + switch feature.kind { + case .door, .window, .gasMeter: + if let bottom = feature.bottom, let top = feature.top, + let outline = geometry.wallQuad(s: feature.span, height: bottom...top, out: 0.01) { + context.fill(outline, with: .color(Palette.signal.opacity(0.18))) + context.stroke(outline, with: .color(color), style: StrokeStyle(lineWidth: 3, lineJoin: .round, dash: [9, 6])) + } + case .driveway, .fence, .acUnit: + let points = feature.points.compactMap(geometry.point) + if points.count >= 2 { + var line = Path() + line.addLines(points) + context.stroke(line, with: .color(.black.opacity(0.3)), style: StrokeStyle(lineWidth: 8, lineCap: .round)) + context.stroke(line, with: .color(color), style: StrokeStyle(lineWidth: 4, lineCap: .round, dash: [10, 7])) + } + } + for point in feature.points.compactMap(geometry.point) { + let dot = CGRect(x: point.x - 8, y: point.y - 8, width: 16, height: 16) + context.fill(Path(ellipseIn: dot.insetBy(dx: -3, dy: -3)), with: .color(.black.opacity(0.25))) + context.fill(Path(ellipseIn: dot), with: .color(.white)) + context.fill(Path(ellipseIn: dot.insetBy(dx: 4, dy: 4)), with: .color(Palette.signal)) + } + } +} diff --git a/ios/HouseScan/UI/Camera/WallProjection.swift b/ios/HouseScan/UI/Camera/WallProjection.swift new file mode 100644 index 000000000..9799f23fc --- /dev/null +++ b/ios/HouseScan/UI/Camera/WallProjection.swift @@ -0,0 +1,88 @@ +import SwiftUI +import simd + +/// Projects wall-frame shapes (s, height, out) onto the camera view. +/// +/// Every overlay is drawn from world points through `CameraProjection`, never from cached +/// screen positions, so when ARKit corrects the meter's anchor the overlays move with the wall. +struct WallProjection { + var projection: CameraProjection + var wall: WallGeometry + var size: CGSize + + func point(_ world: SIMD3) -> CGPoint? { + projection.viewPoint(for: world, in: size) + } + + func point(s: Float, height: Float, out: Float = 0) -> CGPoint? { + point(wall.world(s: s, height: height, out: out)) + } + + /// A quad on the wall face, or nil when any corner is behind the camera. Cells are small + /// (6 in wide), so dropping one that straddles the camera plane loses nothing visible. + func wallQuad(s: ClosedRange, height: ClosedRange, out: Float = 0) -> Path? { + polygon([ + wall.world(s: s.lowerBound, height: height.lowerBound, out: out), + wall.world(s: s.upperBound, height: height.lowerBound, out: out), + wall.world(s: s.upperBound, height: height.upperBound, out: out), + wall.world(s: s.lowerBound, height: height.upperBound, out: out), + ]) + } + + /// A quad on the ground in front of the wall. + func groundQuad(s: ClosedRange, out: ClosedRange, height: Float = 0) -> Path? { + polygon([ + wall.world(s: s.lowerBound, height: height, out: out.lowerBound), + wall.world(s: s.upperBound, height: height, out: out.lowerBound), + wall.world(s: s.upperBound, height: height, out: out.upperBound), + wall.world(s: s.lowerBound, height: height, out: out.upperBound), + ]) + } + + func polygon(_ corners: [SIMD3]) -> Path? { + var points: [CGPoint] = [] + points.reserveCapacity(corners.count) + for corner in corners { + guard let p = point(corner) else { return nil } + points.append(p) + } + var path = Path() + path.addLines(points) + path.closeSubpath() + return path + } + + /// Meters per view point at a world point, for sizing things that should shrink with + /// distance (path dots, pins). + func pointsPerMeter(at world: SIMD3) -> CGFloat? { + let local = projection.cameraSpace(world) + guard local.z < -0.05 else { return nil } + let focal = CGFloat(projection.intrinsics.x) + return focal * projection.scale(in: size) / CGFloat(-local.z) + } + + /// The camera's position along the wall, in meters of s: the nearest point of the wall, + /// round a corner when the walk followed one. + var cameraS: Float { + wall.s(nearest: projection.cameraPosition) + } +} + +extension CameraProjection { + /// Screen direction toward a world point, as a unit vector in view space, usable even when + /// the point is behind the camera. Camera +y (up in the sensor image) is screen right and + /// camera +x is screen down, because the sensor image is shown rotated 90° clockwise. + func screenDirection(toward world: SIMD3) -> CGVector? { + let local = cameraSpace(world) + var dx = CGFloat(local.y) + var dy = CGFloat(local.x) + if local.z > 0 { + // Behind: the point is reached by turning around, so point toward the nearer side. + dx = dx == 0 ? 1 : dx + dy = 0 + } + let length = (dx * dx + dy * dy).squareRoot() + guard length > 1e-5 else { return nil } + return CGVector(dx: dx / length, dy: dy / length) + } +} diff --git a/ios/HouseScan/UI/Camera/WallTape.swift b/ios/HouseScan/UI/Camera/WallTape.swift new file mode 100644 index 000000000..60c4b4c8a --- /dev/null +++ b/ios/HouseScan/UI/Camera/WallTape.swift @@ -0,0 +1,450 @@ +import SwiftUI + +/// The live map of the wall, unrolled into a strip like a tape measure with the meter at zero. +/// +/// Top row is the wall face, bottom row the ground at its foot. Gray is not seen yet, amber +/// seen once, green seen well enough (docs/05 section 2); green is evidence, not approval. +/// Foot ticks run along the bottom edge so distances read at a glance without numbers. +/// +/// The two states that aren't about how well the phone saw get a pattern as well as a color, +/// so they read in grayscale, and a legend line under the strip while any is on it: skipped is +/// slate with a slash, hidden (depth saw something in front) a violet dashed outline round the +/// stretch, as on the camera. On a phone with depth the same line says the map is depth-checked. +/// +/// While ending the wall is on offer, a dashed chalk line shows where the end would land and the +/// strip past it on that side is dimmed: that part would be left out. The line follows the phone +/// (or the reticle) frame by frame, so it doesn't animate. +struct WallTape: View { + var coverage: CoverageStrip + var wall: WallGeometry + var features: [MarkedFeature] + /// The homeowner's position along the wall, meters of s, when known. + var cameraS: Float? + var highlight: GapRequest? + /// The phone has depth, so a cell counts only where depth confirms the wall itself. + var depthChecked = false + /// Where a wall end would land now (`ScanViewState.endPreview`); nil hides the line. + var endPreview: EndPreview? + + /// Glyph size for the meter and feature marks. It follows the text size, like every other + /// glyph in the app (fixed 9 and 10 pt sizes failed the audit's Dynamic Type check), and + /// stops growing at 16 pt so the marks still fit the strip's 58 pt height. + @ScaledMetric(relativeTo: .caption2) private var glyphSize: CGFloat = 10 + @Environment(\.dynamicTypeSize) private var typeSize + @Environment(\.accessibilityReduceMotion) private var reduceMotion + private var glyph: CGFloat { min(glyphSize, 16) } + private var glyphBox: CGFloat { glyph * 1.8 } + + private static let footInMeters: Float = 0.3048 + + var body: some View { + let extent = Self.extent(coverage: coverage, wall: wall, cameraS: cameraS, also: markedS + [endPreview?.s].compactMap(\.self)) + VStack(spacing: 4) { + GeometryReader { proxy in + let map = TapeMap(extent: extent, width: proxy.size.width) + ZStack(alignment: .topLeading) { + Canvas { context, size in + draw(in: &context, size: size, map: map) + } + ForEach(features) { feature in + let center = (feature.span.lowerBound + feature.span.upperBound) / 2 + Image(systemName: ScanCopy.symbol(feature.kind)) + .font(.system(size: glyph, weight: .bold)) + .foregroundStyle(Palette.chalk) + .frame(width: glyphBox, height: glyphBox * 0.9) + .background(Palette.ink.opacity(0.9), in: .rect(cornerRadius: 4)) + .position(x: Self.inside(map.x(center), halfWidth: glyphBox / 2, width: proxy.size.width), y: glyphBox / 2 - 1) + } + Image(systemName: "bolt.fill") + .font(.system(size: glyph * 0.9, weight: .black)) + .foregroundStyle(.white) + .frame(width: glyphBox, height: glyphBox) + .background(Palette.signal, in: .circle) + .position(x: Self.inside(map.x(0), halfWidth: glyphBox / 2, width: proxy.size.width), y: glyphBox / 2 - 1) + } + } + .frame(height: 58) + if showsFooter { + footer + .transition(.opacity) + } + } + .animation(reduceMotion ? nil : .easeOut(duration: 0.2), value: showsFooter) + .padding(.horizontal, 14) + .padding(.vertical, 10) + .background(ScrimShape.rounded(18)) + .accessibilityElement(children: .ignore) + .accessibilityLabel("Map of the wall") + .accessibilityValue(accessibilitySummary) + .accessibilityIdentifier("wallTape") + } + + // MARK: Legend + + private var hiddenSections: [ClosedRange] { sections(of: .hidden) } + private var skippedSections: [ClosedRange] { sections(of: .skipped) } + + private var showsFooter: Bool { + depthChecked || !hiddenSections.isEmpty || !skippedSections.isEmpty || leavesOutWalked != nil + } + + private var leavesOutWalked: Float? { endPreview?.leavesOutWalked } + + /// The legend under the strip. One row when every entry fits on one line; otherwise, and at + /// the accessibility sizes, one entry per line. As one squeezed row, the end preview, the + /// hidden sections and "Depth-checked" were laid over each other on the LiDAR replay, and the + /// audit failed their contrast (PR run 36307476187). + /// + /// Entries come and go without a fade. They change with the walk, and the walk's controls + /// animate their own changes (`WallWalkScreen`, `controlsKey`); inherited here, that animation + /// faded a leaving entry over the one taking its place, text over text again. + private var footer: some View { + Group { + if typeSize.isAccessibilitySize { + VStack(alignment: .leading, spacing: 6) { legendEntries; depthLabel } + } else { + ViewThatFits(in: .horizontal) { + // "Depth-checked" says how the whole strip was measured, so it sits apart, right. + HStack(spacing: 12) { + legendEntries + Spacer(minLength: 0) + depthLabel + } + VStack(alignment: .leading, spacing: 6) { legendEntries; depthLabel } + } + } + } + .frame(maxWidth: .infinity, alignment: .leading) + .transaction { $0.animation = nil } + } + + @ViewBuilder + private var legendEntries: some View { + if let leavesOutWalked { + LegendEntry(title: ScanCopy.endLeavesOut(leavesOutWalked)) { EndPreviewSwatch() } + } + if !hiddenSections.isEmpty { + LegendEntry(title: "Hidden behind something") { HiddenSwatch() } + } + if !skippedSections.isEmpty { + LegendEntry(title: "Skipped") { SkippedSwatch() } + } + } + + @ViewBuilder + private var depthLabel: some View { + if depthChecked { + Label("Depth-checked", systemImage: "cube.transparent") + .font(Typeface.caption) + .foregroundStyle(Palette.chalk) + } + } + + // MARK: Layout + + /// Keeps a glyph of `halfWidth` inside the strip. When the walk covers only one side, the + /// meter sits at the strip's end, and a glyph centred there hung half outside it, which the + /// accessibility audit reported as clipped on the real ADVIO replay. + static func inside(_ x: CGFloat, halfWidth: CGFloat, width: CGFloat) -> CGFloat { + min(max(x, halfWidth), max(halfWidth, width - halfWidth)) + } + + struct TapeMap { + var extent: ClosedRange + var width: CGFloat + + func x(_ s: Float) -> CGFloat { + let span = max(extent.upperBound - extent.lowerBound, 0.01) + return CGFloat((s - extent.lowerBound) / span) * width + } + } + + /// The middle of each marked feature. A mark past an end still counts (the server puts it on + /// the wall's line continued past the end), so the strip draws it where it is rather than + /// pinned to the strip's edge. + private var markedS: [Float] { + features.map { ($0.span.lowerBound + $0.span.upperBound) / 2 } + } + + /// The s range drawn: everything the engine considers worth drawing, the marked ends, the + /// meter, the homeowner and `also`, plus a little air on both sides. + static func extent(coverage: CoverageStrip, wall: WallGeometry, cameraS: Float?, also: [Float] = []) -> ClosedRange { + var lower = min(coverage.visibleRange.lowerBound, 0) + var upper = max(coverage.visibleRange.upperBound, 0) + for value in [wall.leftEnd, wall.rightEnd, cameraS].compactMap(\.self) + also { + lower = min(lower, value) + upper = max(upper, value) + } + let minimumSpan: Float = 3 + if upper - lower < minimumSpan { + let pad = (minimumSpan - (upper - lower)) / 2 + lower -= pad + upper += pad + } + return (lower - 0.25)...(upper + 0.25) + } + + // MARK: Drawing + + private func draw(in context: inout GraphicsContext, size: CGSize, map: TapeMap) { + let wallRow = CGRect(x: 0, y: 20, width: size.width, height: 14) + let groundRow = CGRect(x: 0, y: wallRow.maxY + 3, width: size.width, height: 9) + let tickTop = groundRow.maxY + 3 + + // Base track, so unknown stretches read as "tape" rather than nothing. + context.fill(Path(roundedRect: wallRow, cornerRadius: 3), with: .color(.white.opacity(0.08))) + context.fill(Path(roundedRect: groundRow, cornerRadius: 3), with: .color(.white.opacity(0.08))) + + for band in CoverageBand.allCases { + let row = band == .wall ? wallRow : groundRow + for (index, state) in coverage.cells(band).enumerated() { + let range = coverage.cellRange(index) + // Cells outside the engine's visible range are neither fog nor evidence. + guard FogMemory.interiorsOverlap(range, coverage.visibleRange), isInsideEnds(range) else { continue } + let x0 = map.x(range.lowerBound) + let x1 = map.x(range.upperBound) + let rect = CGRect(x: x0 + 0.5, y: row.minY, width: max(1, x1 - x0 - 1), height: row.height) + // Hidden is a tint under a dashed outline drawn per stretch below: hollow, so it + // never reads as a solid covered cell. + let opacity = switch state { + case .unseen: 0.55 + case .hidden: 0.22 + default: 1.0 + } + context.fill(Path(rect), with: .color(Palette.cell(state).opacity(opacity))) + if state == .skipped { + var slash = Path() + slash.move(to: CGPoint(x: rect.minX, y: rect.maxY)) + slash.addLine(to: CGPoint(x: rect.maxX, y: rect.minY)) + context.stroke(slash, with: .color(.white.opacity(0.7)), lineWidth: 1) + } + } + for run in runs(of: .hidden, in: band) { + let rect = CGRect(x: map.x(run.lowerBound), y: row.minY, width: map.x(run.upperBound) - map.x(run.lowerBound), height: row.height) + .insetBy(dx: 0.75, dy: 0.75) + context.stroke(Path(roundedRect: rect, cornerRadius: 2), with: .color(Palette.hidden), + style: StrokeStyle(lineWidth: 1.5, dash: Palette.hiddenDash)) + } + if let gap = highlight, gap.band == band { + let rect = CGRect(x: map.x(gap.span.lowerBound), y: row.minY - 2, + width: map.x(gap.span.upperBound) - map.x(gap.span.lowerBound), height: row.height + 4) + context.stroke(Path(roundedRect: rect, cornerRadius: 3), with: .color(Palette.caution), lineWidth: 2) + } + } + + // What ending the wall at the preview would leave out: everything past it on that side. + if let preview = endPreview { + let x = map.x(preview.s) + let band = CGRect(x: 0, y: wallRow.minY, width: size.width, height: groundRow.maxY - wallRow.minY) + let veil = preview.side == .left + ? band.divided(atDistance: max(0, x), from: .minXEdge).slice + : band.divided(atDistance: max(0, x), from: .minXEdge).remainder + context.fill(Path(veil), with: .color(.black.opacity(0.55))) + } + + // Foot ticks along the bottom edge; every fifth foot is longer. + let firstFoot = Int((map.extent.lowerBound / Self.footInMeters).rounded(.up)) + let lastFoot = Int((map.extent.upperBound / Self.footInMeters).rounded(.down)) + if firstFoot <= lastFoot { + var ticks = Path() + for foot in firstFoot...lastFoot { + let x = map.x(Float(foot) * Self.footInMeters) + ticks.move(to: CGPoint(x: x, y: tickTop)) + ticks.addLine(to: CGPoint(x: x, y: tickTop + (foot % 5 == 0 ? 7 : 3.5))) + } + context.stroke(ticks, with: .color(Palette.chalk.opacity(0.7)), lineWidth: 1) + } + + // Meter line through both rows. + var meterLine = Path() + meterLine.move(to: CGPoint(x: map.x(0), y: 16)) + meterLine.addLine(to: CGPoint(x: map.x(0), y: groundRow.maxY + 2)) + context.stroke(meterLine, with: .color(Palette.signal), lineWidth: 2) + + // Marked ends: tall chalk caps. + for end in [wall.leftEnd, wall.rightEnd].compactMap(\.self) { + let x = map.x(end) + let cap = CGRect(x: x - 1.5, y: wallRow.minY - 4, width: 3, height: groundRow.maxY - wallRow.minY + 8) + context.fill(Path(roundedRect: cap, cornerRadius: 1.5), with: .color(Palette.chalk)) + } + + // Where the end would land: a dashed chalk line as tall as the end caps, on a dark + // underlay so it reads over bright cells in sun. + if let preview = endPreview { + let x = map.x(preview.s) + var line = Path() + line.move(to: CGPoint(x: x, y: wallRow.minY - 4)) + line.addLine(to: CGPoint(x: x, y: groundRow.maxY + 4)) + context.stroke(line, with: .color(.black.opacity(0.6)), lineWidth: 4) + context.stroke(line, with: .color(Palette.chalk), style: StrokeStyle(lineWidth: 2, dash: [3, 2.5])) + } + + // The homeowner: a white pointer under the strip, like the cursor on a tape. + if let cameraS { + let x = map.x(min(max(cameraS, map.extent.lowerBound), map.extent.upperBound)) + var marker = Path() + marker.move(to: CGPoint(x: x, y: groundRow.maxY + 1)) + marker.addLine(to: CGPoint(x: x - 7, y: groundRow.maxY + 11)) + marker.addLine(to: CGPoint(x: x + 7, y: groundRow.maxY + 11)) + marker.closeSubpath() + context.fill(marker, with: .color(.white)) + context.stroke(marker, with: .color(Palette.signal), lineWidth: 1.5) + } + } + + /// Runs of cells in `state` within one band, as s ranges, over the cells the strip draws. + private func runs(of state: CellState, in band: CoverageBand) -> [ClosedRange] { + var runs: [ClosedRange] = [] + for (index, cell) in coverage.cells(band).enumerated() { + let range = coverage.cellRange(index) + guard cell == state, FogMemory.interiorsOverlap(range, coverage.visibleRange), isInsideEnds(range) else { continue } + if let last = runs.last, last.upperBound >= range.lowerBound - 0.001 { + runs[runs.count - 1] = last.lowerBound...range.upperBound + } else { + runs.append(range) + } + } + return runs + } + + /// Stretches of wall with cells in `state` in either band, overlapping runs merged: a bush + /// that hides both the wall and the ground in front of it is one section, not two. + private func sections(of state: CellState) -> [ClosedRange] { + let all = (runs(of: state, in: .wall) + runs(of: state, in: .ground)).sorted { $0.lowerBound < $1.lowerBound } + var merged: [ClosedRange] = [] + for run in all { + if let last = merged.last, last.upperBound >= run.lowerBound - 0.001 { + merged[merged.count - 1] = last.lowerBound...max(last.upperBound, run.upperBound) + } else { + merged.append(run) + } + } + return merged + } + + private func isInsideEnds(_ range: ClosedRange) -> Bool { + if let left = wall.leftEnd, range.upperBound <= left { return false } + if let right = wall.rightEnd, range.lowerBound >= right { return false } + return true + } + + // MARK: Accessibility + + private var accessibilitySummary: String { + var parts: [String] = [] + let lowerLeft = wall.leftEnd ?? min(coverage.visibleRange.lowerBound, 0) + let upperRight = wall.rightEnd ?? max(coverage.visibleRange.upperBound, 0) + if lowerLeft < -0.05 { + parts.append("Left of your meter: \(percentSeen(in: lowerLeft...0)) percent seen") + } + if upperRight > 0.05 { + parts.append("Right of your meter: \(percentSeen(in: 0...upperRight)) percent seen") + } + if let cameraS { + parts.append("You are \(Distance.spoken(cameraS)) \(cameraS < 0 ? "left" : "right") of your meter") + } + if let hidden = Self.count(hiddenSections.count, "section") { + parts.append("\(hidden) hidden behind something") + } + if let skipped = Self.count(skippedSections.count, "section") { + parts.append("\(skipped) skipped") + } + if let preview = endPreview { + var sentence = preview.atReticle + ? "Wall ends here marks the \(preview.side.rawValue) end \(Self.spokenFromMeter(preview.s))" + : "If you end the wall now, the \(preview.side.rawValue) end goes \(Self.spokenFromMeter(preview.s))" + if let leavesOutWalked { + sentence += ", leaving out \(Distance.spoken(leavesOutWalked)) you walked" + } + parts.append(sentence) + } + for (side, end) in [("Left", wall.leftEnd), ("Right", wall.rightEnd)] { + if let end { parts.append("\(side) end \(Self.spokenFromMeter(end))") } + } + if wall.leftEnd != nil, wall.rightEnd != nil { + parts.append("Both ends marked") + } + if depthChecked { + parts.append("Checked with your phone's depth sensor") + } + return parts.joined(separator: ". ") + } + + /// "5 feet 2 inches left of your meter", or "at your meter" within 3 in, as `Distance.fromMeter`. + static func spokenFromMeter(_ s: Float) -> String { + if abs(s) < Distance.metersPerInch * 3 { return "at your meter" } + return "\(Distance.spoken(s)) \(s < 0 ? "left" : "right") of your meter" + } + + /// "1 section", "2 sections"; nil for none. + private static func count(_ n: Int, _ noun: String) -> String? { + n == 0 ? nil : "\(n) \(noun)\(n == 1 ? "" : "s")" + } + + private func percentSeen(in range: ClosedRange) -> Int { + let states = coverage.wall.indices.filter { range.overlaps(coverage.cellRange($0)) }.map { coverage.wall[$0] } + guard !states.isEmpty else { return 0 } + // Split up: as one expression it took the type checker over 60 ms. + let seen = Double(states.filter { $0 == .seen || $0 == .covered }.count) + let fraction = seen / Double(states.count) + return Int((fraction * 100).rounded()) + } +} + +// MARK: - Legend pieces + +private struct LegendEntry: View { + var title: String + @ViewBuilder var swatch: Swatch + + var body: some View { + HStack(spacing: 6) { + swatch + Text(title) + .font(Typeface.caption) + .foregroundStyle(Palette.chalk) + .fixedSize(horizontal: false, vertical: true) + } + } +} + +/// A hidden cell as the strip draws it: violet tint inside a dashed violet outline. +private struct HiddenSwatch: View { + var body: some View { + RoundedRectangle(cornerRadius: 2) + .fill(Palette.hidden.opacity(0.22)) + .overlay(RoundedRectangle(cornerRadius: 2).strokeBorder(Palette.hidden, style: StrokeStyle(lineWidth: 1.5, dash: [3, 2]))) + .frame(width: 18, height: 12) + } +} + +/// The end preview as the strip draws it: a dashed chalk line with the stretch past it dimmed. +private struct EndPreviewSwatch: View { + var body: some View { + Canvas { context, size in + let rect = CGRect(origin: .zero, size: size) + context.fill(Path(roundedRect: rect, cornerRadius: 2), with: .color(Palette.cell(.covered))) + context.fill(Path(rect.divided(atDistance: size.width / 2, from: .minXEdge).remainder), with: .color(.black.opacity(0.55))) + var line = Path() + line.move(to: CGPoint(x: size.width / 2, y: 0)) + line.addLine(to: CGPoint(x: size.width / 2, y: size.height)) + context.stroke(line, with: .color(Palette.chalk), style: StrokeStyle(lineWidth: 2, dash: [3, 2.5])) + } + .frame(width: 18, height: 12) + } +} + +/// A skipped cell as the strip draws it: slate with a white slash. +private struct SkippedSwatch: View { + var body: some View { + Canvas { context, size in + let rect = CGRect(origin: .zero, size: size) + context.fill(Path(roundedRect: rect, cornerRadius: 2), with: .color(Palette.skipped)) + var slash = Path() + slash.move(to: CGPoint(x: rect.minX + 3, y: rect.maxY)) + slash.addLine(to: CGPoint(x: rect.maxX - 3, y: rect.minY)) + context.stroke(slash, with: .color(.white.opacity(0.7)), lineWidth: 1) + } + .frame(width: 18, height: 12) + } +} diff --git a/ios/HouseScan/UI/Camera/WayfindingOverlay.swift b/ios/HouseScan/UI/Camera/WayfindingOverlay.swift new file mode 100644 index 000000000..43d954f4e --- /dev/null +++ b/ios/HouseScan/UI/Camera/WayfindingOverlay.swift @@ -0,0 +1,139 @@ +import simd +import SwiftUI + +/// Where to stand and where to aim: a blue dotted path on the ground toward the next place to +/// stand, and a ring on the point to aim at (or an edge chevron when it is off screen). +struct WayfindingOverlay: View { + var projection: CameraProjection + var wall: WallGeometry? + var path: [SIMD3] + var target: SIMD3? + + var body: some View { + GeometryReader { proxy in + let size = proxy.size + ZStack { + Canvas { context, size in + drawPath(in: &context, size: size) + } + if let target { + TargetMarker(placement: placement(for: target, in: size)) + } + } + .frame(width: size.width, height: size.height) + } + .ignoresSafeArea() + .allowsHitTesting(false) + .accessibilityHidden(true) + } + + // MARK: Path + + /// Dot spacing on the ground. A display choice: dense enough to read as a line at 3 m, + /// sparse enough that each dot is distinct up close. + private static let dotSpacing: Float = 0.22 + + private func drawPath(in context: inout GraphicsContext, size: CGSize) { + guard path.count >= 2 else { return } + var samples: [SIMD3] = [] + var carry: Float = 0 + for (a, b) in zip(path, path.dropFirst()) { + let length = simd_distance(a, b) + guard length > 0 else { continue } + var d = carry + while d < length { + samples.append(a + (b - a) * (d / length)) + d += Self.dotSpacing + } + carry = d - length + } + samples.append(path[path.count - 1]) + + for (index, sample) in samples.enumerated() { + let local = projection.cameraSpace(sample) + guard local.z < -0.15, let point = projection.viewPoint(for: sample, in: size) else { continue } + let pointsPerMeter = CGFloat(projection.intrinsics.x) * projection.scale(in: size) / CGFloat(-local.z) + // Dots are 7 cm across on the ground, so they shrink with distance like paint would. + let radius = min(14, max(2.5, pointsPerMeter * 0.035)) + let fade = index == samples.count - 1 ? 1 : min(1, 0.55 + Double(index) / Double(max(samples.count, 1))) + let rect = CGRect(x: point.x - radius, y: point.y - radius * 0.62, width: radius * 2, height: radius * 1.24) + context.fill(Path(ellipseIn: rect.insetBy(dx: -2, dy: -1.5)), with: .color(.white.opacity(0.7 * fade))) + context.fill(Path(ellipseIn: rect), with: .color(Palette.signal.opacity(fade))) + } + } + + // MARK: Target + + private func placement(for target: SIMD3, in size: CGSize) -> TargetMarker.Placement { + // The ring shows while its center is comfortably on screen; the chevron takes over + // near the edges, where a half-visible ring would be ambiguous. + let bounds = CGRect(x: 36, y: 150, width: size.width - 72, height: size.height - 330) + if let point = projection.viewPoint(for: target, in: size), bounds.contains(point) { + let scale = wall.flatMap { WallProjection(projection: projection, wall: $0, size: size).pointsPerMeter(at: target) } + let radius = min(64, max(30, (scale ?? 90) * 0.28)) + return .onScreen(point, radius: radius) + } + guard let direction = projection.screenDirection(toward: target) else { + return .hidden + } + // Chevrons stay clear of the instruction card above and the buttons and map below, + // which are drawn over this layer. + let lane = CGRect(x: 40, y: 260, width: size.width - 80, height: max(size.height - 260 - 300, 80)) + let center = CGPoint(x: lane.midX, y: lane.midY) + let halfWidth = lane.width / 2 + let halfHeight = lane.height / 2 + let tx = direction.dx == 0 ? CGFloat.infinity : halfWidth / abs(direction.dx) + let ty = direction.dy == 0 ? CGFloat.infinity : halfHeight / abs(direction.dy) + let t = min(tx, ty) + let edge = CGPoint(x: center.x + direction.dx * t, y: center.y + direction.dy * t) + return .offScreen(edge, angle: .radians(atan2(direction.dy, direction.dx))) + } +} + +/// The aim ring, or a chevron at the screen edge pointing toward it. +private struct TargetMarker: View { + enum Placement: Equatable { + case onScreen(CGPoint, radius: CGFloat) + case offScreen(CGPoint, angle: Angle) + case hidden + } + + var placement: Placement + @Environment(\.accessibilityReduceMotion) private var reduceMotion + + var body: some View { + switch placement { + case .onScreen(let point, let radius): + ZStack { + Circle() + .strokeBorder(.white.opacity(0.9), lineWidth: 7) + Circle() + .strokeBorder(Palette.signal, lineWidth: 4) + Circle() + .fill(Palette.signal) + .frame(width: 8, height: 8) + } + .frame(width: radius * 2, height: radius * 2) + .phaseAnimator(reduceMotion ? [1.0] : [1.0, 1.08]) { ring, scale in + ring.scaleEffect(scale) + } animation: { _ in + .easeInOut(duration: 0.9) + } + .position(point) + .transition(.opacity) + case .offScreen(let point, let angle): + Image(systemName: "chevron.right") + .font(.system(size: 22, weight: .black)) + .foregroundStyle(.white) + .rotationEffect(angle) + .frame(width: 52, height: 52) + .background(Palette.signal, in: .circle) + .overlay(Circle().strokeBorder(.white.opacity(0.85), lineWidth: 2)) + .shadow(color: .black.opacity(0.3), radius: 6, y: 2) + .position(point) + .transition(.opacity) + case .hidden: + EmptyView() + } + } +} diff --git a/ios/HouseScan/UI/Components/AnswerButton.swift b/ios/HouseScan/UI/Components/AnswerButton.swift new file mode 100644 index 000000000..66485ecc3 --- /dev/null +++ b/ios/HouseScan/UI/Components/AnswerButton.swift @@ -0,0 +1,29 @@ +import SwiftUI + +/// One answer to a question on the light review panel: a tinted capsule that fills blue once it +/// is the answer given. Shared by the window question and the ground question so every answer on +/// the panel looks and behaves the same. +struct AnswerButton: View { + let title: String + let selected: Bool + let action: () -> Void + + var body: some View { + Button(action: action) { + Text(title) + .font(Typeface.button) + .foregroundStyle(selected ? .white : Palette.signalText) + .multilineTextAlignment(.center) + .fixedSize(horizontal: false, vertical: true) + .padding(.horizontal, 12) + .padding(.vertical, 8) + .frame(maxWidth: .infinity, minHeight: 46) + // The primary button's darker fill: white on it is about 6.2:1, on Signal 4.9:1. + .background(selected ? Palette.signalFill : Palette.signal.opacity(0.1), in: .capsule) + .contentShape(.capsule) + } + .buttonStyle(PressableStyle()) + .accessibilityAddTraits(selected ? .isSelected : []) + .animation(.easeOut(duration: 0.15), value: selected) + } +} diff --git a/ios/HouseScan/UI/Components/Buttons.swift b/ios/HouseScan/UI/Components/Buttons.swift new file mode 100644 index 000000000..00126013b --- /dev/null +++ b/ios/HouseScan/UI/Components/Buttons.swift @@ -0,0 +1,77 @@ +import SwiftUI + +/// The one big action on a screen: solid blue, white bold text, full width. Solid rather than +/// glass because it has to stay legible over a sunlit wall; white on the fill is about 6:1. +struct PrimaryButtonStyle: ButtonStyle { + @Environment(\.isEnabled) private var isEnabled + + func makeBody(configuration: Configuration) -> some View { + configuration.label + .font(Typeface.button) + .foregroundStyle(.white) + .multilineTextAlignment(.center) + .frame(maxWidth: .infinity, minHeight: Metrics.primaryButtonHeight) + .padding(.horizontal, 20) + .background(Palette.signalFill.opacity(isEnabled ? 1 : 0.45), in: .capsule) + .contentShape(.capsule) + .scaleEffect(configuration.isPressed ? 0.97 : 1) + .animation(.easeOut(duration: 0.12), value: configuration.isPressed) + } +} + +/// Second choice next to a primary action: a dark, heavy material so it reads over any camera +/// image without competing with the blue. +struct SecondaryButtonStyle: ButtonStyle { + var prominent = false + + func makeBody(configuration: Configuration) -> some View { + configuration.label + .font(prominent ? Typeface.button : Typeface.hint.weight(.semibold)) + .foregroundStyle(Palette.chalk) + .multilineTextAlignment(.center) + .frame(minHeight: prominent ? Metrics.primaryButtonHeight : 50) + .padding(.horizontal, 20) + .background(ScrimShape.capsule) + .contentShape(.capsule) + .scaleEffect(configuration.isPressed ? 0.97 : 1) + .animation(.easeOut(duration: 0.12), value: configuration.isPressed) + } +} + +/// Buttons on the light text screens: tinted outline style for the second choice. +struct QuietButtonStyle: ButtonStyle { + func makeBody(configuration: Configuration) -> some View { + configuration.label + .font(Typeface.button) + .foregroundStyle(Palette.signalText) + .frame(maxWidth: .infinity, minHeight: 52) + .padding(.horizontal, 16) + .background(Palette.signal.opacity(configuration.isPressed ? 0.16 : 0.1), in: .capsule) + .contentShape(.capsule) + .scaleEffect(configuration.isPressed ? 0.97 : 1) + .animation(.easeOut(duration: 0.12), value: configuration.isPressed) + } +} + +/// Scale-on-press for custom-drawn buttons. +struct PressableStyle: ButtonStyle { + func makeBody(configuration: Configuration) -> some View { + configuration.label + .scaleEffect(configuration.isPressed ? 0.96 : 1) + .opacity(configuration.isPressed ? 0.85 : 1) + .animation(.easeOut(duration: 0.12), value: configuration.isPressed) + } +} + +extension ButtonStyle where Self == PrimaryButtonStyle { + static var primary: PrimaryButtonStyle { PrimaryButtonStyle() } +} + +extension ButtonStyle where Self == SecondaryButtonStyle { + static var secondary: SecondaryButtonStyle { SecondaryButtonStyle() } + static var secondaryProminent: SecondaryButtonStyle { SecondaryButtonStyle(prominent: true) } +} + +extension ButtonStyle where Self == QuietButtonStyle { + static var quiet: QuietButtonStyle { QuietButtonStyle() } +} diff --git a/ios/HouseScan/UI/Components/CameraChrome.swift b/ios/HouseScan/UI/Components/CameraChrome.swift new file mode 100644 index 000000000..78c03b8b2 --- /dev/null +++ b/ios/HouseScan/UI/Components/CameraChrome.swift @@ -0,0 +1,95 @@ +import SwiftUI + +/// The frame shared by every camera screen: status row and one instruction at the top, the +/// camera open in the middle, actions within thumb reach at the bottom. +struct CameraChrome: View { + var instruction: Instruction + var tone: InstructionCard.Tone = .normal + var reply: InstructionCard.Reply? + /// See `InstructionCard.eyebrow`. + var eyebrow: String? + var photoCount: Int? + var lastCaptureID: Int? + var isReplay: Bool + var isAutopilot: Bool + /// Taps on the open camera area, in the camera view's coordinates (full screen, which is + /// the window's global space). Nil when the screen has nothing to tap. + var onCameraTap: ((CGPoint) -> Void)? + @ViewBuilder var bottom: Bottom + + var body: some View { + // One layout at every text size: the chrome fills the screen and, only when the largest + // text makes it taller than the screen, scrolls instead of squeezing the words. The + // scroll view would swallow taps meant for the camera, so camera taps are caught inside + // it, behind the chrome. + GeometryReader { proxy in + ScrollView { + stack + .frame(width: proxy.size.width) + .frame(minHeight: proxy.size.height) + .background { + if let onCameraTap { + Color.clear + .contentShape(.rect) + .onTapGesture(coordinateSpace: .global) { point in onCameraTap(point) } + .accessibilityHidden(true) + } + } + } + .scrollBounceBehavior(.basedOnSize) + .scrollIndicators(.hidden) + } + } + + private var stack: some View { + VStack(spacing: 10) { + HStack(alignment: .center) { + ModeBadge(isReplay: isReplay, isAutopilot: isAutopilot) + Spacer(minLength: 8) + if let photoCount { + PhotoCounter(count: photoCount, lastCaptureID: lastCaptureID) + } + } + .frame(minHeight: 36) + InstructionCard(instruction: instruction, tone: tone, reply: reply, eyebrow: eyebrow) + Spacer(minLength: 0) + bottom + } + .padding(.horizontal, Metrics.edge) + .padding(.top, 4) + .padding(.bottom, 8) + } +} + +/// Reads the full-screen camera view size so buttons can send "the reticle" (nil point) +/// with the right `viewSize`. +struct CameraSizeReader: View { + @Binding var size: CGSize + + var body: some View { + GeometryReader { proxy in + Color.clear + .onAppear { size = proxy.size } + .onChange(of: proxy.size) { _, newValue in size = newValue } + } + .ignoresSafeArea() + .allowsHitTesting(false) + .accessibilityHidden(true) + } +} + +/// Crosshair ring at the center of the camera: "the phone is pointing here". +struct Reticle: View { + var diameter: CGFloat = 64 + + var body: some View { + ZStack { + Circle().strokeBorder(.black.opacity(0.35), lineWidth: 6) + Circle().strokeBorder(.white, lineWidth: 3) + Circle().fill(.white).frame(width: 6, height: 6) + } + .frame(width: diameter, height: diameter) + .allowsHitTesting(false) + .accessibilityHidden(true) + } +} diff --git a/ios/HouseScan/UI/Components/CenteredScroll.swift b/ios/HouseScan/UI/Components/CenteredScroll.swift new file mode 100644 index 000000000..f3c5fa1a8 --- /dev/null +++ b/ios/HouseScan/UI/Components/CenteredScroll.swift @@ -0,0 +1,18 @@ +import SwiftUI + +/// A scroll view whose content sits in the vertical middle when it fits, and scrolls when a +/// large text size makes it taller than the screen. +struct CenteredScroll: View { + @ViewBuilder var content: Content + + var body: some View { + GeometryReader { proxy in + ScrollView { + content + .frame(maxWidth: .infinity) + .frame(minHeight: proxy.size.height) + } + .scrollBounceBehavior(.basedOnSize) + } + } +} diff --git a/ios/HouseScan/UI/Components/GroundAnswers.swift b/ios/HouseScan/UI/Components/GroundAnswers.swift new file mode 100644 index 000000000..717d68d6c --- /dev/null +++ b/ios/HouseScan/UI/Components/GroundAnswers.swift @@ -0,0 +1,31 @@ +import SwiftUI + +/// The answers to the spot check's ground question: the six ground types in two columns, "Not +/// sure" across the full width under them (one column at accessibility text sizes). +struct GroundAnswers: View { + let answer: (GroundAnswer) -> Void + + @Environment(\.dynamicTypeSize) private var typeSize + + var body: some View { + let columns = typeSize.isAccessibilitySize ? 1 : 2 + let types = GroundType.allCases + Grid(horizontalSpacing: 8, verticalSpacing: 8) { + ForEach(Array(stride(from: 0, to: types.count, by: columns)), id: \.self) { start in + GridRow { + ForEach(types[start.. some View { + AnswerButton(title: ScanCopy.groundAnswer(option), selected: false) { answer(option) } + } +} diff --git a/ios/HouseScan/UI/Components/InstructionCard.swift b/ios/HouseScan/UI/Components/InstructionCard.swift new file mode 100644 index 000000000..ab5bf95de --- /dev/null +++ b/ios/HouseScan/UI/Components/InstructionCard.swift @@ -0,0 +1,119 @@ +import SwiftUI + +/// The single instruction at the top of a camera screen. +/// +/// When the text changes, the words swap at once and the card eases to its new height. A +/// blur or fade between them left both lines half-transparent for a moment, which the +/// accessibility audit reported as low contrast and, for the outgoing line, as clipped text +/// (the marking prompt, the end question). Coaching +/// takes the same slot with an amber icon so a problem replaces the instruction instead of +/// stacking on top of it. +struct InstructionCard: View { + enum Tone: Equatable { + case normal + case coaching(symbol: String) + case refusal + } + + /// A reply to the instruction itself, such as "Can't get there". Lives in the card so it + /// reads as an answer to what the card asks, not as a competing primary action. + struct Reply { + var title: String + var identifier: String + var hint: String + var perform: () -> Void + } + + var instruction: Instruction + var tone: Tone = .normal + var reply: Reply? + /// A short line above the instruction that places it in the flow ("One more view to + /// finish"). Read with the instruction as one VoiceOver element. + var eyebrow: String? + + @Environment(\.accessibilityReduceMotion) private var reduceMotion + + var body: some View { + VStack(alignment: .trailing, spacing: 0) { + message + .id(instruction) + .transition(.identity) + if let reply { + Button(reply.title, action: reply.perform) + .font(Typeface.caption.weight(.bold)) + .foregroundStyle(Palette.chalk) + .padding(.horizontal, 16) + .frame(minHeight: Metrics.minTarget) + .background(Palette.replyFill, in: .capsule) + .contentShape(.capsule) + .buttonStyle(PressableStyle()) + .accessibilityHint(reply.hint) + .accessibilityIdentifier(reply.identifier) + .padding([.horizontal, .bottom], 12) + .padding(.top, -4) + // Appears, goes and changes its words at once, like the message above it: the + // walk's request can change every few seconds, and a fading button spends those + // moments as faint text on the card, which the accessibility audit caught on CI + // (run 36295565916, and the LiDAR replay after the coaching fix). A new identity + // per title: kept as one button, a new title ("Can't get there" to "Can't see + // past it") cross-faded inside the card's animation, both titles half drawn over + // each other (the LiDAR replay at 9b4c57d). + .id(reply.title) + .transition(.identity) + } + } + .frame(maxWidth: .infinity) + .background(ScrimShape.rounded()) + .animation(reduceMotion ? .easeOut(duration: 0.15) : Motion.text, value: instruction) + .animation(Motion.text, value: reply == nil) + } + + private var message: some View { + HStack(alignment: .firstTextBaseline, spacing: 12) { + if let symbol = iconName { + Image(systemName: symbol) + .font(.title3.weight(.bold)) + .foregroundStyle(iconColor) + .accessibilityHidden(true) + } + VStack(alignment: .leading, spacing: 4) { + if let eyebrow { + Label(eyebrow, systemImage: "camera.viewfinder") + .font(Typeface.caption) + // The requested view's amber, as on the camera and the map. + .foregroundStyle(Palette.caution) + .fixedSize(horizontal: false, vertical: true) + .padding(.bottom, 2) + } + Text(instruction.title) + .font(Typeface.instruction) + .foregroundStyle(Palette.chalk) + .fixedSize(horizontal: false, vertical: true) + if let detail = instruction.detail { + Text(detail) + .font(Typeface.hint) + .foregroundStyle(Palette.chalk.opacity(0.92)) + .fixedSize(horizontal: false, vertical: true) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + } + .padding(.horizontal, 18) + .padding(.vertical, 16) + .accessibilityElement(children: .combine) + .accessibilityAddTraits(.updatesFrequently) + .accessibilityIdentifier("instruction") + } + + private var iconName: String? { + switch tone { + case .normal: nil + case .coaching(let symbol): symbol + case .refusal: "exclamationmark.triangle.fill" + } + } + + private var iconColor: Color { + tone == .refusal ? Palette.danger : Palette.caution + } +} diff --git a/ios/HouseScan/UI/Components/OverheadAnswers.swift b/ios/HouseScan/UI/Components/OverheadAnswers.swift new file mode 100644 index 000000000..a1318d588 --- /dev/null +++ b/ios/HouseScan/UI/Components/OverheadAnswers.swift @@ -0,0 +1,29 @@ +import SwiftUI + +/// The two answers to the overhead question (`ScanViewState.overheadQuestion`), shared by the +/// walk's tilt-up step and an overhead gap request. Same shape as the end question: one question, +/// two equal full-width answers that say what they mean. +struct OverheadAnswers: View { + let actions: any ScanActions + + var body: some View { + VStack(spacing: 8) { + Button { + actions.answerOverhead(clear: true) + } label: { + Label(ScanCopy.overheadClear, systemImage: "sun.max") + .frame(maxWidth: .infinity) + } + .buttonStyle(.secondaryProminent) + .accessibilityIdentifier("action.overheadClear") + Button { + actions.answerOverhead(clear: false) + } label: { + Label(ScanCopy.overheadCovered, systemImage: "house") + .frame(maxWidth: .infinity) + } + .buttonStyle(.secondaryProminent) + .accessibilityIdentifier("action.overheadCovered") + } + } +} diff --git a/ios/HouseScan/UI/Components/PhotoCounter.swift b/ios/HouseScan/UI/Components/PhotoCounter.swift new file mode 100644 index 000000000..f2d72420d --- /dev/null +++ b/ios/HouseScan/UI/Components/PhotoCounter.swift @@ -0,0 +1,67 @@ +import SwiftUI + +/// Photo count with a small acknowledgment per capture: the icon flashes and the number changes. +/// The homeowner never presses a shutter, so this is how they learn the phone is taking photos. +/// +/// The number swaps without animation. A rolling digit passes through half-faded frames, and +/// the accessibility audit caught those as contrast failures ("22", "72" on the real replay). +struct PhotoCounter: View { + var count: Int + var lastCaptureID: Int? + + @Environment(\.accessibilityReduceMotion) private var reduceMotion + + var body: some View { + HStack(spacing: 6) { + Image(systemName: "camera.fill") + .font(.subheadline.weight(.bold)) + .phaseAnimator([false, true, false], trigger: lastCaptureID) { icon, lit in + icon + .foregroundStyle(lit ? Palette.covered : Palette.chalk) + .scaleEffect(lit && !reduceMotion ? 1.18 : 1) + } animation: { lit in + lit ? .easeOut(duration: 0.08) : .easeOut(duration: 0.35) + } + Text("\(count)") + .font(Typeface.caption.monospacedDigit()) + .foregroundStyle(Palette.chalk) + } + .padding(.horizontal, 12) + .frame(minWidth: Metrics.minTarget, minHeight: 36) + .background(ScrimShape.capsule) + .accessibilityElement(children: .ignore) + .accessibilityLabel(count == 1 ? "1 photo taken" : "\(count) photos taken") + .accessibilityIdentifier("photoCount") + } +} + +/// "Replay" or "Autopilot": tells a watcher that the camera or the taps are not live. A small +/// quiet pill rather than an amber one: amber means "look at this" on the camera screens, and a +/// test mode shouldn't compete with the instruction. Ink at 0.7 keeps Chalk text about 6.9:1 on +/// the light Canvas; at 0.55 it measured about 4.2:1, under the 4.5:1 small text needs. +struct ModeBadge: View { + var isReplay: Bool + var isAutopilot: Bool + + var body: some View { + if let text { + Text(text) + .font(.caption2.weight(.medium)) + .foregroundStyle(Palette.chalk) + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background(Palette.ink.opacity(0.7), in: .capsule) + .accessibilityLabel(text) + .accessibilityIdentifier("modeBadge") + } + } + + private var text: String? { + switch (isReplay, isAutopilot) { + case (true, true): "Replay · Autopilot" + case (true, false): "Replay" + case (false, true): "Autopilot" + case (false, false): nil + } + } +} diff --git a/ios/HouseScan/UI/Components/SavedMeterPhoto.swift b/ios/HouseScan/UI/Components/SavedMeterPhoto.swift new file mode 100644 index 000000000..9733ee81f --- /dev/null +++ b/ios/HouseScan/UI/Components/SavedMeterPhoto.swift @@ -0,0 +1,25 @@ +import SwiftUI + +/// The meter close-up taken at the start, shown while the phone finds its place again, so +/// "Point at the meter like this." has a picture to match (checklist T7). +struct SavedMeterPhoto: View { + let image: CGImage + + var body: some View { + Image(decorative: image, scale: 1) + .resizable() + .scaledToFill() + .frame(width: 168, height: 168) + .clipShape(.rect(cornerRadius: 24, style: .continuous)) + .overlay { + RoundedRectangle(cornerRadius: 24, style: .continuous) + .strokeBorder(.white, lineWidth: 3) + } + .shadow(color: .black.opacity(0.35), radius: 12, y: 4) + .frame(maxWidth: .infinity, maxHeight: .infinity) + .allowsHitTesting(false) + .accessibilityElement() + .accessibilityLabel("Your photo of the meter from the start of the scan") + .accessibilityIdentifier("relocalize.meterPhoto") + } +} diff --git a/ios/HouseScan/UI/Components/Scrim.swift b/ios/HouseScan/UI/Components/Scrim.swift new file mode 100644 index 000000000..3aeef5e17 --- /dev/null +++ b/ios/HouseScan/UI/Components/Scrim.swift @@ -0,0 +1,44 @@ +import SwiftUI + +/// The surface every piece of chrome over the camera sits on: solid Ink with a faint edge. +/// +/// Solid on purpose. Over a sunlit wall any translucency lets the photo wash out the chrome, and +/// XCUIApplication's accessibility audit flagged chalk text on camera screens even with 94% Ink +/// over the photo; with 100% it passes on every screen. Camera apps keep their controls on solid +/// bars for the same reason. +struct ScrimShape: View { + var shape: S + + var body: some View { + shape + .fill(Palette.ink) + .overlay(shape.strokeBorder(.white.opacity(0.1), lineWidth: 1)) + } +} + +extension ScrimShape where S == Capsule { + static var capsule: ScrimShape { ScrimShape(shape: Capsule()) } +} + +extension ScrimShape where S == RoundedRectangle { + static func rounded(_ radius: CGFloat = Metrics.cardRadius) -> ScrimShape { + ScrimShape(shape: RoundedRectangle(cornerRadius: radius, style: .continuous)) + } +} + +/// Soft dark gradients at the top and bottom of camera screens, under the chrome, so the +/// status bar and the edges of cards keep contrast without boxing the camera in. +struct CameraEdgeShade: View { + var body: some View { + VStack(spacing: 0) { + LinearGradient(colors: [.black.opacity(0.5), .clear], startPoint: .top, endPoint: .bottom) + .frame(height: 180) + Spacer(minLength: 0) + LinearGradient(colors: [.clear, .black.opacity(0.55)], startPoint: .top, endPoint: .bottom) + .frame(height: 260) + } + .ignoresSafeArea() + .allowsHitTesting(false) + .accessibilityHidden(true) + } +} diff --git a/ios/HouseScan/UI/Components/ShareScanButton.swift b/ios/HouseScan/UI/Components/ShareScanButton.swift new file mode 100644 index 000000000..e7ea3f0f0 --- /dev/null +++ b/ios/HouseScan/UI/Components/ShareScanButton.swift @@ -0,0 +1,25 @@ +import SwiftUI + +/// "Share scan": hands the scan bundle (`ScanViewState.shareableScan`) to the share sheet, so a +/// scan can get off the phone by AirDrop, Mail or Files. The keyframe photos leave the phone only +/// this way, so the line under the button says what goes. Always a quiet button: the screen's +/// one prominent action stays the way on. +struct ShareScanButton: View { + let url: URL + + var body: some View { + VStack(spacing: 8) { + ShareLink(item: url, subject: Text("House Scan")) { + Label(ScanCopy.shareScan, systemImage: "square.and.arrow.up") + } + .buttonStyle(.quiet) + .accessibilityIdentifier("action.shareScan") + Text(ScanCopy.shareScanContents) + .font(.footnote) + .foregroundStyle(Palette.muted) + .multilineTextAlignment(.center) + .fixedSize(horizontal: false, vertical: true) + .frame(maxWidth: .infinity) + } + } +} diff --git a/ios/HouseScan/UI/Components/TapRipple.swift b/ios/HouseScan/UI/Components/TapRipple.swift new file mode 100644 index 000000000..cbf65b40d --- /dev/null +++ b/ios/HouseScan/UI/Components/TapRipple.swift @@ -0,0 +1,45 @@ +import SwiftUI + +/// A ring that blooms outward and fades where the homeowner touched: immediate proof the tap +/// landed, independent of how long the engine takes to answer. +struct TapRipple: View { + struct Ripple: Identifiable, Equatable { + let id = UUID() + var point: CGPoint + } + + @Binding var ripples: [Ripple] + @Environment(\.accessibilityReduceMotion) private var reduceMotion + + var body: some View { + ZStack { + ForEach(ripples) { ripple in + RippleRing(reduceMotion: reduceMotion) + .position(ripple.point) + .task { + try? await Task.sleep(for: .milliseconds(650)) + ripples.removeAll { $0.id == ripple.id } + } + } + } + .ignoresSafeArea() + .allowsHitTesting(false) + .accessibilityHidden(true) + } +} + +private struct RippleRing: View { + var reduceMotion: Bool + @State private var expanded = false + + var body: some View { + Circle() + .strokeBorder(.white, lineWidth: 4) + .frame(width: 70, height: 70) + .scaleEffect(reduceMotion ? 1 : (expanded ? 1.35 : 0.7)) + .opacity(expanded ? 0 : 1) + .onAppear { + withAnimation(.easeOut(duration: 0.55)) { expanded = true } + } + } +} diff --git a/ios/HouseScan/UI/Copy/Distance.swift b/ios/HouseScan/UI/Copy/Distance.swift new file mode 100644 index 000000000..ed0cf3522 --- /dev/null +++ b/ios/HouseScan/UI/Copy/Distance.swift @@ -0,0 +1,88 @@ +import Foundation + +/// Lengths in the words a homeowner uses: feet and inches, rounded to the nearest inch. +enum Distance { + static let metersPerInch: Float = 0.0254 + + /// "8 in", "3 ft", "3 ft 4 in". Negative input is treated as its magnitude. + static func feetAndInches(_ meters: Float) -> String { + let totalInches = Int((abs(meters) / metersPerInch).rounded()) + let feet = totalInches / 12 + let inches = totalInches % 12 + switch (feet, inches) { + case (0, let inches): return "\(inches) in" + case (let feet, 0): return "\(feet) ft" + default: return "\(feet) ft \(inches) in" + } + } + + /// Whole feet for rough amounts ("about 12 ft to go"), never below 1 ft. + /// How far is left to walk, in 5 ft steps above 5 ft. The walk card showed the nearest foot, + /// so its text changed about every stride; a coarse count reads calmer and changes rarely. + static func remainingWalk(_ meters: Float) -> String { + let feet = abs(meters) / (metersPerInch * 12) + guard feet > 5 else { return roughFeet(meters) } + return "\(Int((feet / 5).rounded()) * 5) ft" + } + + static func roughFeet(_ meters: Float) -> String { + let feet = max(1, Int((abs(meters) / (metersPerInch * 12)).rounded())) + return "\(feet) ft" + } + + /// Whole feet rounded up, for a distance the homeowner must reach at least: "out to about + /// 7 ft" must not ask for less than the request. + static func feetAtLeast(_ meters: Float) -> String { + "\(max(1, Int((abs(meters) / (metersPerInch * 12) - 0.001).rounded(.up)))) ft" + } + + /// "5 ft either side of your meter", or "from 3 ft left to 5 ft right of your meter" when the + /// stretch is lopsided, for the tilt-up step. + static func stretchAroundMeter(_ span: ClosedRange) -> String { + let left = roughFeet(span.lowerBound), right = roughFeet(span.upperBound) + if span.lowerBound >= 0 { return "from your meter to \(right) right of it" } + if span.upperBound <= 0 { return "from \(left) left of your meter to your meter" } + if left == right { return "\(left) either side of your meter" } + return "from \(left) left to \(right) right of your meter" + } + + /// VoiceOver reads "ft" as letters; spell the units out. + static func spoken(_ meters: Float) -> String { + let totalInches = Int((abs(meters) / metersPerInch).rounded()) + let feet = totalInches / 12 + let inches = totalInches % 12 + let feetWord = feet == 1 ? "foot" : "feet" + let inchWord = inches == 1 ? "inch" : "inches" + switch (feet, inches) { + case (0, let inches): return "\(inches) \(inchWord)" + case (let feet, 0): return "\(feet) \(feetWord)" + default: return "\(feet) \(feetWord) \(inches) \(inchWord)" + } + } + + /// "3 ft 4 in right of your meter" for a position s along the wall. + static func fromMeter(_ s: Float) -> String { + if abs(s) < metersPerInch * 3 { return "at your meter" } + return "\(feetAndInches(s)) \(s < 0 ? "left" : "right") of your meter" + } + + /// "about 5 ft right of your meter" for the middle of a span: a place to walk to, not a + /// measurement to check. + static func aroundFromMeter(_ span: ClosedRange) -> String { + let center = (span.lowerBound + span.upperBound) / 2 + if span.contains(0) || abs(center) < 0.3 { return "around your meter" } + return "about \(roughFeet(center)) \(center < 0 ? "left" : "right") of your meter" + } +} + +extension String { + /// "about 5 ft right of your meter" as the start of a sentence. + var capitalizedFirst: String { prefix(1).uppercased() + dropFirst() } + + /// A title in the middle of a sentence: "Distance from AC units" becomes "distance from AC + /// units". A leading acronym ("AC units nearby") keeps its capitals. + var lowercasedFirst: String { + guard let first, let second = dropFirst().first, second.isLowercase else { return self } + return first.lowercased() + dropFirst() + } +} diff --git a/ios/HouseScan/UI/Copy/ScanCopy.swift b/ios/HouseScan/UI/Copy/ScanCopy.swift new file mode 100644 index 000000000..c0684ca70 --- /dev/null +++ b/ios/HouseScan/UI/Copy/ScanCopy.swift @@ -0,0 +1,513 @@ +import Foundation +import HouseScanKit + +/// One instruction: a short line the homeowner acts on, and an optional second line that says +/// how. Every camera screen shows exactly one of these at a time. +struct Instruction: Hashable { + var title: String + var detail: String? +} + +/// All user-facing words for engine values. The engine sends meanings (contract `GuidanceStep`, +/// `Coaching`, ...); this file is the only place that turns them into copy, so the vocabulary +/// stays consistent: "photos", "the wall", "your meter", feet and inches. +enum ScanCopy { + // MARK: Guidance + + static func guidance(_ step: GuidanceStep) -> Instruction { + switch step { + case .findMeter: + Instruction( + title: "Find your electric meter", + detail: "A gray box with a round glass dial or a small screen, usually on an outside wall." + ) + case .aimAtWallForMeter: + Instruction(title: "Step a little closer to the wall", detail: "Then aim at your meter again.") + case .holdOnMeter: + Instruction(title: "Hold your meter in the circle", detail: "Your phone takes the photo by itself.") + case .walk(let side, let remaining): + Instruction( + title: "Walk slowly to your \(side.rawValue)", + detail: remaining.map { "Keep the wall and the ground in view. About \(Distance.remainingWalk($0)) to go." } + ?? "Keep the wall and the ground in view." + ) + case .markEnd(let side): + Instruction( + title: "Is this the \(side.rawValue) end of the wall?", + detail: "Aim where the wall stops or turns a corner, and tap Wall ends here." + ) + case .aimAtGround(let s): + Instruction(title: "Tilt down to show the ground", detail: "The strip along the wall, \(Distance.fromMeter(s)).") + case .aimAtWall(let s): + // "Around at your meter" read wrong once the walk starts at the meter. + Instruction(title: "Tilt up to show more wall", detail: abs(s) < Distance.metersPerInch * 3 ? "At your meter." : "Around \(Distance.fromMeter(s)).") + case .stepBack: + Instruction(title: "Take a step back", detail: "Your phone needs to see more of the wall at once.") + case .tiltUp(let span): + Instruction( + title: "Tilt up to show above the area by your meter", + detail: "Aim at the wall \(Distance.stretchAroundMeter(span)) and whatever is above it." + ) + case .markNextWall(_, nil): + Instruction(title: "Walk round the corner, then aim at the next wall", detail: "Put the circle on it and tap Mark next wall.") + case .markNextWall(_, let refusal?): + nextWallRefusal(refusal) + case .walkComplete: + Instruction(title: "That's the whole wall", detail: "Tap Done when you're ready.") + case .seeBehind(let s): + Instruction( + title: "Something is in front of the wall here", + // A place to look, not a measurement: "About 5 ft", not "4 ft 11 in". + detail: "\(Distance.aroundFromMeter(s...s).capitalizedFirst). Look at it from the side or step around it." + ) + case .gap: + Instruction(title: "One more view", detail: nil) + } + } + + // MARK: Coaching + + static func coaching(_ coaching: Coaching) -> Instruction { + switch coaching { + case .initializing: + Instruction(title: "Move your phone slowly", detail: "It's getting its bearings.") + case .slowDown: + Instruction(title: "Slow down", detail: "Walk a little slower so the photos stay sharp.") + case .needsTexture: + Instruction(title: "Aim at a corner or somewhere with more texture", detail: "A plain wall or the sky gives your phone nothing to follow.") + case .tooDark: + Instruction(title: "It's too dark to see the wall", detail: "Turn on a porch light, or try again in daylight.") + case .holdSteady: + Instruction(title: "Hold steady", detail: nil) + case .relocalizing: + Instruction(title: "Point at the meter like this.", detail: "Your phone lost its place for a moment.") + case .trackingLost: + Instruction(title: "Your phone lost its place", detail: "Aim back at your meter and move slowly.") + case .pastWallEnd: + Instruction(title: "You're past the end of the wall", detail: "Photos here aren't kept. Walk back toward your meter.") + } + } + + /// Short symbol for a coaching pill. + static func coachingSymbol(_ coaching: Coaching) -> String { + switch coaching { + case .initializing: "iphone.gen3.radiowaves.left.and.right" + case .slowDown: "tortoise.fill" + case .needsTexture: "square.grid.3x3.middle.filled" + case .tooDark: "moon.fill" + case .holdSteady: "hand.raised.fill" + case .relocalizing, .trackingLost: "location.slash.fill" + case .pastWallEnd: "arrow.uturn.backward" + } + } + + // MARK: Wall ends + + /// Under the wall map when ending the wall at the dashed line would cut off part of the walk. + static func endLeavesOut(_ meters: Float) -> String { + "Ending the wall here leaves out \(Distance.roughFeet(meters)) you walked" + } + + /// Over the walk's own prompt after "Done with this wall" was refused and the ends cleared + /// (`ScanViewState.wallTooShort`). + static let wallTooShort = "The ends were too close. Walk along the wall first." + + // MARK: Close-up + + static func closeUpProblem(_ problem: CloseUpProblem) -> String { + switch problem { + case .blurry: "Hold still" + case .tooDark: "Too dark to read. Try your phone's flashlight." + case .tooBright: "Too much glare. Tilt the phone a little." + case .meterNotCentered: "Center the meter in the circle" + case .tooFar: "Move closer to the meter" + case .tracking: "Move slowly" + case .numberTooSmall: "Move closer so the number looks bigger" + case .noNumber: "Couldn't read the number. Hold still for another photo." + } + } + + /// The instruction while the meter number is read and confirmed; nil once it's settled or + /// skipped, when the close-up's own instruction applies. + static func meterNumber(_ state: MeterNumberState?) -> Instruction? { + switch state { + case .reading: + Instruction(title: "Reading your meter number", detail: "One moment.") + case .choose: + Instruction(title: "Which number is on your meter?", detail: "Check it against the meter, then tap it.") + case .confirmed(let number): + Instruction(title: "Meter number saved", detail: number) + case .skipped, nil: + nil + } + } + + static let barcodeMatch = "Matches the barcode" + /// The maker read on the close-up, above the number candidates. + static func meterBrand(_ brand: String) -> String { "\(brand) meter" } + static func notMeterBrand(_ brand: String) -> String { "Not \(brand)" } + static let noneOfThese = "None of these" + + // MARK: Features + + static func name(_ kind: FeatureKind) -> String { + switch kind { + case .gasMeter: "Gas meter" + case .door: "Door" + case .window: "Window" + case .acUnit: "AC unit" + case .driveway: "Driveway" + case .fence: "Fence" + } + } + + /// The name mid-sentence: "Tap the gas meter", "Mark AC unit". Lowercasing `name` read + /// "ac unit" (B-16). + static func noun(_ kind: FeatureKind) -> String { + switch kind { + case .acUnit: "AC unit" + default: name(kind).lowercased() + } + } + + static func symbol(_ kind: FeatureKind) -> String { + switch kind { + case .gasMeter: "flame.fill" + case .door: "door.left.hand.closed" + case .window: "window.vertical.closed" + case .acUnit: "fan.fill" + case .driveway: "car.fill" + case .fence: "square.split.2x1" + } + } + + /// Under a reviewed mark that lies wholly past a marked end (`ScanViewState.featuresPastEnds`). + static let featurePastEnd = "Past the end of your scan" + + static func markingPrompt(_ marking: MarkingState) -> Instruction { + let item = noun(marking.kind) + switch (marking.kind, marking.step) { + case (.door, 0), (.window, 0): + return Instruction(title: "Tap the \(item)'s bottom-left corner", detail: "Put the circle on it and tap Mark, or tap it on screen.") + case (.door, _), (.window, _): + return Instruction(title: "Now tap its top-right corner", detail: nil) + case (.driveway, 0): + return Instruction(title: "Tap one end of the driveway's edge", detail: "Use the edge closest to the wall.") + case (.driveway, _): + return Instruction(title: "Now tap the other end of that edge", detail: nil) + case (.fence, 0): + return Instruction(title: "Tap the bottom of the fence at one end", detail: "Where it meets the ground.") + case (.fence, _): + return Instruction(title: "Now tap the bottom at the other end", detail: nil) + case (.gasMeter, 0): + return Instruction(title: "Tap the gas meter's bottom-left corner", detail: "Where it meets the wall, behind the meter.") + case (.gasMeter, _): + return Instruction(title: "Now tap its top-right corner", detail: "Where it meets the wall.") + case (.acUnit, 0): + return Instruction(title: "Tap the AC unit's front-left corner", detail: "Where it meets the ground.") + case (.acUnit, _): + return Instruction(title: "Now tap its front-right corner", detail: "Where it meets the ground.") + } + } + + static func refusal(_ refusal: MarkRefusal) -> String { + switch refusal { + case .noSurface: "Nothing to pin there. Aim at the wall or the ground and try again." + case .wrongSide: "That spot is behind the wall. Tap something on this side." + case .tooFarFromWall: "That's too far from the wall to matter. Tap something closer." + case .trackingNotReady: "One moment, your phone is still finding its place." + case .fenceAcrossCorner: "That's round the corner. Mark the fence on each side of the corner as its own fence." + case .unitAcrossCorner: "That's round the corner. Tap both front corners on the same side of the house." + } + } + + /// A refused mark of the next wall: what went wrong, then what to do. + static func nextWallRefusal(_ refusal: NextWallRefusal) -> Instruction { + switch refusal { + case .noSurface: Instruction(title: "No wall under the circle", detail: "Step closer and aim at the next wall.") + case .trackingNotReady: Instruction(title: "One moment, your phone is still finding its place", detail: "Then aim at the next wall.") + case .sameWall: Instruction(title: "That looks like the same wall", detail: "Aim at the wall round the corner.") + case .notAtCorner: Instruction(title: "That wall doesn't meet this one at the corner", detail: "Aim at the wall right round the corner.") + } + } + + // MARK: Ground + + /// Names the homeowner would use: "Grass", not the schema's "lawn". + static func groundName(_ type: GroundType) -> String { + switch type { + case .lawn: "Grass" + case .mulch: "Mulch" + case .gravel: "Gravel" + case .concrete: "Concrete" + case .drive: "Driveway" + case .deck: "Deck" + } + } + static let groundNotSure = "Not sure" + static func groundAnswer(_ answer: GroundAnswer) -> String { + switch answer { + case .type(let type): groundName(type) + case .notSure: groundNotSure + } + } + + // MARK: Gap + + static func gap(_ gap: GapRequest) -> Instruction { + let place = Distance.aroundFromMeter(gap.span) + switch gap.reason { + case .groundNearCandidate: + return Instruction(title: "Show the ground \(place)", detail: "This might be a spot for the battery, so the ground there needs a clear look from two places.") + case .wallAboveCandidate: + return Instruction(title: "Show the wall \(place)", detail: "Tilt up so the wall above this spot is in view.") + case .server(let detail): + return Instruction(title: gap.band == .ground ? "Show the ground \(place)" : "Show the wall \(place)", detail: detail) + case .groundOut(let out): + return Instruction( + title: "Show the ground out to about \(Distance.feetAtLeast(out)) from the wall", + detail: "\(place.capitalizedFirst). Step back and tilt down until that much ground is in view." + ) + case .walkOut(let out): + return Instruction( + title: "Walk along this stretch about \(Distance.feetAtLeast(out)) out from the wall", + detail: "\(place.capitalizedFirst). Follow the dotted line. Walking there shows nothing stands in front of the wall." + ) + case .overhead: + return Instruction( + title: "Tilt up here", + detail: "\(place.capitalizedFirst). Show the wall above this spot, up to the roof or the sky." + ) + } + } + + /// The reply on the see-behind step: the homeowner can't get a view past the obstruction. + static let cannotSeeBehind = "Can't see past it" + + // MARK: Follow-up view + + /// The check came back asking for views the camera can take now; the scan goes straight + /// back to the camera for them before the result. Said on the upload screen as it hands + /// over and on the camera card that follows, so the two read as one step. + static func followUp(remaining: Int) -> String { + remaining <= 1 ? "One more view to finish" : "\(remaining) more views to finish" + } + + /// A sample result checked nothing, so it must not say the wall was checked. + static func followUpDetail(sample: Bool) -> String { + sample + ? "The example result asks for another view, so the camera opens again. Nothing leaves this phone." + : "The check needs a view the camera can take now. Your result comes right after." + } + + // MARK: Upload + + /// The question after the tilt-up view. The camera can't tell open sky from an eave. + static let overheadQuestion = Instruction( + title: "What's above that part of the wall?", + detail: "The battery needs clear space above it." + ) + static let overheadClear = "Open sky or nothing overhead" + static let overheadCovered = "A roof edge, porch or stairs" + + /// The question after "Wall ends here". A corner means the wall goes on out of sight, which + /// the result must not treat as the end of usable wall. + static func endQuestion(_ side: WallSide) -> Instruction { + Instruction(title: "What's at the \(side.rawValue) end?", detail: "This tells the installer whether the wall keeps going.") + } + + /// With no server connected nothing is sent, and the words must not say it is. + /// `followUps` is how many views the finished check still wants from the camera. + static func upload(_ upload: UploadState, sample: Bool, followUps: Int = 0) -> Instruction { + if upload == .done, followUps > 0 { + return Instruction(title: followUp(remaining: followUps), detail: followUpDetail(sample: sample)) + } + if sample { + switch upload { + case .idle, .packaging, .uploading, .analyzing: + return Instruction(title: "Making a sample result", detail: "No server is connected, so nothing leaves this phone. The result you'll see is an example, not a check of your wall.") + case .failed, .rejected, .done: + break + } + } + return self.upload(upload) + } + + static func upload(_ upload: UploadState) -> Instruction { + switch upload { + // The upload carries the wall's measurements; the photos stay on the phone. + case .idle, .packaging: + Instruction(title: "Getting your measurements ready", detail: nil) + case .uploading: + Instruction(title: "Sending your measurements", detail: "Keep the app open until this finishes.") + case .analyzing: + Instruction(title: "Checking your wall", detail: "Measuring clearances around your meter.") + case .failed(let message, let offline): + offline + ? Instruction(title: "You're offline", detail: "Your scan is saved on this phone. Try again when you have signal.") + : Instruction(title: "That didn't go through", detail: message) + // The engine's message is already in the homeowner's words and says why. + case .rejected(let message): + Instruction(title: "We couldn't check this scan", detail: message) + case .done: + Instruction(title: "Done", detail: nil) + } + } + + // MARK: Result + + /// The answer in the homeowner's words (`ResultPresentation.answer`). + static func headline(_ answer: ResultReading.Answer) -> String { + switch answer { + case .fits: "A battery fits here" + case .oneMoreLook: "One more look" + case .installer: "An installer will confirm" + case .notHere: "Not on this wall" + } + } + + /// The one line under the headline: where, and how much cable, for a pass. + static func placement(_ result: ResultPresentation) -> String? { + guard let spot = result.spot else { return nil } + let center = (spot.span.lowerBound + spot.span.upperBound) / 2 + var line = Distance.fromMeter(center).capitalizedFirst + if let cable = result.cableLength { + line += ", \(Distance.roughFeet(cable)) of cable" + } + return line + } + + /// Why the closest spot doesn't work, for a result without a spot: "The closest spot, 4 ft + /// left of your meter, fails this check: distance from gas equipment. Measured 2 ft 4 in. The + /// rule is at least 3 ft." The check's title follows a colon because the server's titles name + /// what a passing spot has ("No box or vent above the battery"), so a sentence that used one + /// as the failure would say the opposite for some of them. + static func nearest(_ result: ResultPresentation, spoken: Bool = false) -> String? { + guard let spot = result.nearestSpot, let id = result.nearestFailingCheck, + let row = result.checks.first(where: { $0.id == id }) else { return nil } + let center = (spot.span.lowerBound + spot.span.upperBound) / 2 + let place = spoken && abs(center) >= Distance.metersPerInch * 3 + ? "\(Distance.spoken(center)) \(center < 0 ? "left" : "right") of your meter" + : Distance.fromMeter(center) + var line = "The closest spot, \(place), fails this check: \(row.title.lowercasedFirst)." + if let measurement = measurement(row, spoken: spoken) { + line += " \(measurement)" + } + return line + } + + /// The sentence under a failed or unsure line on the result card: the measurement against the + /// rule, or without a measurement, the server's reason (fail) or who settles it (unsure). + static func cardLine(_ row: CheckRow, spoken: Bool = false) -> String? { + switch row.outcome { + case .pass: nil + case .fail: measurement(row, spoken: spoken) ?? row.reason + case .unsure: measurement(row, spoken: spoken) ?? unsureNote(row) + } + } + + /// "Settles: distance from the gas meter, clear space in front": the checks a requested view + /// would settle, by their titles. A check a person has to judge (a borderline measurement, an + /// unknown attribute) stays off the list: another view doesn't settle it. Nil when none is left. + static func settles(_ item: MissingEvidence, checks: [CheckRow]) -> String? { + let titles = item.checkIDs.compactMap { id in + checks.first { $0.id == id && !$0.needsPerson }?.title.lowercasedFirst + } + guard !titles.isEmpty else { return nil } + return "Settles: \(titles.joined(separator: ", "))" + } + + static let seeOnWall = "See it on your wall" + /// For a spot an installer still has to confirm against the meter's working space. + static let seeClosest = "See the closest spot" + static let showMe = "Show me" + static let scanAnotherWall = "Scan another wall" + static let details = "Details" + + static let installerConfirms = "An installer confirms this on site." + static let rulesNotFinal = "The placement rules aren't final yet, so an installer reviews every result for now." + // The server's result covers where the battery goes, not the panel itself. + static let panelReview = "Your electrical panel still needs an electrician's review. This scan only covers where the battery can go." + + /// Which rules answered, for a reviewer: "Rules 2f52ec35". + static func rulesHash(_ hash: String) -> String { + "Rules \(hash)" + } + + static func unseenSide(_ side: WallSide) -> String { + "A closer spot may exist on the \(side.rawValue) of your meter. The scan didn't reach that side." + } + + static let shareScan = "Share scan" + static let shareScanContents = "Your photos and measurements, for the House Scan team" + + static func unsureNote(_ row: CheckRow) -> String { + row.needsPerson ? "An installer will check this" : "One more photo would settle this" + } + + /// "Measured 3 ft 2 in. The rule is at least 3 ft, and the measurement can be off by about 4 in." + /// The limit says whether it is a minimum or a maximum: without it, the 20 ft cable limit read + /// like a minimum under "Measured 3 ft". + /// `spoken` spells out feet and inches for VoiceOver, which reads "ft" and "in" as letters. + static func measurement(_ row: CheckRow, spoken: Bool = false) -> String? { + guard let measured = row.measured else { return nil } + let length = spoken ? Distance.spoken : Distance.feetAndInches + var parts = [measuredLine(measured, length: length)] + if let threshold = row.threshold { + let limit = ruleLimit(threshold, row.comparison, length: length) + if let plusMinus = row.plusMinus, plusMinus > 0 { + parts.append("The rule is \(limit), and the measurement can be off by about \(length(plusMinus)).") + } else { + parts.append("The rule is \(limit).") + } + } + return parts.joined(separator: " ") + } + + /// "Measured 3 ft 2 in.", or "Overlaps by 1 ft 3 in." below zero. A clearance the server + /// measures to an area (the meter's working space, a box on the wall above) goes negative + /// once the battery is inside it, and the bare magnitude read as clearance (#40). Less than + /// half an inch of overlap stays "Measured 0 in.", not "Overlaps by 0 in.". + static func measuredLine(_ measured: Float, length: (Float) -> String = Distance.feetAndInches) -> String { + if measured <= -Distance.metersPerInch / 2 { + return "Overlaps by \(length(measured))." + } + return "Measured \(length(measured))." + } + + /// "at least 3 ft", "at most 20 ft", or the bare distance when the server didn't say which. + /// A minimum of 0 reads "no overlap": "at least 0 in" says the same thing less plainly. + static func ruleLimit(_ threshold: Float, _ comparison: RuleComparison?, length: (Float) -> String = Distance.feetAndInches) -> String { + let distance = length(threshold) + guard let comparison else { return distance } + switch comparison { + case .atLeast: return threshold < Distance.metersPerInch / 2 ? "no overlap" : "at least \(distance)" + case .atMost: return "at most \(distance)" + } + } + + static func outcomeWord(_ outcome: CheckOutcome) -> String { + switch outcome { + case .pass: "Looks good" + case .unsure: "Not sure yet" + case .fail: "Doesn't work" + } + } + + // MARK: Failure + + static func failure(_ failure: ScanFailure) -> Instruction { + switch failure { + case .cameraDenied: + Instruction(title: "House Scan needs your camera", detail: "It uses the camera to measure the wall around your meter. Turn on Camera for House Scan in Settings.") + case .arUnsupported: + Instruction(title: "This phone can't measure walls", detail: "House Scan needs an iPhone that supports motion tracking with the camera. Try another iPhone from the last few years.") + // The engine's messages for these two are system error text (ARKit's, or the replay + // loader's), not the homeowner's words, so they aren't shown. + case .sessionFailed: + Instruction(title: "The camera stopped", detail: "Something interrupted the camera partway through. Start over to try again.") + case .replayUnreadable: + Instruction(title: "This recording can't be opened", detail: "Some of its files are missing or damaged, so it can't be played back.") + } + } +} diff --git a/ios/HouseScan/UI/Copy/SpotCheckCopy.swift b/ios/HouseScan/UI/Copy/SpotCheckCopy.swift new file mode 100644 index 000000000..f09947359 --- /dev/null +++ b/ios/HouseScan/UI/Copy/SpotCheckCopy.swift @@ -0,0 +1,92 @@ +import Foundation + +/// The spot check's words (`SpotConfirmScreen`, and the guidance log's record of the questions). +extension ScanCopy { + // MARK: What's in the area + + static let spotQuestion = Instruction( + title: "Is anything in the marked area?", + detail: "Look at the wall and the ground inside the blue outline. Photos can miss a low bush, a bin or a step." + ) + static let spotClear = "It's clear" + static let spotSomethingThere = "Something's in the way" + static let spotUnmarked = "A gas meter, AC, window or door" + static let spotClearHint = "Asks what the ground is next." + static let spotSomethingThereHint = "A bush, a bin or a step. Leaves this area out of your scan and checks your wall again. An installer will look at it." + static let spotUnmarkedHint = "One you didn't mark. You'll mark it on the camera." + + static let spotWhich = Instruction( + title: "Which one isn't marked?", + detail: "You'll mark it on the camera, then we'll check your wall again." + ) + /// The kinds the spot check offers to mark: the equipment the server keeps a clearance from. + static let spotUnmarkedKinds: [FeatureKind] = [.gasMeter, .acUnit, .window, .door] + static let spotBack = "Back" + + // MARK: Ground + + /// Asked once the area is clear. The camera can't tell mulch from soil, so without this answer + /// the server's check of the ground under the battery ends unsure. + static let spotGroundQuestion = Instruction( + title: "What's the ground where the battery would stand?", + detail: "Inside the small outline. The battery can only stand on some kinds of ground." + ) + + // MARK: No photo shows it all + + static let spotUnconfirmable = Instruction( + title: "Your photos don't show all of this area", + detail: "So you can't check it here. We'll leave it out and check your wall again, and an installer will look at it." + ) + static let spotContinue = "Continue" + + // MARK: Answered + + static func spotAnswered(_ answer: SpotCheckAnswer, checksAgain: Bool) -> Instruction { + switch answer { + case .clear(.type(let type)): + Instruction(title: "Thanks, it's \(groundName(type).lowercased())", detail: checksAgain ? "Checking your spot again with it." : "Showing your result.") + case .clear(.notSure): + Instruction(title: "Thanks", detail: "Showing your result. An installer will check the ground.") + case .somethingThere: + Instruction(title: "Thanks, we'll leave that area out", detail: "Checking your wall again. An installer will look at what's there.") + case .unmarkedCantMark(let kind): + Instruction( + title: "The \(noun(kind)) can't be marked now", + detail: "Your phone lost its place. We'll leave that area out and check your wall again, and an installer will look at it.") + case .unconfirmed: + Instruction(title: "We'll leave that area out", detail: "Checking your wall again.") + } + } + + /// Said on the result when the spot's area was left out. + static func spotLeftOut(_ answer: SpotCheckAnswer) -> String? { + switch answer { + case .clear: nil + case .somethingThere, .unmarkedCantMark: + "You said something is where this spot is. Your scan leaves that area out, and an installer will check it." + case .unconfirmed: + "Your photos didn't show all of this spot's area, so your scan leaves it out and an installer will check it." + } + } + + // MARK: Place + + /// "From 3 ft to 8 ft 6 in right of your meter": where the area runs. `spoken` spells the + /// units out for VoiceOver. + static func spotArea(_ span: ClosedRange, spoken: Bool = false) -> String { + let length = { (meters: Float) in spoken ? Distance.spoken(meters) : Distance.feetAndInches(meters) } + let low = span.lowerBound, high = span.upperBound + if low >= 0 { return "From \(length(low)) to \(length(high)) right of your meter" } + if high <= 0 { return "From \(length(high)) to \(length(low)) left of your meter" } + return "From \(length(low)) left to \(length(high)) right of your meter" + } + + /// What VoiceOver reads for the photo. + static func spotPhotoDescription(_ check: SpotCheck) -> String { + if check.step == .ground { + return "A small outline marks where the battery would stand. \(spotArea(check.spot, spoken: true))." + } + return "A blue outline marks where the battery would stand and the space around it. \(spotArea(check.area, spoken: true))." + } +} diff --git a/ios/HouseScan/UI/Preview/DemoEngine.swift b/ios/HouseScan/UI/Preview/DemoEngine.swift new file mode 100644 index 000000000..80fd8bfd5 --- /dev/null +++ b/ios/HouseScan/UI/Preview/DemoEngine.swift @@ -0,0 +1,1171 @@ +import HouseScanKit +import SwiftUI + +/// A fake engine for the UI demo: implements `ScanActions` by scripting `ScanViewState` over +/// the made-up wall in `DemoScene`. It mimics the real engine's phases and timing closely enough +/// to exercise every screen; it measures nothing. +@MainActor +final class DemoEngine: ScanActions { + let state = ScanViewState() + + private let freeze: Bool + private let noFeed: Bool + private let offline: Bool + private let passResult: Bool + /// `-uiDemoOverlap`: the sample's spot overlaps the meter's working space (#40). + private let overlapResult: Bool + /// `-uiDemoResultFile ` (debug builds only): a server answer in a JSON file, read + /// through the engine's own mapping, in place of the hand-made samples. + private let resultFile: String? + private let rejectUpload: Bool + /// Which request the gap screen shows (`-uiDemoGap`); the phone's ground request by default. + private let gapKind: String? + /// The tilt-up step was answered or skipped. + private var tiltUpSettled = false + /// Walk-script ticks spent on the tilt-up step, standing in for the phone being tilted up. + private var tiltUpTicks = 0 + private var script: Task? + private var nextCaptureID = 1 + /// How far the walk has seen to each side of the meter, meters. + private var reachedLeft: Float = 0.3 + private var reachedRight: Float = 0.3 + private var skippedSpan: ClosedRange? + /// `-uiDemoEndPreview`: the homeowner walked back 1.5 m, so ending the wall now leaves part + /// of the walk out. + private var walkedBack: Float? + /// Things standing in front of the wall, as depth would find them: a looked-at cell in the + /// span is hidden, or skipped once the homeowner said they can't see past it. + private var obstructions: [(span: ClosedRange, skipped: Bool)] = [] + /// Walk-script ticks spent on the see-behind step, standing in for stepping round the bush. + private var seeBehindTicks = 0 + /// The check's first answer asked for a view and the scan went back to the camera for it, + /// like the real engine does; the next answer goes to the result. + private var followedUp = false + private var followUpSkipped = false + /// The spot check was answered; the next answer goes straight to the result, as the engine's + /// does for a spot an earlier answer settles. + private var spotChecked = false + /// `-uiDemoSpotUnconfirmable`: no photo shows the spot check's whole area. + private var spotUnconfirmable = false + private var failedUploads = 0 + private var rejectedUploads = 0 + + private static let cellWidth: Float = 0.1524 + /// Where the demo wall ends on each side, meters of s. Following a corner moves the end on. + private var demoLeftEnd: Float = -2.9 + private var demoRightEnd: Float = 4.3 + /// How far the made-up wall goes on past a corner the walk follows, meters. + private static let pastCorner: Float = 1.5 + + init(arguments: [String]) { + func value(_ flag: String) -> String? { + guard let index = arguments.firstIndex(of: flag), index + 1 < arguments.count else { return nil } + return arguments[index + 1] + } + freeze = arguments.contains("-uiDemoFreeze") + noFeed = arguments.contains("-uiDemoNoFeed") + offline = arguments.contains("-uiDemoOffline") + passResult = arguments.contains("-uiDemoPass") + overlapResult = arguments.contains("-uiDemoOverlap") + #if DEBUG + resultFile = value("-uiDemoResultFile") + #else + resultFile = nil + #endif + rejectUpload = arguments.contains("-uiDemoRejected") + gapKind = value("-uiDemoGap") + state.feed = DemoScene.image.map(CameraFeed.still) ?? .none + state.isReplay = true + state.tracking = .normal + + if let failure = value("-uiDemoFailure") { + state.failure = switch failure { + case "cameraDenied": .cameraDenied + case "sessionFailed": .sessionFailed("The operation couldn't be completed. (demo error 102.)") + case "replayUnreadable": .replayUnreadable("demo: frames.jsonl not found") + default: .arUnsupported + } + state.phase = .unsupported + return + } + let phase: ScanPhase = value("-uiDemoPhase").flatMap(ScanPhase.init(rawValue:)) ?? .onboarding + spotUnconfirmable = arguments.contains("-uiDemoSpotUnconfirmable") + jump(to: phase) + if let raw = value("-uiDemoMarking"), let kind = FeatureKind(rawValue: raw) { + state.marking = MarkingState(kind: kind, step: 0, refusal: arguments.contains("-uiDemoRefusal") ? .noSurface : nil) + } + if let raw = value("-uiDemoCoaching") { + state.coaching = Self.coaching(raw) + // The real engine reports these two only while the phone has lost its place, which + // also hides the overlays drawn from its position. + switch state.coaching { + case .relocalizing: state.tracking = .limited(.relocalizing) + case .trackingLost: state.tracking = .notAvailable + default: break + } + } + if arguments.contains("-uiDemoEndPreview") { + walkedBack = 1.5 + refreshGuidance() + } + if arguments.contains("-uiDemoEndQuestion") { + state.endQuestion = .left + } + if arguments.contains("-uiDemoNextWall") { + state.wall?.rightEnd = demoRightEnd + reachedRight = demoRightEnd + refreshCoverage() + state.guidance = .markNextWall(side: .right, refusal: arguments.contains("-uiDemoRefusal") ? .notAtCorner : nil) + state.target = nil + state.path = [] + } + if arguments.contains("-uiDemoOverheadQuestion"), state.phase == .gapRequest { + // An overhead request whose tilted-up view just came in. + script?.cancel() + state.overheadQuestion = true + } else if arguments.contains("-uiDemoTiltUp") || arguments.contains("-uiDemoOverheadQuestion") { + enterTiltUp() + state.overheadQuestion = arguments.contains("-uiDemoOverheadQuestion") + } + if arguments.contains("-uiDemoSample") { + state.usesSampleResult = true + } + if arguments.contains("-uiDemoDepth") { + state.depthAvailable = true + } + if arguments.contains("-uiDemoHidden") || arguments.contains("-uiDemoSeeBehind") { + // A bush right of the meter and a bin left of it, on a phone with depth. + state.depthAvailable = true + reachedRight = max(reachedRight, 2.4) + obstructions = [(1.2...1.8, false), (-0.55 ... -0.25, false)] + refreshCoverage() + refreshGuidance() + if arguments.contains("-uiDemoSeeBehind") { + state.guidance = .seeBehind(s: 1.5) + state.target = DemoScene.wall.world(s: 1.5, height: 0.6) + state.path = DemoScene.path(toward: 2.4, out: 1.8) + } + } + if arguments.contains("-uiDemoCorner") { + // The walk followed an outside corner right of the meter, between the battery spot and + // the window, and went on 1.6 m along the next wall: that wall runs away from the + // homeowner and faces right. + let corner: Float = 1.8 + state.wall?.cornerSegments = [WallGeometry.Segment( + span: corner...Float.infinity, along: SIMD3(0, 0, -1), outward: SIMD3(1, 0, 0), + anchor: SIMD3(corner, 0, 0), anchorS: corner)] + state.wall?.rightEnd = corner + 1.6 + } + if state.phase == .spotConfirm { + switch value("-uiDemoSpotStep") { + case "which": state.spotCheck?.step = .which + case "ground": state.spotCheck?.step = .ground + default: break + } + let answer = value("-uiDemoSpotAnswered").flatMap(Self.spotAnswer) + state.spotCheck?.answer = answer + state.spotCheck?.checksAgain = answer.map { $0 != .clear(.notSure) } ?? false + } + if arguments.contains("-uiDemoFollowUp") { + enterFollowUp(at: state.phase) + } + if arguments.contains("-uiDemoNoFeed") { + state.feed = .none + } + if arguments.contains("-uiDemoCloseUpFailed") { + state.closeUpFailedAttempts = 2 + state.closeUp = .aiming(hold: 0.2, problem: .tooBright) + } + if arguments.contains("-uiDemoMeterChoose") { + state.closeUp = .captured(DemoScene.meterThumbnail) + state.meterBrand = Self.demoBrand + state.meterNumber = .choose(Self.demoCandidates) + } + // Frozen, the upload script never runs, so show where it would end. + if freeze, state.phase == .uploading { + if rejectUpload || offline { + state.shareableScan = Self.demoScan + } + if rejectUpload { + state.upload = .rejected(message: Self.rejection) + } else if offline { + state.upload = .failed(message: "No internet connection.", offline: true) + } + } + } + + // MARK: Phase set-up + + /// Puts the state where the real engine would be on arriving at `phase`. + private func jump(to phase: ScanPhase) { + switch phase { + case .onboarding: + state.phase = .onboarding + case .findMeter: + enterFindMeter() + case .meterCloseUp: + enterCloseUp() + case .wallWalk: + placeMeter() + reachedLeft = 0.6 + reachedRight = 0.9 + enterWalk() + case .markFeatures: + placeMeter() + finishedWalkState() + state.phase = .markFeatures + case .gapRequest: + placeMeter() + finishedWalkState() + enterGap() + case .uploading: + placeMeter() + finishedWalkState() + enterUpload() + case .spotConfirm: + placeMeter() + finishedWalkState() + enterSpotCheck() + case .result: + placeMeter() + finishedWalkState() + showResult() + case .resultAR: + placeMeter() + finishedWalkState() + showResult() + state.phase = .resultAR + case .unsupported: + state.failure = .arUnsupported + state.phase = .unsupported + } + } + + private func enterFindMeter() { + state.phase = .findMeter + state.guidance = .findMeter + state.feed = noFeed ? .none : DemoScene.meterImage.map(CameraFeed.still) ?? .none + state.projection = DemoScene.meterProjection + state.wall = nil + state.coverage = .empty + } + + private func placeMeter() { + state.feed = noFeed ? .none : DemoScene.image.map(CameraFeed.still) ?? .none + state.projection = DemoScene.projection + state.wall = DemoScene.wall + state.captureCount = max(state.captureCount, 1) + } + + private func enterCloseUp() { + placeMeter() + state.feed = noFeed ? .none : DemoScene.closeUpImage.map(CameraFeed.still) ?? .none + state.projection = DemoScene.closeUpProjection + state.phase = .meterCloseUp + state.guidance = .holdOnMeter + state.closeUp = .aiming(hold: 0, problem: nil) + state.meterNumber = nil + run { engine in await engine.closeUpScript() } + } + + private func enterWalk() { + state.phase = .wallWalk + state.closeUp = .captured(DemoScene.meterThumbnail) + state.captureCount = max(state.captureCount, 6) + refreshCoverage() + refreshGuidance() + run { engine in await engine.walkScript() } + } + + private func finishedWalkState() { + reachedLeft = -demoLeftEnd + reachedRight = demoRightEnd + state.wall?.leftEnd = demoLeftEnd + state.wall?.rightEnd = demoRightEnd + state.captureCount = max(state.captureCount, 42) + if state.features.isEmpty { + state.features = [Self.demoFeature(.gasMeter), Self.demoFeature(.window)] + } + refreshCoverage() + state.guidance = .walkComplete + state.path = [] + state.target = nil + } + + /// Both ends are marked and answered: the walk asks to tilt up over the stretch by the meter. + private func enterTiltUp() { + placeMeter() + finishedWalkState() + state.phase = .wallWalk + state.closeUp = .captured(DemoScene.meterThumbnail) + refreshGuidance() + run { engine in await engine.walkScript() } + } + + /// A gap request: the phone's own by default, or `serverItem` from the check's answer. + private func enterGap(serverItem: MissingEvidence? = nil) { + // Leave a hole in the ground right of the likely spot, as the phone's planner would find. + state.phase = .gapRequest + state.guidance = .gap + let span: ClosedRange = 1.3...2.1 + setGround(span, to: .seen) + state.gap = GapRequest(id: 1, origin: .phone, reason: .groundNearCandidate, band: .ground, span: span, progress: 0, isSatisfied: false) + state.target = DemoScene.wall.world(s: 1.7, height: 0, out: 0.5) + state.path = DemoScene.path(toward: 1.7) + if let serverItem { + state.gap?.origin = .server + state.gap?.reason = .server(detail: serverItem.text) + run { engine in await engine.gapScript(span: span) } + return + } + // The server's requests, with the numbers its public rules would give (feet in the + // README, meters here): ground out to D + r + e = 5.1 ft, a walk past D + r = 4.83 ft. + switch gapKind { + case "groundOut": + state.gap?.origin = .server + state.gap?.reason = .groundOut(out: 1.56) + state.target = DemoScene.wall.world(s: 1.7, height: 0, out: 1.56) + case "walkOut": + let out: Float = 1.47 + // As the engine draws it: from the homeowner to the stretch's far end, the requested + // distance plus the position error there (0.09 m + 0.16 m per meter from the meter) + // plus 0.3 m out. + let standOut = out + 0.09 + 0.16 * span.upperBound + 0.3 + state.gap?.origin = .server + state.gap?.reason = .walkOut(out: out) + state.target = DemoScene.wall.world(s: (span.lowerBound + span.upperBound) / 2, height: 1.2) + state.path = [span.lowerBound - 0.8, span.upperBound].map { + DemoScene.wall.world(s: $0, height: 0, out: standOut) + } + case "overhead": + state.gap?.origin = .server + state.gap?.reason = .overhead + state.gap?.band = .wall + state.target = DemoScene.wall.world(s: 1.7, height: 3.0) + state.path = [] + default: + break + } + if gapKind == "overhead" { + run { engine in await engine.overheadGapScript() } + } else { + run { engine in await engine.gapScript(span: span) } + } + } + + private func enterUpload() { + state.phase = .uploading + state.gap = nil + state.path = [] + state.target = nil + state.upload = .packaging + run { engine in await engine.uploadScript() } + } + + /// The spot check of the answer's spot, before the result. The area is the footprint and the + /// clear space in front a passing front check would rest on: 3 ft (0.91 m) past the battery. + private func enterSpotCheck() { + let result = sample + state.shareableScan = Self.demoScan + state.upload = .done + state.result = result + guard let spot = result.spot else { return showResult() } + let out = spot.offsetFromWall...(spot.offsetFromWall + spot.depth) + state.spotCheck = SpotCheck( + id: 1, spot: spot.span, spotOut: out, spotHeight: spot.height, area: spot.span, areaDepth: out.upperBound + 0.91, + photo: DemoScene.image.map { SpotCheck.Photo(image: $0, projection: DemoScene.projection) }, + confirmable: !spotUnconfirmable, answer: nil, isSample: result.isSample) + state.phase = .spotConfirm + } + + private func showResult() { + state.shareableScan = Self.demoScan + state.upload = .done + state.result = sample + state.phase = .result + } + + // MARK: Scripts + + /// Runs a timed script unless frozen; a new script replaces the old one. + private func run(_ body: @escaping @MainActor (DemoEngine) async -> Void) { + script?.cancel() + guard !freeze else { return } + script = Task { [weak self] in + guard let self else { return } + await body(self) + } + } + + private func pause(_ seconds: Double) async -> Bool { + try? await Task.sleep(for: .seconds(seconds)) + return !Task.isCancelled + } + + private func closeUpScript() async { + guard await pause(0.6) else { return } + for step in 1...6 { + state.closeUp = .aiming(hold: Double(step) * 0.09, problem: nil) + guard await pause(0.12) else { return } + } + state.closeUp = .aiming(hold: 0.54, problem: .blurry) + guard await pause(1.4) else { return } + for step in 6...11 { + state.closeUp = .aiming(hold: Double(step) * 0.09, problem: nil) + guard await pause(0.12) else { return } + } + capture(.closeUp, thumbnail: DemoScene.meterThumbnail) + state.closeUp = .captured(DemoScene.meterThumbnail) + state.meterNumber = .reading + guard await pause(1.2) else { return } + // Waits here for the homeowner's pick (`chooseMeterNumber`). + state.meterBrand = Self.demoBrand + state.meterNumber = .choose(Self.demoCandidates) + } + + private func walkScript() async { + while !Task.isCancelled { + guard await pause(0.45) else { return } + guard state.marking == nil else { continue } + if case .tiltUp = state.guidance, !state.overheadQuestion { + // About two seconds of tilting up, then the kept view raises the question. + tiltUpTicks += 1 + if tiltUpTicks >= 4 { + capture(.walk) + state.overheadQuestion = true + } + } + if case .seeBehind(let s) = state.guidance { + // A few steps to the side, then depth sees the wall behind the bush. + seeBehindTicks += 1 + if seeBehindTicks >= 5 { + seeBehindTicks = 0 + obstructions.removeAll { $0.span.contains(s) } + capture(.walk) + refreshCoverage() + refreshGuidance() + } + } + if case .walk(let side, _) = state.guidance { + if side == .right { + reachedRight = min(reachedRight + 0.3, demoRightEnd) + } else { + reachedLeft = min(reachedLeft + 0.3, -demoLeftEnd) + } + capture(.walk) + refreshCoverage() + refreshGuidance() + } + } + } + + private func gapScript(span: ClosedRange) async { + guard await pause(1.2) else { return } + let cells = Int(((span.upperBound - span.lowerBound) / Self.cellWidth).rounded(.up)) + for step in 1...cells { + let covered = span.lowerBound...(span.lowerBound + Float(step) * Self.cellWidth) + setGround(covered, to: .covered) + state.gap?.progress = Double(step) / Double(cells) + capture(.gap) + guard await pause(0.45) else { return } + } + state.gap?.isSatisfied = true + guard await pause(1.6) else { return } + enterUpload() + } + + /// About two seconds of tilting up, then the view covers the request and the question comes up. + private func overheadGapScript() async { + guard await pause(2.4) else { return } + capture(.gap) + state.overheadQuestion = true + } + + private func uploadScript() async { + state.upload = .packaging + guard await pause(1.0) else { return } + state.shareableScan = Self.demoScan + for step in 0...10 { + state.upload = .uploading(fraction: Double(step) / 10) + guard await pause(0.22) else { return } + if offline, failedUploads == 0, step == 4 { + failedUploads += 1 + state.upload = .failed(message: "No internet connection.", offline: true) + return + } + } + state.upload = .analyzing + guard await pause(1.8) else { return } + // Refused once; after the review the same scan goes through. + if rejectUpload, rejectedUploads == 0 { + rejectedUploads += 1 + state.upload = .rejected(message: Self.rejection) + return + } + // Like the real engine: an answer that lists a view the camera can take goes back to + // the camera for it, after the upload screen has said so; the next answer is the result. + if !followedUp, let item = sample.missing.first(where: \.capturable) { + state.shareableScan = Self.demoScan + state.result = sample + state.upload = .done + followedUp = true + guard await pause(1.6) else { return } + enterGap(serverItem: item) + return + } + if spotChecked { showResult() } else { enterSpotCheck() } + } + + /// The check's answer, before or after its follow-up view. + private var sample: ResultPresentation { + if passResult { return Self.passSample } + if overlapResult { return Self.overlapSample } + if let fileResult { return fileResult } + guard followedUp else { return Self.reviewSample } + return followUpSkipped ? Self.reviewSample.withFollowUpSkipped : Self.reviewSample.withFollowUpTaken + } + + /// The answer in `-uiDemoResultFile`, mapped as `ScanEngine` maps a server's. A file that + /// doesn't decode stops the demo with the decoding error rather than showing a sample. + private var fileResult: ResultPresentation? { + guard let resultFile else { return nil } + let result: PlacementResult + do { + result = try PlacementResult.decode(Data(contentsOf: URL(fileURLWithPath: resultFile))) + } catch { + fatalError("-uiDemoResultFile \(resultFile): \(error)") + } + let planner = GapPlanner() + let wall = state.wall + return ScanEngine.presentation(of: result, isSample: true, wall: nil) { item in + planner.plan(for: item, leftEnd: wall?.leftEnd, rightEnd: wall?.rightEnd) != nil + } + } + + /// `-uiDemoFollowUp`: the check has answered and asked for a view. On the upload screen, + /// the moment before the camera opens; on the gap screen, the view itself. + private func enterFollowUp(at phase: ScanPhase) { + guard phase == .uploading || phase == .gapRequest, let item = Self.reviewSample.missing.first(where: \.capturable) else { return } + script?.cancel() + state.shareableScan = Self.demoScan + state.result = Self.reviewSample + state.upload = .done + followedUp = true + if phase == .gapRequest { + enterGap(serverItem: item) + } else { + run { engine in + guard await engine.pause(1.6) else { return } + engine.enterGap(serverItem: item) + } + } + } + + // MARK: Coverage + + private func refreshCoverage() { + let wallRange = min(DemoScene.wallRange.lowerBound, demoLeftEnd - 0.3)...max(DemoScene.wallRange.upperBound, demoRightEnd + 0.1) + let count = Int(((wallRange.upperBound - wallRange.lowerBound) / Self.cellWidth).rounded(.up)) + var wallCells: [CellState] = [] + var groundCells: [CellState] = [] + for index in 0..