diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 6d1c66192..24cb5a906 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -28,7 +28,20 @@ concurrency: jobs: ios-build: - name: iOS build + # Three groups of the UI tests, one job each (ios/Tools/ui-test-groups.py holds their selectors + # and checks that together they run every test once). With two groups, journey hit the cap + # below in run 37221532164 (#216 at 6fa43b9b): 12 min of package tests and build, then a + # 75.3 min UI step that stopped after 107 of its 109 tests. That run's per-class times put + # ScreenStatesUITests at 1892 s (its every-state audit 905 s) and the other journey classes at + # about 2475 s, so ScreenStatesUITests has its own job. journey runs the package tests, the + # project check and the Debug device build; photoProcessing runs the unsigned Release build; + # screenStates runs only its UI tests. fail-fast is off so a failure in one group still leaves + # the others' results. + name: iOS build (${{ matrix.group }}) + strategy: + fail-fast: false + matrix: + group: [screenStates, journey, photoProcessing] # A free account runs 5 macOS jobs at once, shared by every open pull request, so drafts and # labels other than full-ui don't take a slot. if: >- @@ -42,7 +55,9 @@ jobs: # freshness UI tests add about 161 s more (run 36744881974), so 60 would cut the # every-state audit short. 75 left room until the 2026-10-03 candidate's 86 UI cases: # run 37121918073 attempt 2 spent 10 min 34 s before the UI step and 64 min 39 s in it, - # and the cap stopped it after 76 cases with 0 failures. 90 runs all of them. + # and the cap stopped it after 76 cases with 0 failures. 90 ran all of them; since photo + # processing's tests moved to their own job, 90 is the cap for each group, and the groups are + # split so that each fits it rather than raising it. timeout-minutes: 90 env: # Newest stable Xcode on macos-26 per images/macos/macos-26-arm64-Readme.md (image 20260907). @@ -55,7 +70,9 @@ jobs: XCODEGEN_VERSION: 2.46.0 XCODEGEN_SHA256: 4d9e34b62172d645eed6457cac13fc222569974098ef4ee9c3368bedf0196806 # The every-state accessibility audit relaunches the app 65 times (about 10 min). Pull - # requests skip it unless labelled full-ui; pushes and manual runs always include it. + # requests skip it unless labelled full-ui; pushes and manual runs always include it. It is + # in the screenStates group; photoProcessing always runs its own audits. + GROUP: ${{ matrix.group }} FULL_UI: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'full-ui') }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -71,6 +88,13 @@ jobs: xcodebuild -version swift --version + # Before anything slow: the groups' selectors must cover every UI test exactly once, and the + # CI tools' own tests (the groups' check and the result snapshot) must pass. + - name: Check the UI test groups + run: | + python3 -m unittest discover -s ios/Tools -p 'test_*.py' + python3 ios/Tools/ui-test-groups.py check + # Booting takes 2 to 3 minutes and needs nothing from the build, so it runs alongside the # steps below; the UI test step waits for it. - name: Boot the Simulator in the background @@ -86,6 +110,7 @@ jobs: fi echo "SIM_UDID=$udid" >> "$GITHUB_ENV" nohup xcrun simctl bootstatus "$udid" -b > "$RUNNER_TEMP/simulator-boot.log" 2>&1 & + echo "SIM_BOOT_PID=$!" >> "$GITHUB_ENV" # The package's drift test compares the vendored server schemas with the server's own copies: # the checkout's files once that branch merges, else the t3/server branch, which the shallow @@ -94,6 +119,7 @@ jobs: # than skip, so CI can't go green without comparing. The checkout counts only when it holds # every compared file: a folder alone isn't enough (main has a server/ without schemas). - name: Fetch the server contract + if: matrix.group == 'journey' run: | fetch_unless_present() { local branch="$1"; shift @@ -111,6 +137,7 @@ jobs: fetch_unless_present t3/server server/schemas/scene.schema.json server/schemas/result.schema.json server/tests/fixtures/example-scene.json - name: Install XcodeGen + if: matrix.group == 'journey' run: | curl -fsSL -o "$RUNNER_TEMP/xcodegen.zip" \ "https://github.com/yonaskolb/XcodeGen/releases/download/${XCODEGEN_VERSION}/xcodegen.zip" @@ -119,9 +146,11 @@ jobs: echo "$RUNNER_TEMP/xcodegen/bin" >> "$GITHUB_PATH" - name: Generate project + if: matrix.group == 'journey' run: xcodegen generate --spec ios/project.yml - name: Check committed project matches project.yml + if: matrix.group == 'journey' run: | if ! git diff --exit-code -- ios; then echo "::error::HouseScan.xcodeproj differs from project.yml. Run 'make ios-project' (XcodeGen ${XCODEGEN_VERSION}) and commit the result." @@ -136,42 +165,73 @@ jobs: - name: Test HouseScanKit # The capture logic is a plain Swift package; its tests run on macOS without a Simulator. + # Once per run: photoProcessing builds the same package sources into its Release build. + if: matrix.group == 'journey' env: HOUSESCAN_REQUIRE_UPSTREAM: "1" run: swift test --package-path ios/HouseScanKit -Xswiftc -warnings-as-errors # CI builds once from clean, so the index Xcode keeps for editing is wasted work. - name: Build + if: matrix.group == 'journey' run: >- xcodebuild -project ios/HouseScan.xcodeproj -scheme HouseScan -configuration Debug -destination "generic/platform=iOS" CODE_SIGNING_ALLOWED=NO COMPILER_INDEX_STORE_ENABLE=NO build + # Proves the Release configuration compiles, the photo-processing setup gate included, whose + # fixture switches exist only in Debug. It doesn't show what Release does at run time. + - name: Build Release, unsigned + if: matrix.group == 'photoProcessing' + run: >- + xcodebuild -project ios/HouseScan.xcodeproj -scheme HouseScan -configuration Release + -destination "generic/platform=iOS" CODE_SIGNING_ALLOWED=NO COMPILER_INDEX_STORE_ENABLE=NO build + + # The background boot first: screenStates reaches this step 3 s after starting it, and a + # second `bootstatus -b` while that boot was under way exited 149 (run 37228728303). Once it + # has finished, `-b` returns at once on a booted device, and boots it if the first try failed. - name: Wait for the Simulator timeout-minutes: 10 run: | - xcrun simctl bootstatus "$SIM_UDID" -b + while kill -0 "$SIM_BOOT_PID" 2>/dev/null; do sleep 2; done + if ! xcrun simctl bootstatus "$SIM_UDID" -b; then + echo "::error::$SIM_NAME on iOS $SIM_OS did not boot. The background boot's log follows." + cat "$RUNNER_TEMP/simulator-boot.log" + exit 1 + fi echo "Booted $SIM_NAME on iOS $SIM_OS ($SIM_UDID)." - - name: UI test the whole flow in the Simulator + - name: UI test the group in the Simulator # Replays the committed synthetic wall (ios/HouseScanUITests/Fixtures/synthetic-wall) with - # -autopilot and the bundled sample result. + # -autopilot and the bundled sample result; photoProcessing adds its in-app fixture. run: | - skip=() - if [ "$FULL_UI" != "true" ]; then - skip=(-skip-testing:HouseScanUITests/ScreenStatesUITests/testEveryStatePassesTheAccessibilityAudit) + if [ "$GROUP" = screenStates ] && [ "$FULL_UI" != "true" ]; then echo "::notice::Skipped the every-state accessibility audit (about 10 min). Add the full-ui label to run it on this pull request." fi + # Read without mapfile: the runner's /bin/bash is 3.2. + list="$(python3 ios/Tools/ui-test-groups.py args "$GROUP" --full-ui "$FULL_UI")" + selectors=() + while IFS= read -r line; do selectors+=("$line"); done <<< "$list" + echo "Selectors: ${selectors[*]}" xcodebuild -project ios/HouseScan.xcodeproj -scheme HouseScan -configuration Debug \ -destination "id=$SIM_UDID" \ -resultBundlePath "$RUNNER_TEMP/HouseScanUITests.xcresult" \ - -only-testing:HouseScanUITests "${skip[@]}" \ + "${selectors[@]}" \ COMPILER_INDEX_STORE_ENABLE=NO test + # A run the cap cancels leaves the bundle half-written, and its Staging/ files vanish while + # they are read: run 37221532164's journey upload stopped at the first one (ENOENT) and kept + # nothing. This copies the bundle first, lets only Staging/ entries vanish, and lists them in + # SNAPSHOT.txt beside it ("complete" or "incomplete"); any other error fails this step. The + # job's own result, a cancellation included, stays as it was. + - name: Snapshot the UI test results + if: always() + run: python3 ios/Tools/snapshot-xcresult.py "$RUNNER_TEMP/HouseScanUITests.xcresult" "$RUNNER_TEMP/ui-test-results" + - name: Keep UI test screenshots if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: ui-test-results - path: ${{ runner.temp }}/HouseScanUITests.xcresult + name: ui-test-results-${{ matrix.group }} + path: ${{ runner.temp }}/ui-test-results if-no-files-found: ignore retention-days: 7 diff --git a/.greptile/config.json b/.greptile/config.json new file mode 100644 index 000000000..9d6735365 --- /dev/null +++ b/.greptile/config.json @@ -0,0 +1,3 @@ +{ + "autoReview": ["open", "push"] +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index feca4fe13..784df22b6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,6 +22,8 @@ State what the old code did before describing the new behavior. Link the reviewe The `size:*` label describes the effective diff. It is a review signal, not a merge gate. Do not set it by hand. +Greptile reviews new commits through [`.greptile/config.json`](.greptile/config.json). The dashboard's open-only trigger left follow-up commits unreviewed. This repository override adds push reviews without changing other repositories' settings. Greptile reads this setting from the PR's source branch, so older branches need the config too. For a branch without it, request a current review with `@greptileai review` after pushing. Use `@greptileai review this draft` for drafts. + ## Checks | Check | Runs on GitHub when | Local command | @@ -36,6 +38,8 @@ The `size:*` label describes the effective diff. It is a review signal, not a me The iOS UI tests skip the every-state accessibility audit on pull requests, because it adds about 10 minutes and macOS runners are scarce. Add the `full-ui` label when a pull request changes screens or copy; pushes to `t3/ios-mvf` and `main` always run it. +The iOS check runs as three jobs, each under its own 90-minute cap: `screenStates` runs `ScreenStatesUITests`, the every-state audit included under `full-ui`; `photoProcessing` runs an unsigned Release build and `PhotoProcessingUITests` with its own audits; and `journey` runs the package tests, the Debug build and every other UI test class. `ios/Tools/ui-test-groups.py` holds the groups' test selectors; `python3 ios/Tools/ui-test-groups.py check` lists the UI tests from source and fails unless the groups run each exactly once. Run it after adding or renaming a UI test class. Each job keeps its result bundle as the artifact `ui-test-results-`, with `SNAPSHOT.txt` saying whether the copy is complete; a job the cap cancelled can leave it incomplete. + `make check` runs the server, web and iOS suites, then `make scoring`, `make measure-lab`, `make evals`, `make recon` and `make meter-closeup` for each of those folders the branch has. They need uv, Node 24 with pnpm, and Xcode 26 or newer; each directory's README has details. No check is required by branch rules yet. Don't call one required until the rules require its status. Keep workflows that run pull-request code away from production credentials and destructive external systems. A green CI run is evidence for the checks it ran, not proof that a capture works on a real house. diff --git a/ios/HouseScan/Contract/ResultPresentation+Reading.swift b/ios/HouseScan/Contract/ResultPresentation+Reading.swift index 19750bb65..7689343b4 100644 --- a/ios/HouseScan/Contract/ResultPresentation+Reading.swift +++ b/ios/HouseScan/Contract/ResultPresentation+Reading.swift @@ -76,10 +76,7 @@ extension ResultPresentation { checks.map { row in ResultReading.Check( id: row.id, outcome: row.outcome.placementOutcome, needsPerson: row.needsPerson, - viewCapturable: viewToTake(for: row) != nil, - margin: ResultReading.margin( - measured: row.measured.map(Double.init), threshold: row.threshold.map(Double.init), - plusMinus: row.plusMinus.map(Double.init), comparison: row.comparison?.placementComparison) + viewCapturable: viewToTake(for: row) != nil ) } } @@ -102,12 +99,3 @@ private extension CheckOutcome { } } } - -private extension RuleComparison { - var placementComparison: PlacementComparison { - switch self { - case .atLeast: .atLeast - case .atMost: .atMost - } - } -} diff --git a/ios/HouseScan/Contract/ScanContract.swift b/ios/HouseScan/Contract/ScanContract.swift index 141a07113..ef617dfa8 100644 --- a/ios/HouseScan/Contract/ScanContract.swift +++ b/ios/HouseScan/Contract/ScanContract.swift @@ -1,5 +1,6 @@ import CoreGraphics import Foundation +import HouseScanKit import Observation import simd import SwiftUI @@ -33,6 +34,9 @@ enum ScanPhase: String, Sendable, CaseIterable { case spotConfirm case result case resultAR + /// A photo-processing scan after it was sent: its progress, then its answer + /// (`ScanViewState.photoProcessing`). Legacy scans never come here. + case processing case unsupported } @@ -567,6 +571,10 @@ enum EndMarkRefusal: Error, Equatable, Sendable { case otherSide(WallSide) /// Tracking isn't normal, as for a feature mark (`MarkRefusal.trackingNotReady`). case trackingNotReady + /// During a past_end request: an end there would leave less wall than the walk's minimum + /// (`CoverageMap.endWouldLeaveTooLittle`, `WallFrame.minWallLength`), which the app can't use. + /// Decided in `aimedEnd`, so the preview and the button agree. + case tooLittleWall } enum MarkRefusal: Equatable, Sendable { @@ -631,6 +639,10 @@ struct GapRequest: Identifiable, Equatable, Sendable { /// Set while a walk-out request's line lies past where the space ends /// (`GapPlanner.walkOutBlock`): no walk can meet it, and the card says so. var spaceEnds: SpaceEnds? = nil + /// A server past_end request's side: the walk stopped there, and the request asks to walk on + /// past it. The screen also offers "Wall ends here" for that end, with its question, as the + /// walk does (B-12). Nil for every other request. + var pastEndSide: WallSide? = nil } // MARK: - Upload and result @@ -957,6 +969,14 @@ final class ScanViewState { var shareableScan: URL? /// A camera permission or session failure the homeowner can act on. var failure: ScanFailure? + /// The backend this scan uses, fixed when it started; nil before a scan starts. + var scanBackend: ProcessingBackend? + /// Photo processing's side of this scan: its consent, progress and answer. Nil on a Legacy + /// scan, and before a scan starts. + var photoProcessing: PhotoProcessingStatus? + /// Photo processing sends a synthetic test capture in place of this scan's photos (DEBUG's + /// `-photoProcessingSyntheticCapture`), so its answer says nothing about the wall on screen. + var photoCaptureIsSynthetic = false init() {} } @@ -1042,6 +1062,14 @@ protocol ScanActions: AnyObject { func showAR() func closeAR() func startOver() + /// The homeowner's answer to sending this scan's photos for processing + /// (`PhotoProcessingStatus.Consent.asking`). + func answerPhotoConsent(_ yes: Bool) + /// A no after a yes: this scan's photos stop going to photo processing. + func stopSendingPhotos() + /// A photo-processing scan this build can't send, stopped before capture: choose Legacy and + /// start a new scan with it. + func scanWithLegacyInstead() /// The app came back to the foreground on the camera-access failure: if access is now on, /// the scan goes on without Start over. Does nothing otherwise. func recheckCameraAccess() diff --git a/ios/HouseScan/Runtime/Autopilot.swift b/ios/HouseScan/Runtime/Autopilot.swift index cfad487a3..dbeec3fb9 100644 --- a/ios/HouseScan/Runtime/Autopilot.swift +++ b/ios/HouseScan/Runtime/Autopilot.swift @@ -39,6 +39,11 @@ final class Autopilot { await engine.waitForGate(.onboarding) engine.finishOnboarding() await prepared + // A photo-processing scan this build can't send stops before the camera: nothing to drive. + if engine.state.phase == .processing { + log("done: photo processing isn't set up, so the scan stopped before capture") + return + } if let window = replay.heldBack { log("holding back frames \(window.frames.lowerBound)..<\(window.frames.upperBound) for the gap loop; gap \(window.gap.band.rawValue) \(format(window.gap.span))") } else { @@ -47,6 +52,12 @@ final class Autopilot { await pause(hold) await engine.waitForGate(.findMeter) + // Sending photos is the homeowner's answer, never the autopilot's: a UI test taps it. + if engine.state.photoProcessing?.consent == .asking { + guard await waitUntil(timeout: 120, { self.engine.state.photoProcessing?.consent != .asking }) else { + return fail("nobody answered whether to send the scan's photos") + } + } engine.markMeter(at: nil, viewSize: viewSize) guard await waitFor(.meterCloseUp, timeout: 10) else { return fail("meter was not marked") } await takeCloseUp() @@ -90,7 +101,10 @@ final class Autopilot { await playGapFrames(replay) } } - guard await waitFor(.uploading, timeout: 150) else { return fail("upload did not start") } + guard await waitUntil(timeout: 150, { [.uploading, .processing].contains(self.engine.state.phase) }) else { + return fail("upload did not start") + } + if engine.state.phase == .processing { return await finishPhotoProcessing() } guard await driveToResult(replay) else { return } writeSceneForTest() await pause(hold) @@ -192,8 +206,9 @@ final class Autopilot { let intervals = map.wallSeenSpans().map(\.span) + map.coveredIntervals(.ground) guard let low = intervals.map(\.lowerBound).min(), let high = intervals.map(\.upperBound).max() else { log("nothing covered; marking ends 1 m either side of the meter") - engine.setEnd(.left, at: -1, kind: .limit) - engine.setEnd(.right, at: 1, kind: .limit) + // These stand in for the homeowner's marks, as the taps below do. + engine.setEnd(.left, at: -1, kind: .limit, source: .homeowner) + engine.setEnd(.right, at: 1, kind: .limit, source: .homeowner) return } for (side, s) in [(WallSide.left, min(low, -0.2)), (.right, max(high, 0.2))] { @@ -204,7 +219,7 @@ final class Autopilot { let marked = side == .left ? engine.coverage?.leftEnd : engine.coverage?.rightEnd if !tapped || marked == nil { log("no replay frame shows the \(side.rawValue) end; setting it directly at s=\(s)") - engine.setEnd(side, at: s, kind: .limit) + engine.setEnd(side, at: s, kind: .limit, source: .homeowner) } if let asked = engine.state.endQuestion { // Every end counts as a corner here: a replay has no fence or gate to report. @@ -305,6 +320,14 @@ final class Autopilot { do { try engine.sceneJSON().write(to: gate.appending(path: "scene.json")) try engine.spotConfirm.lastScene?.write(to: gate.appending(path: "uploaded-scene.json")) + // The last upload's packet marks and guidance log, from the inputs its packet is + // written from (`ScanEngine.packetMarksAndGuidance`), for the UI test to check ends' + // provenance on the real path. + #if DEBUG + if let packet = engine.packetMarksAndGuidance() { + try packet.write(to: gate.appending(path: "packet-marks-guidance.json")) + } + #endif log("wrote scene.json to the gate folder") } catch { log("could not write scene.json to the gate folder: \(error)") @@ -364,6 +387,17 @@ final class Autopilot { log("answered the overhead request: open sky; \(overheadSummary)") } + /// A photo-processing scan has no Legacy steps after the send: wait for its answer or its end, + /// then hold the screen for the UI test. + private func finishPhotoProcessing() async { + guard await waitUntil(timeout: 240, { self.engine.state.photoProcessing?.isFinal == true }) else { + return fail("photo processing never ended") + } + await pause(hold) + await engine.waitForGate(.processing) + log("done") + } + /// After an upload the engine raises the answer's capturable requests one at a time, uploading /// after each, before it shows the result, and checks the answer's spot with the homeowner. /// Drives each request and check, retries a failed upload once, and returns true once the @@ -449,8 +483,17 @@ final class Autopilot { private func driveServerRequest(_ replay: ReplayPlayer) async { guard let request = engine.state.gap else { return } log("server request \(request.id) over \(format(request.span))") + #if DEBUG + if request.pastEndSide != nil, let gate = engine.options.autopilotGate, let packet = engine.packetMarksAndGuidance() { + // The packet of the upload that raised the request, with the end it then cleared, for + // the UI test to compare the end that comes back with. + try? packet.write(to: gate.appending(path: "packet-first.json")) + } + #endif if request.reason == .overhead { await answerOverheadGap() + } else if request.pastEndSide != nil, engine.options.autopilotMarkPastEnd { + await markPastEnd(request, replay: replay) } else { await playGapFrames(replay) } @@ -458,6 +501,45 @@ final class Autopilot { _ = await waitUntil(timeout: 10) { self.engine.state.phase != .gapRequest || self.engine.state.gap?.id != request.id } } + /// `-autopilotMarkPastEnd`: marks a past_end request's end again, 60% of the way from the meter + /// to the end the request cleared, and answers "Something blocks it", so the wall comes out + /// shorter and its end a limit; with `-autopilotPastEndCorner`, "It turns a corner", which + /// leaves that end unexplored and the request cannot_reach. The mark is a tap where the replay shows that place on the + /// wall, not the circle in the middle of the view: it goes through the same `EndAim` check + /// from that point (`ScanEngine.markWallEnd`), but no camera was aimed. Where it marked goes + /// to the gate folder as `past-end-mark.json` (side, s in meters), for the UI test. + private func markPastEnd(_ request: GapRequest, replay: ReplayPlayer) async { + guard let side = request.pastEndSide, let wall = engine.coverage?.wall else { return } + let cleared = side == .left ? request.span.upperBound : request.span.lowerBound + let s = side == .left ? min(cleared * 0.6, -0.5) : max(cleared * 0.6, 0.5) + // The engine plays the request's frames first; showing another frame meanwhile would + // race it. A replay that settled the request by itself leaves nothing to mark. + _ = await waitUntil(timeout: 60) { !replay.isPlaying || self.engine.state.gap?.id != request.id } + await pause(hold) + guard engine.state.phase == .gapRequest, let gap = engine.state.gap, gap.id == request.id, !gap.isSatisfied else { + fail("past_end request \(request.id) settled before its end could be marked again") + return + } + let tapped = await tap(wall.world(s: s, height: 0.5), replay: replay) { viewPoint in + self.engine.markWallEnd(at: viewPoint, viewSize: self.viewSize) + } + guard tapped, engine.state.endQuestion == side else { + let why = engine.state.endMarkRefusal.map { String(describing: $0) } ?? (tapped ? "no question" : "no frame shows it") + fail("could not mark the \(side.rawValue) end again at s=\(s): \(why)") + return + } + let marked = side == .left ? engine.coverage?.leftEnd : engine.coverage?.rightEnd + log("marked the \(side.rawValue) end again at s=\(marked ?? .nan) (cleared at s=\(cleared))") + if let gate = engine.options.autopilotGate, let marked { + let record: [String: Any] = ["side": side.rawValue, "s": Double(marked), "cleared_s": Double(cleared)] + try? JSONSerialization.data(withJSONObject: record).write(to: gate.appending(path: "past-end-mark.json")) + } + await pause(hold) + let corner = engine.options.autopilotPastEndCorner + engine.answerWallEnd(turnsCorner: corner) + log("answered the \(side.rawValue) end: \(corner ? "it turns a corner" : "something blocks it")") + } + /// Waits while the engine plays the replay's frames for the current request. A replay shows /// only what it recorded, so when they don't settle the request the autopilot says "I can't /// get there", as a homeowner would. diff --git a/ios/HouseScan/Runtime/CaptureRecorder.swift b/ios/HouseScan/Runtime/CaptureRecorder.swift index e5c7d4ec5..f48b78cca 100644 --- a/ios/HouseScan/Runtime/CaptureRecorder.swift +++ b/ios/HouseScan/Runtime/CaptureRecorder.swift @@ -35,6 +35,11 @@ final class CaptureRecorder: Sendable { case deviceMotion /// t, pressure in kPa, relative altitude in m. case barometer + /// t, then that ARFrame's own fx, fy, cx, cy and image width and height in pixels of the + /// unrotated sensor image (`FrameCalibrationRow`), written with its trajectory row at the + /// same t. Read by photo processing's 0.4 pose stream only; the 1.1 packet has no column + /// for it (`motionPacketStream`). + case frameIntrinsics var width: Int { switch self { @@ -42,6 +47,20 @@ final class CaptureRecorder: Sendable { case .accelerometer, .gyroscope, .magnetometer: 4 case .deviceMotion: 15 case .barometer: 3 + case .frameIntrinsics: 7 + } + } + + /// The 1.1 packet's stream a motion row goes to; nil for the camera's own streams, which + /// the 1.1 packet writes elsewhere (the trajectory) or not at all (frame intrinsics). + var motionPacketStream: PacketStream? { + switch self { + case .accelerometer: .accelerometer + case .gyroscope: .gyroscope + case .magnetometer: .magnetometer + case .deviceMotion: .deviceMotion + case .barometer: .barometer + case .trajectory, .frameIntrinsics: nil } } @@ -124,34 +143,42 @@ final class CaptureRecorder: Sendable { } } - func recordPose(t: Double, tracking: TrackingCode, cameraToWorld m: simd_float4x4) { + /// One ARFrame's pose and its own calibration (`ARCamera.intrinsics`, `imageResolution`), under + /// one lock, so recording turned off or on between them can't keep one without the other. + func recordPose(t: Double, tracking: TrackingCode, cameraToWorld m: simd_float4x4, intrinsics: simd_float3x3, imageWidth: Double, imageHeight: Double) { let columns = [m.columns.0, m.columns.1, m.columns.2, m.columns.3] let pose = columns.flatMap { [Double($0.x), Double($0.y), Double($0.z), Double($0.w)] } - append(.trajectory, [t, Double(tracking.state), Double(tracking.reason)] + pose) + let calibration = FrameCalibrationRow.row(t: t, intrinsics: intrinsics, imageWidth: imageWidth, imageHeight: imageHeight) + state.withLock { state in + append(.trajectory, [t, Double(tracking.state), Double(tracking.reason)] + pose, &state) + append(.frameIntrinsics, calibration, &state) + } } /// Appends one row. A row not later than the stream's last one is dropped: every stream's `t` /// strictly increases. func append(_ stream: Stream, _ row: [Double]) { + state.withLock { append(stream, row, &$0) } + } + + private func append(_ stream: Stream, _ row: [Double], _ state: inout State) { precondition(row.count == stream.width, "\(stream) row has \(row.count) values, expected \(stream.width)") let t = row[0] - state.withLock { state in - guard state.recording, !state.failed, t >= state.since, t > state.lastT[stream.rawValue] else { return } - state.lastT[stream.rawValue] = t - if stream == .trajectory { - if state.firstUptime == nil { - state.firstUptime = t - // The same instant on the wall clock: the frame is `now - t` seconds old. - state.startedAt = Date(timeIntervalSinceNow: t - ProcessInfo.processInfo.systemUptime) - } - state.lastUptime = t - } - row.withUnsafeBufferPointer { values in - for value in values { withUnsafeBytes(of: value.bitPattern.littleEndian) { state.buffers[stream.rawValue].append(contentsOf: $0) } } - } - if state.buffers[stream.rawValue].count >= Self.flushBytes { - write(stream, &state) + guard state.recording, !state.failed, t >= state.since, t > state.lastT[stream.rawValue] else { return } + state.lastT[stream.rawValue] = t + if stream == .trajectory { + if state.firstUptime == nil { + state.firstUptime = t + // The same instant on the wall clock: the frame is `now - t` seconds old. + state.startedAt = Date(timeIntervalSinceNow: t - ProcessInfo.processInfo.systemUptime) } + state.lastUptime = t + } + var bytes = Data(capacity: row.count * 8) + for value in row { withUnsafeBytes(of: value.bitPattern.littleEndian) { bytes.append(contentsOf: $0) } } + state.buffers[stream.rawValue].append(bytes) + if state.buffers[stream.rawValue].count >= Self.flushBytes { + write(stream, &state) } } @@ -210,6 +237,11 @@ final class CaptureRecorder: Sendable { ) } + /// The id of the session recording now, which `restart` makes new for every world. + var sessionID: String { + state.withLock { $0.sessionID } + } + /// Writes every buffered row and returns the session as it stands. func flush() -> Snapshot { state.withLock { state in diff --git a/ios/HouseScan/Runtime/KeyframeStore.swift b/ios/HouseScan/Runtime/KeyframeStore.swift index c75157660..584b84804 100644 --- a/ios/HouseScan/Runtime/KeyframeStore.swift +++ b/ios/HouseScan/Runtime/KeyframeStore.swift @@ -50,17 +50,25 @@ final class KeyframeStore { private var nextIndex = 1 /// Bumped by `discardKeyframes`, so a write that started before it doesn't land in the list. private var epoch = 0 + /// Hears each photo kept, once its JPEG is on disk: the photo, the still's purpose (nil for a + /// keyframe) and the JPEG. Photo processing seals it into its capture from here. + var onKept: (@MainActor (_ photo: StoredKeyframe, _ purpose: String?, _ jpeg: URL) -> Void)? - /// Makes a new, empty scan folder and deletes every other one: only the current scan is kept - /// on the phone. Covers both a start over (the previous scan's folder) and launch (folders a - /// quit or crashed run left behind), since both make a new store. The folders to delete are - /// listed here, before a newer store can exist, and deleted later off the main actor - /// (`ScanFolderCleanup`), so a deletion that runs late can't take a newer scan's folder. - init() { + /// Makes a new, empty scan folder and deletes the old ones `ScanFolderCleanup` lists: it keeps + /// the two most recent completed scans (Saved scans offers them) and any folder whose bundle + /// is there but not whole. Covers both a start over (the previous scan's folder) and launch + /// (folders a quit or crashed run left behind), since both make a new store. + /// + /// The folders are listed here, before a newer store can exist, so a deletion that runs late + /// can't take a newer scan's folder. They are deleted later off the main actor, once + /// `bundleWrites` ends: the scan bundle writes started before this store, of which the + /// previous scan's may still be assembling its packet in a folder listed as never packaged. + /// Start over doesn't wait for it; only the deletion does. Nil, as at launch, waits for none. + init(deletingAfter bundleWrites: Task? = nil) { let scans = Self.scansRoot directory = scans.appending(path: UUID().uuidString, directoryHint: .isDirectory) try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - Self.delete(ScanFolderCleanup(root: scans, keeping: directory.lastPathComponent)) + Self.delete(ScanFolderCleanup(root: scans, keeping: directory.lastPathComponent), after: bundleWrites) } /// Caches/Scans: one folder per scan. Saved scans lists the completed ones here. @@ -113,6 +121,7 @@ final class KeyframeStore { } keyframes.append(stored) keyframes.sort { $0.id < $1.id } + onKept?(stored, nil, directory.appending(path: stored.fileName)) return (true, thumbnail) } @@ -211,6 +220,7 @@ final class KeyframeStore { return .worldDiscarded } stillCatalog.save(stored, purpose: id, fileName: name) + onKept?(stored, id, directory.appending(path: name)) return .saved } @@ -246,7 +256,8 @@ final class KeyframeStore { /// Zips the packet folder into `scan.zip`, manifest.json at the zip's root, and removes the /// folder. The zip is what "Share scan" offers; the upload sends scene.json alone. It stays on - /// the phone until the next scan unless the homeowner shares it. + /// the phone, listed in Saved scans, until two newer completed scans push it out + /// (`ScanFolderCleanup`). /// /// Streamed to disk one file at a time: a long walk's JPEGs held in memory twice (the entries /// and the archive) is what an in-memory build cost. @@ -267,11 +278,12 @@ final class KeyframeStore { try? files.removeItem(at: folder) } - /// Off the main actor. A write still in flight for a deleted folder fails, because its - /// directory is gone, and is dropped like any failed write. - nonisolated private static func delete(_ cleanup: ScanFolderCleanup) { + /// Off the main actor, after `bundleWrites`. A keyframe write still in flight for a deleted + /// folder fails, because its directory is gone, and is dropped like any failed write; the + /// bundle lists only keyframes already written. + nonisolated private static func delete(_ cleanup: ScanFolderCleanup, after bundleWrites: Task?) { Task.detached(priority: .utility) { - for (url, error) in cleanup.run() { + for (url, error) in await cleanup.run(after: { await bundleWrites?.value }) { RuntimeLog.engine.error("could not delete old scan \(url.lastPathComponent, privacy: .public): \(String(describing: error), privacy: .public)") } } diff --git a/ios/HouseScan/Runtime/LaunchOptions.swift b/ios/HouseScan/Runtime/LaunchOptions.swift index d0e023d60..c5926b554 100644 --- a/ios/HouseScan/Runtime/LaunchOptions.swift +++ b/ios/HouseScan/Runtime/LaunchOptions.swift @@ -1,4 +1,5 @@ import Foundation +import HouseScanKit import OSLog /// Verification hooks from the launch arguments (contract C4). @@ -22,6 +23,12 @@ import OSLog /// marking the ends (`Autopilot.endWalkByCantGetThere`). /// - `-autopilotSomethingThere`: the autopilot answers the first spot check "Something's there" /// instead of "It's clear", so the scan is checked again without that area. +/// - `-autopilotMarkPastEnd`: on a server past_end request the autopilot marks that end again, +/// nearer than the end the request cleared, with a tap where the replay shows that place, and +/// answers "Something blocks it" (`Autopilot.markPastEnd`). Without it the autopilot plays the +/// request's frames and says "I can't get there" when they don't settle it. +/// - `-autopilotPastEndCorner`: with `-autopilotMarkPastEnd`, the autopilot answers "It turns a +/// corner" instead, which a request can't follow (`ScanEngine.answerWallEnd`). /// - `-autopilotCannotCheck`: the autopilot answers the first spot check "I can't check this /// area" instead of "It's clear". It can't be combined with `-autopilotSomethingThere`. /// - `-sampleResultAfterSpotAnswer ` (debug builds only): with the bundled sample, the @@ -44,6 +51,21 @@ import OSLog /// - `-simulateAppStore`: run as an App Store install would, so the developer options and practice /// meter are unavailable whatever the stored switch says (`DeveloperSettings`). It can only take /// the switch away, never offer it. +/// - `-processingBackend ` (debug builds only): the backend choice in +/// Developer options starts at this value and stays in memory for the run, so a UI test's choice +/// never reaches the stored setting the other tests run with (`ProcessingBackendSetting`). +/// - `-photoProcessingFixture ` (debug builds only, with `-replay`): photo processing sends +/// to a capture API answered inside the app (`FixtureCaptureHTTP`), which ends every capture +/// with this answer (`FixtureCaptureHTTP.Answer`). Nothing leaves the phone. Without it, photo +/// processing isn't set up in any build. A replay's own packet always fails the phone's checks, +/// since a replay has no motion. +/// - `-photoProcessingSyntheticCapture` (debug builds only, with `-photoProcessingFixture`): the +/// capture sent is HouseScanKit's `SyntheticCapture`, labelled synthetic in its packet, in place +/// of the replay's photos, so the fixture's answer comes back through the real upload. It is a +/// test of the app's path against the fixture, never a result about the wall on screen. +/// - `-photoProcessingFixtureReadOnlyCapture` (debug builds only): just before "Stop sending +/// photos" takes effect, the capture's folder is made read-only, so the phone can't save the +/// withdrawal and says so (`PhotoProcessingEnd.withdrawn(recorded: false)`). struct LaunchOptions: Equatable { var replayFolder: URL? var autopilot = false @@ -54,11 +76,17 @@ struct LaunchOptions: Equatable { var autopilotCantGetThere = false var autopilotSomethingThere = false var autopilotCannotCheck = false + var autopilotMarkPastEnd = false + var autopilotPastEndCorner = false var sampleResultAfterSpotAnswer: URL? var simulateAppStore = false var injectGroundRise: Float? var answersFromGate = false var failCloseUpSave = false + var processingBackend: ProcessingBackend? + var photoProcessingFixture: FixtureCaptureHTTP.Answer? + var photoProcessingSyntheticCapture = false + var photoProcessingFixtureReadOnlyCapture = false init( arguments: [String] = ProcessInfo.processInfo.arguments, @@ -75,9 +103,26 @@ struct LaunchOptions: Equatable { autopilotCantGetThere = arguments.contains("-autopilotCantGetThere") autopilotSomethingThere = arguments.contains("-autopilotSomethingThere") autopilotCannotCheck = arguments.contains("-autopilotCannotCheck") + autopilotMarkPastEnd = arguments.contains("-autopilotMarkPastEnd") + autopilotPastEndCorner = arguments.contains("-autopilotPastEndCorner") + precondition(!autopilotPastEndCorner || autopilotMarkPastEnd, "-autopilotPastEndCorner answers the end -autopilotMarkPastEnd marks; pass both") precondition(!(autopilotSomethingThere && autopilotCannotCheck), "-autopilotSomethingThere and -autopilotCannotCheck each choose the first spot answer; pass one") #if DEBUG sampleResultAfterSpotAnswer = value(after: "-sampleResultAfterSpotAnswer").map { URL(fileURLWithPath: $0) } + if let text = value(after: "-processingBackend") { + guard let backend = ProcessingBackend(rawValue: text) else { + preconditionFailure("-processingBackend takes legacy or photoProcessing, got \(text)") + } + processingBackend = backend + } + if let text = value(after: "-photoProcessingFixture") { + guard let answer = FixtureCaptureHTTP.Answer(rawValue: text) else { + preconditionFailure("-photoProcessingFixture takes one of \(FixtureCaptureHTTP.Answer.allCases.map(\.rawValue)), got \(text)") + } + photoProcessingFixture = answer + } + photoProcessingSyntheticCapture = arguments.contains("-photoProcessingSyntheticCapture") + photoProcessingFixtureReadOnlyCapture = arguments.contains("-photoProcessingFixtureReadOnlyCapture") #endif simulateAppStore = arguments.contains("-simulateAppStore") failCloseUpSave = arguments.contains("-failCloseUpSave") diff --git a/ios/HouseScan/Runtime/LiveCapture+MeterTap.swift b/ios/HouseScan/Runtime/LiveCapture+MeterTap.swift new file mode 100644 index 000000000..c85f43f28 --- /dev/null +++ b/ios/HouseScan/Runtime/LiveCapture+MeterTap.swift @@ -0,0 +1,39 @@ +import ARKit +import HouseScanKit + +/// The meter tap's one-frame snapshot, for photo processing's capture packet: taken only when the +/// homeowner's meter mark has been accepted, never on a timer (`LiveMode` and the walk's encoding +/// are unchanged). +extension LiveCapture { + /// What one ARFrame gave for the tap: its own camera, the image to encode, or why there is none. + enum MeterTapSnapshot { + case captured(MeterTapFrame, image: LiveSessionDelegate.PixelBufferBox) + case failed(MeterTapFailure) + } + + /// ARKit's current frame, read once, in the same main-actor turn as the accepted raycast. Its + /// time, raw pose, intrinsics, image size, tracking and image all come from that one frame; + /// nothing is taken from the engine's sampled frames. The image is kept only until it is + /// encoded (`encodeMeterTapImage`), as the walk's encode keeps its own. + func meterTapSnapshot() -> MeterTapSnapshot { + guard let frame = arView.session.currentFrame else { return .failed(.noFrame) } + let k = frame.camera.intrinsics + let size = frame.camera.imageResolution + let tracking: PacketTracking = switch frame.camera.trackingState { + case .normal: .normal + case .notAvailable: .notAvailable + case .limited(_): .limited(nil) + } + let camera = MeterTapFrame( + t: frame.timestamp, cameraToWorld: frame.camera.transform, + intrinsics: SIMD4(k.columns.0.x, k.columns.1.y, k.columns.2.x, k.columns.2.y), + width: Int(size.width), height: Int(size.height), tracking: tracking) + return .captured(camera, image: LiveSessionDelegate.PixelBufferBox(buffer: frame.capturedImage)) + } + + /// Encodes the snapshot's image off the main actor with the walk's encoder. Nil when it fails. + func encodeMeterTapImage(_ image: LiveSessionDelegate.PixelBufferBox) async -> Data? { + let delegate = delegate + return await Task.detached(priority: .userInitiated) { delegate.encode(image.buffer) }.value + } +} diff --git a/ios/HouseScan/Runtime/LiveCapture.swift b/ios/HouseScan/Runtime/LiveCapture.swift index c15dcd44e..8208168a0 100644 --- a/ios/HouseScan/Runtime/LiveCapture.swift +++ b/ios/HouseScan/Runtime/LiveCapture.swift @@ -40,7 +40,7 @@ struct VerticalPlaneHit { @MainActor final class LiveCapture { let arView: ARView - private let delegate: LiveSessionDelegate + let delegate: LiveSessionDelegate /// True when this phone gives per-frame LiDAR depth. static var supportsDepth: Bool { ARWorldTrackingConfiguration.supportsFrameSemantics(.sceneDepth) } @@ -393,8 +393,9 @@ final class LiveSessionDelegate: NSObject, ARSessionDelegate, Sendable { } /// The camera image handed to the encode queue. CVPixelBuffer is not Sendable; ARKit doesn't - /// write to a delivered frame's image, and only the encode queue reads it. - private struct PixelBufferBox: @unchecked Sendable { + /// write to a delivered frame's image, and only the encode queue reads it. The meter tap's + /// snapshot hands its one image off the main actor the same way. + struct PixelBufferBox: @unchecked Sendable { let buffer: CVPixelBuffer } @@ -426,8 +427,11 @@ final class LiveSessionDelegate: NSObject, ARSessionDelegate, Sendable { } let shared = shared.withLock { $0 } let tracking = Self.tracking(frame.camera.trackingState) - // Every frame's pose goes to the packet's trajectory, before any sampling. - shared.recorder?.recordPose(t: frame.timestamp, tracking: TrackingCode(tracking), cameraToWorld: frame.camera.transform) + // Every frame's pose goes to the packet's trajectory, before any sampling, with that frame's + // own calibration at the same t. + shared.recorder?.recordPose( + t: frame.timestamp, tracking: TrackingCode(tracking), cameraToWorld: frame.camera.transform, intrinsics: frame.camera.intrinsics, + imageWidth: Double(frame.camera.imageResolution.width), imageHeight: Double(frame.camera.imageResolution.height)) let intrinsics = frame.camera.intrinsics let resolution = frame.camera.imageResolution let camera = CameraFrame( @@ -533,8 +537,9 @@ final class LiveSessionDelegate: NSObject, ARSessionDelegate, Sendable { } } - /// JPEG of the sensor image as captured: landscape, unrotated, matching the intrinsics. - private func encode(_ buffer: CVPixelBuffer) -> Data? { + /// JPEG of the sensor image as captured: landscape, unrotated, matching the intrinsics. The + /// walk's keyframes and the meter tap's snapshot both use it. + func encode(_ buffer: CVPixelBuffer) -> Data? { let image = CIImage(cvPixelBuffer: buffer) let space = CGColorSpace(name: CGColorSpace.sRGB) ?? CGColorSpaceCreateDeviceRGB() let options = [CIImageRepresentationOption(rawValue: kCGImageDestinationLossyCompressionQuality as String): 0.8] diff --git a/ios/HouseScan/Runtime/Practice/ProcessingBackendSetting.swift b/ios/HouseScan/Runtime/Practice/ProcessingBackendSetting.swift new file mode 100644 index 000000000..dc13adf3b --- /dev/null +++ b/ios/HouseScan/Runtime/Practice/ProcessingBackendSetting.swift @@ -0,0 +1,37 @@ +import Foundation +import HouseScanKit +import Observation + +/// The backend choice in Developer options: which remote processing the next scan uses. A scan +/// reads it once, when it starts (`ScanEngine.beginScan`), so a change applies to the next scan. +/// +/// Kept in UserDefaults (`ProcessingBackendPreference`). In debug builds, `-processingBackend` +/// keeps it in memory instead, starting at that value, so a UI test's choice ends with its run. +@MainActor +@Observable +final class ProcessingBackendSetting { + static let shared = ProcessingBackendSetting(options: LaunchOptions()) + + private(set) var selection: ProcessingBackend + /// Who answers a photo-processing scan in this build, for the setting's description. The + /// engine sets it from its own setup when it starts. + var photoProcessingAnswers: ProcessingProfile.Answers = .notSetUp("not known yet") + private let defaults: UserDefaults? + + init(options: LaunchOptions) { + #if DEBUG + if let backend = options.processingBackend { + defaults = nil + selection = backend + return + } + #endif + defaults = .standard + selection = ProcessingBackendPreference.read(.standard) + } + + func choose(_ backend: ProcessingBackend) { + selection = backend + if let defaults { ProcessingBackendPreference.write(backend, to: defaults) } + } +} diff --git a/ios/HouseScan/Runtime/ScanEngine+Actions.swift b/ios/HouseScan/Runtime/ScanEngine+Actions.swift index 100181f83..5061f89ed 100644 --- a/ios/HouseScan/Runtime/ScanEngine+Actions.swift +++ b/ios/HouseScan/Runtime/ScanEngine+Actions.swift @@ -9,6 +9,13 @@ extension ScanEngine: ScanActions { func finishOnboarding() { guard state.phase == .onboarding else { return } startPracticeIfOn() + beginScan() + // A photo-processing scan this build can't send stops here, before the camera or any + // request, and says so with a way to a Legacy scan. + guard !photoScanCantRun else { + go(.processing) + return + } leaveOnboarding() } @@ -65,6 +72,8 @@ extension ScanEngine: ScanActions { // takes the server's error for it. updateCoverage { $0.setWallLineSource(meterLineSource) } setMeterAnchor(live.addMeterAnchor(at: hit.transform), pose: hit.transform) + // In this same turn as the raycast: photo processing's tap, from ARKit's current frame. + takeMeterTap(live, hit: hit) markTimes[MarkKey.meter] = captureClock go(.meterCloseUp) } @@ -138,19 +147,24 @@ extension ScanEngine: ScanActions { } /// Marks a wall end during the walk, or, during a server past_end request, marks that - /// request's end again at wherever the wall is now seen to stop. - /// The end at `point`, or with no point at the circle in the middle of the view: the walk's - /// "Wall ends here" button. The button answers the card, which names a side, so on the walk - /// it marks only that side, only with the phone's place known, and only where the circle is - /// on the wall; otherwise the card says why (`EndMarkRefusal`, B-06). Before, each of those - /// did nothing, and aiming across the meter ended the other side. A tap at a point (the - /// autopilot's) marks the side it lands on, as before. + /// request's end again at wherever the wall really stops (B-12), nearer or farther than the + /// end the request cleared: that end only said how far the walk had seen. + /// The end at `point`, or with no point at the circle in the middle of the view: the + /// "Wall ends here" button. The button answers the card, which names a side, so it marks only + /// that side, only with the phone's place known, and only where the circle is on the wall; + /// otherwise the card says why (`EndMarkRefusal`, B-06). Before, each of those did nothing, + /// and aiming across the meter ended the other side. During a past_end request a tap at a + /// point (the autopilot's) goes through the same check from that point; on the walk it marks + /// the side it lands on, as before. func markWallEnd(at point: CGPoint?, viewSize: CGSize) { - guard state.phase == .wallWalk || (state.phase == .gapRequest && pastEndSide != nil), + guard state.phase == .wallWalk || askedEndSide != nil, + // During a request, once the end is marked its question is what is left to answer. + state.phase != .gapRequest || state.endQuestion == nil, let map = coverage, let frame = currentFrame else { return } let hit: WallPoint - if point == nil, state.phase == .wallWalk, case .markEnd(let asked) = state.guidance { - switch circleEnd(asked: asked, frame: frame, map: map) { + if let asked = askedEndSide, point == nil || state.phase == .gapRequest { + let pixel = point.map { frame.projection.imagePixel(forViewPoint: $0, in: viewSize) } ?? frame.camera.imageSize / 2 + switch aimedEnd(asked: asked, pixel: pixel, frame: frame, map: map) { case .success(let found): hit = found case .failure(let refusal): return refuseEndMark(refusal) } @@ -159,14 +173,16 @@ extension ScanEngine: ScanActions { hit = found } let side: WallSide = hit.s < 0 ? .left : .right - if state.phase == .gapRequest, side != pastEndSide { return } state.endMarkRefusal = nil // Unexplored until the homeowner says something blocks the wall there: an unanswered // question must not tell the server the usable wall stops at this point. state.endQuestion = side state.endQuestionLeavesOut = nil state.endQuestionLeavesOutSeen = false - setEnd(side, at: hit.s, kind: .unexplored) + setEnd(side, at: hit.s, kind: .unexplored, source: .homeowner) + if state.phase == .gapRequest { + RuntimeLog.engine.info("gap \(self.state.gap?.id ?? 0): the \(side.rawValue, privacy: .public) end marked again at s=\(hit.s)") + } } private func refuseEndMark(_ refusal: EndMarkRefusal) { @@ -186,7 +202,24 @@ extension ScanEngine: ScanActions { nextWallSide = side nextWallRefusal = nil } - if state.phase == .gapRequest, side == pastEndSide { settlePastEnd() } + if state.phase == .gapRequest, side == pastEndSide { + if turnsCorner { + // The wall goes on round the corner, so its end stays unexplored, and the server + // settles an unexplored end only by views past it (server/README.md, "unexplored"). + // This request can't follow a corner; the walk does that (`markNextWall`). So it + // closes as cannot_reach, never met: the homeowner's corner stays where they + // marked it (`PastEndSettlement.keepMarked`), and nothing past it counts as seen. + // The next answer can ask again to walk past this side, now planned from the + // corner: the same request this one couldn't serve, so it is recorded as skipped + // too. Only from this end; a past_end from a different end later is a new view. + let end = side == .left ? coverage?.leftEnd : coverage?.rightEnd + skipCurrentGap( + because: "the wall turns a corner at the homeowner's end, and this request can't survey past a corner", + deferring: end.map { gapPlanner.pastEndPlan(side: side.walk, end: $0) }) + } else { + settlePastEnd() + } + } if state.phase == .wallWalk, let frame = currentFrame { resetGuidanceAfterSkip(camera: frame.camera, time: frame.timestamp) } @@ -474,7 +507,9 @@ extension ScanEngine: ScanActions { } func skipGap() { - guard state.phase == .gapRequest else { return } + // With the end just marked, the end question answers the request; the screen hides this + // reply until it is answered. + guard state.phase == .gapRequest, state.endQuestion == nil else { return } // The card said the space ends short of the walk-out line (#164): the same answer, with // why in the log. if let ends = state.gap?.spaceEnds { @@ -485,7 +520,7 @@ extension ScanEngine: ScanActions { } func showResultNow() { - guard state.phase == .gapRequest else { return } + guard state.phase == .gapRequest, state.endQuestion == nil else { return } stopGapRequests() } diff --git a/ios/HouseScan/Runtime/ScanEngine+Ends.swift b/ios/HouseScan/Runtime/ScanEngine+Ends.swift index b61cf80cd..3395eee80 100644 --- a/ios/HouseScan/Runtime/ScanEngine+Ends.swift +++ b/ios/HouseScan/Runtime/ScanEngine+Ends.swift @@ -4,13 +4,35 @@ import OSLog import simd /// Ending the wall during the walk without pointing at the end ("Can't get there", "Wall ends -/// here"), and the preview of where an end would land. +/// here"), marking it at the circle, during the walk or a past-end request, and the preview of +/// where an end would land. extension ScanEngine { + /// The end a card asks the homeowner to mark at the circle: the walk's "Is this the right end + /// of the wall?", or a past_end request's side while it is unsettled (B-12). "Wall ends here" + /// marks only this side, through the same aim check either way (`circleEnd`). + var askedEndSide: WallSide? { + switch state.phase { + case .wallWalk: + if case .markEnd(let side) = state.guidance { return side } + return nil + case .gapRequest: + guard let side = pastEndSide, state.gap?.isSatisfied == false else { return nil } + return side + default: + return nil + } + } + /// The side ending the wall applies to now, and whether the end lands at the reticle. The /// walk's own side while it asks to walk or to mark the end; during a request about the wall /// in front of the homeowner (tilt down, tilt up, step back), the side of the meter the phone - /// is on (`WalkedEnd.side`). Nil when both ends are marked or the walk is doing something else. + /// is on (`WalkedEnd.side`). During a past_end request its side, at the reticle. Nil when both + /// ends are marked or the walk is doing something else. private var endOnOffer: (side: WallSide, atReticle: Bool)? { + if state.phase == .gapRequest { + guard let side = askedEndSide, state.endQuestion == nil, !state.overheadQuestion, coverage != nil else { return nil } + return (side, true) + } guard state.phase == .wallWalk, state.marking == nil, state.endQuestion == nil, !state.overheadQuestion, nextWallSide == nil, let map = coverage else { return nil } switch state.guidance { @@ -103,12 +125,28 @@ extension ScanEngine { /// middle whatever the view's size. Shared by the button and its preview, so the tape never /// shows an end the button would refuse. Only normal tracking counts, as for feature marks. func circleEnd(asked: WallSide, frame: SourceFrame, map: CoverageMap) -> Result { - let hit = map.wall.intersectWall(frame.camera.ray(throughPixel: frame.camera.imageSize / 2)) + aimedEnd(asked: asked, pixel: frame.camera.imageSize / 2, frame: frame, map: map) + } + + /// `circleEnd` for the ray through `pixel` of the sensor image. A tap at a point (the + /// autopilot's) during a past_end request goes through the same check as the circle. + /// + /// During a past_end request only, an end that would leave less wall than the walk's minimum + /// (`WallFrame.minWallLength`, `CoverageMap.endWouldLeaveTooLittle`) is refused here, so the + /// button, its refusal and the tape's preview agree. The walk applies that minimum when it + /// finishes ("Done with this wall") and keeps doing so; a request settles without that step. + /// Whether 0.79 m is the right minimum is an open product question, not settled here. + func aimedEnd(asked: WallSide, pixel: SIMD2, frame: SourceFrame, map: CoverageMap) -> Result { + let hit = map.wall.intersectWall(frame.camera.ray(throughPixel: pixel)) switch EndAim.verdict( hit: hit, camera: frame.camera.position, wall: map.wall, reach: map.config.maxDistance, - groundError: map.heightError, askedLeft: asked == .left, trackingNormal: frame.tracking == .normal + // The frame and the phone now: a pose-only frame after it can report tracking lost + // without replacing the sampled frame whose ray this is. + groundError: map.heightError, askedLeft: asked == .left, trackingNormal: frame.tracking == .normal && state.tracking == .normal ) { - case .end(let point): return .success(point) + case .end(let point): + if state.phase == .gapRequest, map.endWouldLeaveTooLittle(asked.walk, at: point.s) { return .failure(.tooLittleWall) } + return .success(point) case .trackingLimited: return .failure(.trackingNotReady) case .offWall: return .failure(.noWall) case .otherSide: return .failure(.otherSide(asked == .left ? .right : .left)) @@ -122,7 +160,7 @@ extension ScanEngine { // A refusal stands until the circle is on the end the card asks for, or the card moves on. if state.endMarkRefusal != nil { var fixed = true - if case .markEnd(let side) = state.guidance { fixed = preview?.atReticle == true && preview?.side == side } + if let side = askedEndSide { fixed = preview?.atReticle == true && preview?.side == side } if fixed { state.endMarkRefusal = nil } } } @@ -141,7 +179,8 @@ extension ScanEngine { state.endQuestionLeavesOut = leavesOut state.endQuestionLeavesOutSeen = leavesOut != nil && WalkedEnd.leftOutIsSeen(preview.side.walk, s: preview.s, walked: map.walkedPositions, wall: map.wall, seen: seen) - setEnd(preview.side, at: preview.s, kind: .unexplored) + // The homeowner said the wall ends where they stand: their mark. + setEnd(preview.side, at: preview.s, kind: .unexplored, source: .homeowner) } /// "Can't get there" while the walk asks to walk `side` or to mark its end: the end goes where @@ -151,7 +190,8 @@ extension ScanEngine { guard let s = walkedEnd(side), let map = coverage else { return } logEnd(refused ? "can't get there" : "the wall keeps going", side: side, at: s) let walked = map.walkedFarthest(side.walk) - setEnd(side, at: s, kind: .unexplored) + // Where the walk reached, not a place the homeowner marked: inferred (B-12). + setEnd(side, at: s, kind: .unexplored, source: .inferred) guard refused else { return } walkRefusals.ended(side.walk, at: s, walked: walked, time: ScanEngine.refusalClock) if walkRefusals.wasRefused(side.walk, end: s) { diff --git a/ios/HouseScan/Runtime/ScanEngine+GuidanceLog.swift b/ios/HouseScan/Runtime/ScanEngine+GuidanceLog.swift index fb6bbdb59..c25302ab6 100644 --- a/ios/HouseScan/Runtime/ScanEngine+GuidanceLog.swift +++ b/ios/HouseScan/Runtime/ScanEngine+GuidanceLog.swift @@ -44,7 +44,7 @@ extension ScanEngine { ) case .spotConfirm: return spotCheckGuidance - case .onboarding, .findMeter, .markFeatures, .uploading, .result, .resultAR, .unsupported: + case .onboarding, .findMeter, .markFeatures, .uploading, .result, .resultAR, .processing, .unsupported: return nil } } @@ -112,7 +112,10 @@ extension ScanEngine { // before it leaves. return nextWallSide == nil ? .met : .superseded case .markEnd(let side): - return wallEndKinds[side] != nil ? .met : .superseded + // Met only by the homeowner's mark: an end the walk inferred answered another request + // ("Can't get there", or "The wall keeps going", closed as cannot_reach) (B-12). + guard wallEndKinds[side] != nil, let stamp = endStamps[side] else { return .superseded } + return stamp.markEndOutcome case .stepBack: guard let camera = currentFrame?.camera, let wall = coverage?.wall else { return .superseded } return wall.wallPoint(camera.position).out > GuidancePlanner().config.tooClose ? .met : .superseded diff --git a/ios/HouseScan/Runtime/ScanEngine+Packet.swift b/ios/HouseScan/Runtime/ScanEngine+Packet.swift index 1e515631b..dda3f27b7 100644 --- a/ios/HouseScan/Runtime/ScanEngine+Packet.swift +++ b/ios/HouseScan/Runtime/ScanEngine+Packet.swift @@ -114,6 +114,23 @@ extension ScanEngine { return inputs } + /// The marks and guidance log of the last upload's packet, as `writePacket` writes them into + /// manifest.json (`{"marks": [...], "guidance": [...]}`): `withCurrentGuidance` of the inputs + /// the upload captured, the same projection `saveBundle` is given. Only the autopilot's UI-test + /// gate reads it (`Autopilot.writeSceneForTest`); nil before an upload. Debug builds only. + #if DEBUG + func packetMarksAndGuidance() -> Data? { + guard let inputs = spotConfirm.lastPacket.map(withCurrentGuidance) else { return nil } + struct Projection: Encodable { + var marks: [PacketMark] + var guidance: [PacketGuidanceEntry] + } + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] + return try? encoder.encode(Projection(marks: inputs.marks, guidance: inputs.guidance)) + } + #endif + /// `utsname.machine`, such as "iPhone16,1": the hardware, never the phone's name. "arm64" in /// the Simulator. nonisolated static func hardwareModel() -> String { @@ -137,7 +154,9 @@ extension ScanEngine { guard let s else { continue } marks.append(.wallEnd( id: "wall_end_\(side.rawValue)", side: side == .left ? .left : .right, - endKind: wallEndKinds[side] == .limit ? .limit : .unexplored, s: s, wall: wall, frame: frame, t: markTimes[MarkKey.end(side)] + endKind: wallEndKinds[side] == .limit ? .limit : .unexplored, s: s, wall: wall, frame: frame, + // No stamp is never read as the homeowner's mark (B-12). + stamp: endStamps[side] ?? .inferred )) } for feature in state.features { @@ -170,7 +189,6 @@ extension ScanEngine { /// Keys of `markTimes`. enum MarkKey { static let meter = "meter" - static func end(_ side: WallSide) -> String { "end.\(side.rawValue)" } static func feature(_ id: UUID) -> String { id.uuidString } } @@ -433,17 +451,12 @@ extension ScanEngine { /// Core Motion's streams, rows inside the capture only. Returns how many rows were refused. private nonisolated static func writeMotion(_ inputs: PacketInputs, window: ClosedRange, into writer: inout PacketWriter) -> Int { var refused = 0 - for stream in CaptureRecorder.Stream.allCases where stream != .trajectory && inputs.motionStreams.contains(stream) { + // Motion streams only: the trajectory is written elsewhere, and frame intrinsics have no + // column in the 1.1 packet (`CaptureRecorder.Stream.motionPacketStream`). + for stream in CaptureRecorder.Stream.allCases where inputs.motionStreams.contains(stream) { + guard let packetStream = stream.motionPacketStream else { continue } let rows = inputs.recorder.rows(stream).filter { window.contains($0[0]) } guard !rows.isEmpty else { continue } - let packetStream: PacketStream = switch stream { - case .trajectory: .trajectory - case .accelerometer: .accelerometer - case .gyroscope: .gyroscope - case .magnetometer: .magnetometer - case .deviceMotion: .deviceMotion - case .barometer: .barometer - } // CMAltimeter sets its own rate (about 1 Hz), so the barometer claims none. if stream != .barometer { try? writer.setNominalRate(MotionSource.rate, for: packetStream) } for r in rows { @@ -458,7 +471,7 @@ extension ScanEngine { userAcceleration: SIMD3(r[8], r[9], r[10]), rotationRate: SIMD3(r[11], r[12], r[13]), headingDegrees: r[14] )) case .barometer: try writer.appendBarometer(t: r[0], pressureKPa: r[1], relativeAltitudeM: r[2]) - case .trajectory: break + case .trajectory, .frameIntrinsics: break } } catch { refused += 1 diff --git a/ios/HouseScan/Runtime/ScanEngine+Processing.swift b/ios/HouseScan/Runtime/ScanEngine+Processing.swift new file mode 100644 index 000000000..9c5967029 --- /dev/null +++ b/ios/HouseScan/Runtime/ScanEngine+Processing.swift @@ -0,0 +1,290 @@ +import Foundation +import HouseScanKit +import OSLog + +/// The scan's backend, and photo processing (HouseScanKit's `PhotoProcessingController`). +/// +/// A scan reads the backend setting once, when it starts (`beginScan`), into its `ScanContext`, +/// and keeps that profile to the end: a change in Developer options applies to the next scan. +/// Start over ends the scan; a world reset keeps the scan and its backend and starts a new spatial +/// session. Sending (`startUpload`) goes to the scan's own backend: a photo-processing scan never +/// reaches the Legacy checker, not even from a resend. +extension ScanEngine { + /// The homeowner started a scan. Runs once per scan, before anything of it is kept. + func beginScan() { + guard scanContext == nil else { return } + // App Store installs have no developer options, so they ignore a stored choice. + let backend = DeveloperSettings.shared.isAvailable ? ProcessingBackendSetting.shared.selection : .operationalDefault + let profile = switch backend { + case .legacy: legacyProfile + case .photoProcessing: photoProcessing.profile + } + let context = ScanContext(scanID: store.directory.lastPathComponent, profile: profile, recordingSessionID: recorder.sessionID) + scanContext = context + state.scanBackend = backend + RuntimeLog.engine.info("scan \(context.scanID, privacy: .public) uses \(backend.rawValue, privacy: .public) processing") + guard backend == .photoProcessing else { return } + // A synthetic capture is sent in place of the scan's photos, never mixed with them. + if syntheticCapture == nil { attachKeptPhotos() } + photoProcessing.beginScan(context, recording: photoRecording) + if case .ended(.notSetUp(let reason))? = state.photoProcessing?.stage { + RuntimeLog.engine.info("photo processing not set up: \(reason, privacy: .public)") + } + } + + /// The scan uses photo processing and this build can't send it. + var photoScanCantRun: Bool { + guard let profile = scanContext?.profile, profile.backend == .photoProcessing, case .notSetUp = profile.answers else { return false } + return true + } + + /// The homeowner's way out of a photo-processing scan this build can't run, before anything + /// was captured: Legacy becomes the choice in Developer options, and a new scan starts with + /// it. The old scan isn't rerouted; it ends. + func scanWithLegacyInstead() { + guard state.phase == .processing, photoScanCantRun else { return } + RuntimeLog.engine.info("photo processing isn't set up: the homeowner chose a Legacy scan instead") + ProcessingBackendSetting.shared.choose(.legacy) + startOver() + finishOnboarding() + } + + /// Start over: the scan ends, and whatever photo processing it had ends with it. `recorder` + /// is already the next scan's. + func endScan() { + scanContext = nil + meterTapTask = nil + state.scanBackend = nil + photoProcessing.endScan(recording: photoRecording) + } + + /// A world reset: the same scan, backend and consent in a new spatial session. `recorder` has + /// already restarted for the new world. + func scanWorldReset() { + guard let context = scanContext else { return } + let next = context.inNewWorld(recordingSessionID: recorder.sessionID) + scanContext = next + // The old world's tap, if still encoding, is dropped when it finishes (`takeMeterTap`). + meterTapTask = nil + if next.profile.backend == .photoProcessing { + photoProcessing.worldReset(next, recording: photoRecording) + } + } + + /// The photo-processing scan was sent: once the photos still being written are kept, its + /// packet is frozen, and the screen follows the upload to the answer. With DEBUG's synthetic + /// capture, its photos are kept now instead, and its own close-up is the accepted one. + func startPhotoProcessing() { + go(.processing) + let context = scanContext + let synthetic = syntheticCapture + Task { + let saves = await drainPendingSaves() + guard saves != .scanChanged, scanContext == context, state.phase == .processing else { return } + if saves == .stillWriting { + RuntimeLog.engine.error("photo processing: photos were still being saved after 10 s; the capture isn't sealed") + photoProcessing.capturePreparationFailed() + return + } + guard let synthetic else { + // A live capture is sealed only with its meter tap; a replay has none to give. + if replay == nil { + await meterTapTask?.value + guard scanContext == context, state.phase == .processing else { return } + guard photoProcessing.meterTap == .forwarded else { + RuntimeLog.engine.error("photo processing: the capture has no meter tap, so it isn't sealed") + photoProcessing.capturePreparationFailed() + return + } + } + let closeUp = state.closeUp == .skipped ? nil : store.stillFrames["meter_close"]?.t + photoProcessing.captureEnded(acceptedCloseUpAt: closeUp) + return + } + let folder = FileManager.default.temporaryDirectory.appending(path: "PhotoProcessingFixture/synthetic-photos", directoryHint: .isDirectory) + let photos = (try? await Task.detached(priority: .userInitiated) { try synthetic.photos(in: folder) }.value) ?? [] + guard scanContext == context, state.phase == .processing else { return } + if photos.isEmpty { RuntimeLog.engine.error("photo processing: the synthetic capture's photos couldn't be written") } + for photo in photos { photoProcessing.kept(photo) } + photoProcessing.captureEnded(acceptedCloseUpAt: synthetic.acceptedCloseUpAt) + } + } + + /// Photo processing's meter tap for an accepted mark: one snapshot of ARKit's current frame, + /// taken now, then its image encoded off the main actor and the tap forwarded. Legacy scans, + /// the synthetic capture and a packet that wants no tap take none. A snapshot that can't give + /// a tap ends the packet as not prepared (`PhotoProcessingController.meterTapFailed`). The + /// result is used only while the scan and its world are the ones the mark was made in. + func takeMeterTap(_ live: LiveCapture, hit: VerticalPlaneHit) { + guard let context = scanContext, context.profile.backend == .photoProcessing, syntheticCapture == nil, + photoProcessing.context == context, photoProcessing.wantsMeterTap else { return } + let position = hit.position + let estimated = hit.source == .estimatedPlane + switch live.meterTapSnapshot() { + case .failed(let reason): + photoProcessing.meterTapFailed(reason, scan: context) + case .captured(let frame, let image): + // The geometry is checked before anything is encoded. + if case .failure(let reason) = frame.observation(hit: position, jpeg: { nil }) { + photoProcessing.meterTapFailed(reason, scan: context) + return + } + meterTapTask = Task { [weak self, live] in + let jpeg = await live.encodeMeterTapImage(image) + guard let self else { return } + guard self.scanContext == context, self.photoProcessing.context == context else { + RuntimeLog.engine.info("photo processing: a meter tap from a scan or world that is gone was dropped") + return + } + guard let jpeg else { + self.photoProcessing.meterTapFailed(.encodeFailed, scan: context) + return + } + switch frame.observation(hit: position, jpeg: { jpeg }) { + case .success(let tap): self.photoProcessing.meterTapped(tap, hit: frame.tapHit(position, estimatedPlane: estimated), scan: context) + case .failure(let reason): self.photoProcessing.meterTapFailed(reason, scan: context) + } + } + } + } + + func answerPhotoConsent(_ yes: Bool) { + guard state.photoProcessing?.consent == .asking else { return } + RuntimeLog.engine.info("photo processing consent: \(yes ? "yes" : "no", privacy: .public)") + photoProcessing.answerConsent(yes) + } + + func stopSendingPhotos() { + guard state.phase == .processing else { return } + #if DEBUG + if options.photoProcessingFixtureReadOnlyCapture, photoProcessing.profile.answers == .standIn, let folder = photoProcessing.captureFolder { + try? FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) + } + #endif + photoProcessing.stopSending() + if case .withdrawn(let recorded)? = state.photoProcessing?.consent { + RuntimeLog.engine.info("photo processing: sending stopped; the phone \(recorded ? "saved" : "could not save", privacy: .public) the withdrawal") + } + } + + // MARK: Translation + + private var legacyProfile: ProcessingProfile { + if let client = resultClient as? HTTPResultClient { + return ProcessingProfile(backend: .legacy, answers: .service, origin: client.serverURL) + } + return ProcessingProfile(backend: .legacy, answers: .standIn, origin: nil) + } + + /// What photo processing records from: the synthetic capture's recording when there is one, + /// otherwise the current world's. + var photoRecording: RecordingSource { + syntheticCapture?.recording ?? recordingSource + } + + /// The current world's recording as the capture packet reads it: live, each ARFrame's pose with + /// its own intrinsics at the same t, which the packet joins by exact time + /// (`Packet04Streams.poseRows`). A replay records no poses, intrinsics or motion, so its packet + /// can't pass its checks and the phone refuses it; nothing is made up to fill the gap. + var recordingSource: RecordingSource { + let recorder = recorder + return RecordingSource( + start: { + let snapshot = recorder.flush() + guard let uptime = snapshot.firstUptime, let date = snapshot.startedAt else { return nil } + return (uptime, date) + }, + rows: { + _ = recorder.flush() + return RecorderRows( + trajectory: recorder.rows(.trajectory), intrinsics: recorder.rows(.frameIntrinsics), + accelerometer: recorder.rows(.accelerometer), gyroscope: recorder.rows(.gyroscope)) + }) + } + + private func attachKeptPhotos() { + let current = store + current.onKept = { [weak self, weak current] photo, purpose, jpeg in + guard let self, let current, current === self.store, self.scanContext?.profile.backend == .photoProcessing else { return } + self.photoProcessing.kept(ScanEngine.keptPhoto(photo, purpose: purpose, jpeg: jpeg, storeDirectory: current.directory)) + } + } + + private static func keptPhoto(_ photo: StoredKeyframe, purpose: String?, jpeg: URL, storeDirectory: URL) -> KeptPhoto { + let stored = photo.depth + return KeptPhoto( + // The pose as ARKit reported it; anchor corrections belong to scene.json and the 1.1 packet. + t: photo.t, cameraToWorld: photo.camera.cameraToWorld, cameraIntrinsics: photo.camera.intrinsics, + cameraImageSize: photo.camera.imageSize, width: photo.width, height: photo.height, + tracking: TrackingCode(photo.tracking).packetTracking, + exposure: photo.exposure.map { Packet04.Exposure(duration: $0.durationS, offset: $0.offsetEV, iso: $0.iso, fNumber: $0.fNumber) }, + jpeg: jpeg, purpose: purpose, + depth: { + // Only ARKit's own depth with its confidence; a replay's depth isn't ARKit's. + guard let stored, let depth = KeyframeStore.loadDepth(stored, in: storeDirectory), let confidence = depth.confidence, + depth.source == .arkitSceneDepth || depth.source == .arkitSmoothedSceneDepth else { return nil } + return Packet04Depth(meters: depth.meters, confidence: confidence, width: depth.width, height: depth.height) + }) + } + + // MARK: Setup + + /// What photo processing can do in this build. Sending a capture off the phone is off in this + /// slice: the 0.4 clock and privacy contract, a scoped credential and the deployment aren't + /// settled. Per-frame intrinsics and the meter tap from the accepted mark's own frame are + /// recorded but unchecked on a phone. The one way to run it is DEBUG's fixture on a replay, + /// which answers on the phone (`CaptureFixtureLaunch`); a release build can't, since + /// `debugBuild` is decided at compile time and the launch options aren't even parsed there. + static func photoProcessingSetup(_ options: LaunchOptions) -> (setup: PhotoProcessingController.Setup, synthetic: SyntheticCapture?) { + #if DEBUG + let debugBuild = true + #else + let debugBuild = false + #endif + let launch = CaptureFixtureLaunch.resolve( + debugBuild: debugBuild, onReplay: options.replayFolder != nil, answer: options.photoProcessingFixture, + syntheticCapture: options.photoProcessingSyntheticCapture) + switch launch { + case .off(let reason): + return (.notSetUp(reason), nil) + case .on(let answer, let synthetic): + #if DEBUG + return (.ready(fixtureEnvironment(answer, synthetic: synthetic, gate: options.autopilotGate), standIn: true), synthetic ? SyntheticCapture.standard : nil) + #else + return (.notSetUp("a release build has no capture fixture"), nil) + #endif + } + } + + #if DEBUG + /// The capture fixture: every request answered in this process (`FixtureCaptureHTTP`), its + /// routes listed one per line in `/photo-transport.log` for the UI tests. + private static func fixtureEnvironment(_ answer: FixtureCaptureHTTP.Answer, synthetic: Bool, gate: URL?) -> CaptureSessionCoordinator.Environment { + let log = gate?.appending(path: "photo-transport.log") + let http = FixtureCaptureHTTP(answer: answer) { routes in + guard let log else { return } + try? Data(routes.joined(separator: "\n").utf8).write(to: log, options: .atomic) + } + var policy = CaptureUploader.Policy() + // A retry against a fixture costs nothing, and the UI tests shouldn't wait out the + // service's back-off. + policy.firstDelay = 0.1 + policy.maxDelay = 0.5 + let device = CaptureAPI.Device(model: "fixture", systemVersion: "fixture", appVersion: "fixture") + let folder = FileManager.default.temporaryDirectory.appending(path: "PhotoProcessingFixture", directoryHint: .isDirectory) + return .init( + endpoint: FixtureCaptureHTTP.base, sends: true, http: http, capturesFolder: folder, + sessionInfo: { packetID, video in + // The synthetic capture says so in its packet; a replay's photos say replay. + if synthetic { return SyntheticCapture.sessionInfo(packetID: packetID, video: video, appVersion: device.appVersion) } + return Packet04SessionInfo( + packetID: packetID, sessionID: packetID, source: .replay, appVersion: device.appVersion, deviceModel: device.model, + systemVersion: device.systemVersion, lidarAvailable: false, sceneDepthEnabled: false, meshReconstructionSupported: nil, + sceneReconstruction: nil, planeDetection: nil, videoWidth: Int(video.x), videoHeight: Int(video.y), framesPerSecond: nil, + timeZone: nil) + }, + device: device, tier: .arkit, policy: policy, + log: { line in RuntimeLog.engine.info("\(line, privacy: .public)") }) + } + #endif +} diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index d43244723..98512e833 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -128,14 +128,16 @@ final class ScanEngine { /// new one (`keepOverheadView`); counting keyframes would not do, since the walk keeps them too. private var overheadViewsAtGapStart = 0 /// Requests the homeowner skipped or answered with something overhead: they go to installer - /// review, and the result doesn't offer them as captures again. + /// review, and the result doesn't offer them as captures again. With them, the past_end + /// request from a corner a request couldn't follow (`skipCurrentGap(deferring:)`). private(set) var skippedGaps: [GapPlan] = [] /// The side of a server past_end request being captured: that end was cleared, and marking - /// it again settles the request (see `markWallEnd`). + /// it again as where the wall stops settles the request (see `markWallEnd`, `answerWallEnd`). var pastEndSide: WallSide? - /// The end the past_end request cleared: where it was, its kind and when it was marked, put - /// back or moved on when the request ends without it marked again (`settleClearedEnd`). - private var clearedEnd: (s: Float, kind: EndKind, t: Double?)? + /// The end the past_end request cleared: where it was, its kind and its stamp (marked, with + /// when, or inferred), put back or moved on when the request ends without it marked again + /// (`settleClearedEnd`). + private var clearedEnd: (s: Float, kind: EndKind, stamp: WallEndStamp)? /// Server requests raised without a tap since the review was confirmed (`automaticGapQueue`), /// oldest first. Each is raised once, whether its view was taken or the homeowner couldn't /// get there; the result still offers it as a capture. @@ -188,6 +190,16 @@ final class ScanEngine { private var resultBuiltFor: WallGeometry? // Upload + /// The scan under way: its id, backend profile and world, fixed when it starts + /// (`beginScan`); nil before a scan starts. + var scanContext: ScanContext? + /// The meter tap's encode and forward, from an accepted mark on a live photo-processing scan + /// (`takeMeterTap`); the send waits for it. + var meterTapTask: Task? + /// Photo processing's capture and answer, for scans that use it (`ScanEngine+Processing`). + let photoProcessing: PhotoProcessingController + /// DEBUG's synthetic capture, sent in place of the scan's photos; nil otherwise. + let syntheticCapture: SyntheticCapture? let resultClient: any ResultClient private var uploadTask: Task? /// The current upload's scene has fixed its ground; keep this true through answer pacing. @@ -232,8 +244,12 @@ final class ScanEngine { } } } - /// When each mark was made, on the capture clock (`MarkKey`). + /// When each mark was made, on the capture clock (`MarkKey`). Wall ends keep theirs in + /// `endStamps`, with whether the homeowner marked them at all. var markTimes: [String: Double] = [:] + /// Whether each wall end was marked by the homeowner, with when, or inferred from the walk + /// (`WallEndStamp`, B-12). Written by every `setEnd`, so it never outlives its end's source. + var endStamps: [WallSide: WallEndStamp] = [:] /// The packet's clock for guidance and marks: the latest frame's time, ARFrame.timestamp /// live. A replay plays parts of its recording more than once, so its clock is the latest /// frame time seen and never runs back. Nil until the first frame. @@ -262,9 +278,16 @@ final class ScanEngine { } else { resultClient = SampleResultClient(pace: options.autopilot ? options.autopilotHold : 1.2) } + let photoSetup = Self.photoProcessingSetup(options) + photoProcessing = PhotoProcessingController(setup: photoSetup.setup) + syntheticCapture = photoSetup.synthetic + ProcessingBackendSetting.shared.photoProcessingAnswers = photoProcessing.profile.answers + photoProcessing.onChange = { [weak self] status in self?.state.photoProcessing = status } + photoProcessing.meterAnchor = { [weak self] in self?.meterTracking?.pose } state.isAutopilot = options.autopilot state.isReplay = options.replayFolder != nil state.usesSampleResult = resultClient.isSample + state.photoCaptureIsSynthetic = syntheticCapture != nil } // MARK: Start @@ -362,7 +385,7 @@ final class ScanEngine { replay.play(range: range, speed: replaySpeed) } } - case .markFeatures, .uploading, .spotConfirm, .result: + case .markFeatures, .uploading, .spotConfirm, .result, .processing: live?.setMode(.idle) replay?.stop() case .resultAR: @@ -532,11 +555,11 @@ final class ScanEngine { case .failed, .rejected, .unusableAnswer: false case .idle, .packaging, .uploading, .analyzing, .done: true } - case .onboarding, .spotConfirm, .result, .resultAR, .unsupported: false + case .onboarding, .spotConfirm, .result, .resultAR, .processing, .unsupported: false } let depthFrames = switch state.phase { case .meterCloseUp, .wallWalk, .gapRequest: true - case .onboarding, .findMeter, .markFeatures, .uploading, .spotConfirm, .result, .resultAR, .unsupported: false + case .onboarding, .findMeter, .markFeatures, .uploading, .spotConfirm, .result, .resultAR, .processing, .unsupported: false } recorder.setRecording(capturing, depthFrames: depthFrames) guard live != nil else { return } @@ -558,6 +581,10 @@ final class ScanEngine { if state.tracking == .normal { breakWalkedPath(because: "tracking left normal") } state.tracking = frame.tracking live?.setResultVisible(frame.tracking == .normal) + // The circle's end checks this tracking (`aimedEnd`), and a pose-only frame returns + // below before guidance republishes the preview, so the tape would keep an end that + // "Wall ends here" now refuses (review of #213). + publishEndPreview() } noteMeterAnchor(frame) // A frame made before the meter was anchored again carries the old anchor's pose. First, @@ -1383,7 +1410,9 @@ final class ScanEngine { } else { store.keyframes.count > keyframesAtGapStart } - let satisfied = gapPlanner.isSatisfied(plan, map) && fresh + // Not while the end question is up: the homeowner marked the end and is saying what is + // there, and that answer closes the request (`answerWallEnd`). + let satisfied = gapPlanner.isSatisfied(plan, map) && fresh && state.endQuestion == nil state.guidance = .gap let center = (plan.span.lowerBound + plan.span.upperBound) / 2 let cue = gapCue(plan, map, center: center) @@ -1545,7 +1574,7 @@ final class ScanEngine { // "Add something", which taps into the world frame, waits for tracking to return // (`beginMarking`). Nothing is thrown away. break - case .uploading, .spotConfirm, .result, .resultAR, .onboarding, .unsupported: + case .uploading, .spotConfirm, .result, .resultAR, .processing, .onboarding, .unsupported: // The bundle is already packed and the server's answer does not depend on the live // world frame, so the scan and the result stay. The AR result hides its overlay while // tracking is not normal and shows it again if ARKit does relocalize. @@ -1567,7 +1596,7 @@ final class ScanEngine { case .cameraDenied: // As for a failed session: once the scan is sent, the answer stays on screen. switch state.phase { - case .uploading, .spotConfirm, .result, .resultAR: + case .uploading, .spotConfirm, .result, .resultAR, .processing: RuntimeLog.engine.error("camera access lost after capture") _ = sourceState.sourceFailed(.recoverable, afterCapture: true) loseSpatialResult() @@ -1579,7 +1608,7 @@ final class ScanEngine { // longer need the camera: keep them on screen. Only the AR view needs it, and it // already hides the battery while the camera isn't tracking. switch state.phase { - case .uploading, .spotConfirm, .result, .resultAR: + case .uploading, .spotConfirm, .result, .resultAR, .processing: RuntimeLog.engine.error("camera session failed after capture: \(message, privacy: .public)") _ = sourceState.sourceFailed(.recoverable, afterCapture: true) loseSpatialResult() @@ -1600,6 +1629,7 @@ final class ScanEngine { generation += 1 // A new world frame is a new packet session: what was recorded is in the old frame. recorder.restart() + scanWorldReset() resetPacketLog() relocalizingSince = nil planes = PlaneSnapshot() @@ -1675,6 +1705,7 @@ final class ScanEngine { coverage?.heightError = groundMeasured ? 0 : Self.estimatedGroundError self.groundMeasured = groundMeasured endKinds = [:] + endStamps = [:] state.endQuestion = nil state.wallTooShort = false nextWallSide = nil @@ -1704,6 +1735,9 @@ final class ScanEngine { /// Takes the answer down when a ground change leaves it describing a ground the phone no /// longer has (`GroundFreshness`), before `publishWall` can draw it again on the new one. private func answerAfter(_ change: GroundFreshness.Change) { + // A photo-processing packet doesn't follow the phone's ground after it is sent, and + // nothing here may send that scan to the Legacy checker. + guard scanContext?.profile.backend != .photoProcessing else { return } let screen: GroundFreshness.Screen = switch state.phase { case .uploading: switch state.upload { @@ -1867,9 +1901,16 @@ final class ScanEngine { ) } - func setEnd(_ side: WallSide, at s: Float, kind: EndKind) { + /// Sets the end on `side`. `source` says who put it there: the homeowner's mark is stamped + /// with the capture clock; an end inferred from the walk has no mark time (B-12). + func setEnd(_ side: WallSide, at s: Float, kind: EndKind, source: WallEndSource) { + setEnd(side, at: s, kind: kind, stamp: source == .homeowner ? .marked(at: captureClock) : .inferred) + } + + /// Sets the end on `side` with a given stamp: a restored end keeps the one it had. + func setEnd(_ side: WallSide, at s: Float, kind: EndKind, stamp: WallEndStamp) { guard var map = coverage else { return } - markTimes[MarkKey.end(side)] = captureClock + endStamps[side] = stamp map.setEnd(side == .left ? .left : .right, at: s) // Ground past a limit end still counts toward clearances (server contract, "Ends and // corners"); past an unexplored end it doesn't. @@ -1880,7 +1921,7 @@ final class ScanEngine { state.endMarkRefusal = nil publishWall() publishCoverage() - RuntimeLog.engine.info("end \(side.rawValue, privacy: .public) at s=\(s) (\(kind == .limit ? "limit" : "unexplored", privacy: .public))") + RuntimeLog.engine.info("end \(side.rawValue, privacy: .public) at s=\(s) (\(kind == .limit ? "limit" : "unexplored", privacy: .public), \(stamp.isInferred ? "inferred" : "marked", privacy: .public))") if let camera = lastFrame?.camera, state.phase == .wallWalk { updateGuidance(camera: camera, time: lastFrame?.timestamp ?? 0) } @@ -1894,7 +1935,7 @@ final class ScanEngine { } func clearEnd(_ side: WallSide) { - markTimes[MarkKey.end(side)] = nil + endStamps[side] = nil updateCoverage { $0.clearEnd(side == .left ? .left : .right) } endKinds[side] = nil if state.endQuestion == side { state.endQuestion = nil } @@ -1939,7 +1980,9 @@ final class ScanEngine { keyframesAtGapStart = store.keyframes.count overheadViewsAtGapStart = coverage?.overheadCameras.count ?? 0 let progress = coverage.map { gapPlanner.progress(of: plan, $0) } ?? 0 - state.gap = GapRequest(id: gapCounter, origin: origin, reason: reason, band: plan.band == .ground ? .ground : .wall, span: plan.span, progress: progress, isSatisfied: false) + state.gap = GapRequest( + id: gapCounter, origin: origin, reason: reason, band: plan.band == .ground ? .ground : .wall, span: plan.span, + progress: progress, isSatisfied: false, pastEndSide: pastEndSide) state.guidance = .gap go(.gapRequest) updateGap(camera: lastFrame?.camera) @@ -1949,6 +1992,15 @@ final class ScanEngine { /// with the new evidence (the closed loop: gap, instruction, capture, updated result). The /// answer then leads to the next capturable request or to the result (`upload`). private func afterGapResolved() { + // A request left with its end question unanswered (the phone lost its place, say): the + // end stays the homeowner's mark, unexplored as for any unanswered end, and the question + // must not outlive the request and hold the next one. + if let side = state.endQuestion, side == pastEndSide { + state.endQuestion = nil + state.endQuestionLeavesOut = nil + state.endQuestionLeavesOutSeen = false + } + state.endMarkRefusal = nil settleClearedEnd() gapPlan = nil pastEndSide = nil @@ -1958,8 +2010,9 @@ final class ScanEngine { startUpload() } - /// The past_end request's end was marked again and its question answered: the request is - /// settled, so the scan goes to the upload like a closed gap. + /// The past_end request's end was marked again and the homeowner said the wall stops there: + /// the request is settled, so the scan goes to the upload like a closed gap. A corner doesn't + /// settle it (`answerWallEnd`). func settlePastEnd() { guard state.phase == .gapRequest, var request = state.gap, !request.isSatisfied else { return } resolveGuidance(.met) @@ -2007,7 +2060,14 @@ final class ScanEngine { /// overhead, or "Show my result"): recorded for installer review, then on to the upload. The /// answer that follows raises the next item it lists, never this one again /// (`automaticGapQueue`); only "Show my result" (`stopGapRequests`) ends the requests. - func skipCurrentGap(because reason: String = "the homeowner can't get there", refused: Bool = true) { + /// + /// `deferring` is a request this one stands for that nobody was shown: the past_end request + /// from the corner the homeowner just marked (`answerWallEnd`), which this request can't + /// follow. Recorded with the skipped requests only, so the next answer doesn't raise it again + /// (`GapPlan.asksForSameView`); it logs no guidance and marks no cells, since no view was + /// asked for or taken there. A past_end from a different end later, and other requests, can + /// still be raised. + func skipCurrentGap(because reason: String = "the homeowner can't get there", refused: Bool = true, deferring: GapPlan? = nil) { guard let plan = gapPlan else { return } // Something overhead is an answer: the request goes to review without its view. resolveGuidance(refused ? .cannotReach : .skipped) @@ -2015,6 +2075,7 @@ final class ScanEngine { // nothing about the band the strip draws. if plan.need == .cells { coverage?.markSkipped(plan.band, plan.span) } skippedGaps.append(plan) + if let deferring { skippedGaps.append(deferring) } publishCoverage() RuntimeLog.engine.info("gap \(self.gapCounter) left for installer review: \(reason, privacy: .public)") afterGapResolved() @@ -2031,7 +2092,26 @@ final class ScanEngine { // MARK: Upload + enum PendingSaves { case done, scanChanged, stillWriting } + + /// Waits, for at most 10 s, for the keyframe writes this world started, so the capture holds + /// every photo kept before the send. `stillWriting` when some are still being written then: a + /// capture sealed now would leave them out, and their late `onKept` would be refused. + func drainPendingSaves() async -> PendingSaves { + let scan = generation + for _ in 0..<200 where (pendingSaves[scan] ?? 0) > 0 { + try? await Task.sleep(for: .milliseconds(50)) + } + guard scan == generation else { return .scanChanged } + return (pendingSaves[scan] ?? 0) > 0 ? .stillWriting : .done + } + func startUpload() { + // The scan's own backend only: a photo-processing scan never reaches the Legacy checker. + if scanContext?.profile.backend == .photoProcessing { + startPhotoProcessing() + return + } scenePackaged = false // "Try again" sends from the upload screen; anything else is a new send of this scan. if state.phase != .uploading { unusableAnswers = 0 } @@ -2187,7 +2267,8 @@ final class ScanEngine { let wallSide: WallSide = side == .left ? .left : .right pastEnd = wallSide let old = wallSide == .left ? coverage?.leftEnd : coverage?.rightEnd - clearedEnd = old.map { (s: $0, kind: endKinds[wallSide] ?? EndKind.unexplored, t: markTimes[MarkKey.end(wallSide)]) } + // An end without a stamp comes back as inferred: no source is ever made up for it. + clearedEnd = old.map { (s: $0, kind: endKinds[wallSide] ?? EndKind.unexplored, stamp: endStamps[wallSide] ?? .inferred) } clearEnd(wallSide) } // Set first, so the guidance log records the request as a past-end one. @@ -2195,21 +2276,24 @@ final class ScanEngine { beginGap(plan, origin: .server, reason: .server(detail: item.message)) } - /// A past_end request ending without its end marked again (the gap screen offers only "I - /// can't get there") must not leave that side without an end: the export would run the wall - /// out to whatever the fog saw, and the next past_end request would be planned from the - /// meter (issue #35). Met, the end moves on past the ground the request showed, still - /// unexplored (`GapPlanner.endAfterPastEnd`); skipped, the end it cleared comes back with its - /// kind and mark time. + /// A past_end request leaving the screen settles the end it cleared (`PastEndSettlement`). + /// Marked again during the request ("Wall ends here"), the homeowner's end stands, nearer or + /// farther than the cleared one, which only said how far the walk had seen. Met by views, + /// the end moves on past the ground the request showed, unexplored and inferred + /// (`GapPlanner.endAfterPastEnd`). Skipped, the cleared end comes back with its kind, source + /// and mark time. Left without an end, the export would run the wall out to whatever the fog + /// saw, and the next past_end request would be planned from the meter (issue #35). private func settleClearedEnd() { guard let side = pastEndSide, let old = clearedEnd, let plan = gapPlan else { return } clearedEnd = nil - guard (side == .left ? coverage?.leftEnd : coverage?.rightEnd) == nil else { return } - if state.gap?.isSatisfied == true { - setEnd(side, at: gapPlanner.endAfterPastEnd(plan, side: side == .left ? .left : .right, clearedAt: old.s), kind: .unexplored) - } else { - setEnd(side, at: old.s, kind: old.kind) - markTimes[MarkKey.end(side)] = old.t + let marked = (side == .left ? coverage?.leftEnd : coverage?.rightEnd) != nil + switch PastEndSettlement.when(endMarkedDuringRequest: marked, met: state.gap?.isSatisfied == true) { + case .keepMarked: + return + case .moveOn: + setEnd(side, at: gapPlanner.endAfterPastEnd(plan, side: side == .left ? .left : .right, clearedAt: old.s), kind: .unexplored, source: .inferred) + case .restore: + setEnd(side, at: old.s, kind: old.kind, stamp: old.stamp) } } @@ -2220,7 +2304,18 @@ final class ScanEngine { /// place, so it is not offered while a write is under way, and writes run one after another: /// a retry's write waits for the last one, and a write already superseded is skipped. /// `inputs` were captured with the upload's scene (`captureUpload`); this rereads nothing. + /// + /// Inputs read from an earlier store write into that store's folder, which the current + /// store's cleanup may delete; they are refused before they change what this scan offers. + /// The callers already drop a capture from before Start over; this keeps that true here. + /// + /// Start over doesn't wait for a write in flight: the next store's cleanup waits for + /// `bundleTask` before deleting the folder, so the write's bundle lands and is kept. func saveBundle(_ inputs: PacketInputs?) { + if let inputs, inputs.storeDirectory != store.directory { + RuntimeLog.engine.error("scan bundle not written: its capture belongs to a scan already started over") + return + } state.shareableScan = nil bundleSerial += 1 let serial = bundleSerial @@ -2271,9 +2366,14 @@ final class ScanEngine { meterTracking = nil meterAnchorPresence = MeterAnchorPresence() meterPlaneSource = .detectedPlane - store = KeyframeStore() + // The old scan's bundle may still be packing in the folder the new store's cleanup lists + // as never packaged. Start over doesn't wait for it; the cleanup's deletion does, for this + // chain head (every write waits for the one before it). `generation` was raised above, + // so a queued write for the old scan still skips, and a finished one isn't offered here. + store = KeyframeStore(deletingAfter: bundleTask) recorder = Self.makeRecorder(store) live?.setRecorder(recorder) + endScan() motion.stop() resetPacketLog() keptSourceIDs = [] @@ -2388,6 +2488,7 @@ final class ScanEngine { private func resetPacketLog() { guidanceLog = GuidanceLog() markTimes = [:] + endStamps = [:] captureClock = nil } diff --git a/ios/HouseScan/UI/Components/EndAimCircle.swift b/ios/HouseScan/UI/Components/EndAimCircle.swift new file mode 100644 index 000000000..7cef095aa --- /dev/null +++ b/ios/HouseScan/UI/Components/EndAimCircle.swift @@ -0,0 +1,16 @@ +import SwiftUI + +extension View { + /// The circle a mark lands under (`Reticle`), as one accessibility element: "Aiming circle". + /// The reticle itself is hidden from VoiceOver as decoration. Here it says where "Wall ends + /// here" and the other marks land, and UI tests read its real frame to check that nothing + /// covers it (`PastEndRecoveryUITests`, `WalkRecoveryUITests`). Apply it before the reticle + /// is stretched to the screen, so the element's frame is the circle's. Show the circle only + /// while it is offered: `.accessibilityHidden` on this element left it in the tree at zero + /// opacity (CI run 37171342665), so VoiceOver would stop on a circle that isn't there. + func endAimCircle() -> some View { + accessibilityElement(children: .ignore) + .accessibilityLabel("Aiming circle") + .accessibilityIdentifier("aim.circle") + } +} diff --git a/ios/HouseScan/UI/Components/EndQuestionAnswers.swift b/ios/HouseScan/UI/Components/EndQuestionAnswers.swift new file mode 100644 index 000000000..068b175d2 --- /dev/null +++ b/ios/HouseScan/UI/Components/EndQuestionAnswers.swift @@ -0,0 +1,42 @@ +import SwiftUI + +/// The three answers to "What's at the right end?" (`ScanViewState.endQuestion`), shared by the +/// walk and a past-end request, where marking the end again asks the same question (B-12). One +/// question, three equal full-width answers that say what they mean (checklist I4). +struct EndQuestionAnswers: View { + let actions: any ScanActions + + var body: some View { + VStack(spacing: 8) { + Button { + actions.answerWallEnd(turnsCorner: true) + } label: { + Label("It turns a corner", systemImage: "arrow.turn.up.right") + .frame(maxWidth: .infinity) + } + .buttonStyle(.secondaryProminent) + .accessibilityIdentifier("action.endCorner") + Button { + actions.answerWallEnd(turnsCorner: false) + } label: { + Label("Something blocks it", systemImage: "xmark.octagon") + .frame(maxWidth: .infinity) + } + .buttonStyle(.secondaryProminent) + .accessibilityHint("A fence, gate, or your neighbor's yard") + .accessibilityIdentifier("action.endBlocked") + // A garden wall or a fence can just stop, with no corner and nothing in the way, + // and neither answer above fits it (#70). Ends the wall as "Something blocks it" + // does: the usable wall stops here. + Button { + actions.answerWallEnd(turnsCorner: false) + } label: { + Label("The wall just ends", systemImage: "arrow.right.to.line") + .frame(maxWidth: .infinity) + } + .buttonStyle(.secondaryProminent) + .accessibilityHint("No corner and nothing in the way. The wall stops here.") + .accessibilityIdentifier("action.endEnds") + } + } +} diff --git a/ios/HouseScan/UI/Components/MovedReplyButton.swift b/ios/HouseScan/UI/Components/MovedReplyButton.swift new file mode 100644 index 000000000..312cc0d48 --- /dev/null +++ b/ios/HouseScan/UI/Components/MovedReplyButton.swift @@ -0,0 +1,37 @@ +import SwiftUI + +/// A card's reply ("I can't get there", "The wall keeps going") shown among the actions instead, +/// while the card folds around the circle at the largest text sizes (`GapRequestScreen`, +/// `WallWalkScreen`). It keeps the card reply's input lock: after its task changes it takes no +/// taps for `InstructionCard.replyLock`, so a tap meant for the step that just left can't answer +/// this one (#82). Coaching or a refusal within the same task keeps it unlocked, as on the card. +/// The secondary style doesn't read `isEnabled`, so the lock doesn't change how it looks; +/// VoiceOver and UI tests read it as dimmed. +struct MovedReplyButton: View { + let title: String + let hint: String + let identifier: String + /// The step the reply answers; a new one locks the button again. + let task: Instruction + let perform: () -> Void + + @State private var unlockedTask: Instruction? + + var body: some View { + Button(action: perform) { + Text(title) + .frame(maxWidth: .infinity) + } + .buttonStyle(.secondary) + .disabled(unlockedTask != task) + .accessibilityHint(hint) + .accessibilityIdentifier(identifier) + .task(id: task) { + // Cleared first, so a task that comes back within the lock is locked again. + let shown = task + unlockedTask = nil + do { try await Task.sleep(for: InstructionCard.replyLock) } catch { return } + unlockedTask = shown + } + } +} diff --git a/ios/HouseScan/UI/Components/PhotoConsentCard.swift b/ios/HouseScan/UI/Components/PhotoConsentCard.swift new file mode 100644 index 000000000..ffffd735e --- /dev/null +++ b/ios/HouseScan/UI/Components/PhotoConsentCard.swift @@ -0,0 +1,60 @@ +import SwiftUI + +/// Asks, once per scan and before anything is sent, whether this scan's photos may go to photo +/// processing. It covers the camera and blocks it until answered, because photos kept from the +/// meter close-up on would go up under the answer. Both choices are large, and "Don't send" +/// is a real answer: the scan goes on and ends saying it wasn't processed. +struct PhotoConsentCard: View { + var standIn: Bool + var synthetic: Bool + let actions: any ScanActions + + var body: some View { + ZStack(alignment: .bottom) { + Palette.ink.opacity(0.6) + .ignoresSafeArea() + .accessibilityHidden(true) + ScrollView { + VStack(alignment: .leading, spacing: 14) { + Text(ProcessingCopy.screenLabel) + .font(Typeface.caption) + .textCase(.uppercase) + .tracking(0.6) + .foregroundStyle(Palette.chalk.opacity(0.8)) + Text(ProcessingCopy.consentTitle) + .font(Typeface.instruction) + .foregroundStyle(Palette.chalk) + .fixedSize(horizontal: false, vertical: true) + .accessibilityAddTraits(.isHeader) + .accessibilityIdentifier("photoConsent.title") + Text(ProcessingCopy.consentBody(standIn: standIn, synthetic: synthetic)) + .font(Typeface.hint) + .foregroundStyle(Palette.chalk) + .fixedSize(horizontal: false, vertical: true) + Text(ProcessingCopy.consentDeclineNote) + .font(Typeface.hint) + .foregroundStyle(Palette.chalk.opacity(0.85)) + .fixedSize(horizontal: false, vertical: true) + VStack(spacing: 12) { + Button(ProcessingCopy.consentSend) { actions.answerPhotoConsent(true) } + .buttonStyle(.primary) + .accessibilityIdentifier("photoConsent.send") + Button(ProcessingCopy.consentDecline) { actions.answerPhotoConsent(false) } + .buttonStyle(.secondaryProminent) + .frame(maxWidth: .infinity) + .accessibilityIdentifier("photoConsent.decline") + } + .padding(.top, 6) + } + .padding(22) + .background(Palette.ink, in: .rect(cornerRadius: Metrics.cardRadius)) + .padding(Metrics.edge) + } + .scrollBounceBehavior(.basedOnSize) + .defaultScrollAnchor(.bottom) + } + .accessibilityElement(children: .contain) + .accessibilityAddTraits(.isModal) + .accessibilityIdentifier("photoConsent") + } +} diff --git a/ios/HouseScan/UI/Copy/ProcessingCopy.swift b/ios/HouseScan/UI/Copy/ProcessingCopy.swift new file mode 100644 index 000000000..2fdc004fb --- /dev/null +++ b/ios/HouseScan/UI/Copy/ProcessingCopy.swift @@ -0,0 +1,235 @@ +import HouseScanKit + +/// Words for the backend choice and photo processing. The service's own messages and prompts are +/// shown as written, never rewritten here; every answer stays provisional, as on the Legacy +/// result (`ScanCopy.installerConfirms`). +enum ProcessingCopy { + static func name(_ backend: ProcessingBackend) -> String { + switch backend { + case .legacy: "Legacy (remote checker)" + case .photoProcessing: "Photo processing (beta)" + } + } + + static func summary(_ backend: ProcessingBackend) -> String { + switch backend { + case .legacy: + "Sends the scan's measurements to the House Scan server, which checks the placement rules. Photos stay on this phone." + case .photoProcessing: + "Sends the scan's photos and how the phone moved to House Scan's photo processing service, which suggests a spot. Asks before sending." + } + } + + /// What this build can do with a photo-processing scan, under its option; nil when a service + /// answers. + static func setupNote(_ answers: ProcessingProfile.Answers) -> String? { + switch answers { + case .service: nil + case .standIn: "Test build: answers come from a stand-in on this phone, and nothing leaves it." + case .notSetUp: "Not set up in this build. A scan with it stops before the camera starts and offers a Legacy scan instead." + } + } + + static func choiceFooter(scanBackend: ProcessingBackend?) -> String { + guard let scanBackend else { return "Applies to the next scan you start." } + return "This scan uses \(name(scanBackend)). A change applies to the next scan." + } + + // MARK: Consent + + static let consentTitle = "Send this scan's photos?" + + static func consentBody(standIn: Bool, synthetic: Bool) -> String { + let destination = standIn + ? "In this test build, a stand-in on this phone receives them, so nothing leaves it." + : "They go to House Scan's processing service while you scan." + let body = "Photo processing uses the photos you take of the wall around your meter, and how the phone moved, to suggest a battery spot. \(destination)" + return synthetic ? body + " " + syntheticCapture : body + } + + /// Said wherever a synthetic capture stands in for the scan's photos. + static let syntheticCapture = "This test sends a synthetic capture made on the phone instead of this scan's photos, so its answer says nothing about your wall." + + + static let consentDeclineNote = "If you don't send them, nothing leaves this phone and this scan won't get an answer." + static let consentSend = "Send photos" + static let consentDecline = "Don't send" + + // MARK: Processing screen + + static let screenLabel = "Photo processing (beta)" + static let standInBadge = "Test answer from this phone, not the processing service" + static let serviceSaid = "The service says" + static let stopSending = "Stop sending photos" + static let stopQuestion = "Stop sending this scan's photos?" + /// Stopping can come after the photos reached the service and its processing began, and the app + /// can't cancel the service's job, so this says only what House Scan stops doing. + static let stopDetail = "House Scan will stop sending from this scan and won't resume it. Photos already sent may still be processed by the service." + static let confirmStop = "Stop sending" + static let keepSending = "Keep sending" + static let setupRefusedDetail = "House Scan couldn't complete its connection to the processing service. This is a setup problem, not a problem with your photos or marks." + static let scanWithLegacy = "Scan with Legacy instead" + static let scanWithLegacyHint = "Chooses Legacy in Developer options and starts a new scan with it." + static let closedAppLimit = "House Scan can't come back to this scan after you close the app." + static let retrying = "Connection trouble. Trying again." + + struct Screen: Equatable { + /// A stable name for the state, for the UI tests (`photo.state.`). + var id: String + var title: String + var detail: String + /// `detail` is the service's message, shown quoted, not House Scan's words. + var fromService = false + var working = false + var tone: Tone = .neutral + var symbol: String + /// Further facts, each its own line. + var notes: [String] = [] + } + + enum Tone { + case neutral, proposed, attention + } + + static func screen(_ status: PhotoProcessingStatus?, isReplay: Bool) -> Screen { + guard let status else { + return Screen( + id: "none", title: "Nothing to show", detail: "This scan has no photo processing to show. Start over to scan again.", + tone: .attention, symbol: "questionmark.circle") + } + switch status.stage { + case .capturing, .queued: + return Screen( + id: "queued", title: "Sending your scan", detail: "Waiting for the processing service to open your scan.", working: true, + symbol: "arrow.up.circle") + case .uploading(let received, let kept): + return Screen( + id: "uploading", title: "Sending your scan", detail: "\(received) of \(kept) files received by the processing service.", + working: true, symbol: "arrow.up.circle") + case .processing: + return Screen( + id: "processing", title: "Processing your scan", detail: "The service has every file and is working on it.", working: true, + symbol: "cube.transparent") + case .answered(let answer): + return screen(answer) + case .ended(let end): + return screen(end, sentSome: status.consent != .declined && status.consent != .asking, isReplay: isReplay) + } + } + + private static func screen(_ answer: PhotoProcessingAnswer) -> Screen { + switch answer { + case .candidate(let candidate): + let ar = switch candidate.ar { + case .withheld: "Showing it on your wall isn't available for this answer." + case .notShownInThisBuild: "Showing it on your wall isn't available for photo processing yet." + } + let preview = candidate.previewMade ? "The service made a preview, which this beta doesn't show yet." : "The service made no preview for this scan." + return Screen( + id: "candidate", title: "A possible battery spot", detail: candidate.message, fromService: true, tone: .proposed, + symbol: "mappin.and.ellipse", + notes: [ScanCopy.installerConfirms, ar, preview]) + case .needsMorePhotos(let more): + return Screen( + id: "needsMorePhotos", title: "One more look needed", + detail: more.message ?? "The processing service needs more views of the wall.", fromService: more.message != nil, + tone: .attention, + symbol: "camera.viewfinder", + notes: ["Adding photos to a scan isn't supported in this beta yet. To try again, start over and include these views."]) + case .installerReview(let message): + return Screen( + id: "installerReview", title: "Needs an installer's review", detail: message, fromService: true, + symbol: "person.crop.circle.badge.checkmark") + case .noCandidate(let message): + return Screen( + id: "noCandidate", title: "No possible spot found", detail: message, fromService: true, symbol: "mappin.slash", + notes: [ScanCopy.installerConfirms]) + case .unrecognized(_, let message): + return Screen( + id: "unrecognized", title: "An answer this version can't read", + detail: message.isEmpty ? "The processing service sent a kind of answer House Scan doesn't know yet." : message, + fromService: !message.isEmpty, tone: .attention, symbol: "questionmark.bubble", + notes: ["House Scan doesn't know this kind of answer yet, so it shows the service's words as they are."]) + } + } + + private static func screen(_ end: PhotoProcessingEnd, sentSome: Bool, isReplay: Bool) -> Screen { + let kept = "Photos already sent stay with the processing service." + switch end { + case .notSetUp: + return Screen( + id: "notSetUp", title: "Photo processing isn't set up in this build", + detail: "Nothing has been captured or sent. Scan with \(name(.legacy)) instead, or choose another method in Developer options.", + tone: .attention, symbol: "wrench.and.screwdriver") + case .consentNotGiven: + return Screen( + id: "consentNotGiven", title: "Photos weren't sent", + detail: "You didn't agree to send this scan's photos, so nothing left this phone and the scan wasn't processed.", + symbol: "hand.raised") + case .withdrawn(let recorded): + guard !recorded else { + return Screen( + id: "withdrawn", title: "You stopped sending photos", + // Stopping can follow sending, and the app can't cancel the service's job. + detail: "House Scan won't send anything more from this scan. Photos already sent may still be processed by the service.", + symbol: "hand.raised", notes: [kept]) + } + return Screen( + id: "withdrawalNotRecorded", title: "You stopped sending photos", + detail: "Sending stopped, but House Scan couldn't save your choice on this phone. Don't use this scan again: start over for a new one.", + tone: .attention, symbol: "exclamationmark.triangle", notes: [kept]) + case .notPrepared: + var notes = sentSome ? [kept] : [] + if isReplay { notes.insert("Replays carry no motion data, which photo processing needs.", at: 0) } + return Screen( + id: "notPrepared", title: "This scan couldn't be prepared", + // Some photos may already have been sent; the phone knows only that it couldn't send the scan. + detail: "House Scan couldn't put this scan together for photo processing on this phone, so it couldn't be sent for processing.", + tone: .attention, symbol: "exclamationmark.triangle", notes: notes) + case .processingFailed: + return Screen( + id: "processingFailed", title: "Processing didn't finish", detail: "The processing service couldn't finish this scan.", + tone: .attention, symbol: "exclamationmark.triangle") + case .expired: + return Screen( + id: "expired", title: "This scan expired", detail: "The processing service stopped waiting before this scan was finished.", + tone: .attention, symbol: "clock.badge.exclamationmark") + case .refused: + return Screen( + id: "refused", title: "The processing service refused this scan", detail: "Sending it again wouldn't change that.", + tone: .attention, symbol: "exclamationmark.triangle") + case .setupRefused: + // The boundary's four codes read the same to the homeowner: this build's connection + // isn't right, nothing they did. The code itself goes to the log. It can come at the + // events or result step, after the service already has the scan, so the words don't + // say whether it was processed, and an unavailable credential can be temporary. + return Screen( + id: "setupRefused", title: "Photo processing isn't connected correctly", + detail: setupRefusedDetail, tone: .attention, symbol: "wrench.and.screwdriver", + notes: ["The connection needs to be fixed before you try again. Start over to scan again, or choose \(name(.legacy)) in Developer options first."]) + case .uploadStateUnsaved: + // The phone, not the service: it couldn't save how far the upload had got, so it + // stopped sending. That stops nothing on the service, and this app doesn't reopen a + // saved capture after it closes, so the way on is a new scan. + return Screen( + id: "uploadStateUnsaved", title: "This phone stopped sending this scan", + detail: "House Scan couldn't save how far this scan's upload had got, so it stopped sending from this scan. Photos already sent may still be processed by the service.", + tone: .attention, symbol: "exclamationmark.triangle", + notes: ["Start over to scan again. If your phone's storage is nearly full, freeing some space may help."]) + case .answerUnreadable: + return Screen( + id: "answerUnreadable", title: "House Scan couldn't read the answer", + detail: "The processing service answered, but not in a form this version can read.", tone: .attention, + symbol: "exclamationmark.bubble") + case .answerNotReady: + return Screen( + id: "answerNotReady", title: "The answer wasn't ready", + detail: "The service finished, but its answer didn't become available in time.", tone: .attention, symbol: "clock") + case .answerMismatch: + return Screen( + id: "answerMismatch", title: "The answer didn't match this scan", + detail: "House Scan received an answer for a different capture, so it isn't shown.", tone: .attention, + symbol: "exclamationmark.triangle") + } + } +} diff --git a/ios/HouseScan/UI/Copy/ScanCopy.swift b/ios/HouseScan/UI/Copy/ScanCopy.swift index 24f23609e..a898a6eb0 100644 --- a/ios/HouseScan/UI/Copy/ScanCopy.swift +++ b/ios/HouseScan/UI/Copy/ScanCopy.swift @@ -71,7 +71,8 @@ enum ScanCopy { case .markEnd(let side): Instruction( title: "Is this the \(side.rawValue) end of the wall?", - detail: "Aim where it stops or turns a corner and tap Wall ends here. If it goes on, tap The wall keeps going." + detail: "Aim where it stops or turns a corner and tap Wall ends here. If it goes on, tap The wall keeps going.", + folded: endAimFold("Aim at the \(side.rawValue) end", "Is this the \(side.rawValue) end of the wall? Aim where it stops or turns a corner and tap Wall ends here. If it goes on, tap The wall keeps going.") ) case .aimAtGround(let s): // A cell counts once seen from two places at least 0.25 m apart (`coveringBaseline`), @@ -251,6 +252,9 @@ enum ScanCopy { /// Over the walk's own prompt after "Done with this wall" was refused and the ends cleared /// (`ScanViewState.wallTooShort`). static let wallTooShort = "The ends were too close. Walk along the wall first." + /// `wallTooShort` on a card folded around the circle, as the walk's end can be when both ends + /// were cleared: two lines beside the refusal triangle, the rest under Details. + static let wallTooShortLead = "Walk the wall first" // MARK: Close-up @@ -389,14 +393,73 @@ enum ScanCopy { static let wallKeepsGoing = "The wall keeps going" /// Why "Wall ends here" marked nothing, and what to do instead (`EndMarkRefusal`). + /// Folded at the largest text sizes, each refusal leads with what to do about it, and its + /// reason and the full words go under Details (`endAimFold`). static func endMarkRefusal(_ refusal: EndMarkRefusal, asked side: WallSide) -> Instruction { switch refusal { case .noWall: - Instruction(title: "The circle isn't on the wall", detail: "Aim it at the wall where it stops or turns, then tap Wall ends here.") + Instruction( + title: "The circle isn't on the wall", detail: "Aim it at the wall where it stops or turns, then tap Wall ends here.", + folded: endAimFold("Aim at the wall", "The circle isn't on the wall. Aim it at the wall where it stops or turns, then tap Wall ends here.")) case .otherSide(let landed): - Instruction(title: "That's the \(landed.rawValue) side of your meter", detail: "Turn to the \(side.rawValue) end, then tap Wall ends here.") + Instruction( + title: "That's the \(landed.rawValue) side of your meter", detail: "Turn to the \(side.rawValue) end, then tap Wall ends here.", + folded: endAimFold("Face the \(side.rawValue) end", "That's the \(landed.rawValue) side of your meter. Turn to the \(side.rawValue) end, then tap Wall ends here.")) case .trackingNotReady: - Instruction(title: "One moment, your phone is still finding its place", detail: "Then aim at the \(side.rawValue) end.") + Instruction( + title: "One moment, your phone is still finding its place", detail: "Then aim at the \(side.rawValue) end.", + folded: endAimFold("Wait a moment", "Your phone is still finding its place. Then aim at the \(side.rawValue) end.")) + case .tooLittleWall: + // Says what the app can't do, not where to aim: the homeowner must not be nudged to + // mark an end farther than the wall really goes. "I can't get there" stays the way on. + // The app's own minimum (`WallFrame.minWallLength`), not a measured installation rule: + // no reason is given for it, and rule distances belong to the server. Folded, the lead + // says only that; aiming elsewhere is right only if the circle was off the real end, + // so it doesn't lead. + Instruction( + title: "House Scan can't use a wall that short", detail: "If this is where the wall ends, tap I can't get there to continue.", + folded: endAimFold("Too short to use", "House Scan can't use a wall that short. If this is where the wall ends, tap I can't get there to continue.")) + } + } + + /// The words of a step that marks the wall's end at the circle, folded at the largest text + /// sizes (`InstructionCard.foldsDetail`): a lead short enough to stay two lines, even beside + /// the refusal triangle, and every word of the unfolded card under Details. Unfolded, the + /// card covered the circle in the middle of the camera at AX5 (CI run 37165788075, iPhone + /// 17): the circle's top is 28 pt above the middle, and a two-line card without its reply + /// ends above it, where one with a third line or the reply doesn't (`GapRequestScreen`, + /// `WallWalkScreen` move the reply to their actions at those sizes). + static func endAimFold(_ lead: String, _ everything: String) -> Instruction.Folded { + Instruction.Folded(title: lead, detail: everything) + } + + /// A folded end-aim card with coaching riding along (`withCoaching`). Its note under the lead + /// would reach the circle again: the long dark note is several lines at AX5, and even one + /// line uses the room above the circle. So the coaching leads instead, in a few words beside + /// its symbol, and its whole note opens Details. With a refusal, the refusal's correction + /// keeps the lead and the coaching opens Details. Only for a card that folds; coaching that + /// replaces the task isn't folded at all. + static func foldedAroundCircle(_ card: Instruction, coaching: Coaching?, refused: Bool) -> Instruction { + guard let coaching, !coachingReplacesTask(coaching), var folded = card.folded else { return card } + var out = card + out.note = nil + folded.detail = "\(coachingNote(coaching)) \(folded.detail)" + if !refused { folded.title = coachingLead(coaching) } + out.folded = folded + return out + } + + /// Coaching in a few words, for a lead that must stay two lines beside its symbol at AX5: what + /// to do, since the symbol already says what is wrong and darkness can last a whole walk. + static func coachingLead(_ coaching: Coaching) -> String { + switch coaching { + case .slowDown: "Slow down" + case .needsTexture: "Find more texture" + case .tooDark: "Try your flashlight" + case .tooDarkToMeasure: "Try in daylight" + case .holdSteady: "Hold steady" + case .turnSlowly: "Turn more slowly" + case .initializing, .relocalizing, .trackingLost, .pastWallEnd: ScanCopy.coaching(coaching).title } } @@ -441,6 +504,16 @@ enum ScanCopy { // MARK: Gap + /// A server request's lead when folded: the band and which side of the meter, in a few words + /// that stay a short block at AX5. The stretch's two ends are under Details, and the camera and + /// the map mark the stretch itself. Named against the meter, as the full card names it. + static func serverLead(_ gap: GapRequest) -> String { + let band = gap.band == .ground ? "ground" : "wall" + if gap.span.lowerBound >= 0 { return "Show the \(band) right of your meter" } + if gap.span.upperBound <= 0 { return "Show the \(band) left of your meter" } + return "Show the \(band) by your meter" + } + /// The card for a gap request. A server request can run along much of the wall, so its /// stretch is named by its two ends, all of it (issue #75, `Distance.range`); the phone's own /// requests are short and named by their middle. @@ -453,7 +526,19 @@ enum ScanCopy { case .wallAboveCandidate: return Instruction(title: "Show the wall \(place)", detail: "Tilt up so the wall above this spot is in view.") case .server(let detail): - return Instruction(title: gap.band == .ground ? "Show the ground \(stretch)" : "Show the wall \(stretch)", detail: detail) + let title = gap.band == .ground ? "Show the ground \(stretch)" : "Show the wall \(stretch)" + // A past_end request asks to walk on past an end; the wall may really stop before + // that, and the screen offers the walk's own "Wall ends here" for it (B-12). + // Folded at the largest text sizes on a follow-up (`GapRequestScreen.followUpFolds`): + // where to aim leads, and the whole request goes under Details. + guard let side = gap.pastEndSide else { + return Instruction(title: title, detail: detail, folded: Instruction.Folded(title: serverLead(gap), detail: "\(title). \(detail)")) + } + // Folded, walking on leads, as the request asks; marking the end is under Details, + // with the circle and "Wall ends here" in view. + return Instruction( + title: title, detail: "\(detail) \(pastEndAlternative)", + folded: endAimFold("Keep walking \(side.rawValue)", "\(title). \(detail) \(pastEndAlternative)")) case .groundOut(let out): return Instruction( title: "Show the ground out to about \(Distance.feetAtLeast(out)) from the wall", @@ -489,6 +574,9 @@ enum ScanCopy { /// The request a gap card asks for, without the walk-out's reading from where the phone is /// now: what the card's reply answers (`InstructionCard.Reply.task`) and what the guidance log /// keeps. The reading changes as the phone moves, and each change would lock the reply again. + /// After a past_end request's own words: the other true answer, in the walk's vocabulary. + static let pastEndAlternative = "If the wall stops sooner, aim where it stops and tap Wall ends here." + static func gapTask(_ gap: GapRequest) -> Instruction { var steady = gap steady.walkOut = nil @@ -627,10 +715,11 @@ enum ScanCopy { // MARK: Result - /// The answer in the homeowner's words (`ResultPresentation.answer`). + /// The answer in the homeowner's words (`ResultPresentation.answer`). A passing spot is a + /// possible one, never "fits": see `candidateNote`. static func headline(_ answer: ResultReading.Answer) -> String { switch answer { - case .fits: "A battery fits here" + case .candidate: "A possible battery spot" case .oneMoreLook: "One more look" case .installer: "Needs an installer's review" case .notHere: "Not on this wall" @@ -690,7 +779,18 @@ enum ScanCopy { return "Settles: \(titles.joined(separator: ", "))" } - static let seeOnWall = "See it on your wall" + /// Under a possible spot's headline and placement. The server's checks pass on space the scan + /// recorded, some of it beyond what the homeowner confirmed in the spot check, so the app + /// can't say all the space a battery needs is clear (`ResultReading`, B17). It says what the + /// fit still needs, never that a review was sent: the app contacts nobody. + static let candidateNote = "The scan suggests this spot, but it couldn't confirm all the space a battery needs. An installer needs to check the fit on site." + + /// The AR title over a possible spot, and the words for what the AR view draws there. + static let proposedSpotOverlay = "A proposed battery spot, drawn on your wall" + static let proposedSpotThisWay = "The proposed spot is this way" + static let proposedSpotOffScreen = "The proposed spot is off screen. Turn the phone toward the arrow." + + static let seeOnWall = "See this spot on your wall" /// For a spot an installer still has to confirm against the meter's working space. static let seeClosest = "See the closest spot" static let showMe = "Show me" @@ -703,12 +803,12 @@ enum ScanCopy { static let wallNotMeasuredDetail = "The scan stopped before you walked along the wall on either side of your meter, so we can't tell where a battery would fit. Scan again and walk a few steps each way." static let scanAgain = "Scan again" - /// Shown on every result, with or without a spot ("Not on this wall" has none), so it names - /// no spot. + /// Shown on every result but a possible spot, whose `candidateNote` says it already, with or + /// without a spot ("Not on this wall" has none), so it names no spot. static let installerConfirms = "Before any battery goes in, an installer has to confirm where it goes on site." static let rulesNotFinal = "The placement rules aren't final yet, so every result needs an installer's review for now." // The server's result covers where the battery goes, not the panel itself. - static let panelReview = "Your electrical panel still needs an electrician's review. This scan only covers where the battery can go." + static let panelReview = "Your electrical panel still needs an electrician's review. This scan only suggests where the battery could go." /// Which rules answered, for a reviewer: "Rules 2f52ec35". static func rulesHash(_ hash: String) -> String { @@ -807,9 +907,14 @@ enum ScanCopy { } } + /// The heading over every check in Details, and what a passing one means there: the server's + /// calculation on the scan's measurements, not space anyone confirmed clear. + static let calculatedTitle = "What the server calculated" + static let calculatedNote = "From what your scan recorded. A passing check means those measurements meet the rule, not that the space is confirmed clear." + static func outcomeWord(_ outcome: CheckOutcome) -> String { switch outcome { - case .pass: "Looks good" + case .pass: "Passes on recorded data" case .unsure: "Not sure yet" case .fail: "Doesn't work" } diff --git a/ios/HouseScan/UI/Practice/DeveloperOptions.swift b/ios/HouseScan/UI/Practice/DeveloperOptions.swift index 6eb0dacb0..21a7dace3 100644 --- a/ios/HouseScan/UI/Practice/DeveloperOptions.swift +++ b/ios/HouseScan/UI/Practice/DeveloperOptions.swift @@ -1,13 +1,41 @@ import HouseScanKit import SwiftUI -/// The way into the developer options, on the first screen. Only development and TestFlight -/// installs show it (`DeveloperSettings.isAvailable`); an App Store install shows nothing here. -/// While practice meter is on it says so, so nobody starts a scan without knowing. +/// The way into the developer options, on the first screen and the photo-processing screen. Only +/// development and TestFlight installs show it (`DeveloperSettings.isAvailable`); an App Store +/// install shows nothing here. While practice meter or photo processing is on it says so, so +/// nobody starts a scan without knowing. struct DeveloperOptionsButton: View { + /// The backend of the scan under way, for the sheet to say which one it keeps; nil before a + /// scan starts. + let scanBackend: ProcessingBackend? @AppStorage(DeveloperSettings.practiceMeterKey) private var practiceMeter = false @State private var showing = false private let settings = DeveloperSettings.shared + private let processing = ProcessingBackendSetting.shared + + init(scanBackend: ProcessingBackend? = nil) { + self.scanBackend = scanBackend + } + + /// The practice meter's words stay as they were; photo processing adds its own sentence. + private var spokenLabel: String { + var parts: [String] = [] + if practiceMeter { parts.append("Practice meter is on.") } + if processing.selection == .photoProcessing { parts.append("Photo processing is chosen.") } + return parts.isEmpty ? "Developer options" : (["Developer options."] + parts).joined(separator: " ") + } + + /// What the chip says next to the wrench, when anything differs from an ordinary scan. + private var notice: String? { + let photo = processing.selection == .photoProcessing + return switch (practiceMeter, photo) { + case (true, true): "Practice meter on · Photo processing" + case (true, false): "Practice meter on" + case (false, true): "Photo processing" + case (false, false): nil + } + } var body: some View { if settings.isAvailable { @@ -16,35 +44,36 @@ struct DeveloperOptionsButton: View { } label: { HStack(spacing: 6) { Image(systemName: "wrench.and.screwdriver.fill") - if practiceMeter { + if let notice { // Wraps at the largest text sizes rather than dropping out. - Text("Practice meter on") + Text(notice) .fixedSize(horizontal: false, vertical: true) } } .font(Typeface.caption) - .foregroundStyle(practiceMeter ? Palette.ink : Palette.muted) - .padding(.horizontal, practiceMeter ? 10 : 0) + .foregroundStyle(notice != nil ? Palette.ink : Palette.muted) + .padding(.horizontal, notice != nil ? 10 : 0) .padding(.vertical, 5) .background { - if practiceMeter { Capsule().fill(Palette.caution) } + if notice != nil { Capsule().fill(Palette.caution) } } .frame(minWidth: Metrics.minTarget, minHeight: Metrics.minTarget) .contentShape(.rect) } .buttonStyle(PressableStyle()) - .accessibilityLabel(practiceMeter ? "Developer options. Practice meter is on." : "Developer options") + .accessibilityLabel(spokenLabel) .accessibilityIdentifier("action.developerOptions") .sheet(isPresented: $showing) { - DeveloperOptionsSheet(environment: settings.environment) + DeveloperOptionsSheet(environment: settings.environment, scanBackend: scanBackend) } } } } -/// One switch for now: practice meter. +/// Practice meter, and which remote processing the next scan uses. private struct DeveloperOptionsSheet: View { var environment: PracticeMeter.InstallEnvironment + var scanBackend: ProcessingBackend? @AppStorage(DeveloperSettings.practiceMeterKey) private var practiceMeter = false @Environment(\.dismiss) private var dismiss @@ -57,6 +86,7 @@ private struct DeveloperOptionsSheet: View { } footer: { Text("For trying the scan where there's no electric meter. Tap any spot on a wall and a sample meter is drawn there. A photo of it stands in for the meter close-up, and the rest of the scan runs for real. Each screen and the scan's stamp say it's practice. Starts with the next scan.") } + ProcessingChoice(scanBackend: scanBackend) Section { LabeledContent("Build", value: Self.name(environment)) } @@ -82,3 +112,52 @@ private struct DeveloperOptionsSheet: View { } } } + +/// The two remote backends as a radio list. Both are services off the phone; neither decides on +/// it. A scan keeps the backend it started with, so the footer says which one a scan under way +/// uses and that a change waits for the next scan. +private struct ProcessingChoice: View { + var scanBackend: ProcessingBackend? + private let setting = ProcessingBackendSetting.shared + + var body: some View { + Section { + ForEach(ProcessingBackend.allCases, id: \.self) { backend in + let selected = setting.selection == backend + Button { + setting.choose(backend) + } label: { + HStack(alignment: .firstTextBaseline, spacing: 12) { + VStack(alignment: .leading, spacing: 4) { + Text(ProcessingCopy.name(backend)) + .foregroundStyle(.primary) + Text(ProcessingCopy.summary(backend)) + .font(.footnote) + .foregroundStyle(.secondary) + if backend == .photoProcessing, let note = ProcessingCopy.setupNote(setting.photoProcessingAnswers) { + Text(note) + .font(.footnote.weight(.semibold)) + .foregroundStyle(.primary) + } + } + .fixedSize(horizontal: false, vertical: true) + Spacer(minLength: 0) + Image(systemName: "checkmark") + .font(.body.weight(.semibold)) + .foregroundStyle(.tint) + .opacity(selected ? 1 : 0) + .accessibilityHidden(true) + } + .contentShape(.rect) + } + .accessibilityAddTraits(selected ? .isSelected : []) + .accessibilityIdentifier("developer.processing.\(backend.rawValue)") + } + } header: { + Text("Processing") + } footer: { + Text(ProcessingCopy.choiceFooter(scanBackend: scanBackend)) + .accessibilityIdentifier("developer.processing.footer") + } + } +} diff --git a/ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift b/ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift new file mode 100644 index 000000000..80170fdbc --- /dev/null +++ b/ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift @@ -0,0 +1,83 @@ +import Foundation +import HouseScanKit + +/// Photo processing in the UI demo. `-uiDemoPhase processing -uiDemoPhotoState ` shows one of +/// the screen's states (ids as in `ProcessingCopy.Screen.id`), and `-uiDemoPhotoConsent` on a +/// camera phase shows the consent question. The states are the types the engine publishes, with +/// the capture fixture's words, so every screen says its answer is a test one. +extension DemoEngine { + func showPhotoProcessing(photoState id: String?, consent: Bool) { + if consent { + state.scanBackend = .photoProcessing + state.photoProcessing = PhotoProcessingStatus(stage: .capturing(sent: 0), consent: .asking, standIn: true) + } + guard state.phase == .processing else { return } + state.scanBackend = .photoProcessing + state.photoProcessing = Self.photoExamples[id ?? "processing"] ?? Self.photoExamples["processing"] + } + + /// The scan was sent: a yes ends on the fixture's possible spot, anything else on photos not + /// sent, as the engine would. + func finishPhotoProcessing() { + let sent = state.photoProcessing?.consent == .granted + state.gap = nil + state.phase = .processing + state.photoProcessing = Self.photoExamples[sent ? "candidate" : "consentNotGiven"] + } + + func answerPhotoConsent(_ yes: Bool) { + guard var status = state.photoProcessing, status.consent == .asking else { return } + status.consent = yes ? .granted : .declined + state.photoProcessing = status + } + + func scanWithLegacyInstead() { + startOver() + finishOnboarding() + state.scanBackend = .legacy + } + + func stopSendingPhotos() { + guard var status = state.photoProcessing, status.consent == .granted, !status.isFinal else { return } + status.consent = .withdrawn(recorded: true) + status.stage = .ended(.withdrawn(recorded: true)) + state.photoProcessing = status + } + + static let photoExamples: [String: PhotoProcessingStatus] = { + func sent(_ stage: PhotoProcessingStatus.Stage) -> PhotoProcessingStatus { + PhotoProcessingStatus(stage: stage, consent: .granted, standIn: true) + } + let prompt = try? JSONDecoder().decode( + CaptureResult.Prompt.self, + from: Data(#"{"title": "Fixture: show the ground below the meter", "body": "Fixture: step back until the ground below the meter is in view."}"#.utf8)) + let answers = FixtureCaptureHTTP.Answer.self + return [ + "queued": sent(.queued), + "uploading": sent(.uploading(received: 9, kept: 14)), + "processing": sent(.processing), + "candidate": sent(.answered(.candidate(.init( + message: answers.candidate.message ?? "", previewMade: false, ar: .withheld(.analysisUnverified))))), + "needsMorePhotos": sent(.answered(.needsMorePhotos(.init( + prompts: prompt.map { [$0] } ?? [], message: answers.needsViews.message, parent: nil)))), + "installerReview": sent(.answered(.installerReview(message: answers.manualReview.message ?? ""))), + "noCandidate": sent(.answered(.noCandidate(message: answers.notEligible.message ?? ""))), + "unrecognized": sent(.answered(.unrecognized(kind: "fixture_kind", message: "Fixture answer: a kind of answer this app doesn't know."))), + "notSetUp": PhotoProcessingStatus(stage: .ended(.notSetUp("demo")), consent: .notNeeded, standIn: false), + "consentNotGiven": PhotoProcessingStatus(stage: .ended(.consentNotGiven), consent: .declined, standIn: true), + "withdrawn": PhotoProcessingStatus(stage: .ended(.withdrawn(recorded: true)), consent: .withdrawn(recorded: true), standIn: true), + "withdrawalNotRecorded": PhotoProcessingStatus( + stage: .ended(.withdrawn(recorded: false)), consent: .withdrawn(recorded: false), standIn: true), + "notPrepared": sent(.ended(.notPrepared)), + "processingFailed": sent(.ended(.processingFailed)), + "expired": sent(.ended(.expired)), + "refused": sent(.ended(.refused(step: "create"))), + // At the result step: the service may already have processed the scan. + "setupRefused": sent(.ended(.setupRefused(step: "result", .credentialUnavailable))), + "uploadStateUnsaved": sent(.ended(.uploadStateUnsaved)), + "answerUnreadable": sent(.ended(.answerUnreadable)), + "answerNotReady": sent(.ended(.answerNotReady)), + "answerMismatch": sent(.ended(.answerMismatch(.run))), + ] + }() +} diff --git a/ios/HouseScan/UI/Preview/DemoEngine.swift b/ios/HouseScan/UI/Preview/DemoEngine.swift index 506d3dae4..f60b86629 100644 --- a/ios/HouseScan/UI/Preview/DemoEngine.swift +++ b/ios/HouseScan/UI/Preview/DemoEngine.swift @@ -26,7 +26,15 @@ final class DemoEngine: ScanActions { /// `-uiDemoUnusableAnswer `: how many answers in a row House Scan can't use; 0 when off. private let unusableAnswers: Int /// Which request the gap screen shows (`-uiDemoGap`); the phone's ground request by default. + /// `pastEnd` is a server past_end request on the right, with "Wall ends here" (B-12); + /// `pastEndLeft` the same on the left. private let gapKind: String? + /// `-uiDemoEndMarkRefusal`: "Wall ends here" was pressed with the circle off the wall, and the + /// demo's circle stays off it, so pressing again refuses again. + private let endMarkRefused: Bool + /// `-uiDemoEndMarkOtherSide`: as `-uiDemoEndMarkRefusal`, but the circle is on the wall on + /// the meter's other side (`EndMarkRefusal.otherSide`). + private let endMarkOtherSide: Bool /// The tilt-up step was answered or skipped. private var tiltUpSettled = false /// Walk-script ticks spent on the tilt-up step, standing in for the phone being tilted up. @@ -84,6 +92,8 @@ final class DemoEngine: ScanActions { rejectUpload = arguments.contains("-uiDemoRejected") unusableAnswers = arguments.contains("-uiDemoUnusableAnswer") ? max(value("-uiDemoUnusableAnswer").flatMap(Int.init) ?? 1, 1) : 0 gapKind = value("-uiDemoGap") + endMarkOtherSide = arguments.contains("-uiDemoEndMarkOtherSide") + endMarkRefused = arguments.contains("-uiDemoEndMarkRefusal") || endMarkOtherSide state.feed = DemoScene.image.map(CameraFeed.still) ?? .none state.isReplay = true state.tracking = .normal @@ -103,6 +113,7 @@ final class DemoEngine: ScanActions { let opensReview = arguments.contains("-uiDemoGroundQuestion") || groundAnswer != nil let phase: ScanPhase = opensReview ? .markFeatures : value("-uiDemoPhase").flatMap(ScanPhase.init(rawValue:)) ?? .onboarding jump(to: phase) + showPhotoProcessing(photoState: value("-uiDemoPhotoState"), consent: arguments.contains("-uiDemoPhotoConsent")) state.groundAnswer = groundAnswer if let raw = value("-uiDemoMarking"), let kind = FeatureKind(rawValue: raw) { state.marking = MarkingState(kind: kind, step: 0, refusal: arguments.contains("-uiDemoRefusal") ? .noSurface : nil) @@ -123,9 +134,9 @@ final class DemoEngine: ScanActions { reachedRight = demoRightEnd refreshCoverage() refreshGuidance() - if arguments.contains("-uiDemoEndMarkRefusal") { - // As in the real engine, a circle off the wall has no end preview (`circleEnd`). - state.endMarkRefusal = .noWall + if endMarkRefused { + // As in the real engine, a refused circle has no end preview (`circleEnd`). + state.endMarkRefusal = demoRefusal(asked: .right) state.endPreview = nil } } @@ -293,6 +304,10 @@ final class DemoEngine: ScanActions { finishedWalkState() showResult() state.phase = .resultAR + case .processing: + placeMeter() + finishedWalkState() + state.phase = .processing case .unsupported: state.failure = .arUnsupported state.phase = .unsupported @@ -417,6 +432,35 @@ final class DemoEngine: ScanActions { state.gap?.band = .wall state.target = DemoScene.wall.world(s: 1.7, height: 3.0) state.path = [] + case "pastEnd", "pastEndLeft": + // The check asks to walk on past an end, which it clears, as the real engine's + // `beginServerGap` does; the wall may stop sooner, so "Wall ends here" is offered. + let side: WallSide = gapKind == "pastEndLeft" ? .left : .right + let sign: Float = side == .right ? 1 : -1 + let end = side == .right ? demoRightEnd : demoLeftEnd + let past: ClosedRange = side == .right ? end...(end + 0.6) : (end - 0.6)...end + if side == .right { state.wall?.rightEnd = nil } else { state.wall?.leftEnd = nil } + state.gap?.origin = .server + state.gap?.reason = .server(detail: side == .right + ? "Keep walking past the right end of the scan (14 ft 1 in right of the meter): a spot within reach may be there." + : "Keep walking past the left end of the scan (9 ft 6 in left of the meter): a spot within reach may be there.") + state.gap?.span = past + state.gap?.pastEndSide = side + // The check sends a past_end request with its answer, so the card says "One more view + // to finish" over it, as the real engine's does (`GapRequestScreen.followUps`). + state.result = sample + state.followUps = 1 + state.target = DemoScene.wall.world(s: end + sign * 0.3, height: 0, out: 0.5) + state.path = DemoScene.path(toward: end + sign * 0.3) + if endMarkRefused { + // As in the real engine, a refused circle has no end preview (`circleEnd`). + state.endMarkRefusal = demoRefusal(asked: side) + state.endPreview = nil + } else { + // The circle on the wall a little before the old end: where the wall really stops. + state.endPreview = EndPreview(side: side, s: end - sign * 0.4, atReticle: true) + } + return default: break } @@ -428,6 +472,11 @@ final class DemoEngine: ScanActions { } private func enterUpload() { + // A photo-processing scan never takes the Legacy upload, in the demo as in the engine. + if state.scanBackend == .photoProcessing { + finishPhotoProcessing() + return + } state.phase = .uploading state.gap = nil state.path = [] @@ -854,7 +903,27 @@ final class DemoEngine: ScanActions { } } + /// Why the demo's "Wall ends here" marks nothing: the circle off the wall, or on the wall on + /// the other side of the meter from the end `asked` (`-uiDemoEndMarkOtherSide`). + private func demoRefusal(asked: WallSide) -> EndMarkRefusal { + endMarkOtherSide ? .otherSide(asked == .right ? .left : .right) : .noWall + } + func markWallEnd(at point: CGPoint?, viewSize: CGSize) { + if state.phase == .gapRequest, let side = state.gap?.pastEndSide, state.gap?.isSatisfied == false, state.endQuestion == nil { + if endMarkRefused { + state.endMarkRefusal = demoRefusal(asked: side) + state.endPreview = nil + return + } + let s = state.endPreview?.s ?? (side == .right ? demoRightEnd : demoLeftEnd) + if side == .right { state.wall?.rightEnd = s } else { state.wall?.leftEnd = s } + state.endMarkRefusal = nil + state.endQuestion = side + state.endQuestionLeavesOut = nil + state.endQuestionLeavesOutSeen = false + return + } guard case .markEnd(let side) = state.guidance else { return } state.endMarkRefusal = nil if side == .right { @@ -891,6 +960,26 @@ final class DemoEngine: ScanActions { state.endQuestion = nil state.endQuestionLeavesOut = nil state.endQuestionLeavesOutSeen = false + if state.phase == .gapRequest, var gap = state.gap, gap.pastEndSide == side { + // A corner the request can't follow: no check mark, straight on to the upload, as + // `ScanEngine.skipCurrentGap` does. + if turnsCorner { + enterUpload() + return + } + // The end marked again settles the request, as `ScanEngine.settlePastEnd` does; the + // check runs again after the request's check mark has been seen. + gap.isSatisfied = true + gap.progress = 1 + state.gap = gap + let id = gap.id + Task { [weak self] in + try? await Task.sleep(for: .seconds(1)) + guard let self, self.state.phase == .gapRequest, self.state.gap?.id == id else { return } + self.enterUpload() + } + return + } let followed = state.wall?.cornerSegments.contains { side == .right ? $0.span.lowerBound > 0 : $0.span.upperBound < 0 } ?? true if turnsCorner, !followed { state.guidance = .markNextWall(side: side, refusal: nil) @@ -1127,6 +1216,8 @@ final class DemoEngine: ScanActions { state.endScanQuestion = false state.endScanTooShort = false state.groundAnswer = nil + state.scanBackend = nil + state.photoProcessing = nil } func liveCameraView() -> AnyView { @@ -1212,7 +1303,7 @@ final class DemoEngine: ScanActions { /// The offline sample result. Placeholder rules, so the decision is manual review. static let reviewSample = ResultPresentation( decision: .manualReview, - summary: "A spot 3 ft right of your meter fits, but it's close to a window, so it needs an installer's review.", + summary: "The best spot, 3 ft right of your meter, is close to a window, so it needs an installer's review.", policyApproved: false, spot: BatterySpot(span: 0.55...1.34, depth: 0.56, height: 1.1, offsetFromWall: 0.03), cableRoute: [SIMD2(0.16, 1.45), SIMD2(0.95, 1.45), SIMD2(0.95, 1.1)], diff --git a/ios/HouseScan/UI/Preview/UIDemo.swift b/ios/HouseScan/UI/Preview/UIDemo.swift index fbcfed2e9..0141045ea 100644 --- a/ios/HouseScan/UI/Preview/UIDemo.swift +++ b/ios/HouseScan/UI/Preview/UIDemo.swift @@ -57,6 +57,10 @@ import SwiftUI /// answered "I can't check this area", so the result says so beside its spot's notice. /// - `-uiDemoFollowUp`: with `-uiDemoPhase uploading` or `gapRequest`, the check has answered /// and asked for one more view: the upload screen as it hands over, or the view itself. +/// - `-uiDemoPhase processing -uiDemoPhotoState `: a photo-processing scan after it was sent, +/// in one of its states (`ProcessingCopy.Screen.id`, such as `uploading`, `candidate` or +/// `withdrawalNotRecorded`; `processing` by default), with the capture fixture's words. +/// - `-uiDemoPhotoConsent`: on a camera phase, the question whether to send the scan's photos. /// /// Unfrozen, the demo goes back to the camera once after the first answer, as the engine does /// when the answer lists a view the camera can take. diff --git a/ios/HouseScan/UI/Result/ResultScene3D.swift b/ios/HouseScan/UI/Result/ResultScene3D.swift index a95a45813..678583e7f 100644 --- a/ios/HouseScan/UI/Result/ResultScene3D.swift +++ b/ios/HouseScan/UI/Result/ResultScene3D.swift @@ -220,7 +220,7 @@ struct ResultScene3D: View { } for (index, zone) in result.clearances.enumerated() { - var material = UnlitMaterial(color: SceneColor.outcome(zone.outcome)) + var material = UnlitMaterial(color: SceneColor.zone(zone.outcome)) material.blending = .transparent(opacity: .init(floatLiteral: 0.35)) // Stacked zones sit a few millimeters apart so overlapping ones don't flicker. let lift = ResultMarkLayout.zoneLift(index: index, base: Self.zoneBase) @@ -467,11 +467,11 @@ struct ResultScene3D: View { } var parts: [String] if spot.span.lowerBound > 0 { - parts = ["Battery \(Distance.spoken(spot.span.lowerBound)) right of your meter"] + parts = ["Proposed battery spot \(Distance.spoken(spot.span.lowerBound)) right of your meter"] } else if spot.span.upperBound < 0 { - parts = ["Battery \(Distance.spoken(-spot.span.upperBound)) left of your meter"] + parts = ["Proposed battery spot \(Distance.spoken(-spot.span.upperBound)) left of your meter"] } else { - parts = ["Battery below your meter"] + parts = ["Proposed battery spot below your meter"] } if let cable = result.cableLength { parts.append("cable \(Distance.spoken(cable))") @@ -549,7 +549,7 @@ enum ResultARModel { wall.world(s: s, height: height, out: out) - wall.meter } for (index, zone) in result.clearances.enumerated() { - var material = UnlitMaterial(color: SceneColor.outcome(zone.outcome)) + var material = UnlitMaterial(color: SceneColor.zone(zone.outcome)) material.blending = .transparent(opacity: .init(floatLiteral: 0.35)) let width = zone.span.upperBound - zone.span.lowerBound let middle = zone.span.lowerBound + width / 2 @@ -659,9 +659,11 @@ private enum SceneColor { } } - static func outcome(_ outcome: CheckOutcome) -> UIColor { + /// A clearance zone's tint, as `Palette.zone`: where the server's sweep passes, the + /// proposal's blue, not a green that reads as ground confirmed clear. + static func zone(_ outcome: CheckOutcome) -> UIColor { switch outcome { - case .pass: rgb(0x2FC273) + case .pass: signal case .unsure: rgb(0xF5B53D) case .fail: rgb(0xFF5A4E) } diff --git a/ios/HouseScan/UI/ScanRootView.swift b/ios/HouseScan/UI/ScanRootView.swift index 9cc143054..2048254b4 100644 --- a/ios/HouseScan/UI/ScanRootView.swift +++ b/ios/HouseScan/UI/ScanRootView.swift @@ -40,7 +40,25 @@ struct ScanRootView: View { screen .id(state.phase) .transition(Self.showsCamera(state.phase) ? .opacity : .identity) + // While the photo question shows, the screen under it is out of the accessibility + // tree: its words show dimmed through the card's scrim, and the audit failed them + // on contrast (#216, run 37219668935). Only ever set to hidden, never to + // `.accessibilityHidden(false)`, which overrode the hiding the camera's decoration + // sets on itself and left an element with no description on every camera screen + // (run 37213636011). The camera feed and practice overlay hide themselves already. + .hiddenFromAccessibility(while: asksForPhotoConsent) + if asksForPhotoConsent { + PhotoConsentCard(standIn: state.photoProcessing?.standIn == true, synthetic: state.photoCaptureIsSynthetic, actions: actions) + .transition(.opacity) + } } + .animation(Motion.screen, value: asksForPhotoConsent) + } + + /// The photo-processing question covers the camera until it is answered, on every capture + /// screen, so no photo is kept without an answer in view. + private var asksForPhotoConsent: Bool { + state.photoProcessing?.consent == .asking && Self.showsCamera(state.phase) && state.phase != .resultAR } @ViewBuilder @@ -76,6 +94,9 @@ struct ScanRootView: View { case .resultAR: ResultARScreen(state: state, actions: actions) .screenIdentifier(.resultAR) + case .processing: + PhotoProcessingScreen(state: state, actions: actions) + .screenIdentifier(.processing) case .unsupported: UnsupportedScreen(state: state, actions: actions) .screenIdentifier(.unsupported) @@ -85,7 +106,7 @@ struct ScanRootView: View { static func showsCamera(_ phase: ScanPhase) -> Bool { switch phase { case .findMeter, .meterCloseUp, .wallWalk, .gapRequest, .resultAR, .markFeatures: true - case .onboarding, .uploading, .spotConfirm, .result, .unsupported: false + case .onboarding, .uploading, .spotConfirm, .result, .processing, .unsupported: false } } } @@ -93,15 +114,20 @@ struct ScanRootView: View { /// Haptics for the moments that matter, fired on the same state change the screen animates: /// a deliberate photo, the meter pinned, a mark placed or refused, a requested view done, the /// result. +/// +/// The result arrives with a light tap, whatever it says. `.success` there celebrated every +/// answer alike, a rejected wall included, and on a possible spot it said "done, it fits" when +/// the scan couldn't confirm the space (B17, B26). Coming back from the AR view taps nothing. private struct ScanHaptics: ViewModifier { let state: ScanViewState - // Two steps: the five-modifier chain took 215 ms to type-check on Swift 6.4, and CI's - // Swift 6.2 has failed on slower expressions before. + // Steps of at most three: the five-modifier chain took 215 ms to type-check on Swift 6.4, and + // CI's Swift 6.2 has failed on slower expressions before. func body(content: Content) -> some View { let captures = content .sensoryFeedback(.impact(weight: .light, intensity: 0.7), trigger: deliberateCaptureID) { _, new in new != nil } - .sensoryFeedback(.success, trigger: state.phase, condition: Self.isMilestone) + .sensoryFeedback(.success, trigger: state.phase, condition: Self.meterPinned) + .sensoryFeedback(.impact(weight: .light), trigger: state.phase, condition: Self.resultArrived) return captures .sensoryFeedback(.success, trigger: state.gap?.isSatisfied ?? false) { _, new in new } .sensoryFeedback(.impact(weight: .medium), trigger: state.features.count) { old, new in new > old } @@ -116,8 +142,12 @@ private struct ScanHaptics: ViewModifier { return capture.id } - private static func isMilestone(_ old: ScanPhase, _ new: ScanPhase) -> Bool { - (old == .findMeter && new == .meterCloseUp) || (old != .resultAR && new == .result) + private static func meterPinned(_ old: ScanPhase, _ new: ScanPhase) -> Bool { + old == .findMeter && new == .meterCloseUp + } + + private static func resultArrived(_ old: ScanPhase, _ new: ScanPhase) -> Bool { + old != .resultAR && new == .result } } @@ -127,3 +157,16 @@ private extension View { .accessibilityIdentifier("screen.\(phase.rawValue)") } } + +private extension View { + /// `.accessibilityHidden(true)` while `hidden`, and no accessibility modifier at all otherwise, + /// so the hiding a subview sets on itself still holds. The view is rebuilt when `hidden` + /// changes; here that happens once, when the photo question is answered. + @ViewBuilder func hiddenFromAccessibility(while hidden: Bool) -> some View { + if hidden { + accessibilityHidden(true) + } else { + self + } + } +} diff --git a/ios/HouseScan/UI/Screens/GapRequestScreen.swift b/ios/HouseScan/UI/Screens/GapRequestScreen.swift index 41417cb76..e03f2d9f7 100644 --- a/ios/HouseScan/UI/Screens/GapRequestScreen.swift +++ b/ios/HouseScan/UI/Screens/GapRequestScreen.swift @@ -8,6 +8,11 @@ import SwiftUI /// review) carries "One more view to finish" above the instruction, the words the upload screen /// just said. "I can't get there" skips only this view and goes on to the check's next one; /// "Show my result", on every such request, stops asking and leads to the result. +/// +/// A past_end request (the walk stopped at an end, and the check asks to walk on past it) also +/// offers the walk's own "Wall ends here" for that end, with the circle, the end's preview on the +/// tape, the refusal when the circle isn't on that end, and the end question (B-12). Walking on +/// stays the request; marking the end is the other true answer. struct GapRequestScreen: View { let state: ScanViewState let actions: any ScanActions @@ -27,6 +32,24 @@ struct GapRequestScreen: View { SavedMeterPhoto(image: meterPhoto) .transition(.opacity) } + // The end lands under the circle, as on the walk's "Is this the right end?". It fades + // with "Wall ends here" on the same spring, so the pair comes and goes together; + // scoped to the circle, so nothing else on the camera animates with it. Under Reduce + // Motion only its opacity changes, briefly: nothing moves. It is there only while + // offered, rather than kept at zero opacity: kept, its "Aiming circle" stayed in the + // accessibility tree under the end question, hidden or not (CI run 37171342665). + if state.gap?.pastEndSide != nil { + ZStack { + if offersEndMark { + Reticle(diameter: 56) + .endAimCircle() + .transition(.opacity) + } + } + .frame(maxWidth: .infinity, maxHeight: .infinity) + .ignoresSafeArea() + .animation(reduceMotion ? .easeOut(duration: 0.15) : Motion.settle, value: offersEndMark) + } if state.gap?.isSatisfied == true { SuccessBadge() .frame(maxWidth: .infinity, maxHeight: .infinity) @@ -35,8 +58,8 @@ struct GapRequestScreen: View { } CameraChrome( instruction: instruction, - tone: asking ? .normal : coaching.map { .coaching(symbol: ScanCopy.coachingSymbol($0)) } ?? .normal, - reply: state.gap?.isSatisfied == true || asking ? nil : InstructionCard.Reply( + tone: tone, + reply: state.gap?.isSatisfied == true || asking || askingEnd || folds ? nil : InstructionCard.Reply( title: "I can't get there", identifier: "action.skipGap", hint: skipHint, @@ -45,7 +68,7 @@ struct GapRequestScreen: View { // as the phone moves: neither locks the reply again (InstructionCard.replyLock). task: state.gap.map { ScanCopy.gapTask($0) } ), - eyebrow: followUps > 0 && state.gap?.isSatisfied != true ? ScanCopy.followUp(remaining: followUps) : nil, + eyebrow: followUps > 0 && state.gap?.isSatisfied != true && !folds ? ScanCopy.followUp(remaining: followUps) : nil, photoCount: state.captureCount, lastCaptureID: state.lastCapture?.id, isReplay: state.isReplay, @@ -58,6 +81,23 @@ struct GapRequestScreen: View { OverheadAnswers(actions: actions) .transition(.opacity) } + // The end's controls fade on their own: the rows below them move into place at + // once rather than sliding with the stack, and under Reduce Motion nothing + // animates. + VStack(spacing: 10) { + if askingEnd { + EndQuestionAnswers(actions: actions) + .transition(.opacity) + } + if offersEndMark { + markEndButton + } + if folds { + declineButton + } + } + .animation(reduceMotion ? nil : Motion.settle, value: askingEnd) + .animation(reduceMotion ? nil : Motion.settle, value: offersEndMark) if !typeSize.isAccessibilitySize { showResult } @@ -71,7 +111,8 @@ struct GapRequestScreen: View { features: state.features, cameraS: state.projection.map { WallProjection(projection: $0, wall: wall, size: cameraSize).cameraS }, highlight: state.gap, - depthChecked: state.depthAvailable + depthChecked: state.depthAvailable, + endPreview: offersEndMark ? state.endPreview : nil ) } if typeSize.isAccessibilitySize { @@ -79,6 +120,7 @@ struct GapRequestScreen: View { } } .animation(reduceMotion ? .easeOut(duration: 0.15) : Motion.settle, value: asking) + } } .animation(reduceMotion ? .easeOut(duration: 0.15) : Motion.pin, value: state.gap?.isSatisfied) @@ -96,7 +138,7 @@ struct GapRequestScreen: View { /// sizes it goes below the tape, the last thing on the screen, reached by scrolling like /// everything else below the card at that size. @ViewBuilder private var showResult: some View { - if followUps > 0, state.gap?.isSatisfied != true, !asking { + if followUps > 0, state.gap?.isSatisfied != true, !asking, !askingEnd { Button { actions.showResultNow() } label: { @@ -113,6 +155,86 @@ struct GapRequestScreen: View { /// until answered, as on the walk. private var asking: Bool { state.overheadQuestion && state.gap?.isSatisfied != true } + /// The end question after "Wall ends here" on a past_end request: it replaces the request, + /// "I can't get there" and "Show my result" until answered. An end the wall stops at settles the + /// request; a corner, which the request can't follow, moves on without settling it. + private var askingEnd: Bool { + guard let side = state.endQuestion, let gap = state.gap else { return false } + return gap.pastEndSide == side && !gap.isSatisfied + } + + /// "Wall ends here" with the circle, on a past_end request not yet settled or answered. + private var offersEndMark: Bool { + guard let gap = state.gap, gap.pastEndSide != nil, !gap.isSatisfied else { return false } + return !asking && !askingEnd + } + + /// The walk's "Wall ends here" at the circle, the same words and flag. Beside walking on, which + /// stays the request, it is the other answer, so it is not the screen's primary button. + private var markEndButton: some View { + Button { + actions.markWallEnd(at: nil, viewSize: cameraSize) + } label: { + Label("Wall ends here", systemImage: "flag.fill") + .frame(maxWidth: .infinity) + } + .buttonStyle(.secondaryProminent) + .accessibilityHint("Marks the end of the wall at the circle in the middle of the screen") + .accessibilityIdentifier("action.markEnd") + .transition(.opacity) + } + + /// "I can't get there" among the actions, first, or under "Wall ends here", while the card is + /// folded (`folds`): the same words, hint and action as the card's reply. + @ViewBuilder private var declineButton: some View { + if let gap = state.gap { + MovedReplyButton( + title: "I can't get there", hint: skipHint, identifier: "action.skipGap", + task: ScanCopy.gapTask(gap), perform: { actions.skipGap() }) + .transition(.opacity) + } + } + + /// At the largest text sizes while "Wall ends here" is offered, the card folds to a two-line + /// lead, as on the walk's end, so the circle in the middle of the camera stays open: the + /// request's words and its "One more view to finish" go under Details, and "I can't get + /// there" moves from the card to the actions. With either still on the card it reached past + /// the circle (`ScanCopy.endAimFold`). Not while coaching replaces the request: then the card + /// says why nothing can be aimed yet. + private var endAimFolds: Bool { + guard offersEndMark, typeSize.isAccessibilitySize else { return false } + return !(coaching.map(ScanCopy.coachingReplacesTask) ?? false) + } + + /// At the largest text sizes, a request the finished check sent back folds the same way: its + /// "One more view to finish", the stretch's two ends and the check's own words filled the + /// camera, with "I can't get there" cut off at the bottom edge (CI runs 37221532164 and + /// 37231825488). Folded, the side to aim at leads (`ScanCopy.serverLead`), everything else is + /// under Details, "I can't get there" moves to the actions, and the camera keeps + /// `CameraChromeLayout.minCameraWindow`. Only a server request in its own words: a past_end + /// one folds around the circle instead, and coaching that replaces the request keeps the card. + private var followUpFolds: Bool { + guard typeSize.isAccessibilitySize, followUps > 0, let gap = state.gap, !gap.isSatisfied, gap.pastEndSide == nil, + !asking, !askingEnd, case .server = gap.reason else { return false } + return !(coaching.map(ScanCopy.coachingReplacesTask) ?? false) + } + + /// Whether the card is folded and its reply among the actions. + private var folds: Bool { endAimFolds || followUpFolds } + + /// A refusal on "Wall ends here" keeps its red triangle, as on the walk; otherwise coaching + /// marks the card. + private var tone: InstructionCard.Tone { + if asking || askingEnd { return .normal } + if refusal != nil, !(coaching.map(ScanCopy.coachingReplacesTask) ?? false) { return .refusal } + return coaching.map { .coaching(symbol: ScanCopy.coachingSymbol($0)) } ?? .normal + } + + /// Why the last "Wall ends here" marked nothing, while the request still asks for that end. + private var refusal: EndMarkRefusal? { + offersEndMark ? state.endMarkRefusal : nil + } + /// Views the finished check still wants, counting this one, when this request is one of /// them: a server request while the check's answer is in. Zero otherwise. private var followUps: Int { @@ -138,9 +260,13 @@ struct GapRequestScreen: View { /// needs" reading or "tap I can't get there", tilting up to the roof or sky, and a server's /// own words, which this screen can't judge. Nor the overhead question, a request already /// seen, or coaching that replaces the request. + /// + /// A past_end request offering "Wall ends here" aims too, refusal or not (`endAimFolds`). private var aims: Bool { - guard let gap = state.gap, !asking, !gap.isSatisfied else { return false } + guard let gap = state.gap, !asking, !askingEnd, !gap.isSatisfied else { return false } if let coaching, ScanCopy.coachingReplacesTask(coaching) { return false } + if offersEndMark || followUpFolds { return true } + guard refusal == nil else { return false } if case .groundNearCandidate = gap.reason { return true } return false } @@ -150,12 +276,30 @@ struct GapRequestScreen: View { /// can stay up for a whole night request, and replacing the card hid the request and "Got it, /// thanks" for as long as it did (field test 4.1, run 3). private var instruction: Instruction { + var card = requestCard + guard folds else { return card } + // Folded, ride-along coaching leads or opens Details (`ScanCopy.foldedAroundCircle`), and + // "One more view to finish" leaves the card's top and leads the words under Details. + card = ScanCopy.foldedAroundCircle(card, coaching: coaching, refused: refusal != nil) + if followUps > 0, let folded = card.folded { + card.folded?.detail = "\(ScanCopy.followUp(remaining: followUps)). \(folded.detail)" + } + return card + } + + /// The card's words before folding around the circle moves the eyebrow (`instruction`). + private var requestCard: Instruction { if asking { return ScanCopy.overheadQuestion } + if askingEnd, let side = state.endQuestion { return ScanCopy.endQuestion(side) } if let coaching, ScanCopy.coachingReplacesTask(coaching) { return ScanCopy.coaching(coaching, meterPhoto: meterPhoto != nil) } guard let gap = state.gap else { return ScanCopy.withCoaching(ScanCopy.guidance(.gap), coaching) } if gap.isSatisfied { return Instruction(title: "Got it, thanks", detail: followUps > 0 ? "Updating your result." : "That's the view we needed.") } + // "Wall ends here" marked nothing: why, and what to do (B-06, B-12). + if let refusal, let side = gap.pastEndSide { + return ScanCopy.withCoaching(ScanCopy.endMarkRefusal(refusal, asked: side), coaching) + } return ScanCopy.withCoaching(ScanCopy.gap(gap), coaching) } diff --git a/ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift b/ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift new file mode 100644 index 000000000..7ae68a8ab --- /dev/null +++ b/ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift @@ -0,0 +1,228 @@ +import HouseScanKit +import SwiftUI + +/// A photo-processing scan after it was sent: how far the upload got, then the service's answer or +/// why there is none. Separate facts, never a percentage or a time estimate. The service's words +/// are quoted as written and every answer stays provisional. Nothing here offers AR, more photos or +/// sending again, because this build can do none of them for photo processing; the screen says so +/// instead of showing a button that leads nowhere. +struct PhotoProcessingScreen: View { + let state: ScanViewState + let actions: any ScanActions + @State private var confirmingStop = false + + init(state: ScanViewState, actions: any ScanActions) { + self.state = state + self.actions = actions + } + + var body: some View { + let status = state.photoProcessing + let copy = ProcessingCopy.screen(status, isReplay: state.isReplay) + CenteredScroll { + VStack(spacing: 24) { + ProcessingEmblem(symbol: copy.symbol, tone: copy.tone, working: copy.working) + VStack(spacing: 10) { + Text(ProcessingCopy.screenLabel) + .font(Typeface.caption) + .textCase(.uppercase) + .tracking(0.6) + .foregroundStyle(Palette.muted) + Text(copy.title) + .font(Typeface.screenTitle) + .multilineTextAlignment(.center) + .fixedSize(horizontal: false, vertical: true) + .accessibilityAddTraits(.isHeader) + .accessibilityIdentifier("photo.state.\(copy.id)") + if !copy.fromService { + Text(copy.detail) + .font(Typeface.hint) + .foregroundStyle(Palette.muted) + .multilineTextAlignment(.center) + .fixedSize(horizontal: false, vertical: true) + } + } + if status?.standIn == true { + StandInNote() + } + if copy.fromService { + ServiceWords(text: copy.detail) + } + if case .answered(.needsMorePhotos(let more))? = status?.stage, !more.prompts.isEmpty { + RequestedViews(prompts: more.prompts) + } + if !copy.notes.isEmpty || copy.working || state.photoCaptureIsSynthetic { + ProcessingNotes(lines: notes(copy, retrying: status?.retrying == true)) + } + VStack(spacing: 16) { + if copy.working, status?.consent == .granted { + Button(ProcessingCopy.stopSending) { confirmingStop = true } + .buttonStyle(.quiet) + .accessibilityIdentifier("action.stopSendingPhotos") + } + // The scan is kept: shared from here only when its archive exists. + if copy.id == "setupRefused", let scan = state.shareableScan { + ShareScanButton(url: scan) + } + if copy.id == "notSetUp" { + Button(ProcessingCopy.scanWithLegacy) { actions.scanWithLegacyInstead() } + .buttonStyle(.primary) + .accessibilityHint(ProcessingCopy.scanWithLegacyHint) + .accessibilityIdentifier("action.scanWithLegacy") + } + Group { + if copy.working || copy.id == "notSetUp" { + Button("Start over") { actions.startOver() }.buttonStyle(.quiet) + } else { + Button("Start over") { actions.startOver() }.buttonStyle(.primary) + } + } + // Start over can come while the service is processing, and the app can't cancel that. + .accessibilityHint("Starts a new scan. House Scan stops sending from this one and won't show its answer. Photos already sent may still be processed by the service.") + .accessibilityIdentifier("action.startOver") + } + } + .padding(24) + .frame(maxWidth: 520) + } + .safeAreaInset(edge: .top) { + // Stacked, each on its own full-width row, so the options chip's words can wrap rather + // than be squeezed beside the badge (the audit found them clipped there). + VStack(alignment: .leading, spacing: 4) { + ModeBadge(isReplay: state.isReplay, isAutopilot: state.isAutopilot) + DeveloperOptionsButton(scanBackend: state.scanBackend) + .frame(maxWidth: .infinity, alignment: .leading) + } + .padding(.horizontal, 24) + } + .background(Palette.canvas.ignoresSafeArea()) + .confirmationDialog(ProcessingCopy.stopQuestion, isPresented: $confirmingStop, titleVisibility: .visible) { + Button(ProcessingCopy.confirmStop, role: .destructive) { actions.stopSendingPhotos() } + .accessibilityIdentifier("photo.confirmStop") + Button(ProcessingCopy.keepSending, role: .cancel) {} + } message: { + Text(ProcessingCopy.stopDetail) + } + } + + private func notes(_ copy: ProcessingCopy.Screen, retrying: Bool) -> [String] { + var lines = copy.notes + if retrying { lines.insert(ProcessingCopy.retrying, at: 0) } + let sent = switch state.photoProcessing?.consent { + case .granted?, .withdrawn(_)?: true + default: false + } + if state.photoCaptureIsSynthetic, sent { lines.insert(ProcessingCopy.syntheticCapture, at: 0) } + if copy.working { lines.append(ProcessingCopy.closedAppLimit) } + return lines + } +} + +/// The state's symbol in a tinted disc, as on the upload screen. It pulses only while the service +/// is working, and not at all with Reduce Motion. +private struct ProcessingEmblem: View { + var symbol: String + var tone: ProcessingCopy.Tone + var working: Bool + @Environment(\.accessibilityReduceMotion) private var reduceMotion + + var body: some View { + ZStack { + Circle() + .fill(tint.opacity(0.12)) + .frame(width: 120, height: 120) + Image(systemName: symbol) + .font(.system(size: 46, weight: .semibold)) + .foregroundStyle(tint) + .symbolEffect(.pulse, options: .repeating, isActive: working && !reduceMotion) + .contentTransition(.symbolEffect(.replace)) + } + .accessibilityHidden(true) + } + + private var tint: Color { + switch tone { + case .attention: Palette.caution + case .neutral, .proposed: Palette.signal + } + } +} + +/// Says the answer is a fixture's, wherever one is on screen, as the Legacy sample says it is one. +private struct StandInNote: View { + var body: some View { + Label(ProcessingCopy.standInBadge, systemImage: "testtube.2") + .font(Typeface.caption) + .foregroundStyle(Palette.ink) + .padding(.horizontal, 12) + .padding(.vertical, 8) + .background(Palette.caution, in: .rect(cornerRadius: 12)) + .fixedSize(horizontal: false, vertical: true) + .accessibilityElement(children: .combine) + .accessibilityIdentifier("photo.standIn") + } +} + +/// The service's message, quoted and attributed, so it never reads as House Scan's own judgment. +private struct ServiceWords: View { + var text: String + + var body: some View { + VStack(alignment: .leading, spacing: 6) { + Text(ProcessingCopy.serviceSaid) + .font(Typeface.caption) + .foregroundStyle(Palette.muted) + Text(text) + .font(Typeface.hint) + .foregroundStyle(Palette.ink) + .fixedSize(horizontal: false, vertical: true) + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(16) + .background(Palette.surface, in: .rect(cornerRadius: 16)) + .accessibilityElement(children: .combine) + .accessibilityIdentifier("photo.serviceMessage") + } +} + +/// The views the service asked for, each prompt as written. +private struct RequestedViews: View { + var prompts: [CaptureResult.Prompt] + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + ForEach(Array(prompts.enumerated()), id: \.offset) { _, prompt in + VStack(alignment: .leading, spacing: 4) { + Text(prompt.title) + .font(Typeface.sectionTitle) + .fixedSize(horizontal: false, vertical: true) + Text(prompt.body) + .font(Typeface.hint) + .foregroundStyle(Palette.muted) + .fixedSize(horizontal: false, vertical: true) + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(16) + .background(Palette.surface, in: .rect(cornerRadius: 16)) + .accessibilityElement(children: .combine) + } + } + .accessibilityIdentifier("photo.requestedViews") + } +} + +private struct ProcessingNotes: View { + var lines: [String] + + var body: some View { + VStack(alignment: .leading, spacing: 10) { + ForEach(lines, id: \.self) { line in + Text(line) + .font(Typeface.hint) + .foregroundStyle(Palette.muted) + .fixedSize(horizontal: false, vertical: true) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + } +} diff --git a/ios/HouseScan/UI/Screens/ResultARScreen.swift b/ios/HouseScan/UI/Screens/ResultARScreen.swift index ef697d885..fa37ca3b5 100644 --- a/ios/HouseScan/UI/Screens/ResultARScreen.swift +++ b/ios/HouseScan/UI/Screens/ResultARScreen.swift @@ -41,7 +41,8 @@ struct ResultARScreen: View { CameraChrome( instruction: instruction, isReplay: state.isReplay, - isAutopilot: state.isAutopilot + isAutopilot: state.isAutopilot, + aims: showsSpot ) { Button("Done") { actions.closeAR() } .buttonStyle(.primary) @@ -53,10 +54,20 @@ struct ResultARScreen: View { } } + /// Whether the spot is on the wall in view: the phone tracks and the result has a spot. Then + /// the homeowner is looking at the spot, so at the accessibility text sizes the card folds + /// its detail (the placement, and what the fit still needs) under Details and leaves the + /// camera open (`CameraChrome.aims`). At AX-XXXL the full card covered nearly the whole + /// camera (run 37162669527, resultAR-candidate-settled-AX5). "Point at your meter" while + /// tracking is lost keeps every word in view: it is what to do, and nothing is drawn yet. + private var showsSpot: Bool { + state.tracking == .normal && state.result?.spot != nil + } + private static func overlayLabel(_ result: ResultPresentation) -> String { guard result.spot != nil else { return "The cable run and clearances, drawn on your wall" } return result.spotIsClean - ? "The battery, drawn on your wall at its spot" + ? ScanCopy.proposedSpotOverlay : "An outline of the spot an installer needs to check, drawn on your wall" } @@ -67,17 +78,17 @@ struct ResultARScreen: View { guard let result = state.result, result.spot != nil else { return Instruction(title: "Point at your meter", detail: nil) } - // Only a pass under approved rules may sound settled; a spot an installer still has to - // check says so, as the result screen does. - let title = result.decision == .pass && result.policyApproved && result.spotIsClean - ? "Your battery could go here" - : "The spot an installer needs to check" + // The same answer as the result screen: a possible spot says it's possible and what it + // still needs; any other spot an installer still has to check says so. + let candidate = result.answer == .candidate + let title = candidate ? ScanCopy.headline(.candidate) : "The spot an installer needs to check" let placement = ScanCopy.placement(result) guard !result.isSample else { // A sample spot drawn on the homeowner's real wall must not pass for their result. return Instruction(title: "Example spot, not your result", detail: ["No server checked this scan.", placement].compactMap { $0 }.joined(separator: " ")) } - return Instruction(title: title, detail: placement) + let onSite = candidate ? "An installer needs to check the fit on site." : nil + return Instruction(title: title, detail: [placement.map { "\($0)." }, onSite].compactMap { $0 }.joined(separator: " ")) } } @@ -107,7 +118,7 @@ private struct SpotDirection: View { if let chevron = chevronPlacement(in: size) { ZStack { TargetMarker(placement: .offScreen(chevron.point, angle: chevron.angle)) - Text("Your battery spot is this way") + Text(ScanCopy.proposedSpotThisWay) .font(.subheadline.weight(.semibold)) .foregroundStyle(Palette.chalk) .multilineTextAlignment(.center) @@ -122,7 +133,7 @@ private struct SpotDirection: View { } .frame(width: size.width, height: size.height) .accessibilityElement(children: .ignore) - .accessibilityLabel("Your battery spot is off screen. Turn the phone toward the arrow.") + .accessibilityLabel(ScanCopy.proposedSpotOffScreen) .accessibilityIdentifier("ar.spotDirection") } } @@ -187,7 +198,7 @@ struct BatteryOverlay: View, Animatable { private func drawClearances(in context: inout GraphicsContext, _ geometry: WallProjection) { for zone in result.clearances { guard let quad = geometry.groundQuad(s: zone.span, out: 0...zone.depth, height: 0.01) else { continue } - let color = Palette.outcome(zone.outcome) + let color = Palette.zone(zone.outcome) context.fill(quad, with: .color(color.opacity(0.28 * rise))) context.stroke(quad, with: .color(color.opacity(0.9 * rise)), style: StrokeStyle(lineWidth: 2, dash: [8, 6])) } diff --git a/ios/HouseScan/UI/Screens/ResultScreen.swift b/ios/HouseScan/UI/Screens/ResultScreen.swift index 9b8c88ae3..e5b3e7ad1 100644 --- a/ios/HouseScan/UI/Screens/ResultScreen.swift +++ b/ios/HouseScan/UI/Screens/ResultScreen.swift @@ -127,7 +127,9 @@ struct ResultScreen: View { /// Plain small print, never boxes: none of it changes what the homeowner does next. private func footnotes(_ result: ResultPresentation) -> some View { VStack(alignment: .leading, spacing: 6) { - footnote(ScanCopy.installerConfirms, id: "result.installerConfirms") + if result.answer != .candidate { + footnote(ScanCopy.installerConfirms, id: "result.installerConfirms") + } if let notice = result.rulesNotice { footnote(notice, id: "result.rulesNotice") } @@ -160,7 +162,12 @@ struct ResultScreen: View { private func details(_ result: ResultPresentation) -> some View { DisclosureGroup(isExpanded: $detailsExpanded) { VStack(alignment: .leading, spacing: 22) { - if !result.summary.isEmpty { + // A pass's summary, and a possible spot's under manual review, is the server's own + // pass ("fits every check"), which the card's candidate note contradicts on + // purpose; a pass the reading sends to an installer contradicts itself and says the + // same. Manual review and reject summaries explain what failed or is unsure, so + // they stay. + if !result.summary.isEmpty, result.decision != .pass, result.answer != .candidate { Text(result.summary) .font(Typeface.hint) .foregroundStyle(Palette.muted) @@ -300,6 +307,12 @@ private struct AnswerCard: View { .foregroundStyle(Palette.signalText) .fixedSize(horizontal: false, vertical: true) .accessibilityIdentifier("result.placement") + if answer == .candidate { + Text(ScanCopy.candidateNote) + .font(Typeface.hint) + .fixedSize(horizontal: false, vertical: true) + .accessibilityIdentifier("result.candidateNote") + } } else if let nearest = ScanCopy.nearest(result) { Text(nearest) .font(Typeface.hint) @@ -420,7 +433,7 @@ private struct CardCheckLine: View { HStack(alignment: .firstTextBaseline, spacing: 12) { Image(systemName: CheckSymbol.name(row.outcome)) .font(.body.weight(.semibold)) - .foregroundStyle(Palette.outcomeInk(row.outcome)) + .foregroundStyle(CheckSymbol.ink(row.outcome)) .frame(width: 22) VStack(alignment: .leading, spacing: 3) { Text(row.title) @@ -469,13 +482,19 @@ private struct Reveal: ViewModifier { } private enum CheckSymbol { + /// A pass is a muted outline: it met the rule on what the scan recorded, and a filled green + /// badge read as space confirmed clear. static func name(_ outcome: CheckOutcome) -> String { switch outcome { - case .pass: "checkmark.circle.fill" + case .pass: "checkmark.circle" case .unsure: "questionmark.circle.fill" case .fail: "xmark.circle.fill" } } + + static func ink(_ outcome: CheckOutcome) -> Color { + outcome == .pass ? Palette.muted : Palette.outcomeInk(outcome) + } } // MARK: - Details @@ -487,9 +506,16 @@ private struct ChecksList: View { var body: some View { VStack(alignment: .leading, spacing: 12) { - Text("What we checked") - .font(Typeface.sectionTitle) - .accessibilityAddTraits(.isHeader) + VStack(alignment: .leading, spacing: 4) { + Text(ScanCopy.calculatedTitle) + .font(Typeface.sectionTitle) + .accessibilityAddTraits(.isHeader) + Text(ScanCopy.calculatedNote) + .font(.subheadline) + .foregroundStyle(Palette.muted) + .fixedSize(horizontal: false, vertical: true) + .accessibilityIdentifier("result.calculatedNote") + } VStack(spacing: 0) { ForEach(Array(checks.enumerated()), id: \.element.id) { index, row in CheckRowView(row: row, photoOffered: photoOffered.contains(row.id)) @@ -507,20 +533,29 @@ private struct CheckRowView: View { var row: CheckRow var photoOffered: Bool + /// The server's reason, except for a pass: its reasons ("Nothing limits the clear space…") + /// state as fact what is only a calculation on recorded space. A pass keeps its measurement + /// against the rule. + private var reason: String? { + row.outcome == .pass ? nil : row.reason + } + var body: some View { HStack(alignment: .top, spacing: 12) { Image(systemName: CheckSymbol.name(row.outcome)) .font(.title3.weight(.semibold)) - .foregroundStyle(Palette.outcomeInk(row.outcome)) + .foregroundStyle(CheckSymbol.ink(row.outcome)) .frame(width: 28) .accessibilityHidden(true) VStack(alignment: .leading, spacing: 4) { Text(row.title) .font(Typeface.hint.weight(.semibold)) - Text(row.reason) - .font(.subheadline) - .foregroundStyle(Palette.muted) - .fixedSize(horizontal: false, vertical: true) + if let reason { + Text(reason) + .font(.subheadline) + .foregroundStyle(Palette.muted) + .fixedSize(horizontal: false, vertical: true) + } if let measurement = ScanCopy.measurement(row) { Text(measurement) .font(.subheadline.weight(.medium)) @@ -537,7 +572,7 @@ private struct CheckRowView: View { .padding(14) .accessibilityElement(children: .combine) .accessibilityLabel("\(row.title): \(ScanCopy.outcomeWord(row.outcome))") - .accessibilityValue([row.reason, ScanCopy.measurement(row, spoken: true), row.outcome == .unsure ? ScanCopy.unsureNote(photoOffered: photoOffered) : nil] + .accessibilityValue([reason, ScanCopy.measurement(row, spoken: true), row.outcome == .unsure ? ScanCopy.unsureNote(photoOffered: photoOffered) : nil] .compactMap(\.self).joined(separator: ". ")) // The card shows the deciding checks as `check.`; this is the full list. .accessibilityIdentifier("detail.check.\(row.id)") diff --git a/ios/HouseScan/UI/Screens/WallWalkScreen.swift b/ios/HouseScan/UI/Screens/WallWalkScreen.swift index c030f71b9..699259dd0 100644 --- a/ios/HouseScan/UI/Screens/WallWalkScreen.swift +++ b/ios/HouseScan/UI/Screens/WallWalkScreen.swift @@ -34,6 +34,7 @@ struct WallWalkScreen: View { // asks for it) lands under the circle. if state.marking != nil || controlsKey == .nextWall || controlsKey == .markEnd { Reticle(diameter: 56) + .endAimCircle() .frame(maxWidth: .infinity, maxHeight: .infinity) .ignoresSafeArea() TapRipple(ripples: $taps) @@ -68,7 +69,10 @@ struct WallWalkScreen: View { ) } } - .animation(reduceMotion ? .easeOut(duration: 0.15) : Motion.settle, value: controlsKey) + // Under Reduce Motion the controls swap at once: the swap changes the stack's height + // (the end's step adds its reply at the largest sizes), and an eased resize still + // moves the map under it. + .animation(reduceMotion ? nil : Motion.settle, value: controlsKey) } } .animation(.easeOut(duration: 0.2), value: showsMeterPhoto) @@ -80,6 +84,14 @@ struct WallWalkScreen: View { // MARK: Instruction private var instruction: Instruction { + // Folded around the circle, ride-along coaching leads or opens Details instead of sitting + // under the lead (`ScanCopy.foldedAroundCircle`). + guard endAimFolds else { return stepCard } + return ScanCopy.foldedAroundCircle(stepCard, coaching: coaching, refused: state.endMarkRefusal != nil || state.wallTooShort) + } + + /// The card's words for the step, before folding around the circle (`instruction`). + private var stepCard: Instruction { if let marking = state.marking { let prompt = ScanCopy.markingPrompt(marking) if let refusal = marking.refusal { @@ -99,7 +111,11 @@ struct WallWalkScreen: View { // Coaching about how the photos come out (the capture gate's, and too little texture) // rides along with the task (`ScanCopy.withCoaching`), and its symbol marks the card (`tone`). if state.wallTooShort { - return ScanCopy.withCoaching(Instruction(title: ScanCopy.wallTooShort, detail: ScanCopy.guidance(state.guidance).title), coaching) + let step = ScanCopy.guidance(state.guidance).title + // Folded around the circle (the walk can ask for the end again after the too-close + // ends were cleared), the long title would cover it: the correction leads instead. + let folded = Instruction.Folded(title: ScanCopy.wallTooShortLead, detail: "\(ScanCopy.wallTooShort) \(step)") + return ScanCopy.withCoaching(Instruction(title: ScanCopy.wallTooShort, detail: step, folded: folded), coaching) } return ScanCopy.withCoaching(ScanCopy.guidance(state.guidance, hint: state.guidanceHint), coaching) } @@ -136,21 +152,32 @@ struct WallWalkScreen: View { /// place, and a step to the side moves into the title when it is needed (`ScanCopy.aim`). /// Looking past an obstruction folds the other way: its action ("Look around it") leads and /// the situation and how go under Details (`Instruction.folded`), since unfolded at AX5 it - /// covered the camera and the spot it asks about. Not where the second line is the action - /// and nothing leads with it: a step back added to an aim step, the wall's end, whose title - /// only asks ("Aim where the wall stops"), marking the next wall. Nor a question, a mark, the - /// tray, coaching that replaces the task, or a refusal. + /// covered the camera and the spot it asks about. The wall's end folds the same way, around + /// the circle (`endAimFolds`). Not where the second line is the action and nothing leads with + /// it: a step back added to an aim step, marking the next wall. Nor a question, a mark, the + /// tray, coaching that replaces the task, or a refusal other than the end's. private var aims: Bool { switch controlsKey { case .walking, .finish: if let coaching, ScanCopy.coachingReplacesTask(coaching) { return false } if state.guidanceHint?.stepBack == true { return false } return tone != .refusal - case .markEnd, .nextWall, .marking, .endScanQuestion, .endQuestion, .overheadQuestion, .nextWallConfirm, .tray: + case .markEnd: + // Folds around the circle, refusal or not (`endAimFolds`). + if let coaching, ScanCopy.coachingReplacesTask(coaching) { return false } + return true + case .nextWall, .marking, .endScanQuestion, .endQuestion, .overheadQuestion, .nextWallConfirm, .tray: return false } } + /// At the largest text sizes on "Is this the right end of the wall?", the card folds to a + /// two-line lead ("Aim at the right end", or a refusal's correction) so the circle in the + /// middle of the camera stays open, and "The wall keeps going" moves from the card to the + /// actions under "Wall ends here": with it on the card, or the unfolded words, the card + /// reached past the circle (`ScanCopy.endAimFold`). As on a past_end request. + private var endAimFolds: Bool { controlsKey == .markEnd && aims && typeSize.isAccessibilitySize } + // MARK: Controls private enum ControlsKey: Hashable { @@ -254,39 +281,8 @@ struct WallWalkScreen: View { } .transition(.opacity) case .endQuestion: - // One question, three equal full-width answers that say what they mean (checklist I4). - VStack(spacing: 8) { - Button { - actions.answerWallEnd(turnsCorner: true) - } label: { - Label("It turns a corner", systemImage: "arrow.turn.up.right") - .frame(maxWidth: .infinity) - } - .buttonStyle(.secondaryProminent) - .accessibilityIdentifier("action.endCorner") - Button { - actions.answerWallEnd(turnsCorner: false) - } label: { - Label("Something blocks it", systemImage: "xmark.octagon") - .frame(maxWidth: .infinity) - } - .buttonStyle(.secondaryProminent) - .accessibilityHint("A fence, gate, or your neighbor's yard") - .accessibilityIdentifier("action.endBlocked") - // A garden wall or a fence can just stop, with no corner and nothing in the way, - // and neither answer above fits it (#70). Ends the wall as "Something blocks it" - // does: the usable wall stops here. - Button { - actions.answerWallEnd(turnsCorner: false) - } label: { - Label("The wall just ends", systemImage: "arrow.right.to.line") - .frame(maxWidth: .infinity) - } - .buttonStyle(.secondaryProminent) - .accessibilityHint("No corner and nothing in the way. The wall stops here.") - .accessibilityIdentifier("action.endEnds") - } - .transition(.opacity) + EndQuestionAnswers(actions: actions) + .transition(.opacity) case .overheadQuestion: OverheadAnswers(actions: actions) .transition(.opacity) @@ -369,6 +365,13 @@ struct WallWalkScreen: View { .accessibilityHint("Marks the end of the wall at the circle in the middle of the screen") .accessibilityIdentifier("action.markEnd") .transition(.opacity) + if endAimFolds, let copy = ScanCopy.reply(for: state.guidance) { + // The card's reply, moved here while the card folds around the circle. + MovedReplyButton( + title: copy.title, hint: copy.hint, identifier: "action.cannotAccess", + task: replyTask, perform: { actions.cannotAccessArea() }) + .transition(.opacity) + } case .walking where offersEndHere: Button { actions.endWallHere() @@ -434,7 +437,7 @@ struct WallWalkScreen: View { /// every step. private var reply: InstructionCard.Reply? { guard state.marking == nil, !state.endScanQuestion, state.endQuestion == nil, !state.overheadQuestion, state.nextWallConfirm == nil, !coachingHidesReply, !trayOpen, - let copy = ScanCopy.reply(for: state.guidance) else { return nil } + !endAimFolds, let copy = ScanCopy.reply(for: state.guidance) else { return nil } return InstructionCard.Reply( title: copy.title, identifier: "action.cannotAccess", diff --git a/ios/HouseScan/UI/Theme/Theme.swift b/ios/HouseScan/UI/Theme/Theme.swift index 2313e425f..e11f9ba2d 100644 --- a/ios/HouseScan/UI/Theme/Theme.swift +++ b/ios/HouseScan/UI/Theme/Theme.swift @@ -68,6 +68,13 @@ enum Palette { } } + /// A result's clearance zone in AR. Where the server's sweep passes, the proposal's blue + /// rather than `covered` green, which read as ground confirmed clear: the checks pass there + /// on recorded space, not on space anyone confirmed (B17). + static func zone(_ outcome: CheckOutcome) -> Color { + outcome == .pass ? signal : self.outcome(outcome) + } + static func outcomeInk(_ outcome: CheckOutcome) -> Color { switch outcome { case .pass: passInk diff --git a/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResult.swift b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResult.swift new file mode 100644 index 000000000..e04617f41 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResult.swift @@ -0,0 +1,166 @@ +import Foundation + +/// The capture API's device result, `GET /v1/captures/{id}/result`, written from the server's +/// documented wire contract. The whole body is camelCase. +/// +/// Only the fields the app shows or gates on are decoded. The verdict and its URL are left out on +/// purpose: the server masks a reject as manual review for the device, and the app must not read +/// around that mask. Unknown keys are ignored, because the server serves a result even when it has +/// drifted from its own schema. A status or outcome kind the app doesn't know decodes as +/// `.unknown`, so a new server value never hides the message and views that came with it. +public enum CaptureResult { + public struct Response: Decodable, Sendable, Equatable { + /// Nil before the server has started a run for the capture. + public var runId: String? + public var status: Status + public var viewsNeeded: [ViewNeeded] + public var memberActions: [String] + /// Nil until the run writes its result: while uploading, awaiting files or processing. + public var outcome: Outcome? + /// Nil when the server sent no criteria. The app shows none rather than inventing rows. + public var criteria: [Criterion]? + /// A signed URL that expires; nil when the run made no preview. + public var previewUrl: String? + /// A path on the API, not a full URL. + public var reviewUrl: String? + } + + public enum Status: Sendable, Equatable, Decodable { + case uploading, awaitingFiles, processing, needsViews, complete, manualReview, failed, expired + case unknown(String) + + public init(rawValue: String) { + self = switch rawValue { + case "uploading": .uploading + case "awaiting_files": .awaitingFiles + case "processing": .processing + case "needs_views": .needsViews + case "complete": .complete + case "manual_review": .manualReview + case "failed": .failed + case "expired": .expired + default: .unknown(rawValue) + } + } + + public init(from decoder: any Decoder) throws { + self.init(rawValue: try decoder.singleValueContainer().decode(String.self)) + } + } + + /// The decision shown to the homeowner. It is already masked for the device. + public struct Outcome: Decodable, Sendable, Equatable { + public var kind: OutcomeKind + public var profile: String + /// Text for the homeowner, shown as written. + public var message: String + public var viewsNeeded: [ViewNeeded] + /// Empty when the server shows a reject as manual review. + public var reasons: [Reason] + /// The ARKit world the spatial fields are in. Nil when the scene was not built in one. + public var arkitEpoch: String? + public var recommendedPlacement: RecommendedPlacement? + } + + public enum OutcomeKind: Sendable, Equatable, Decodable { + case eligible, needsMorePhotos, notEligible, manualReview + case unknown(String) + + public init(rawValue: String) { + self = switch rawValue { + case "eligible": .eligible + case "needs_more_photos": .needsMorePhotos + case "not_eligible": .notEligible + case "manual_review": .manualReview + default: .unknown(rawValue) + } + } + + public init(from decoder: any Decoder) throws { + self.init(rawValue: try decoder.singleValueContainer().decode(String.self)) + } + } + + public struct RecommendedPlacement: Decodable, Sendable, Equatable { + public var wallId: String + public var startSM: Double + public var confidence: Double + /// Nil when the scene was not built in an ARKit world. + public var boxArkitWorld: BoxArkitWorld? + } + + /// The box in ARKit world coordinates, as sent. The numbers are checked only when the app asks + /// for an AR placement (`Record.placement`), so a malformed box withholds AR and leaves the + /// rest of the result readable. + public struct BoxArkitWorld: Decodable, Sendable, Equatable { + /// Box centre to world, 16 numbers column by column. Local +x runs along the wall (left to + /// right seen from outside), +y up, +z out of the wall. + public var pose: [Double] + /// Width, height and depth in ARKit metres. + public var size: [Double] + } + + public struct ViewNeeded: Decodable, Sendable, Equatable { + public var id: String + /// The server's view kind, kept as text so a new kind still shows its prompt. + public var kind: String + public var criteria: [String] + public var why: String + public var promptId: String + /// Rendered text; show it verbatim. + public var prompt: Prompt + public var band: String? + public var wallId: String? + public var target: String? + /// Set in the photo flow instead of ARKit anchors. + public var referenceCrop: ReferenceCrop? + } + + public struct Prompt: Decodable, Sendable, Equatable { + public var title: String + public var body: String + } + + public struct ReferenceCrop: Decodable, Sendable, Equatable { + public var image: String + /// Pixel box in `image`: x0, y0, x1, y1. + public var boxPx: [Double] + } + + public struct Reason: Decodable, Sendable, Equatable { + public var criterion: String + public var text: String + public var measuredFt: Double? + public var plusMinusFt: Double? + public var thresholdFt: Double? + public var comparison: String? + } + + public struct Criterion: Decodable, Sendable, Equatable { + public var id: String + public var outcome: CriterionOutcome + public var unsureCause: String? + public var measuredFt: Double? + public var plusMinusFt: Double? + public var thresholdFt: Double? + public var coverage: String? + } + + public enum CriterionOutcome: Sendable, Equatable, Decodable { + case pass, fail, unsure + case unknown(String) + + public init(rawValue: String) { + self = switch rawValue { + case "pass": .pass + case "fail": .fail + case "unsure": .unsure + default: .unknown(rawValue) + } + } + + public init(from decoder: any Decoder) throws { + self.init(rawValue: try decoder.singleValueContainer().decode(String.self)) + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultPlacement.swift b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultPlacement.swift new file mode 100644 index 000000000..118c246b2 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultPlacement.swift @@ -0,0 +1,182 @@ +import simd + +extension CaptureResult { + /// The local identities a result belongs to. The response carries a run id and an ARKit epoch + /// but not the session or capture, and epochs repeat ("e1" starts every session), so only the + /// caller can say which scan a result is for. + public struct Association: Sendable, Equatable, Hashable { + /// The app's id for one run of the ARKit session. A new session gets a new id, even when it + /// reuses the epoch string. + public var sessionID: String + public var captureID: String + /// The run that finalize started for the capture. + public var runID: String + /// The ARKit epoch the packet was captured in. + public var epoch: String + + public init(sessionID: String, captureID: String, runID: String, epoch: String) { + self.sessionID = sessionID + self.captureID = captureID + self.runID = runID + self.epoch = epoch + } + } + + /// Whether the caller knows the server ran real analysis for this result. A stubbed server + /// returns a result that passes the contract without reconstructing anything, and nothing in + /// the JSON tells the two apart, so only the caller can grant `.verified`. + public enum Analysis: Sendable, Equatable { + case unverified + case verified + } + + /// A decoded result kept with the local scan it was fetched for. + public struct Record: Sendable, Equatable { + public let response: Response + public let association: Association + public let analysis: Analysis + + public init(response: Response, association: Association, analysis: Analysis = .unverified) { + self.response = response + self.association = association + self.analysis = analysis + } + + /// The recommended box in the meter anchor's frame, or the reason there is none. + /// + /// - Parameters: + /// - world: the session, capture, finalized run and epoch on screen now. + /// - meterAnchor: the meter anchor's anchor-to-world pose, in the same world coordinates + /// as the uploaded packet. Attaching the returned box to the anchor then carries it + /// through ARKit's later corrections. + public func placement(in world: Association, meterAnchor: simd_float4x4) -> Placement { + guard let outcome = response.outcome else { return .unavailable(.noOutcome(response.status)) } + guard outcome.kind == .eligible else { return .unavailable(.notEligible(outcome.kind)) } + guard response.status == .complete else { return .unavailable(.statusNotComplete(response.status)) } + guard analysis == .verified else { return .unavailable(.analysisUnverified) } + + guard world.sessionID == association.sessionID else { return .unavailable(.mismatch(.session)) } + guard world.captureID == association.captureID else { return .unavailable(.mismatch(.capture)) } + guard world.runID == association.runID, response.runId == association.runID else { + return .unavailable(.mismatch(.run)) + } + guard let epoch = outcome.arkitEpoch, + let box = outcome.recommendedPlacement?.boxArkitWorld + else { + return .unavailable(outcome.recommendedPlacement == nil ? .noPlacement : .notInARKitWorld) + } + guard world.epoch == association.epoch, epoch == association.epoch else { + return .unavailable(.mismatch(.epoch)) + } + + guard box.pose.count == 16 else { return .unavailable(.malformedBox(.poseCount(box.pose.count))) } + guard box.size.count == 3 else { return .unavailable(.malformedBox(.sizeCount(box.size.count))) } + guard (box.pose + box.size).allSatisfy(\.isFinite) else { return .unavailable(.malformedBox(.nonFinite)) } + guard box.size.allSatisfy({ $0 > 0 }) else { return .unavailable(.malformedBox(.nonPositiveSize)) } + let worldFromBox = Self.matrix(columnMajor: box.pose) + // The server divides the world transform's scale out of the pose's rotation and + // multiplies it into `size` instead. A pose that still carries a scale was built some + // other way, and using it would scale the box a second time. + guard Self.isRigid(worldFromBox) else { return .unavailable(.malformedBox(.notRigid)) } + + let worldFromAnchor = Self.double(meterAnchor) + guard Self.isRigid(worldFromAnchor) else { return .unavailable(.invalidMeterAnchor) } + + let anchorFromBox = Self.float(worldFromAnchor.inverse * worldFromBox) + let size = SIMD3(Float(box.size[0]), Float(box.size[1]), Float(box.size[2])) + // Finite Doubles can overflow Float, and positive sizes can round to zero. + let columns = [anchorFromBox.columns.0, anchorFromBox.columns.1, anchorFromBox.columns.2, anchorFromBox.columns.3] + guard columns.allSatisfy({ c in (0..<4).allSatisfy { c[$0].isFinite } }), + (0..<3).allSatisfy({ size[$0].isFinite }) + else { return .unavailable(.malformedBox(.nonFinite)) } + guard (0..<3).allSatisfy({ size[$0] > 0 }) else { return .unavailable(.malformedBox(.nonPositiveSize)) } + return .box(AnchoredBox(anchorFromBox: anchorFromBox, size: size)) + } + + private static func matrix(columnMajor v: [Double]) -> simd_double4x4 { + simd_double4x4( + SIMD4(v[0], v[1], v[2], v[3]), + SIMD4(v[4], v[5], v[6], v[7]), + SIMD4(v[8], v[9], v[10], v[11]), + SIMD4(v[12], v[13], v[14], v[15])) + } + + private static func double(_ m: simd_float4x4) -> simd_double4x4 { + simd_double4x4( + SIMD4(m.columns.0), SIMD4(m.columns.1), + SIMD4(m.columns.2), SIMD4(m.columns.3)) + } + + private static func float(_ m: simd_double4x4) -> simd_float4x4 { + simd_float4x4( + SIMD4(m.columns.0), SIMD4(m.columns.1), + SIMD4(m.columns.2), SIMD4(m.columns.3)) + } + + /// A finite rotation with no scale or shear, a translation, and a bottom row of 0 0 0 1. + /// The server writes each entry to six decimals, so its rotations are orthonormal to about + /// 1e-6; 1e-3 accepts that and ARKit's float poses, and rejects a 0.1% scale (whose + /// squared column length is off by 2e-3). + private static func isRigid(_ m: simd_double4x4) -> Bool { + let tolerance = 1e-3 + let columns = [m.columns.0, m.columns.1, m.columns.2, m.columns.3] + guard columns.allSatisfy({ c in (0..<4).allSatisfy { c[$0].isFinite } }) else { return false } + let bottom = SIMD4(m.columns.0.w, m.columns.1.w, m.columns.2.w, m.columns.3.w) + guard simd_reduce_max(simd_abs(bottom - SIMD4(0, 0, 0, 1))) <= tolerance else { return false } + let r = simd_double3x3( + SIMD3(m.columns.0.x, m.columns.0.y, m.columns.0.z), + SIMD3(m.columns.1.x, m.columns.1.y, m.columns.1.z), + SIMD3(m.columns.2.x, m.columns.2.y, m.columns.2.z)) + let gram = r.transpose * r + let identity = matrix_identity_double3x3 + for i in 0..<3 where simd_reduce_max(simd_abs(gram[i] - identity[i])) > tolerance { return false } + return r.determinant > 0 + } + } + + public enum Placement: Sendable, Equatable { + case box(AnchoredBox) + case unavailable(Unavailable) + } + + public struct AnchoredBox: Sendable, Equatable { + /// Box centre to meter anchor, rigid. Local +x along the wall, +y up, +z out of the wall. + public var anchorFromBox: simd_float4x4 + /// Width, height and depth in metres, as the server stated them. The server has already + /// applied the scene-to-ARKit scale, so the app uses these as they are. + public var size: SIMD3 + } + + /// Why a result has no AR placement. The rest of the result stays readable in every case. + public enum Unavailable: Sendable, Equatable { + /// The run has not written a result yet. + case noOutcome(Status) + /// Manual review, more photos, not eligible, or a kind this app doesn't know. + case notEligible(OutcomeKind) + case statusNotComplete(Status) + /// The caller has not confirmed that real analysis produced the result. + case analysisUnverified + /// The result belongs to another session, capture, run or ARKit world than the one on + /// screen. + case mismatch(IdentityField) + /// An eligible outcome without a recommended placement, which breaks the contract. + case noPlacement + /// The scene was not built in an ARKit world, so there is no box to anchor. + case notInARKitWorld + case malformedBox(BoxProblem) + /// The caller's meter anchor pose is not finite and rigid. + case invalidMeterAnchor + } + + public enum IdentityField: Sendable, Equatable, CaseIterable { + case session, capture, run, epoch + } + + public enum BoxProblem: Sendable, Equatable { + case poseCount(Int) + case sizeCount(Int) + case nonFinite + case nonPositiveSize + case notRigid + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultSummary.swift b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultSummary.swift new file mode 100644 index 000000000..65e156101 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultSummary.swift @@ -0,0 +1,51 @@ +import Foundation +import simd + +/// What the integration build shows of a capture-API result: where it came from, what the server +/// decided, its own words, the views it asks for, and why there is no AR box. Built from the +/// decoded response only; nothing is inferred, and an absent part stays absent. +public struct CaptureResultSummary: Sendable, Equatable { + public enum State: Sendable, Equatable { + /// The run has no outcome yet. + case working(CaptureResult.Status) + case decided(CaptureResult.OutcomeKind) + /// The capture is over without an outcome: it failed or expired. + case ended(CaptureResult.Status) + } + + public var state: State + /// The server's message for the homeowner, verbatim; nil before an outcome. + public var message: String? + /// Each requested view's prompt, verbatim, title then body. + public var prompts: [CaptureResult.Prompt] + /// The criteria rows the server sent; nil when it sent none. Never filled in by the app. + public var criteria: [CaptureResult.Criterion]? + /// False until someone has verified the server ran real analysis for this result. + public var analysisVerified: Bool + /// Why AR shows nothing for this result on the scan now on screen. + public var arUnavailable: CaptureResult.Unavailable? + + /// `world` is the scan on screen now; `meterAnchor` its meter anchor's pose in that world, nil + /// when the scan has none to place a box on (AR then says so, after the other checks). + public init(_ record: CaptureResult.Record, world: CaptureResult.Association, meterAnchor: simd_float4x4?) { + let response = record.response + state = if let outcome = response.outcome { + .decided(outcome.kind) + } else { + switch response.status { + case .failed, .expired: .ended(response.status) + default: .working(response.status) + } + } + message = response.outcome?.message + let views = response.viewsNeeded.isEmpty ? response.outcome?.viewsNeeded ?? [] : response.viewsNeeded + prompts = views.map(\.prompt) + criteria = response.criteria + analysisVerified = record.analysis == .verified + // A zero matrix is not a pose, so without an anchor the gate ends on `.invalidMeterAnchor`. + arUnavailable = switch record.placement(in: world, meterAnchor: meterAnchor ?? simd_float4x4()) { + case .box: nil + case .unavailable(let reason): reason + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Guidance/GapPlanner.swift b/ios/HouseScanKit/Sources/HouseScanKit/Guidance/GapPlanner.swift index c90e98838..972cb7fd1 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Guidance/GapPlanner.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Guidance/GapPlanner.swift @@ -374,18 +374,25 @@ extension GapPlanner { case .pastEnd: guard let side = item.side else { return nil } switch side { - case .left: - let end = leftEnd ?? 0 - return GapPlan(band: .ground, span: (end - 2)...end, reason: .server) - case .right: - let end = rightEnd ?? 0 - return GapPlan(band: .ground, span: end...(end + 2), reason: .server) + case .left: return pastEndPlan(side: .left, end: leftEnd ?? 0) + case .right: return pastEndPlan(side: .right, end: rightEnd ?? 0) } case .unknown: return nil } } + /// The request a past_end item makes from the end at `end` on `side`: the ground 2 m beyond + /// it (`plan(for:leftEnd:rightEnd:limitEnds:)`). The one place that extent is decided, so a + /// request recorded against an end the homeowner marked (`ScanEngine.skipCurrentGap`) is the + /// same view the next answer's past_end item plans from that end. + public func pastEndPlan(side: WalkSide, end: Float) -> GapPlan { + switch side { + case .left: GapPlan(band: .ground, span: (end - 2)...end, reason: .server) + case .right: GapPlan(band: .ground, span: end...(end + 2), reason: .server) + } + } + /// Where the end a past_end request cleared goes once the request is met without the end /// marked again: past the stretch the request showed, at its far edge (never back inside /// `clearedAt`), so the wall runs on over ground that was seen and a next past_end request diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Guidance/WalkedEnd.swift b/ios/HouseScanKit/Sources/HouseScanKit/Guidance/WalkedEnd.swift index cad0bfea1..32a616ecd 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Guidance/WalkedEnd.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Guidance/WalkedEnd.swift @@ -174,6 +174,16 @@ extension CoverageMap { return rightEnd - leftEnd < WallFrame.minWallLength } + /// True when an end on `side` at `s` would leave the wall shorter than + /// `WallFrame.minWallLength` from the other end, or from the meter while the other side has + /// none: the same policy as `endsTooClose`, asked before the end is set (B-12, a past-end + /// request's end marked again nearer). The other end is never moved. + public func endWouldLeaveTooLittle(_ side: WalkSide, at s: Float) -> Bool { + let other: Float = side == .left ? (rightEnd ?? 0) : (leftEnd ?? 0) + let length = side == .left ? other - s : s - other + return length < WallFrame.minWallLength + } + /// The unexplored end the camera stands beyond while its view shows nothing between the marked /// ends, or nil. A photo taken there adds nothing: cells past an end are never observed /// (`isWithinEnds`). Past a limit end it is nil, since the ground seen there still counts diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/FrameCalibrationRow.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/FrameCalibrationRow.swift new file mode 100644 index 000000000..f786520fe --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/FrameCalibrationRow.swift @@ -0,0 +1,13 @@ +import simd + +/// One ARFrame's own calibration as the recorder stores it beside that frame's pose: the row +/// `Packet04Streams.poseRows(trajectory:intrinsics:tracking:)` joins to the pose with the same `t`. +public enum FrameCalibrationRow { + /// `[t, fx, fy, cx, cy, width, height]`: ARKit's `camera.intrinsics` (column-major, so fx is + /// column 0's x, fy column 1's y, cx and cy column 2's x and y) in pixels of the unrotated + /// landscape sensor image, whose size `camera.imageResolution` gives. Values are kept as ARKit + /// reported them, with no rotation, scaling or rounding. + public static func row(t: Double, intrinsics k: simd_float3x3, imageWidth: Double, imageHeight: Double) -> [Double] { + [t, Double(k.columns.0.x), Double(k.columns.1.y), Double(k.columns.2.x), Double(k.columns.2.y), imageWidth, imageHeight] + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Gzip.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Gzip.swift new file mode 100644 index 000000000..e4cd01dff --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Gzip.swift @@ -0,0 +1,19 @@ +import Foundation + +/// gzip (RFC 1952) around the raw DEFLATE stream Apple's `.zlib` compression writes. The packet's +/// streams travel as `.csv.gz`, and Foundation has no gzip writer of its own. +public enum Gzip { + public static func compress(_ data: Data) throws -> Data { + // NSData's .zlib is raw DEFLATE (RFC 1951) with no zlib header, which is what gzip wraps. + let deflated = try (data as NSData).compressed(using: .zlib) as Data + var out = Data([0x1F, 0x8B, 0x08, 0x00, 0, 0, 0, 0, 0x00, 0xFF]) + out.append(deflated) + appendLE(ZipCRC32.checksum(data), to: &out) + appendLE(UInt32(truncatingIfNeeded: data.count), to: &out) + return out + } + + private static func appendLE(_ value: UInt32, to data: inout Data) { + withUnsafeBytes(of: value.littleEndian) { data.append(contentsOf: $0) } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04.swift new file mode 100644 index 000000000..f7ce75d4d --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04.swift @@ -0,0 +1,306 @@ +import Foundation + +/// The capture packet 0.4 that the capture API's finalize call takes as its body: `packet.json`, +/// camelCase, in the ARKit world frame of one epoch. This is the subset House Scan produces, +/// written from the published wire contract, not copied from the server's models. Optional +/// fields are left out when nil, which the server reads as absent. +/// +/// Unlike the 1.1 packet (`PacketManifest`), poses here are the raw ARKit camera-to-world +/// transforms, never moved into the meter frame. +public enum Packet04 { + public static let format = "house-capture-packet" + public static let formatVersion = "0.4" + + public struct Packet: Codable, Sendable, Equatable { + public var format = Packet04.format + public var formatVersion = Packet04.formatVersion + public var packetId: String + public var createdAt: String + public var units = Units() + public var conventions = Conventions() + public var source: Source + public var session: Session + public var epochs: [Epoch] + public var keyframes: [Keyframe] + public var stills: [Still]? + public var taps: [Tap]? + public var tracking: [TrackingEvent] + public var streams: Streams + public var scaleReference: ScaleReference + public var files: [FileEntry] + public var ext: [String: String]? + } + + public struct Units: Codable, Sendable, Equatable { + public var length = "meters" + public var angle = "degrees" + public var time = "seconds of device uptime (ARFrame.timestamp and Core Motion timestamp)" + public var image = "pixels of the saved, unrotated landscape JPEG" + public var rotationRate = "radians per second" + public var acceleration = "g (9.80665 m/s^2)" + } + + public struct Conventions: Codable, Sendable, Equatable { + public var world = "ARKit world, worldAlignment .gravity: right-handed, +y up, origin where the epoch started" + public var camera = "ARKit camera: +x right, +y up in the unrotated landscape image, looking along -z" + public var pose = "camera-to-world 4x4, 16 numbers column by column (simd_float4x4 layout), meters" + public var intrinsics = "[fx, fy, cx, cy] in pixels of the saved, unrotated landscape image" + public var pixel = "[u, v] continuous: (0, 0) is the top-left corner of the top-left pixel, v grows down" + } + + public struct Source: Codable, Sendable, Equatable { + public enum Kind: String, Codable, Sendable { case device, replay, synthetic } + public var kind: Kind + public var appBuild: String? + + public init(kind: Kind, appBuild: String?) { + self.kind = kind + self.appBuild = appBuild + } + } + + public enum Tier: String, Codable, Sendable { case arkit, arkitLidar = "arkit_lidar" } + + public struct VideoFormat: Codable, Sendable, Equatable { + public var imageResolution: [Int] + public var framesPerSecond: Double? + + public init(width: Int, height: Int, framesPerSecond: Double?) { + imageResolution = [width, height] + self.framesPerSecond = framesPerSecond + } + } + + public struct Session: Codable, Sendable, Equatable { + public var id: String + public var startedAt: String + public var startedAtUptime: Double + public var endedAtUptime: Double? + public var timeZone: String? + public var appVersion: String + public var deviceModel: String + public var systemVersion: String + public var tier: Tier + public var lidarAvailable: Bool + public var meshReconstructionSupported: Bool? + public var sceneDepthEnabled: Bool + public var sceneReconstruction: String? + public var worldAlignment = "gravity" + public var planeDetection: [String]? + public var videoFormat: VideoFormat + public var motionReferenceFrame: String? + } + + public struct Epoch: Codable, Sendable, Equatable { + public var id: String + public var startTime: Double + public var reason: String + + public init(id: String, startTime: Double, reason: String) { + self.id = id + self.startTime = startTime + self.reason = reason + } + } + + public struct Exposure: Codable, Sendable, Equatable { + public var duration: Double? + public var offset: Double? + public var iso: Double? + public var fNumber: Double? + + public init(duration: Double?, offset: Double?, iso: Double?, fNumber: Double?) { + self.duration = duration + self.offset = offset + self.iso = iso + self.fNumber = fNumber + } + } + + public struct DepthRef: Codable, Sendable, Equatable { + public var file: String + public var confidenceFile: String? + public var w: Int + public var h: Int + } + + public struct Keyframe: Codable, Sendable, Equatable { + public var id: String + public var img: String + public var w: Int + public var h: Int + public var intrinsics: [Double] + public var pose: [Double] + public var timestamp: Double + public var tracking: String + public var epoch: String + public var reason: String + public var exposure: Exposure? + public var depth: DepthRef? + public var purpose: String? + } + + public struct Still: Codable, Sendable, Equatable { + public var id: String + public var purpose: String + public var img: String + public var w: Int + public var h: Int + public var timestamp: Double + /// EXIF orientation of the stored pixels. House Scan stores the sensor image unrotated. + public var orientation: Int + public var keyframe: String? + public var intrinsics: [Double]? + public var intrinsicsSource: String? + } + + public struct TapHit: Codable, Sendable, Equatable { + public var position: [Double] + public var target: String + public var alignment: String? + public var distance: Double? + + public init(position: [Double], target: String, alignment: String?, distance: Double?) { + self.position = position + self.target = target + self.alignment = alignment + self.distance = distance + } + } + + public struct Tap: Codable, Sendable, Equatable { + public var id: String + public var time: Double + public var label: String + public var keyframe: String + public var pixel: [Double] + public var rayOrigin: [Double] + public var rayDirection: [Double] + public var hit: TapHit? + public var epoch: String? + } + + public struct TrackingEvent: Codable, Sendable, Equatable { + public var time: Double + public var state: String + public var epoch: String? + + public init(time: Double, state: String, epoch: String?) { + self.time = time + self.state = state + self.epoch = epoch + } + } + + public struct StreamRef: Codable, Sendable, Equatable { + public var file: String + public var columns: [String] + public var rateHz: Double + } + + public struct Streams: Codable, Sendable, Equatable { + public var imuRaw: StreamRef + public var arkitPoses: StreamRef + } + + public struct CloseUp: Codable, Sendable, Equatable { + public var captured: Bool + public var still: String? + public var arkitDistanceM: Double? + public var side: String? + + public init(captured: Bool, still: String?, arkitDistanceM: Double?, side: String?) { + self.captured = captured + self.still = still + self.arkitDistanceM = arkitDistanceM + self.side = side + } + + public static let notCaptured = CloseUp(captured: false, still: nil, arkitDistanceM: nil, side: nil) + } + + public struct ScaleExt: Codable, Sendable, Equatable { + public var method = "a9-meter-model" + public var obliqueCloseUp: CloseUp + /// Head-on views of doors on the meter side. House Scan does not detect doors yet, so this + /// is empty, which the contract allows. + public var doorViews: [String] = [] + } + + public struct ScaleReference: Codable, Sendable, Equatable { + public var meterCloseUp: CloseUp + public var ext: ScaleExt + } + + public enum Role: String, Codable, Sendable, CaseIterable { + case keyframe, depth, confidence, features, still, stream, other + } + + public struct FileEntry: Codable, Sendable, Equatable { + public var path: String + public var bytes: Int + public var sha256: String + public var role: Role + } + + /// The `meta` a register call sends with an image: exactly one record, the same one the final + /// packet lists. + public enum ImageRecord: Codable, Sendable, Equatable { + case keyframe(Keyframe) + case still(Still) + + public var path: String { + switch self { + case .keyframe(let k): k.img + case .still(let s): s.img + } + } + + private enum CodingKeys: String, CodingKey { case keyframe, still } + + public func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .keyframe(let k): try container.encode(k, forKey: .keyframe) + case .still(let s): try container.encode(s, forKey: .still) + } + } + + public init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + if let k = try container.decodeIfPresent(Keyframe.self, forKey: .keyframe) { + self = .keyframe(k) + } else { + self = .still(try container.decode(Still.self, forKey: .still)) + } + } + } + + // MARK: Id and path patterns the server enforces + + /// `^[A-Za-z0-9_-]{8,64}$`: packetId. + public static func isStorageID(_ s: String) -> Bool { + (8...64).contains(s.count) && s.unicodeScalars.allSatisfy { isAlnum($0) || $0 == "_" || $0 == "-" } + } + + /// `^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$`: keyframe and still ids. + public static func isSegmentID(_ s: String) -> Bool { + guard let first = s.unicodeScalars.first, isAlnum(first), s.count <= 128 else { return false } + return s.unicodeScalars.allSatisfy { isAlnum($0) || $0 == "_" || $0 == "-" } + } + + /// Every segment starts with an alphanumeric, and a dot always starts a new alphanumeric token. + public static func isRelPath(_ s: String) -> Bool { + guard !s.isEmpty else { return false } + return s.split(separator: "/", omittingEmptySubsequences: false).allSatisfy { segment in + segment.split(separator: ".", omittingEmptySubsequences: false).allSatisfy { token in + guard let first = token.unicodeScalars.first, isAlnum(first) else { return false } + return token.unicodeScalars.allSatisfy { isAlnum($0) || $0 == "_" || $0 == "-" } + } + } + } + + private static func isAlnum(_ c: Unicode.Scalar) -> Bool { + ("a"..."z").contains(c) || ("A"..."Z").contains(c) || ("0"..."9").contains(c) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Check.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Check.swift new file mode 100644 index 000000000..3ea995149 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Check.swift @@ -0,0 +1,115 @@ +import Foundation +import simd + +/// House Scan's own checks of a 0.4 packet against the rules the capture API documents as +/// rejections: id and path patterns, the ARKit tier's required sections, listed files that exist +/// with their stated size and SHA-256, rigid poses, plausible intrinsics, still-to-keyframe links +/// and the tap replay. A packet that passes can still fail a server check this does not repeat. +public enum Packet04Check { + /// The server's tap replay tolerance on origin and direction. + public static let tapReplayTolerance = 1e-3 + + public static func problems(_ p: Packet04.Packet, folder: URL?) -> [String] { + var out: [String] = [] + if !Packet04.isStorageID(p.packetId) { out.append("packetId \(p.packetId) is not a StorageId") } + if p.formatVersion != Packet04.formatVersion { out.append("formatVersion \(p.formatVersion)") } + if p.keyframes.isEmpty { out.append("ARKit tier needs at least one keyframe") } + if p.session.tier == .arkitLidar, !p.session.lidarAvailable { out.append("arkit_lidar needs lidarAvailable") } + if p.session.worldAlignment != "gravity" { out.append("worldAlignment must be gravity") } + if p.streams.imuRaw.rateHz < Packet04Streams.minimumIMUHz { out.append("imuRaw rateHz \(p.streams.imuRaw.rateHz) < 50") } + if p.streams.arkitPoses.rateHz < Packet04Streams.minimumPoseHz { out.append("arkitPoses rateHz \(p.streams.arkitPoses.rateHz) < 10") } + + let epochs = Set(p.epochs.map(\.id)) + var listed: [String: Packet04.FileEntry] = [:] + for entry in p.files { + if !Packet04.isRelPath(entry.path) { out.append("file path \(entry.path) is not a RelPath") } + if listed.updateValue(entry, forKey: entry.path) != nil { out.append("file \(entry.path) listed twice") } + } + func requireListed(_ path: String, _ role: Packet04.Role, _ place: String) { + guard let entry = listed[path] else { return out.append("\(place) \(path) is not in files") } + if entry.role != role { out.append("\(place) \(path) has role \(entry.role.rawValue), not \(role.rawValue)") } + } + + var frames: [String: Packet04.Keyframe] = [:] + var byTime: [Double: Packet04.Keyframe] = [:] + for k in p.keyframes { + if !Packet04.isSegmentID(k.id) { out.append("keyframe id \(k.id) is not a SegmentId") } + if frames.updateValue(k, forKey: k.id) != nil { out.append("keyframe \(k.id) listed twice") } + if !epochs.contains(k.epoch) { out.append("keyframe \(k.id) names unknown epoch \(k.epoch)") } + requireListed(k.img, .keyframe, "keyframe \(k.id) image") + if let depth = k.depth { + requireListed(depth.file, .depth, "keyframe \(k.id) depth") + if let confidence = depth.confidenceFile { requireListed(confidence, .confidence, "keyframe \(k.id) confidence") } + if let entry = listed[depth.file], entry.bytes != depth.w * depth.h * 4 { out.append("keyframe \(k.id) depth size") } + } + guard let pose = matrix(k.pose) else { out.append("keyframe \(k.id) pose is not 16 finite numbers"); continue } + if !PacketPose.isRigid(pose) { out.append("keyframe \(k.id) pose is not rigid") } + if k.intrinsics.count != 4 { + out.append("keyframe \(k.id) intrinsics") + } else if let problem = PacketWriter.intrinsicsProblem( + SIMD4(k.intrinsics.map(Float.init)), width: k.w, height: k.h) { + out.append("keyframe \(k.id) intrinsics: \(problem)") + } + if let other = byTime[k.timestamp], other.epoch == k.epoch, other.pose != k.pose { + out.append("keyframes \(other.id) and \(k.id) share timestamp \(k.timestamp) with different poses") + } + byTime[k.timestamp] = k + } + var stillIDs = Set() + for s in p.stills ?? [] { + if !Packet04.isSegmentID(s.id) { out.append("still id \(s.id) is not a SegmentId") } + stillIDs.insert(s.id) + requireListed(s.img, .still, "still \(s.id) image") + if let link = s.keyframe { + guard let frame = frames[link] else { out.append("still \(s.id) names unknown keyframe \(link)"); continue } + if frame.w != s.w || frame.h != s.h { out.append("still \(s.id) and keyframe \(link) sizes differ") } + } + } + for tap in p.taps ?? [] { + guard let frame = frames[tap.keyframe] else { out.append("tap \(tap.id) names unknown keyframe \(tap.keyframe)"); continue } + if tap.pixel.count != 2 || !(0...Double(frame.w)).contains(tap.pixel[0]) || !(0...Double(frame.h)).contains(tap.pixel[1]) { + out.append("tap \(tap.id) pixel is outside keyframe \(frame.id)") + continue + } + if let epoch = tap.epoch, epoch != frame.epoch { out.append("tap \(tap.id) epoch differs from its keyframe's") } + // Compared pairwise below, a missing or extra component, or a NaN, would go unseen. + guard tap.rayOrigin.count == 3, tap.rayDirection.count == 3, (tap.rayOrigin + tap.rayDirection).allSatisfy(\.isFinite) else { + out.append("tap \(tap.id) ray is not two finite 3-vectors") + continue + } + let replay = ray(pose: frame.pose, intrinsics: frame.intrinsics, pixel: tap.pixel) + let error = zip(replay.origin + replay.direction, tap.rayOrigin + tap.rayDirection).map { abs($0 - $1) }.max() ?? .infinity + if !(error <= tapReplayTolerance) { out.append("tap \(tap.id) ray replays with error \(error)") } + } + for (name, close) in [("meterCloseUp", p.scaleReference.meterCloseUp), ("obliqueCloseUp", p.scaleReference.ext.obliqueCloseUp)] { + if close.captured, !(close.still.map(stillIDs.contains) ?? false) { out.append("\(name) names a still that is not in the packet") } + } + requireListed(p.streams.imuRaw.file, .stream, "imuRaw") + requireListed(p.streams.arkitPoses.file, .stream, "arkitPoses") + + if let folder { + for entry in p.files { + guard let data = try? Data(contentsOf: folder.appending(path: entry.path)) else { out.append("file \(entry.path) is missing"); continue } + if data.count != entry.bytes { out.append("file \(entry.path) is \(data.count) bytes, listed \(entry.bytes)") } + if PacketFiles.sha256(data) != entry.sha256 { out.append("file \(entry.path) SHA-256 differs from its listing") } + } + } + return out + } + + /// The ray through `pixel` as the server replays it: origin the camera position, direction + /// the unit vector of R · ((u − cx)/fx, −(v − cy)/fy, −1). + public static func ray(pose: [Double], intrinsics k: [Double], pixel: [Double]) -> (origin: [Double], direction: [Double]) { + guard pose.count == 16, k.count == 4, pixel.count == 2 else { return ([], []) } + let d = SIMD3((pixel[0] - k[2]) / k[0], -(pixel[1] - k[3]) / k[1], -1) + let c0 = SIMD3(pose[0], pose[1], pose[2]), c1 = SIMD3(pose[4], pose[5], pose[6]), c2 = SIMD3(pose[8], pose[9], pose[10]) + let world = simd_normalize(c0 * d.x + c1 * d.y + c2 * d.z) + return ([pose[12], pose[13], pose[14]], [world.x, world.y, world.z]) + } + + static func matrix(_ values: [Double]) -> simd_float4x4? { + guard values.count == 16, values.allSatisfy(\.isFinite) else { return nil } + let f = values.map(Float.init) + return simd_float4x4(SIMD4(f[0], f[1], f[2], f[3]), SIMD4(f[4], f[5], f[6], f[7]), SIMD4(f[8], f[9], f[10], f[11]), SIMD4(f[12], f[13], f[14], f[15])) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Producer.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Producer.swift new file mode 100644 index 000000000..7691ebe16 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Producer.swift @@ -0,0 +1,385 @@ +import CryptoKit +import Foundation +import ImageIO +import simd + +/// A file of the 0.4 packet that is on disk and will never change: its bytes, digests and, for +/// an image, the record that describes it. The same value registers the file with the capture API +/// and lists it in the final packet, so the two can't disagree. +public struct SealedFile: Codable, Sendable, Equatable { + public var path: String + public var bytes: Int + public var sha256: String + /// Base64 of the 16-byte MD5, as the register call and the storage PUT's Content-MD5 want it. + public var md5: String + public var role: Packet04.Role + public var contentType: String + public var meta: Packet04.ImageRecord? + /// Lower goes first: the meter close-ups before keyframes before streams. + public var priority: Int + + public var entry: Packet04.FileEntry { .init(path: path, bytes: bytes, sha256: sha256, role: role) } + + static func seal(_ data: Data, path: String, role: Packet04.Role, contentType: String, meta: Packet04.ImageRecord?, priority: Int) -> SealedFile { + SealedFile( + path: path, bytes: data.count, sha256: PacketFiles.sha256(data), + md5: Data(Insecure.MD5.hash(data: data)).base64EncodedString(), role: role, contentType: contentType, meta: meta, + priority: priority) + } +} + +/// What the capture observed for one saved camera image, read from its own ARFrame when the +/// pixels were taken: the raw ARKit camera-to-world pose, never one corrected into the meter frame. +public struct Packet04Observation: Sendable, Equatable { + public var t: Double + public var cameraToWorld: simd_float4x4 + /// [fx, fy, cx, cy] in pixels of the stored JPEG. + public var intrinsics: SIMD4 + public var width: Int + public var height: Int + public var tracking: PacketTracking + public var exposure: Packet04.Exposure? + + public init(t: Double, cameraToWorld: simd_float4x4, intrinsics: SIMD4, width: Int, height: Int, tracking: PacketTracking, exposure: Packet04.Exposure? = nil) { + self.t = t + self.cameraToWorld = cameraToWorld + self.intrinsics = intrinsics + self.width = width + self.height = height + self.tracking = tracking + self.exposure = exposure + } +} + +/// ARKit depth for one keyframe: Float32 meters and UInt8 confidence, row by row. +public struct Packet04Depth: Sendable { + public var meters: [Float] + public var confidence: [UInt8] + public var width: Int + public var height: Int + + public init(meters: [Float], confidence: [UInt8], width: Int, height: Int) { + self.meters = meters + self.confidence = confidence + self.width = width + self.height = height + } +} + +/// Fixed facts about the capture, known when the ARKit session starts and the homeowner consents, +/// so the create request can be sent before the first photo. +public struct Packet04SessionInfo: Codable, Sendable, Equatable { + public var packetID: String + public var sessionID: String + public var source: Packet04.Source.Kind + public var appVersion: String + public var deviceModel: String + public var systemVersion: String + public var lidarAvailable: Bool + public var sceneDepthEnabled: Bool + public var meshReconstructionSupported: Bool? + public var sceneReconstruction: String? + public var planeDetection: [String]? + public var videoWidth: Int + public var videoHeight: Int + public var framesPerSecond: Double? + public var timeZone: String? + + public init( + packetID: String, sessionID: String, source: Packet04.Source.Kind, appVersion: String, deviceModel: String, systemVersion: String, + lidarAvailable: Bool, sceneDepthEnabled: Bool, meshReconstructionSupported: Bool?, sceneReconstruction: String?, + planeDetection: [String]?, videoWidth: Int, videoHeight: Int, framesPerSecond: Double?, timeZone: String? + ) { + self.packetID = packetID + self.sessionID = sessionID + self.source = source + self.appVersion = appVersion + self.deviceModel = deviceModel + self.systemVersion = systemVersion + self.lidarAvailable = lidarAvailable + self.sceneDepthEnabled = sceneDepthEnabled + self.meshReconstructionSupported = meshReconstructionSupported + self.sceneReconstruction = sceneReconstruction + self.planeDetection = planeDetection + self.videoWidth = videoWidth + self.videoHeight = videoHeight + self.framesPerSecond = framesPerSecond + self.timeZone = timeZone + } + + /// `arkit_lidar` needs depth the session actually recorded, not a phone that could have. + public var tier: Packet04.Tier { lidarAvailable && sceneDepthEnabled ? .arkitLidar : .arkit } +} + +public enum Packet04Error: Error, Equatable, CustomStringConvertible { + case invalidID(String) + case sealedAfterFinish(String) + case duplicate(String) + case invalidImage(id: String, reason: String) + case notRigid(String) + case unknownKeyframe(String) + case streamTooSlow(stream: String, measuredHz: Double, minimumHz: Double) + case streamEmpty(String) + case noKeyframes + case invalidPacket([String]) + + public var description: String { + switch self { + case .invalidID(let id): "\(id) does not fit the server's id pattern" + case .sealedAfterFinish(let what): "\(what) arrived after the packet was finished" + case .duplicate(let id): "\(id) was sealed twice" + case .invalidImage(let id, let reason): "image \(id): \(reason)" + case .notRigid(let place): "\(place): pose is not a rotation plus a translation" + case .unknownKeyframe(let id): "no keyframe \(id)" + case .streamTooSlow(let stream, let hz, let minimum): "\(stream) measured \(hz) Hz; the contract requires at least \(minimum) Hz" + case .streamEmpty(let stream): "\(stream) has no rows" + case .noKeyframes: "an ARKit packet needs at least one keyframe" + case .invalidPacket(let problems): "packet fails its own checks: \(problems.joined(separator: "; "))" + } + } +} + +/// Builds one 0.4 packet while the capture runs. Each image is written into `folder` and sealed +/// the moment it is observed, so it can be uploaded during the capture; `finish` adds the streams +/// and returns the exact `packet.json` bytes, which are frozen from then on. +/// +/// One producer is one ARKit epoch. A world reset ends the packet: the app starts a new producer +/// with a new packet id rather than mixing two worlds under one epoch. +public actor Packet04Producer { + public nonisolated let folder: URL + public nonisolated let info: Packet04SessionInfo + public nonisolated let epoch = "e1" + private let startedAtUptime: Double + private let startedAt: Date + private var keyframes: [Packet04.Keyframe] = [] + private var stills: [Packet04.Still] = [] + private var taps: [Packet04.Tap] = [] + private var files: [SealedFile] = [] + private var finished: Data? + + /// `startedAtUptime` is the uptime of the session's first ARFrame and `startedAt` the wall clock + /// at that instant. + public init(folder: URL, info: Packet04SessionInfo, startedAtUptime: Double, startedAt: Date) throws { + guard Packet04.isStorageID(info.packetID) else { throw Packet04Error.invalidID(info.packetID) } + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + self.folder = folder + self.info = info + self.startedAtUptime = startedAtUptime + self.startedAt = startedAt + } + + public var sealedFiles: [SealedFile] { files } + public var keyframeCount: Int { keyframes.count } + public var isFinished: Bool { finished != nil } + + /// The keyframe sealed from the frame at `t`, if there is one: a still taken from a frame that + /// was already kept links to it instead of repeating it. + public func keyframeID(at t: Double) -> String? { keyframes.first { $0.timestamp == t }?.id } + + /// "k00001" for 1. + public static func keyframeID(_ number: Int) -> String { + let digits = String(number) + return "k" + String(repeating: "0", count: max(0, 5 - digits.count)) + digits + } + + // MARK: Images + + /// Writes the JPEG as keyframes/.jpg with, when given, its depth and confidence, and seals + /// them. `reason` is the contract's keyframe reason (motion, tap, still, request, auto). + public func sealKeyframe( + jpeg: Data, observation o: Packet04Observation, reason: String, purpose: String? = nil, depth: Packet04Depth? = nil + ) throws -> [SealedFile] { + guard finished == nil else { throw Packet04Error.sealedAfterFinish("keyframe") } + let id = Self.keyframeID(keyframes.count + 1) + try check(o, id: id, jpeg: jpeg) + guard let tracking = Self.trackingText(o.tracking) else { + throw Packet04Error.invalidImage(id: id, reason: "tracking is limited for a reason the contract has no name for") + } + var sealed: [SealedFile] = [] + var depthRef: Packet04.DepthRef? + if let depth, depth.meters.count == depth.width * depth.height, depth.confidence.count == depth.meters.count, depth.width > 0 { + let map = try write(PacketFiles.depthData(meters: depth.meters), "keyframes/\(id).depth.f32", role: .depth, type: "application/octet-stream", priority: 20) + let confidence = try write(Data(depth.confidence), "keyframes/\(id).confidence.u8", role: .confidence, type: "application/octet-stream", priority: 20) + sealed += [map, confidence] + depthRef = .init(file: map.path, confidenceFile: confidence.path, w: depth.width, h: depth.height) + } + let record = Packet04.Keyframe( + id: id, img: "keyframes/\(id).jpg", w: o.width, h: o.height, intrinsics: Self.numbers(o.intrinsics), + pose: PacketPose.columnMajor(o.cameraToWorld), timestamp: o.t, tracking: tracking, epoch: epoch, + reason: reason, exposure: o.exposure, depth: depthRef, purpose: purpose) + let image = try write(jpeg, record.img, role: .keyframe, type: "image/jpeg", meta: .keyframe(record), priority: 10) + keyframes.append(record) + return [image] + sealed + } + + /// A checklist still taken from an ARKit keyframe already sealed: the same pixels under + /// stills/.jpg, linked to the keyframe so it keeps its pose. Stills go first. + /// + /// A retake of the same purpose is a new still ("meter_close-2"): sealed files never change, + /// and `finish` says which one the scale reference names. + public func sealStill(purpose: String, keyframe keyframeID: String) throws -> SealedFile { + guard finished == nil else { throw Packet04Error.sealedAfterFinish("still") } + guard Packet04.isSegmentID(purpose) else { throw Packet04Error.invalidID(purpose) } + guard let frame = keyframes.first(where: { $0.id == keyframeID }) else { throw Packet04Error.unknownKeyframe(keyframeID) } + if let same = stills.last(where: { $0.purpose == purpose }), same.keyframe == keyframeID { + throw Packet04Error.duplicate(same.id) + } + let taken = stills.filter { $0.purpose == purpose }.count + let id = taken == 0 ? purpose : "\(purpose)-\(taken + 1)" + let jpeg = try Data(contentsOf: folder.appending(path: frame.img)) + let still = Packet04.Still( + id: id, purpose: purpose, img: "stills/\(id).jpg", w: frame.w, h: frame.h, timestamp: frame.timestamp, + orientation: 1, keyframe: frame.id, intrinsics: frame.intrinsics, intrinsicsSource: "arkit") + let file = try write(jpeg, still.img, role: .still, type: "image/jpeg", meta: .still(still), priority: 0) + stills.append(still) + return file + } + + /// A tap on a sealed keyframe's pixel, with the ray rebuilt from that keyframe's own pose and + /// intrinsics, so the server's replay reproduces it exactly. + public func addTap(id: String, label: String, keyframe keyframeID: String, pixel: SIMD2, time: Double, hit: Packet04.TapHit?) throws { + guard finished == nil else { throw Packet04Error.sealedAfterFinish("tap") } + guard let frame = keyframes.first(where: { $0.id == keyframeID }) else { throw Packet04Error.unknownKeyframe(keyframeID) } + let ray = Packet04Check.ray(pose: frame.pose, intrinsics: frame.intrinsics, pixel: [pixel.x, pixel.y]) + taps.append(.init( + id: id, time: time, label: label, keyframe: keyframeID, pixel: [pixel.x, pixel.y], rayOrigin: ray.origin, + rayDirection: ray.direction, hit: hit, epoch: epoch)) + } + + /// The frame the homeowner tapped, kept as a keyframe (reason `tap`) unless it already is one, + /// and the tap on it: `pixel` in that frame's stored image, the ray rebuilt from its own pose + /// and intrinsics, and what the phone's raycast hit. + public func sealTap( + id: String, label: String, jpeg: Data, observation: Packet04Observation, pixel: SIMD2, hit: Packet04.TapHit? + ) throws -> [SealedFile] { + var files: [SealedFile] = [] + if keyframeID(at: observation.t) == nil { + files = try sealKeyframe(jpeg: jpeg, observation: observation, reason: "tap") + } + guard let keyframe = keyframeID(at: observation.t) else { throw Packet04Error.unknownKeyframe("at \(observation.t)") } + try addTap(id: id, label: label, keyframe: keyframe, pixel: pixel, time: observation.t, hit: hit) + return files + } + + // MARK: Finish + + /// Writes the two required streams, then returns their sealed files and the exact packet + /// bytes. The packet is checked against the contract's rules first; a failure names each + /// problem instead of sending a packet the server would reject. + /// `acceptedCloseUpAt` is the frame time of the head-on close-up the scan accepted, nil when + /// none was (skipped, or every shot refused): only that still is the scale reference. + public func finish( + poses: [Packet04Streams.PoseRow], accelerometer: [Packet04Streams.MotionRow], gyroscope: [Packet04Streams.MotionRow], + endedAtUptime: Double, acceptedCloseUpAt: Double?, closeUpDistanceM: Double? = nil, createdAt: Date = Date() + ) throws -> (streams: [SealedFile], packet: Data) { + if let finished { return (files.filter { $0.role == .stream }, finished) } + guard !keyframes.isEmpty else { throw Packet04Error.noKeyframes } + let posesCSV = try Packet04Streams.arkitPoses(poses, epoch: epoch) + let imu = try Packet04Streams.imuRaw(accelerometer: accelerometer, gyroscope: gyroscope) + let poseFile = try write(try Gzip.compress(posesCSV.csv), "streams/arkit_poses.csv.gz", role: .stream, type: "application/gzip", priority: 30) + let imuFile = try write(try Gzip.compress(imu.csv), "streams/imu_raw.csv.gz", role: .stream, type: "application/gzip", priority: 30) + // Each sensor as Core Motion timed it, so the server can re-pair them its own way. + let accelFile = try write( + try Gzip.compress(try Packet04Streams.rawMotionCSV(accelerometer, columns: ["t", "ax", "ay", "az"])), + "streams/accelerometer_raw.csv.gz", role: .stream, type: "application/gzip", priority: 31) + let gyroFile = try write( + try Gzip.compress(try Packet04Streams.rawMotionCSV(gyroscope, columns: ["t", "gx", "gy", "gz"])), + "streams/gyroscope_raw.csv.gz", role: .stream, type: "application/gzip", priority: 31) + var ext = imu.pairingNote + ext["rawAccelerometerFile"] = accelFile.path + ext["rawGyroscopeFile"] = gyroFile.path + + let close = acceptedCloseUpAt.flatMap { t in stills.last { $0.purpose == "meter_close" && $0.timestamp == t } } + // The app takes no oblique close-up yet, so none is ever accepted. + let oblique: Packet04.Still? = nil + let packet = Packet04.Packet( + packetId: info.packetID, createdAt: PacketWriter.iso8601(createdAt), + source: .init(kind: info.source, appBuild: info.appVersion), + session: .init( + id: info.sessionID, startedAt: PacketWriter.iso8601(startedAt), startedAtUptime: startedAtUptime, + endedAtUptime: max(endedAtUptime, startedAtUptime), timeZone: info.timeZone, appVersion: info.appVersion, + deviceModel: info.deviceModel, systemVersion: info.systemVersion, tier: info.tier, lidarAvailable: info.lidarAvailable, + meshReconstructionSupported: info.meshReconstructionSupported, sceneDepthEnabled: info.sceneDepthEnabled, + sceneReconstruction: info.sceneReconstruction, planeDetection: info.planeDetection, + videoFormat: .init(width: info.videoWidth, height: info.videoHeight, framesPerSecond: info.framesPerSecond), + motionReferenceFrame: nil), + epochs: [.init(id: epoch, startTime: startedAtUptime, reason: "sessionStart")], + keyframes: keyframes, stills: stills.isEmpty ? nil : stills, taps: taps.isEmpty ? nil : taps, + tracking: Packet04Streams.trackingChanges(poses, epoch: epoch), + streams: .init( + imuRaw: .init(file: imuFile.path, columns: Packet04Streams.imuColumns, rateHz: imu.rateHz), + arkitPoses: .init(file: poseFile.path, columns: Packet04Streams.poseColumns, rateHz: posesCSV.rateHz)), + scaleReference: .init( + meterCloseUp: close.map { .init(captured: true, still: $0.id, arkitDistanceM: closeUpDistanceM, side: nil) } ?? .notCaptured, + ext: .init(obliqueCloseUp: oblique.map { .init(captured: true, still: $0.id, arkitDistanceM: nil, side: nil) } ?? .notCaptured)), + files: files.map(\.entry), + ext: ext) + let problems = Packet04Check.problems(packet, folder: folder) + guard problems.isEmpty else { throw Packet04Error.invalidPacket(problems) } + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys, .withoutEscapingSlashes] + let data = try encoder.encode(packet) + try data.write(to: folder.appending(path: "packet.json"), options: .atomic) + finished = data + return ([poseFile, imuFile, accelFile, gyroFile], data) + } + + // MARK: Helpers + + private func check(_ o: Packet04Observation, id: String, jpeg: Data) throws { + guard PacketPose.isRigid(o.cameraToWorld) else { throw Packet04Error.notRigid("keyframe \(id)") } + guard o.t.isFinite, o.t >= 0 else { throw Packet04Error.invalidImage(id: id, reason: "timestamp \(o.t)") } + if let problem = PacketWriter.intrinsicsProblem(o.intrinsics, width: o.width, height: o.height) { + throw Packet04Error.invalidImage(id: id, reason: problem) + } + if let problem = Self.jpegProblem(jpeg, width: o.width, height: o.height) { + throw Packet04Error.invalidImage(id: id, reason: problem) + } + if let same = keyframes.first(where: { $0.timestamp == o.t }) { + throw Packet04Error.invalidImage(id: id, reason: "keyframe \(same.id) already has timestamp \(o.t)") + } + } + + private func write(_ data: Data, _ path: String, role: Packet04.Role, type: String, meta: Packet04.ImageRecord? = nil, priority: Int) throws -> SealedFile { + guard Packet04.isRelPath(path) else { throw Packet04Error.invalidID(path) } + guard !files.contains(where: { $0.path == path }) else { throw Packet04Error.duplicate(path) } + let url = folder.appending(path: path) + try FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true) + try data.write(to: url, options: .atomic) + let file = SealedFile.seal(data, path: path, role: role, contentType: type, meta: meta, priority: priority) + files.append(file) + return file + } + + /// The bytes must be a JPEG of the stated size, stored unrotated. + static func jpegProblem(_ jpeg: Data, width: Int, height: Int) -> String? { + guard let source = CGImageSourceCreateWithData(jpeg as CFData, nil), CGImageSourceGetType(source) as String? == "public.jpeg", + let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any] + else { return "not a readable JPEG" } + let w = properties[kCGImagePropertyPixelWidth] as? Int, h = properties[kCGImagePropertyPixelHeight] as? Int + guard w == width, h == height else { return "the JPEG is \(w ?? 0)x\(h ?? 0), not \(width)x\(height)" } + if let orientation = properties[kCGImagePropertyOrientation] as? Int, orientation != 1 { + return "EXIF orientation \(orientation); store the unrotated sensor image" + } + return nil + } + + static func numbers(_ k: SIMD4) -> [Double] { [k.x, k.y, k.z, k.w].map(PacketNumber.double) } + + /// The contract's tracking state. Nil for a limited state whose reason the contract has no + /// name for (a reason added after iOS 26): no value there would be an observation. + public static func trackingText(_ tracking: PacketTracking) -> String? { + switch tracking { + case .normal: "normal" + case .notAvailable: "notAvailable" + case .limited(let why): + switch why { + case .initializing?: "limited.initializing" + case .excessiveMotion?: "limited.excessiveMotion" + case .insufficientFeatures?: "limited.insufficientFeatures" + case .relocalizing?: "limited.relocalizing" + case nil: nil + } + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Streams.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Streams.swift new file mode 100644 index 000000000..be5cabecc --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Streams.swift @@ -0,0 +1,147 @@ +import Foundation +import simd + +/// The 0.4 packet's two required streams as CSV, and the tracking changes read from the poses. +public enum Packet04Streams { + public static let poseColumns = ["t", "epoch", "tracking", "tx", "ty", "tz", "qx", "qy", "qz", "qw", "fx", "fy", "cx", "cy"] + public static let imuColumns = ["t", "ax", "ay", "az", "gx", "gy", "gz"] + /// The contract's floors for `rateHz`. + public static let minimumPoseHz = 10.0 + public static let minimumIMUHz = 50.0 + + /// One ARFrame: raw camera-to-world and that frame's intrinsics at the video format's size. + public struct PoseRow: Sendable, Equatable { + public var t: Double + public var tracking: PacketTracking + public var cameraToWorld: simd_float4x4 + public var intrinsics: SIMD4 + + public init(t: Double, tracking: PacketTracking, cameraToWorld: simd_float4x4, intrinsics: SIMD4) { + self.t = t + self.tracking = tracking + self.cameraToWorld = cameraToWorld + self.intrinsics = intrinsics + } + } + + /// One Core Motion sample with its own timestamp: g for the accelerometer, rad/s for the gyro. + public struct MotionRow: Sendable, Equatable { + public var t: Double + public var value: SIMD3 + + public init(t: Double, value: SIMD3) { + self.t = t + self.value = value + } + } + + public static func arkitPoses(_ rows: [PoseRow], epoch: String) throws -> (csv: Data, rateHz: Double) { + guard !rows.isEmpty else { throw Packet04Error.streamEmpty("arkitPoses") } + var text = poseColumns.joined(separator: ",") + "\n" + var times: [Double] = [] + for row in rows where row.t > (times.last ?? -.infinity) { + guard PacketPose.isRigid(row.cameraToWorld) else { continue } + let p = PacketPose.translation(row.cameraToWorld) + let q = PacketPose.quaternion(PacketPose.rotation(row.cameraToWorld)) + let k = row.intrinsics + // The column is free text: a limited state whose reason the contract can't name is + // still limited, and the pose it came with is still a measurement. + let cells = [row.t.description, epoch, Packet04Producer.trackingText(row.tracking) ?? "limited"] + + [p.x, p.y, p.z].map { PacketNumber.double($0).description } + + [q.x, q.y, q.z, q.w].map(\.description) + + [k.x, k.y, k.z, k.w].map { PacketNumber.double($0).description } + text += cells.joined(separator: ",") + "\n" + times.append(row.t) + } + let rate = measuredRate(times) + guard rate >= minimumPoseHz else { throw Packet04Error.streamTooSlow(stream: "arkitPoses", measuredHz: rate, minimumHz: minimumPoseHz) } + return (Data(text.utf8), rate) + } + + /// Accelerometer and gyro rows for `imuRaw`. Core Motion times the two sensors separately, and + /// the contract's row holds both. Each row is an accelerometer sample paired with the gyro + /// sample nearest in time, only when that gyro sample is within `maxPairingOffset`; an + /// accelerometer sample with no gyro sample that close is left out rather than given one. + /// How the server wants the two clocks joined is an open question to the server team, so the + /// rule and its measured worst offset travel in the packet's `ext`. + public static func imuRaw( + accelerometer: [MotionRow], gyroscope: [MotionRow], maxPairingOffset: Double = 0.005 + ) throws -> (csv: Data, rateHz: Double, pairingNote: [String: String]) { + let accel = accelerometer.sorted { $0.t < $1.t } + let gyro = gyroscope.sorted { $0.t < $1.t } + guard !accel.isEmpty, !gyro.isEmpty else { throw Packet04Error.streamEmpty("imuRaw") } + var text = imuColumns.joined(separator: ",") + "\n" + var times: [Double] = [] + var worst = 0.0 + var g = 0 + for a in accel where a.t > (times.last ?? -.infinity) { + while g + 1 < gyro.count, abs(gyro[g + 1].t - a.t) <= abs(gyro[g].t - a.t) { g += 1 } + let offset = abs(gyro[g].t - a.t) + guard offset <= maxPairingOffset else { continue } + worst = max(worst, offset) + let cells = [a.t, a.value.x, a.value.y, a.value.z, gyro[g].value.x, gyro[g].value.y, gyro[g].value.z] + text += cells.map(\.description).joined(separator: ",") + "\n" + times.append(a.t) + } + guard !times.isEmpty else { throw Packet04Error.streamEmpty("imuRaw") } + let rate = measuredRate(times) + guard rate >= minimumIMUHz else { throw Packet04Error.streamTooSlow(stream: "imuRaw", measuredHz: rate, minimumHz: minimumIMUHz) } + let note = [ + "imuRawRowClock": "accelerometer", + "imuRawGyroPairing": "nearest gyro sample within \(maxPairingOffset) s; accelerometer samples without one are left out", + "imuRawGyroOffsetMaxS": worst.description, + "imuRawRowsKept": "\(times.count) of \(accel.count)", + "imuRawPairingStatus": "proposal from the phone; the server has not agreed how accelerometer and gyro samples join into one row", + ] + return (Data(text.utf8), rate, note) + } + + /// The first state and every change. A limited state with no name in the contract is not + /// listed; the rows around it still are. + public static func trackingChanges(_ rows: [PoseRow], epoch: String) -> [Packet04.TrackingEvent] { + var events: [Packet04.TrackingEvent] = [] + for row in rows { + guard let state = Packet04Producer.trackingText(row.tracking), state != events.last?.state else { continue } + events.append(.init(time: row.t, state: state, epoch: epoch)) + } + return events + } + + /// One sensor's samples exactly as Core Motion timed them, `t,x,y,z`: the raw evidence behind + /// `imuRaw`, kept because the pairing rule there is a proposal the server has not agreed to. + public static func rawMotionCSV(_ rows: [MotionRow], columns: [String]) throws -> Data { + guard !rows.isEmpty else { throw Packet04Error.streamEmpty(columns.joined(separator: ",")) } + var text = columns.joined(separator: ",") + "\n" + for row in rows.sorted(by: { $0.t < $1.t }) { + text += [row.t, row.value.x, row.value.y, row.value.z].map(\.description).joined(separator: ",") + "\n" + } + return Data(text.utf8) + } + + /// Pose rows from the recorder's raw rows: trajectory rows `[t, tracking state, tracking + /// reason, 16 camera-to-world values column by column]` and intrinsics rows `[t, fx, fy, cx, + /// cy, width, height]` written for the same frame at the same t. A pose with no intrinsics row + /// at its t is left out rather than given another frame's calibration. + public static func poseRows(trajectory: [[Double]], intrinsics: [[Double]], tracking: (Int, Int) -> PacketTracking) -> [PoseRow] { + var calibration: [Double: SIMD4] = [:] + for row in intrinsics where row.count >= 5 { calibration[row[0]] = SIMD4(Float(row[1]), Float(row[2]), Float(row[3]), Float(row[4])) } + return trajectory.compactMap { row in + guard row.count == 19, let k = calibration[row[0]] else { return nil } + let f = row[3...].map(Float.init) + let m = simd_float4x4(SIMD4(f[0], f[1], f[2], f[3]), SIMD4(f[4], f[5], f[6], f[7]), SIMD4(f[8], f[9], f[10], f[11]), SIMD4(f[12], f[13], f[14], f[15])) + return PoseRow(t: row[0], tracking: tracking(Int(row[1]), Int(row[2])), cameraToWorld: m, intrinsics: k) + } + } + + /// Motion rows from raw `[t, x, y, z]` rows. + public static func motionRows(_ rows: [[Double]]) -> [MotionRow] { + rows.filter { $0.count == 4 }.map { MotionRow(t: $0[0], value: SIMD3($0[1], $0[2], $0[3])) } + } + + /// 1 / the median interval between rows, the rate actually recorded. 0 with fewer than two. + public static func measuredRate(_ times: [Double]) -> Double { + let deltas = zip(times.dropFirst(), times).map { $0 - $1 }.filter { $0 > 0 }.sorted() + guard !deltas.isEmpty else { return 0 } + return 1 / deltas[deltas.count / 2] + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/PacketMarks.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/PacketMarks.swift index deb42bab3..64f097e99 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Packet/PacketMarks.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/PacketMarks.swift @@ -54,17 +54,20 @@ public struct PacketMark: Codable, Sendable, Equatable { public var kind: Kind /// Meter frame, meters. public var points: [SIMD3] - /// Uptime when it was marked. + /// Uptime when the homeowner marked it. An inferred wall end has none (`inferred`). public var t: Double? public var photoIDs: [String]? public var side: Side? public var endKind: EndKind? /// `attrs.operable` of an opening; nil when the homeowner was not asked. public var operable: Bool? + /// `attrs.inferred` of a wall end the app placed from the walk rather than one the homeowner + /// marked (`WallEndSource`). Written only when true. + public var inferred: Bool private init( id: String, kind: Kind, points: [SIMD3], t: Double?, photoIDs: [String]?, - side: Side? = nil, endKind: EndKind? = nil, operable: Bool? = nil + side: Side? = nil, endKind: EndKind? = nil, operable: Bool? = nil, inferred: Bool = false ) { self.id = id self.kind = kind @@ -74,6 +77,7 @@ public struct PacketMark: Codable, Sendable, Equatable { self.side = side self.endKind = endKind self.operable = operable + self.inferred = inferred } /// The meter itself: the meter frame's origin. @@ -81,13 +85,16 @@ public struct PacketMark: Codable, Sendable, Equatable { PacketMark(id: id, kind: .meter, points: [.zero], t: t, photoIDs: photoIDs) } - /// Where the wall was marked as ending, at `s` meters along `wall`'s chain. The point is on - /// the wall face at the meter's height, as S4's synthetic fixture places wall ends. + /// Where the wall ends, at `s` meters along `wall`'s chain. The point is on the wall face at + /// the meter's height, as S4's synthetic fixture places wall ends. `stamp` says whether the + /// homeowner marked it, with the time, or the app inferred it from the walk, with no time. public static func wallEnd( - id: String, side: Side, endKind: EndKind, s: Float, wall: SceneWall, frame: MeterFrame, t: Double? = nil + id: String, side: Side, endKind: EndKind, s: Float, wall: SceneWall, frame: MeterFrame, stamp: WallEndStamp ) -> PacketMark { let point = frame.point(on: wall, s: s, height: wall.meter.y - wall.groundY, out: 0) - return PacketMark(id: id, kind: .wallEnd, points: [point], t: t, photoIDs: nil, side: side, endKind: endKind) + return PacketMark( + id: id, kind: .wallEnd, points: [point], t: stamp.markedAt, photoIDs: nil, side: side, endKind: endKind, + inferred: stamp.isInferred) } /// A door, window or garage door: the bottom-left and top-right corners on the wall face, @@ -147,6 +154,8 @@ public struct PacketMark: Codable, Sendable, Equatable { if points.count != kind.pointCount { return "\(points.count) points, a \(kind.rawValue) has \(kind.pointCount)" } if !points.allSatisfy(PacketNumber.isFinite) { return "points must be finite" } if kind == .wallEnd, side == nil || endKind == nil { return "a wall end needs side and end_kind" } + if inferred, kind != .wallEnd { return "only a wall end can be inferred" } + if inferred, t != nil { return "an inferred wall end has no mark time: nobody marked it" } return nil } @@ -158,6 +167,7 @@ public struct PacketMark: Codable, Sendable, Equatable { private struct Attrs: Codable { var operable: Bool? + var inferred: Bool? } public init(from decoder: any Decoder) throws { @@ -174,7 +184,9 @@ public struct PacketMark: Codable, Sendable, Equatable { photoIDs = try c.decodeIfPresent([String].self, forKey: .photoIDs) side = try c.decodeIfPresent(Side.self, forKey: .side) endKind = try c.decodeIfPresent(EndKind.self, forKey: .endKind) - operable = try c.decodeIfPresent(Attrs.self, forKey: .attrs)?.operable + let attrs = try c.decodeIfPresent(Attrs.self, forKey: .attrs) + operable = attrs?.operable + inferred = attrs?.inferred ?? false } public func encode(to encoder: any Encoder) throws { @@ -186,7 +198,9 @@ public struct PacketMark: Codable, Sendable, Equatable { try c.encodeIfPresent(photoIDs, forKey: .photoIDs) try c.encodeIfPresent(side, forKey: .side) try c.encodeIfPresent(endKind, forKey: .endKind) - try c.encodeIfPresent(operable.map { Attrs(operable: $0) }, forKey: .attrs) + if operable != nil || inferred { + try c.encode(Attrs(operable: operable, inferred: inferred ? true : nil), forKey: .attrs) + } } } diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/WallEndProvenance.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/WallEndProvenance.swift new file mode 100644 index 000000000..d3c3001a7 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/WallEndProvenance.swift @@ -0,0 +1,64 @@ +import Foundation + +/// Whether the homeowner put a wall end where it is, or the app inferred it from the walk. +/// +/// Before, every end carried a mark time, so the packet listed ends the app placed as ends the +/// homeowner marked: "Can't get there" and "The wall keeps going" put the end where the walk +/// reached (`WalkedEnd`), and a past-end request met by walking past moved it on to the far edge +/// of what the request showed (`GapPlanner.endAfterPastEnd`). Field analysts read those as marks +/// (run3-2055, A41-2055-06). Either kind of end keeps its place and kind in scene.json and the +/// packet; only the packet says which it is (packet/README.md, `attrs.inferred`). +public enum WallEndSource: Sendable, Equatable { + /// "Wall ends here", at the circle or where the homeowner stands. + case homeowner + /// Placed from the walk: where it reached, or past what a past-end request showed. + case inferred +} + +/// A wall end's source and, for a homeowner's mark, when it was made. +public struct WallEndStamp: Sendable, Equatable { + public let source: WallEndSource + /// Uptime of the homeowner's mark. Nil for an inferred end, which nobody marked, and nil for + /// a mark made before the packet's clock started; absence never stands for a homeowner. + public let markedAt: Double? + + private init(source: WallEndSource, markedAt: Double?) { + self.source = source + self.markedAt = markedAt + } + + /// The homeowner marked the end at uptime `time`. + public static func marked(at time: Double?) -> WallEndStamp { + WallEndStamp(source: .homeowner, markedAt: time) + } + + /// The app placed the end from the walk; it has no mark time. + public static let inferred = WallEndStamp(source: .inferred, markedAt: nil) + + public var isInferred: Bool { source == .inferred } + + /// How a "mark the end" request closes once this end is set: met only when the homeowner + /// marked it. An end the walk inferred answers a different request ("Can't get there", which + /// closes as `cannot_reach` before the end is placed), so it never counts as the mark. + public var markEndOutcome: PacketGuidanceEntry.Outcome { + source == .homeowner ? .met : .superseded + } +} + +/// What becomes of the end a past-end request cleared when the request leaves the screen. +public enum PastEndSettlement: Sendable, Equatable { + /// The homeowner marked the end during the request: that mark stands, nearer or farther than + /// the cleared end, since the cleared end only said how far the walk had seen. + case keepMarked + /// The request was met by views and nobody marked the end: it moves on past the ground the + /// request showed, still unexplored, as an inferred end (`GapPlanner.endAfterPastEnd`). + case moveOn + /// The homeowner said "I can't get there": the cleared end comes back with its place, kind, + /// source and mark time, and nothing past it counts as seen. + case restore + + public static func when(endMarkedDuringRequest: Bool, met: Bool) -> PastEndSettlement { + if endMarkedDuringRequest { return .keepMarked } + return met ? .moveOn : .restore + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/FixtureCaptureHTTP.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/FixtureCaptureHTTP.swift new file mode 100644 index 000000000..be965ee27 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/FixtureCaptureHTTP.swift @@ -0,0 +1,271 @@ +import Foundation +import Synchronization + +/// The capture API, answered inside this process, for synthetic runs: the DEBUG app's +/// `-photoProcessingFixture` and the package tests. No request leaves the process, so nothing can +/// be uploaded anywhere. It takes what the uploader sends as the documented API does (create, +/// register with PUT targets, PUT, commit as the only acknowledgement, finalize, events, result) +/// and ends every capture with the answer it was made with. Each answer's words say they come +/// from a fixture. +/// +/// Written from the public API types in `CaptureAPI` and `CaptureResult`; it is not the service, +/// and an answer from it says nothing about any wall. The type is compiled into every build of +/// HouseScanKit; only the app's activation of it is limited to debug builds on a replay +/// (`CaptureFixtureLaunch`). +public final class FixtureCaptureHTTP: CaptureHTTP, Sendable { + public enum Answer: String, Sendable, CaseIterable { + case candidate, needsViews, manualReview, notEligible, failed, expired + /// The run finishes, but its answer never becomes readable. + case notReady + /// The result read gets a body that isn't a result. + case unreadable + /// The service refuses to open the capture. + case refuseCreate + /// The run never finishes, so the capture stays in processing. + case hold + /// The result names another run than the one finalize started, which the uploader refuses. + case wrongRun + + /// The homeowner message the answer carries, when it has an outcome. + public var message: String? { + switch self { + case .candidate: "Fixture answer: a possible spot to the left of the meter." + case .needsViews: "Fixture answer: one more view of the wall, please." + case .manualReview: "Fixture answer: an installer needs to look at this wall." + case .notEligible: "Fixture answer: no spot on this wall." + case .wrongRun: "Fixture answer: this answer is for another run." + case .failed, .expired, .notReady, .unreadable, .refuseCreate, .hold: nil + } + } + + /// The capture's status once every listed file is committed. + var endStatus: String { + switch self { + case .candidate, .notEligible, .notReady, .unreadable, .wrongRun: "complete" + case .needsViews: "needs_views" + case .manualReview: "manual_review" + case .failed: "failed" + case .expired: "expired" + case .refuseCreate, .hold: "processing" + } + } + } + + /// The API base the fixture answers for. A `.invalid` host can't resolve, so a request that + /// somehow reached a real transport would fail rather than land somewhere. + public static let base = URL(string: "https://capture-fixture.invalid/v1")! + + /// The ARKit epoch a candidate's box is in: the name the coordinator gives every packet's + /// world (`CaptureSessionCoordinator.epoch`), which is main-actor isolated. + public static let arkitEpoch = "e1" + + private struct Capture { + var id: String + var packetID: String + var uploaded: Set = [] + var committed: Set = [] + var listed: [String]? + var runID: String? + + var allCommitted: Bool { listed.map { $0.allSatisfy(committed.contains) } ?? false } + } + + private struct State { + var captures: [String: Capture] = [:] + var byPacket: [String: String] = [:] + var tokens: [String: (capture: String, path: String)] = [:] + var routes: [String] = [] + var next = 1 + } + + public let answer: Answer + private let state = Mutex(State()) + private let onRequest: @Sendable ([String]) -> Void + + /// `onRequest` hears every route so far each time a request arrives, with the fixture's lock + /// held, so calls never overlap. It must not call back into the fixture. + public init(answer: Answer, onRequest: @escaping @Sendable ([String]) -> Void = { _ in }) { + self.answer = answer + self.onRequest = onRequest + } + + /// Every request so far as "METHOD route", ids left out: "POST captures", "PUT upload", + /// "POST captures/files:commit", "GET captures/result". + public var routes: [String] { state.withLock { $0.routes } } + + public func send(_ request: URLRequest) async throws -> HTTPReply { + reply(to: request) + } + + public func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + // The bytes stay where they are; a file that isn't there fails as a real upload would. + guard FileManager.default.fileExists(atPath: file.path) else { throw URLError(.fileDoesNotExist) } + return reply(to: request) + } + + // MARK: Routing + + private func reply(to request: URLRequest) -> HTTPReply { + let method = request.httpMethod ?? "GET" + let url = request.url ?? Self.base + var parts = url.path.split(separator: "/").map(String.init) + if parts.first == "v1" { parts.removeFirst() } + let route = Self.route(method: method, parts: parts) + state.withLock { s in + s.routes.append(route) + onRequest(s.routes) + } + let body = request.httpBody ?? Data() + return state.withLock { s in + if parts.first == "upload" { return put(token: parts.last ?? "", &s) } + if method == "POST", parts == ["captures"] { return create(body, &s) } + guard parts.count >= 3, parts[0] == "captures", var capture = s.captures[parts[1]] else { return error(404, "capture_not_found") } + defer { s.captures[capture.id] = capture } + switch (method, parts[2...].joined(separator: "/")) { + case ("POST", "files"): return register(body, &capture, &s) + case ("POST", "files:commit"): return commit(body, &capture) + case ("POST", "finalize"): return finalize(body, &capture, &s) + case ("GET", "events"): return events(url, capture) + case ("GET", "result"): return result(capture) + default: return error(404, "route") + } + } + } + + static func route(method: String, parts: [String]) -> String { + if parts.first == "upload" { return "\(method) upload" } + if parts.count <= 2 { return "\(method) captures" } + return "\(method) captures/\(parts[2...].joined(separator: "/"))" + } + + // MARK: API + + private func create(_ body: Data, _ s: inout State) -> HTTPReply { + guard let request = try? JSONDecoder().decode(CaptureAPI.CreateRequest.self, from: body) else { return error(422, "schema") } + if answer == .refuseCreate { return error(422, "fixture_refused") } + let id: String + let status: Int + if let existing = s.byPacket[request.packetId] { + (id, status) = (existing, 200) + } else { + id = "cap_fixture_\(s.next)" + s.next += 1 + s.captures[id] = Capture(id: id, packetID: request.packetId) + s.byPacket[request.packetId] = id + status = 201 + } + return json(status, ["captureId": id, "status": "uploading", "upload": ["maxBatch": 50]]) + } + + private func register(_ body: Data, _ c: inout Capture, _ s: inout State) -> HTTPReply { + guard let request = try? JSONDecoder().decode(CaptureAPI.RegisterRequest.self, from: body) else { return error(422, "schema") } + var files: [[String: Any]] = [] + for file in request.files { + if c.committed.contains(file.path) { + files.append(["path": file.path, "state": "committed"]) + continue + } + let token = UUID().uuidString + s.tokens[token] = (c.id, file.path) + files.append([ + "path": file.path, "state": "pending", + "upload": [ + "method": "PUT", "url": "https://capture-fixture.invalid/upload/\(token)", + "headers": ["Content-Type": file.contentType], "expiresAt": "2099-01-01T00:00:00Z", + ], + ]) + } + return json(200, ["files": files]) + } + + private func put(token: String, _ s: inout State) -> HTTPReply { + guard let target = s.tokens[token], var capture = s.captures[target.capture] else { return HTTPReply(status: 403, body: Data()) } + capture.uploaded.insert(target.path) + s.captures[capture.id] = capture + return HTTPReply(status: 200, body: Data()) + } + + private func commit(_ body: Data, _ c: inout Capture) -> HTTPReply { + guard let request = try? JSONDecoder().decode(CaptureAPI.CommitRequest.self, from: body) else { return error(422, "schema") } + let paths = request.files.map(\.path) + let committed = paths.filter(c.uploaded.contains) + c.committed.formUnion(committed) + return json(200, ["committed": committed, "notFound": paths.filter { !c.uploaded.contains($0) }, "mismatch": [String]()]) + } + + private func finalize(_ body: Data, _ c: inout Capture, _ s: inout State) -> HTTPReply { + guard let packet = try? JSONDecoder().decode(Packet04.Packet.self, from: body) else { return error(422, "schema") } + guard packet.packetId == c.packetID else { return error(422, "packet_id_mismatch") } + if c.runID == nil { + c.listed = packet.files.map(\.path) + c.runID = "run_fixture_\(s.next)" + s.next += 1 + } + let missing = (c.listed ?? []).filter { !c.committed.contains($0) } + return json(202, ["status": missing.isEmpty ? "processing" : "awaiting_files", "missing": missing, "runId": c.runID ?? ""]) + } + + private func events(_ url: URL, _ c: Capture) -> HTTPReply { + let after = URLComponents(url: url, resolvingAgainstBaseURL: false)?.queryItems?.first(where: { $0.name == "after" })?.value.flatMap(Int.init) ?? 0 + guard let runID = c.runID else { return json(200, ["status": "uploading", "next": after, "events": [Any]()]) } + guard c.allCommitted, answer != .hold else { + return json(200, ["status": c.allCommitted ? "processing" : "awaiting_files", "next": after, "events": [Any]()]) + } + let ended = answer.endStatus == "failed" || answer.endStatus == "expired" + let event: [String: Any] = [ + "seq": 1, "type": ended ? "failed" : "verdict_ready", "at": "2026-01-01T00:00:00Z", "data": ["runId": runID], + ] + let list: [[String: Any]] = after < 1 ? [event] : [] + return json(200, ["status": answer.endStatus, "next": 1, "events": list]) + } + + private func result(_ c: Capture) -> HTTPReply { + guard let runID = c.runID, c.allCommitted, answer != .hold else { + return json(200, ["runId": c.runID ?? "", "status": "processing", "viewsNeeded": [Any](), "memberActions": [Any](), "outcome": NSNull()]) + } + switch answer { + case .notReady: return error(404, "result_not_ready") + case .unreadable: return HTTPReply(status: 200, body: Data("not a result".utf8)) + default: break + } + let views: [[String: Any]] = answer == .needsViews ? [[ + "id": "view_fixture_1", "kind": "wall_band", "criteria": [String](), "why": "unseen", "promptId": "prompt_fixture_1", + "prompt": ["title": "Fixture: show the ground below the meter", "body": "Fixture: step back until the ground below the meter is in view."], + ]] : [] + var outcome: Any = NSNull() + if let message = answer.message { + let kind = switch answer { + case .candidate: "eligible" + case .needsViews: "needs_more_photos" + case .manualReview: "manual_review" + default: "not_eligible" + } + var decided: [String: Any] = ["kind": kind, "profile": "fixture", "message": message, "viewsNeeded": [Any](), "reasons": [Any]()] + if answer == .candidate { + // A box in the packet's ARKit world, so AR is withheld for the right reasons + // (analysis nobody verified) rather than for a missing placement. + let identity: [Double] = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, -0.9, 0.6, 0.2, 1] + decided["arkitEpoch"] = Self.arkitEpoch + decided["recommendedPlacement"] = [ + "wallId": "wall_fixture", "startSM": -1.2, "confidence": 0.5, + "boxArkitWorld": ["pose": identity, "size": [0.7, 1.2, 0.3]], + ] + } + outcome = decided + } + return json(200, [ + "runId": answer == .wrongRun ? "run_fixture_other" : runID, "status": answer.endStatus, "viewsNeeded": views, "memberActions": [Any](), "outcome": outcome, + ]) + } + + // MARK: Plumbing + + private func json(_ status: Int, _ object: [String: Any]) -> HTTPReply { + let body = (try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys])) ?? Data() + return HTTPReply(status: status, body: body) + } + + private func error(_ status: Int, _ code: String) -> HTTPReply { + json(status, ["errors": [["code": code]]]) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/MeterTapObservation.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/MeterTapObservation.swift new file mode 100644 index 000000000..0dcd59fb5 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/MeterTapObservation.swift @@ -0,0 +1,58 @@ +import Foundation +import simd + +/// Why the homeowner's accepted meter mark gave the capture packet no tap. Each ends the packet as +/// not prepared rather than letting it go out without its tap (`PhotoProcessingController.meterTapFailed`). +public enum MeterTapFailure: String, Error, Sendable, Equatable, CaseIterable { + /// ARKit had no current frame to take. + case noFrame + /// The frame's tracking wasn't normal, so its pose can't place the tap. + case trackingLimited + /// The raycast's hit is behind that frame's camera or outside its image. + case hitOutsideImage + /// The frame's image couldn't be encoded as a JPEG. + case encodeFailed +} + +/// One ARFrame's own camera, read once from that frame: its time, raw pose, intrinsics and image +/// size in pixels of the unrotated landscape sensor image, and its tracking. +public struct MeterTapFrame: Sendable, Equatable { + public var t: Double + public var cameraToWorld: simd_float4x4 + /// [fx, fy, cx, cy]: ARKit's pinhole K, focal lengths and principal point in pixels from the + /// image's top-left corner. + public var intrinsics: SIMD4 + public var width: Int + public var height: Int + public var tracking: PacketTracking + + public init(t: Double, cameraToWorld: simd_float4x4, intrinsics: SIMD4, width: Int, height: Int, tracking: PacketTracking) { + self.t = t + self.cameraToWorld = cameraToWorld + self.intrinsics = intrinsics + self.width = width + self.height = height + self.tracking = tracking + } + + /// The tap on this frame: the raycast's world hit projected through this frame's own camera + /// (`TapObservation.pointing(at:)`), so pixel, pose and intrinsics all belong to one frame. + /// `jpeg` returns this frame's image, already encoded. + public func observation(hit: SIMD3, jpeg: @escaping @Sendable () -> Data?) -> Result { + guard tracking == .normal else { return .failure(.trackingLimited) } + let unaimed = TapObservation( + t: t, cameraToWorld: cameraToWorld, intrinsics: intrinsics, width: width, height: height, tracking: tracking, + pixel: SIMD2(Double(width) / 2, Double(height) / 2), jpeg: jpeg) + guard let aimed = unaimed.pointing(at: hit) else { return .failure(.hitOutsideImage) } + return .success(aimed) + } + + /// What the packet records about the raycast's hit, its distance measured from this frame's + /// camera. + public func tapHit(_ hit: SIMD3, estimatedPlane: Bool) -> Packet04.TapHit { + let camera = SIMD3(cameraToWorld.columns.3.x, cameraToWorld.columns.3.y, cameraToWorld.columns.3.z) + return Packet04.TapHit( + position: [hit.x, hit.y, hit.z].map(Double.init), target: estimatedPlane ? "estimatedPlane" : "existingPlaneGeometry", + alignment: "vertical", distance: Double(simd_distance(hit, camera))) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoCaptureBinding.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoCaptureBinding.swift new file mode 100644 index 000000000..404b9e11a --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoCaptureBinding.swift @@ -0,0 +1,81 @@ +import Foundation + +/// What a photo-processing answer has to match before the app shows it, taken when the scan's +/// capture ended. +/// +/// This is a local association, not a binding the service declares. The answer +/// (`CaptureResult.Response`) names only its run; the session, packet and capture it is held +/// against are this phone's own record of what it sent, and the service returns no digest of the +/// input it processed and no attestation. So a match means "the answer the uploader fetched for +/// the capture this scan sent, for the run that capture started", not "the service says it read +/// these bytes". Until the capture API states its input, photo processing can't be called +/// accepted for live use. +public struct PhotoCaptureBinding: Sendable, Equatable { + public var spatial: SpatialSession + public var origin: URL + /// The id of the coordinator session whose packet was frozen (`CaptureSessionCoordinator.Session.localID`). + public var captureSessionID: String + public var packetID: String + /// The ARKit epoch name the packet used (`CaptureSessionCoordinator.epoch`). + public var epoch: String + + public init(spatial: SpatialSession, origin: URL, captureSessionID: String, packetID: String, epoch: String) { + self.spatial = spatial + self.origin = origin + self.captureSessionID = captureSessionID + self.packetID = packetID + self.epoch = epoch + } +} + +/// The scan on screen when an answer arrives. Nil fields mean there is nothing of that kind now. +public struct PhotoCaptureObservation: Sendable, Equatable { + public var spatial: SpatialSession? + public var origin: URL? + public var captureSessionID: String? + public var packetID: String? + /// The server's capture and run for the upload now, from its latest status. + public var captureID: String? + public var runID: String? + + public init( + spatial: SpatialSession?, origin: URL?, captureSessionID: String?, packetID: String?, captureID: String?, runID: String? + ) { + self.spatial = spatial + self.origin = origin + self.captureSessionID = captureSessionID + self.packetID = packetID + self.captureID = captureID + self.runID = runID + } +} + +/// The first identity an answer and the scan on screen disagree on. All but `run` are compared +/// with the phone's own records (`PhotoCaptureBinding`). +public enum PhotoBindingMismatch: Sendable, Equatable, CaseIterable { + /// Another ARKit world, or none: an answer for a world that is gone. + case world + /// Another destination than the one the scan's profile sends to. + case origin + /// Another coordinator session than the one whose packet was frozen. + case session + case packet + case capture + case run + case epoch +} + +extension PhotoCaptureBinding { + /// The first thing `record` and the scan on screen disagree on, or nil when the answer is this + /// capture's. + public func mismatch(_ record: CaptureResult.Record, observed: PhotoCaptureObservation) -> PhotoBindingMismatch? { + guard observed.spatial == spatial else { return .world } + guard observed.origin == origin else { return .origin } + guard observed.captureSessionID == captureSessionID, record.association.sessionID == captureSessionID else { return .session } + guard observed.packetID == packetID else { return .packet } + guard let captureID = observed.captureID, record.association.captureID == captureID else { return .capture } + guard let runID = observed.runID, record.association.runID == runID, record.response.runId == runID else { return .run } + guard record.association.epoch == epoch else { return .epoch } + return nil + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift new file mode 100644 index 000000000..4b6a5e38b --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift @@ -0,0 +1,355 @@ +import Foundation +import simd + +/// Photo processing for the scan under way. The app's capture events go to the process's one +/// `CaptureSessionCoordinator`, and its status and result come back as one +/// `PhotoProcessingStatus`, checked against the scan they belong to. +/// +/// Each photo-processing scan starts a new coordinator session (`beginScan`), which asks for +/// consent again. Start over, or a Legacy scan starting, ends it (`endScan`); a world reset starts +/// the scan's next packet with the same answer (`worldReset`). The coordinator drops status and +/// answers from a session that ended, and `PhotoCaptureBinding` refuses an answer whose run, or +/// whose local association (the recording's world, the coordinator session, packet and capture +/// the phone sent), isn't this scan's. The service declares no input digest, so this is the +/// phone's association, not proof of what the service read. +/// +/// A capture saved on disk is never resumed from here. Folder ownership lasts only this process, +/// so a relaunch asks for consent again rather than act on a yes it can't prove. +@MainActor +public final class PhotoProcessingController { + public enum Setup: Sendable { + /// Nothing can be sent, for the reason given. + case notSetUp(String) + /// Captures go to `environment`, which must allow sending. `standIn`: a stand-in on the + /// phone answers, such as the DEBUG capture fixture. + case ready(CaptureSessionCoordinator.Environment, standIn: Bool) + } + + private let setup: Setup + private let coordinator: CaptureSessionCoordinator? + private let log: @Sendable (String) -> Void + private var begun = false + /// The world the current session records, so the session can be ended in place + /// (`capturePreparationFailed`). + private var recording: RecordingSource? + /// The photo-processing scan under way; nil when there is none. + public private(set) var context: ScanContext? + public private(set) var status: PhotoProcessingStatus? { + didSet { if status != oldValue { onChange?(status) } } + } + public var onChange: (@MainActor (PhotoProcessingStatus?) -> Void)? + /// The meter anchor's pose now, for the AR check on an answer. Nil while there is none. + public var meterAnchor: @MainActor () -> simd_float4x4? = { nil } + private var binding: PhotoCaptureBinding? + private var upload: CaptureUploadStatus? + private var captureEnded = false + + /// What became of the current packet's meter tap. Every packet needs its own: a world reset + /// starts a packet with none. + public enum MeterTap: Sendable, Equatable { + case none + case forwarded + case failed(MeterTapFailure) + } + + public private(set) var meterTap: MeterTap = .none + + /// Whether the current packet still wants its meter tap: it is being captured, the homeowner + /// hasn't said no, and no tap result came yet. + public var wantsMeterTap: Bool { acceptsCapture && meterTap == .none } + + public init(setup: Setup) { + if case .ready(let environment, _) = setup, !environment.sends { + self.setup = .notSetUp("the capture endpoint may not receive this scan") + } else { + self.setup = setup + } + if case .ready(let environment, _) = self.setup { + coordinator = CaptureSessionCoordinator(environment: environment) + log = environment.log + } else { + coordinator = nil + log = { _ in } + } + coordinator?.onStatus = { [weak self] in self?.uploadChanged($0) } + coordinator?.onResult = { [weak self] in self?.answered($0) } + } + + /// The profile a photo-processing scan starting now gets. + public var profile: ProcessingProfile { + switch setup { + case .notSetUp(let reason): + ProcessingProfile(backend: .photoProcessing, answers: .notSetUp(reason), origin: nil) + case .ready(let environment, let standIn): + ProcessingProfile(backend: .photoProcessing, answers: standIn ? .standIn : .service, origin: environment.endpoint) + } + } + + /// The current capture's folder on the phone, while a session is open. + public var captureFolder: URL? { coordinator?.session?.folder } + + // MARK: Scan events + + /// A scan that uses photo processing starts, recording into `recording`'s world. + public func beginScan(_ context: ScanContext, recording: RecordingSource) { + precondition(context.profile.backend == .photoProcessing, "a Legacy scan has no photo processing") + // Whatever the last scan left open ends here, whether or not this one can send. + startSession("new scan", recording: recording, newScan: true) + self.context = context + resetCapture() + guard coordinator != nil, context.profile == profile, case .ready(_, let standIn) = setup else { + let reason: String = switch context.profile.answers { + case .notSetUp(let why): why + case .service, .standIn: "the scan's processing setup is not this build's" + } + status = PhotoProcessingStatus(stage: .ended(.notSetUp(reason)), consent: .notNeeded, standIn: false) + return + } + status = PhotoProcessingStatus(stage: .capturing(sent: 0), consent: .asking, standIn: standIn) + } + + /// Start over, or a Legacy scan starting: the photo-processing scan's session ends, and + /// nothing it sent or receives later reaches the screen. `recording` is the next world's. + public func endScan(recording: RecordingSource) { + if begun { coordinator?.newWorld("start over", recording: recording, newScan: true) } + context = nil + resetCapture() + status = nil + } + + /// The scan's ARKit world was thrown away: its packet can't be finished, and the next one is + /// recorded in `next`'s world under the same consent. A scan whose processing already ended + /// (not set up, consent withdrawn) stays ended. + public func worldReset(_ next: ScanContext, recording: RecordingSource) { + guard let current = context, next.scanID == current.scanID, next.profile == current.profile else { return } + context = next + resetCapture() + guard var state = status, !state.isFinal else { return } + startSession("world reset", recording: recording, newScan: false) + state.stage = .capturing(sent: 0) + state.retrying = false + status = state + } + + public func answerConsent(_ yes: Bool) { + guard context != nil, let coordinator, var state = status, state.consent == .asking else { return } + // A yes after this packet lost its tap: the packet can't be sent, so it ends now as not + // prepared, and the coordinator never hears the yes, so no upload starts for it. + if yes, case .failed = meterTap { + state.consent = .granted + status = state + capturePreparationFailed() + return + } + // Before any yes there is no upload to withdraw, so a no can't fail here. + coordinator.answerConsent(yes) + state.consent = yes ? .granted : .declined + status = state + } + + /// A no after a yes: nothing more from this scan is sent, and the scan can't be processed + /// again. The stage says whether the phone saved the no. + public func stopSending() { + guard let coordinator, var state = status, state.consent == .granted, !state.isFinal else { return } + let recorded = switch coordinator.answerConsent(false) { + case .success: true + case .failure: false + } + state.consent = .withdrawn(recorded: recorded) + state.stage = .ended(.withdrawn(recorded: recorded)) + state.retrying = false + status = state + } + + public func kept(_ photo: KeptPhoto) { + guard acceptsCapture else { return } + coordinator?.kept(photo) + } + + /// The homeowner's accepted meter mark, as the frame it was taken on saw it, for the scan in + /// `scan` as it stood at the mark. Nothing is forwarded when that scan or its world is gone: + /// a tap from an old world belongs to a packet that no longer exists. `tap`'s image is already + /// encoded, so the coordinator's own read of it can't come back empty. + public func meterTapped(_ tap: TapObservation, hit: Packet04.TapHit?, scan: ScanContext) { + guard scan == context, meterTap == .none, acceptsCapture else { + log("photo processing: a meter tap for another scan, world or packet was dropped") + return + } + meterTap = .forwarded + coordinator?.meterTapped(tap, hit: hit) + } + + /// The accepted mark gave no tap, for the scan in `scan` as it stood at the mark. The packet + /// can't be sent without it: with a yes it ends as not prepared now, so photos of a packet that + /// can't be finished stop going up; before an answer the failure is kept and a later yes ends + /// it the same way; a no or a withdrawal keeps its own end. Dropped when that scan or world is + /// gone, so it never ends a newer packet. + public func meterTapFailed(_ reason: MeterTapFailure, scan: ScanContext) { + guard scan == context, meterTap == .none, acceptsCapture else { + log("photo processing: a meter tap failure for another scan, world or packet was dropped") + return + } + meterTap = .failed(reason) + log("photo processing: the meter mark gave no tap (\(reason.rawValue)); the capture can't be prepared") + if status?.consent == .granted { capturePreparationFailed() } + } + + /// The scan was sent: the packet is frozen and processing starts, if the homeowner said yes. + /// `acceptedCloseUpAt` is the frame time of the close-up the scan accepted, nil after a skip. + public func captureEnded(acceptedCloseUpAt: Double?) { + guard let context, var state = status, !captureEnded, case .capturing = state.stage else { return } + if state.consent == .granted, case .failed = meterTap { + capturePreparationFailed() + return + } + captureEnded = true + switch state.consent { + case .granted: + guard let coordinator, let session = coordinator.session, session.uploader != nil, let origin = profile.origin else { + state.stage = .ended(.notPrepared) + status = state + return + } + binding = PhotoCaptureBinding( + spatial: context.spatial, origin: origin, captureSessionID: session.localID, packetID: session.packetID, + epoch: CaptureSessionCoordinator.epoch) + coordinator.captureEnded(acceptedCloseUpAt: acceptedCloseUpAt) + if let upload { + state.stage = .progress(upload, captureEnded: true) + } else { + state.stage = .queued + } + case .asking, .declined: + state.stage = .ended(.consentNotGiven) + case .withdrawn(let recorded): + state.stage = .ended(.withdrawn(recorded: recorded)) + case .notNeeded: + return + } + status = state + } + + /// The scan was sent, but the phone couldn't finish saving every photo it kept before the send, + /// so the capture would leave one out: it is never sealed, and the scan ends as not prepared. + /// + /// The capture's coordinator session ends too, the way Start over ends one (`newWorld` with a + /// new scan): its uploader stops sending and is abandoned, so it starts no new request and + /// retries nothing, and anything it reports later never reaches the screen. A request already + /// on its way may still be answered by the service; photos already uploaded stay there. The + /// empty session that replaces it has no consent, so it sends nothing. + public func capturePreparationFailed() { + guard context != nil, var state = status, !captureEnded, case .capturing = state.stage else { return } + captureEnded = true + switch state.consent { + case .granted: state.stage = .ended(.notPrepared) + case .asking, .declined: state.stage = .ended(.consentNotGiven) + case .withdrawn(let recorded): state.stage = .ended(.withdrawn(recorded: recorded)) + case .notNeeded: return + } + state.retrying = false + status = state + if begun, let recording { coordinator?.newWorld("capture not prepared", recording: recording, newScan: true) } + } + + /// Waits until the current session's queued work and uploads are idle. For tests. + public func settle() async { + await coordinator?.settle() + } + + // MARK: Coordinator events + + private var acceptsCapture: Bool { + guard context != nil, !captureEnded, let status, case .capturing = status.stage else { return false } + return status.consent == .asking || status.consent == .granted + } + + private func startSession(_ reason: String, recording: RecordingSource, newScan: Bool) { + guard let coordinator else { return } + self.recording = recording + if begun { + coordinator.newWorld(reason, recording: recording, newScan: newScan) + } else { + coordinator.begin(recording: recording) + begun = true + } + } + + private func resetCapture() { + binding = nil + upload = nil + captureEnded = false + meterTap = .none + } + + /// The coordinator reports only its current session; nil means that session ended, and + /// whoever ended it has already set the stage. + private func uploadChanged(_ next: CaptureUploadStatus?) { + guard let next, context != nil else { return } + upload = next + guard var state = status, !state.isFinal else { return } + state.retrying = next.retryingAt != nil + switch next.phase { + case .creating, .uploading, .processing: + state.stage = .progress(next, captureEnded: captureEnded) + case .finished: + // The answer follows through `answered`, once the coordinator has decoded it. + state.stage = .processing + case .abandoned: + // Start over and world resets end the session before its uploader stops, so an + // abandoned upload still current is the packet the phone couldn't finish. + guard captureEnded else { return } + state.stage = .ended(.notPrepared) + state.retrying = false + case .failed: + readRefusal() + return + } + status = state + } + + private func readRefusal() { + guard let session = coordinator?.session, let uploader = session.uploader else { return } + Task { [weak self] in + let end = await uploader.snapshot.end + // A withdrawal already set its own end, which a refusal read afterwards never replaces. + guard let self, self.coordinator?.session === session, var state = self.status, !state.isFinal, + case .failed(let step, let codes, let status)? = end else { return } + state.stage = .refusal(step: step, codes: codes, status: status) + if case .ended(.setupRefused(_, .storageAuthorization)) = state.stage { + self.log("photo processing: a service or contract setup failure: the service signed a storage upload carrying Authorization (step \(step))") + } else if case .ended(.setupRefused(_, let problem)) = state.stage { + self.log("photo processing: refused at the credential boundary, \(problem.code) (step \(step))") + } else if case .ended(.uploadStateUnsaved) = state.stage { + self.log("photo processing: this phone couldn't save the upload's progress; sending stopped") + } else if case .ended(.notPrepared) = state.stage { + self.log("photo processing: the phone lost a photo or tap the scan accepted (step \(step)); the capture can't be prepared") + } + state.retrying = false + self.status = state + } + } + + private func answered(_ record: CaptureResult.Record) { + guard let binding, var state = status, !state.isFinal, let session = coordinator?.session else { return } + let observed = PhotoCaptureObservation( + spatial: context?.spatial, origin: profile.origin, captureSessionID: session.localID, packetID: session.packetID, + captureID: upload?.captureID, runID: upload?.runID) + if let mismatch = binding.mismatch(record, observed: observed) { + // An answer for a world that is gone was never this scan's to show or to end. + guard mismatch != .world else { return } + state.stage = .ended(.answerMismatch(mismatch)) + } else { + let world = CaptureResult.Association( + sessionID: session.localID, captureID: upload?.captureID ?? "", runID: upload?.runID ?? "", epoch: binding.epoch) + state.stage = .answer(record, world: world, meterAnchor: meterAnchor(), parent: amendmentParent(record, session: session)) + } + state.retrying = false + status = state + } + + private func amendmentParent(_ record: CaptureResult.Record, session: CaptureSessionCoordinator.Session) -> PhotoAmendmentParent? { + guard let context, let captureID = upload?.captureID, let runID = upload?.runID else { return nil } + let views = record.response.viewsNeeded.isEmpty ? record.response.outcome?.viewsNeeded ?? [] : record.response.viewsNeeded + return PhotoAmendmentParent(captureID: captureID, profile: context.profile, runID: runID, viewIDs: views.map(\.id), folder: session.folder) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift new file mode 100644 index 000000000..3ae95d264 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift @@ -0,0 +1,252 @@ +import Foundation +import simd + +/// Photo processing's side of one scan, as the screens show it: separate facts, never a +/// percentage or a time estimate, and nothing the service didn't say. +public struct PhotoProcessingStatus: Sendable, Equatable { + public enum Consent: Sendable, Equatable { + /// Not answered yet. Nothing is sent before a yes. + case asking + case granted + case declined + /// A no after a yes. `recorded` is false when sending stopped in this process but the phone + /// couldn't save the no (`CaptureSessionCoordinator.ConsentWithdrawalError.notRecorded`). + case withdrawn(recorded: Bool) + /// This scan can't send anything, so nobody is asked. + case notNeeded + } + + public enum Stage: Sendable, Equatable { + /// The scan is still being taken. `sent` counts files the service has acknowledged. + case capturing(sent: Int) + /// The capture ended and the service hasn't opened it yet. + case queued + /// Files still going up: `received` of `kept` acknowledged by the service. + case uploading(received: Int, kept: Int) + /// Every file is in and the service is working on it. + case processing + case answered(PhotoProcessingAnswer) + case ended(PhotoProcessingEnd) + } + + public var stage: Stage + public var consent: Consent + /// A stand-in on the phone answers rather than a service (`ProcessingProfile.Answers.standIn`). + public var standIn: Bool + /// The upload is waiting to try again after a failure a later try may fix. + public var retrying: Bool + + public init(stage: Stage, consent: Consent, standIn: Bool, retrying: Bool = false) { + self.stage = stage + self.consent = consent + self.standIn = standIn + self.retrying = retrying + } + + /// Nothing more will change for this scan. + public var isFinal: Bool { + switch stage { + case .answered, .ended: true + case .capturing, .queued, .uploading, .processing: false + } + } +} + +/// What the service answered. Every answer is provisional: an installer confirms any spot on site. +public enum PhotoProcessingAnswer: Sendable, Equatable { + /// The service proposed a spot ("eligible"), which is still only a candidate. + case candidate(Candidate) + /// The service wants more views. This build can't add photos to a capture. + case needsMorePhotos(NeedsMorePhotos) + /// The service sends the scan to an installer. + case installerReview(message: String) + /// The service found no spot. + case noCandidate(message: String) + /// An outcome this build doesn't know. Its message is shown as written. + case unrecognized(kind: String, message: String) + + public struct Candidate: Sendable, Equatable { + /// The service's words for the homeowner, verbatim. + public var message: String + /// Whether the service made a preview image. This build shows none either way. + public var previewMade: Bool + public var ar: ARAvailability + + public init(message: String, previewMade: Bool, ar: ARAvailability) { + self.message = message + self.previewMade = previewMade + self.ar = ar + } + } + + public struct NeedsMorePhotos: Sendable, Equatable { + /// Each requested view's prompt, verbatim. + public var prompts: [CaptureResult.Prompt] + /// The service's words for the homeowner, when it sent an outcome. + public var message: String? + /// What a follow-up capture would add to; nil when the upload had no capture or run yet. + public var parent: PhotoAmendmentParent? + + public init(prompts: [CaptureResult.Prompt], message: String?, parent: PhotoAmendmentParent?) { + self.prompts = prompts + self.message = message + self.parent = parent + } + } + + /// Whether the spot can be shown on the wall in AR. + public enum ARAvailability: Sendable, Equatable { + /// The result failed a check AR needs (`CaptureResult.Record.placement`), such as analysis + /// nobody verified, or another session, run or epoch than the recording's. + case withheld(CaptureResult.Unavailable) + /// Every check passed, but this build has no AR view for photo-processing results. + case notShownInThisBuild + } +} + +/// How photo processing ended without an answer to show. +public enum PhotoProcessingEnd: Sendable, Equatable { + /// This build can't send the scan for photo processing, for the reason given. Nothing was sent. + case notSetUp(String) + /// The homeowner didn't agree to send the photos, so none went. + case consentNotGiven + /// The homeowner stopped sending part way. `recorded`: whether the phone saved that. + case withdrawn(recorded: Bool) + /// The phone couldn't build a capture the service can take: the packet failed its checks on + /// the phone, or no photo was kept. + case notPrepared + /// The service's run failed. + case processingFailed + /// The capture expired before it finished. + case expired + /// The service refused a request at `step`, which sending the same thing again won't change. + case refused(step: String) + /// The phone refused to send a request at `step` because this build's connection to the + /// service isn't set up right (`ScopedCaptureHTTPError`): nothing about the scan caused it, and + /// sending again won't change it. + case setupRefused(step: String, PhotoSetupProblem) + /// This phone couldn't save the upload's progress (`CaptureUploader.unsavedStateCode`, step + /// `save`, status 0), so the uploader stopped sending from this scan. A local problem: no + /// server refused anything, and what the service already has may still be processed. + case uploadStateUnsaved + /// The service answered with something this build can't read as a result. + case answerUnreadable + /// The run finished, but its answer wasn't readable before the upload stopped asking. + case answerNotReady + /// The answer belongs to another capture than this scan's, so it isn't shown. + case answerMismatch(PhotoBindingMismatch) +} + +/// The credential boundary's refusals (`ScopedCaptureHTTPError`), read back from the stable code +/// the uploader ends with at status 0, since no server answered. +public enum PhotoSetupProblem: Sendable, Equatable, CaseIterable { + /// A request's URL is outside the capture API's configured scope. + case destinationRefused + /// The credential provider couldn't supply a credential. + case credentialUnavailable + /// The credential isn't a well-formed bearer token. + case credentialMalformed + /// The service signed a storage upload that carries an Authorization header: a service or + /// contract setup failure, not the phone's credential. + case storageAuthorization + + /// The code the uploader ends with, taken from `ScopedCaptureHTTPError.code` so the two can't + /// drift apart. + public var code: String { + switch self { + case .destinationRefused: ScopedCaptureHTTPError.destinationNotAllowed("").code + case .credentialUnavailable: ScopedCaptureHTTPError.credentialUnavailable("").code + case .credentialMalformed: ScopedCaptureHTTPError.credentialMalformed.code + case .storageAuthorization: ScopedCaptureHTTPError.storageRequestHasAuthorization.code + } + } + + /// The problem an upload refusal names: one of the boundary's codes at status 0. + public init?(codes: [String], status: Int) { + guard status == 0, let problem = Self.allCases.first(where: { codes.contains($0.code) }) else { return nil } + self = problem + } +} + +/// What a follow-up capture would carry from the capture it adds views to: plain values only. +/// Nothing in this build sends a follow-up, so a request for more photos ends saying so; an +/// amendment client starts from this. The follow-up keeps its own session and epoch: nothing here +/// joins its poses to the parent's. +public struct PhotoAmendmentParent: Sendable, Equatable { + /// The server's id for the parent capture. + public var captureID: String + /// Where the parent went: the scan's profile, which a follow-up must keep. + public var profile: ProcessingProfile + /// The finished run that asked for more. + public var runID: String + /// The ids of the views the answer asked for (`CaptureResult.ViewNeeded.id`). + public var viewIDs: [String] + /// The parent's capture folder. One uploader at a time owns a folder (`CaptureUploader`'s + /// registry), so a follow-up has to take it through that rather than around it. + public var folder: URL + + public init(captureID: String, profile: ProcessingProfile, runID: String, viewIDs: [String], folder: URL) { + self.captureID = captureID + self.profile = profile + self.runID = runID + self.viewIDs = viewIDs + self.folder = folder + } +} + +extension PhotoProcessingStatus.Stage { + /// The stage for a decoded answer already bound to the scan. `world` and `meterAnchor` are the + /// scan on screen now, for the AR check; `parent` is what a follow-up would add to. + public static func answer( + _ record: CaptureResult.Record, world: CaptureResult.Association, meterAnchor: simd_float4x4?, + parent: PhotoAmendmentParent? + ) -> Self { + let summary = CaptureResultSummary(record, world: world, meterAnchor: meterAnchor) + let message = summary.message ?? "" + switch summary.state { + case .decided(.eligible): + let ar: PhotoProcessingAnswer.ARAvailability = summary.arUnavailable.map { .withheld($0) } ?? .notShownInThisBuild + return .answered(.candidate(.init(message: message, previewMade: record.response.previewUrl != nil, ar: ar))) + case .decided(.needsMorePhotos), .working(.needsViews): + return .answered(.needsMorePhotos(.init(prompts: summary.prompts, message: summary.message, parent: parent))) + case .decided(.manualReview), .working(.manualReview): + return .answered(.installerReview(message: message)) + case .decided(.notEligible): + return .answered(.noCandidate(message: message)) + case .decided(.unknown(let kind)): + return .answered(.unrecognized(kind: kind, message: message)) + case .ended(.expired): + return .ended(.expired) + case .ended: + return .ended(.processingFailed) + case .working: + // The uploader only keeps an answer once it has an outcome or the capture failed or + // expired, so this is an answer without one: unreadable as a result. + return .ended(.answerUnreadable) + } + } + + /// The stage for an upload that stopped on a refusal (`CaptureUploadState.End.failed`). + public static func refusal(step: String, codes: [String], status: Int) -> Self { + // Only the uploader's own end (`CaptureUploader.unsavedEnd`): step `save`, status 0, its code. + if step == "save", status == 0, codes.contains(CaptureUploader.unsavedStateCode) { return .ended(.uploadStateUnsaved) } + // The coordinator's own end when the phone lost a photo or tap the scan accepted + // (`CaptureSessionCoordinator.inputLostCode`): the capture can't be prepared, so it is + // never sealed and the scan ends the way a capture that couldn't be prepared does. + if step == "prepare", status == 0, codes.contains(CaptureSessionCoordinator.inputLostCode) { return .ended(.notPrepared) } + if let problem = PhotoSetupProblem(codes: codes, status: status) { return .ended(.setupRefused(step: step, problem)) } + guard step == "result" else { return .ended(.refused(step: step)) } + if codes.contains("result_not_ready") { return .ended(.answerNotReady) } + if codes.contains("result_for_another_run") { return .ended(.answerMismatch(.run)) } + if codes.contains("result_unreadable") { return .ended(.answerUnreadable) } + return .ended(.refused(step: step)) + } + + /// Where an upload in progress stands. Before the capture ends, only how much was sent. + public static func progress(_ upload: CaptureUploadStatus, captureEnded: Bool) -> Self { + guard captureEnded else { return .capturing(sent: upload.committed) } + if upload.phase == .creating { return .queued } + if upload.committed < upload.retained { return .uploading(received: upload.committed, kept: upload.retained) } + return upload.phase == .processing ? .processing : .uploading(received: upload.committed, kept: upload.retained) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/ProcessingBackend.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/ProcessingBackend.swift new file mode 100644 index 000000000..2a26417ed --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/ProcessingBackend.swift @@ -0,0 +1,61 @@ +import Foundation + +/// The remote service that decides where a scan's battery could go. Both run off the phone: +/// Legacy sends scene.json to the placement checker, and photo processing sends the scan's photos +/// and camera path to the capture API. Neither decides on the phone. +public enum ProcessingBackend: String, Sendable, Equatable, Hashable, CaseIterable, Codable { + case legacy + case photoProcessing + + /// What a scan uses when nobody chose. Legacy stays the default until photo processing passes + /// a real end-to-end acceptance; until then the beta offers it as a choice. + public static let operationalDefault: ProcessingBackend = .legacy +} + +/// The developer setting that picks the next scan's backend, in UserDefaults. A scan reads it once, +/// when it starts (`ScanContext`), so a change applies to the next scan. +public enum ProcessingBackendPreference { + public static let key = "processingBackend" + + /// The stored choice. Nothing stored, or a value this build doesn't know, reads as the + /// operational default rather than guessing. + public static func read(_ defaults: UserDefaults) -> ProcessingBackend { + defaults.string(forKey: key).flatMap(ProcessingBackend.init(rawValue:)) ?? .operationalDefault + } + + public static func write(_ backend: ProcessingBackend, to defaults: UserDefaults) { + defaults.set(backend.rawValue, forKey: key) + } +} + +/// The backend a scan uses, as it stood when the scan started: which one, who answers, where the +/// scan's data may go, and whose credential it may present. The scan keeps it to its end, so a +/// later change to the setting or the endpoint can't send an existing capture somewhere else. +public struct ProcessingProfile: Sendable, Equatable, Hashable { + public enum Answers: Sendable, Equatable, Hashable { + /// A service off the phone. + case service + /// A stand-in on this phone: the bundled sample result, or the DEBUG capture fixture. + /// Nothing leaves the phone. + case standIn + /// Nobody: this build has no usable setup for the backend, for the reason given. Nothing + /// is sent. + case notSetUp(String) + } + + public var backend: ProcessingBackend + public var answers: Answers + /// The API base this scan's data may go to. Nil when it goes nowhere. + public var origin: URL? + /// Who issues the scoped credential this scan may present, never the credential itself. Nil + /// when the scan presents none. A token refreshed from the same issuer leaves the profile as + /// it is. + public var credentialIssuer: String? + + public init(backend: ProcessingBackend, answers: Answers, origin: URL?, credentialIssuer: String? = nil) { + self.backend = backend + self.answers = answers + self.origin = origin + self.credentialIssuer = credentialIssuer + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/ScanContext.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/ScanContext.swift new file mode 100644 index 000000000..fbea1dee7 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/ScanContext.swift @@ -0,0 +1,41 @@ +import Foundation + +/// One ARKit world within a scan. Results and AR bind to this, not to the scan's id: a world reset +/// keeps the scan but makes every pose and answer from the old world meaningless. +public struct SpatialSession: Sendable, Equatable, Hashable { + /// The packet recorder's session id, which it makes new for every world. + public var recordingSessionID: String + /// Worlds this scan threw away before this one: 0 for its first. + public var worldEpoch: Int + + public init(recordingSessionID: String, worldEpoch: Int) { + self.recordingSessionID = recordingSessionID + self.worldEpoch = worldEpoch + } +} + +/// What a scan is, fixed when it starts: its identity, its backend profile and its current world. +/// +/// The engine's work counter isn't an identity: it moves on every reset, inside a scan as well as +/// between scans, so it can't say which scan a capture or an answer belongs to. This can. +public struct ScanContext: Sendable, Equatable { + /// The scan's id, which the app takes from its scan folder's name. Start over makes a new scan + /// and so a new id; a world reset keeps it. + public let scanID: String + public let profile: ProcessingProfile + public private(set) var spatial: SpatialSession + + public init(scanID: String, profile: ProcessingProfile, recordingSessionID: String) { + self.scanID = scanID + self.profile = profile + spatial = SpatialSession(recordingSessionID: recordingSessionID, worldEpoch: 0) + } + + /// The same scan in a new ARKit world: the id and the backend stay, the spatial session is + /// new. + public func inNewWorld(recordingSessionID: String) -> ScanContext { + var next = self + next.spatial = SpatialSession(recordingSessionID: recordingSessionID, worldEpoch: spatial.worldEpoch + 1) + return next + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/SyntheticCapture.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/SyntheticCapture.swift new file mode 100644 index 000000000..b88b57abf --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/SyntheticCapture.swift @@ -0,0 +1,151 @@ +import CoreGraphics +import Foundation +import ImageIO +import simd + +/// A made-up capture whose packet says so (`source.kind: synthetic`), for running photo +/// processing end to end without a phone's sensors: the DEBUG app's +/// `-photoProcessingSyntheticCapture` and the package tests. +/// +/// The numbers are #192's synthetic recording (`NativeCaptureFixture` in the package tests): a +/// camera 2 m from a wall at 1.5 m height walking +x at 0.1 m/s for 12 s, a 60 Hz trajectory with +/// each frame's intrinsics beside it, and accelerometer and gyro at about 100 Hz on their own +/// clocks with a fixed jitter. Photos are pattern images, no house, taken at times on that +/// trajectory with its poses and intrinsics. Everything is computed from fixed constants, so two +/// runs give the same rows and the same JPEG bytes, and no wall clock enters the data. +/// +/// It is never a recording of the scan on screen: a replay keeps no motion, and nothing here is +/// added to one. Like `FixtureCaptureHTTP`, the type is in every build; only the app's activation +/// is limited to debug builds on a replay. +public struct SyntheticCapture: Sendable { + public static let standard = SyntheticCapture() + + /// Sensor image size and its intrinsics [fx, fy, cx, cy]. + public static let imageSize = SIMD2(1920, 1440) + public static let intrinsics = SIMD4(1445, 1445, 960, 720) + /// The stored JPEGs are a quarter of the sensor size each way, with the intrinsics scaled to + /// match (`KeptPhoto.observation`), which keeps writing them fast. + public static let jpegSize = (width: 480, height: 360) + /// A fixed wall-clock start, so the packet holds no date from the phone running it. + public static let startedAt = Date(timeIntervalSince1970: 1_790_000_000) + + public let start = 5000.0 + public let end = 5012.0 + + public init() {} + + public func pose(_ t: Double) -> simd_float4x4 { + var m = matrix_identity_float4x4 + m.columns.3 = SIMD4(Float(t - start) * 0.1, 1.5, 0, 1) + return m + } + + /// The close-up the scan accepted, as the coordinator wants it at the capture's end. + public var acceptedCloseUpAt: Double { start + 2.5 } + + /// The photos, in the order they are kept: the meter close-up, then five walk frames. + public var photoTimes: [(t: Double, purpose: String?)] { + var times: [(t: Double, purpose: String?)] = [(start + 2.5, "meter_close")] + for i in 0..<5 { times.append((start + 3 + Double(i) * 1.5, nil)) } + return times + } + + public var recording: RecordingSource { + let capture = self + return RecordingSource(start: { (capture.start, Self.startedAt) }, rows: { capture.rows }) + } + + public var rows: RecorderRows { + var trajectory: [[Double]] = [], intrinsics: [[Double]] = [] + let k = Self.intrinsics + for i in 0...Int((end - start) * 60) { + let t = start + Double(i) / 60 + let m = pose(t) + let columns: [SIMD4] = [m.columns.0, m.columns.1, m.columns.2, m.columns.3] + let values: [Double] = columns.flatMap { (c: SIMD4) -> [Double] in [Double(c.x), Double(c.y), Double(c.z), Double(c.w)] } + trajectory.append([t, 0, 0] + values) + intrinsics.append([t, Double(k.x), Double(k.y), Double(k.z), Double(k.w), Double(Self.imageSize.x), Double(Self.imageSize.y)]) + } + // The fixed jitter keeps the two sensors from ever sharing a timestamp. + var accel: [[Double]] = [], gyro: [[Double]] = [] + for i in 0.., appVersion: String) -> Packet04SessionInfo { + Packet04SessionInfo( + packetID: packetID, sessionID: packetID, source: .synthetic, appVersion: appVersion, deviceModel: "synthetic", + systemVersion: "synthetic", lidarAvailable: false, sceneDepthEnabled: false, meshReconstructionSupported: nil, + sceneReconstruction: nil, planeDetection: nil, videoWidth: Int(video.x), videoHeight: Int(video.y), framesPerSecond: nil, + timeZone: nil) + } + + /// Writes the photos' JPEGs into `folder` (again only when missing) and returns them as kept + /// photos. + public func photos(in folder: URL) throws -> [KeptPhoto] { + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + return try photoTimes.enumerated().map { index, photo in + let url = folder.appending(path: "synthetic-\(index).jpg") + if !FileManager.default.fileExists(atPath: url.path) { try Self.writeJPEG(pattern: index, to: url) } + return KeptPhoto( + t: photo.t, cameraToWorld: pose(photo.t), cameraIntrinsics: Self.intrinsics, cameraImageSize: Self.imageSize, + width: Self.jpegSize.width, height: Self.jpegSize.height, tracking: .normal, + exposure: .init(duration: 0.004, offset: 0, iso: 64, fNumber: 1.6), jpeg: url, purpose: photo.purpose) + } + } + + public enum ImageError: Error, Equatable { + case notWritten + } + + /// A colour pattern, shifted per photo so no two are the same image. + static func writeJPEG(pattern: Int, to url: URL) throws { + let (width, height) = jpegSize + guard let context = CGContext( + data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 4 * width, + space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue), + let data = context.data + else { throw ImageError.notWritten } + let pixels = data.bindMemory(to: UInt8.self, capacity: 4 * width * height) + for y in 0.. Self { + guard debugBuild else { return .off("a release build has no capture fixture") } + guard let answer else { + return .off(syntheticCapture ? "the synthetic capture needs the capture fixture" : "sending captures is off in this build") + } + guard onReplay else { return .off("the capture fixture runs only on a replay, never the live camera") } + return .on(answer, syntheticCapture: syntheticCapture) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultCardActions.swift b/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultCardActions.swift index 1a77f1aad..cfb7d8f88 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultCardActions.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultCardActions.swift @@ -23,7 +23,7 @@ public enum ResultCardActions { public static func primary(answer: ResultReading.Answer, hasSpot: Bool, spotIsClean: Bool, hasViewToTake: Bool, sourceAvailable: Bool) -> Primary? { switch answer { - case .fits: + case .candidate: return hasSpot && sourceAvailable ? .showAR(clean: true) : nil case .oneMoreLook: return offersView(capturable: hasViewToTake, sourceAvailable: sourceAvailable) ? .takeView : nil diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultReading.swift b/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultReading.swift index 8330d35cd..563d9aa32 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultReading.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultReading.swift @@ -3,8 +3,14 @@ import Foundation // What the result screen leads with, read from the server's answer. The app's screen turns these // into words; the rules that pick them live here so each one has a test against a decoded answer // (ResultReadingTests). They read the checks as well as `decision`: a manual_review held back -// only by rules that aren't approved yet still fits, and a spot that might stand in the meter's -// working space goes to an installer whatever the decision says. +// only by rules that aren't approved yet is still a candidate, and a spot that might stand in the +// meter's working space goes to an installer whatever the decision says. +// +// No answer says a battery fits. A spot whose checks all pass is a candidate: the server's checks +// read observed space beyond the area the homeowner confirms in the spot check, some of it +// inferred rather than seen (a walked path, a photo's view; `CoverageMap`, "Bounded +// exceptions"), and the answer doesn't say which. Until it does, the app can't tell that all the +// space a battery needs was confirmed (B17). The answer itself is unchanged: this reads it. public enum ResultReading { /// The server's check for the NEC 110.26 working space in front of the meter @@ -14,8 +20,9 @@ public enum ResultReading { /// What the screen says first. public enum Answer: String, Equatable, Sendable { - /// The spot passes every check; at most the rules' approval holds it back. - case fits + /// A proposed spot: every check passes on what the scan recorded, and at most the rules' + /// approval holds it back. Not a confirmed fit: an installer checks the space on site. + case candidate /// An unsure check that a view the camera can take now would settle. case oneMoreLook /// A person has to decide: a borderline measurement, an unknown attribute, a spot that @@ -33,16 +40,12 @@ public enum ResultReading { public var needsPerson: Bool /// True when a view the camera can take now would settle it. public var viewCapturable: Bool - /// How far the measurement clears its limit (`margin(measured:threshold:plusMinus:comparison:)`); - /// nil without both a measurement and a limit. - public var margin: Double? - public init(id: String, outcome: PlacementOutcome, needsPerson: Bool = false, viewCapturable: Bool = false, margin: Double? = nil) { + public init(id: String, outcome: PlacementOutcome, needsPerson: Bool = false, viewCapturable: Bool = false) { self.id = id self.outcome = outcome self.needsPerson = needsPerson self.viewCapturable = viewCapturable - self.margin = margin } } @@ -58,29 +61,34 @@ public enum ResultReading { if hasSpot, !spotIsClean(hasSpot: hasSpot, checks: checks) { return .installer } switch decision { case .pass: - return .fits + // A candidate needs what makes one: a spot, and checks that all pass. A pass without a + // spot, with no checks, or with a check that didn't pass contradicts itself, and a + // person reads it; its checks stay as the server sent them. + return hasSpot && allPass(checks) ? .candidate : .installer case .reject: return .notHere case .manualReview: - // Needs at least one check: over none, "every check passes" is vacuously true. - if hasSpot, !policyApproved, !checks.isEmpty, checks.allSatisfy({ $0.outcome == .pass }) { return .fits } + if hasSpot, !policyApproved, allPass(checks) { return .candidate } if checks.contains(where: { $0.outcome == .unsure && !$0.needsPerson && $0.viewCapturable }) { return .oneMoreLook } return .installer } } - /// Indices of the checks the result card shows, in order: every FAIL, then every UNSURE, then - /// the two PASSes closest to their limits (smallest `margin`; a pass without one sorts last). - /// At most `limit`. + /// At least one check, and every one passed. Over none, "every check passes" is vacuously + /// true and says nothing. + private static func allPass(_ checks: [Check]) -> Bool { + !checks.isEmpty && checks.allSatisfy { $0.outcome == .pass } + } + + /// Indices of the checks the result card shows, in order: every FAIL, then every UNSURE, at + /// most `limit`. No PASS: on the card a passing line read as space confirmed clear, which the + /// scan can't show (see the top of this file). Every check, passing ones included, stays in + /// the result's Details. public static func cardLines(_ checks: [Check], limit: Int = 3) -> [Int] { let indices = checks.indices let fails = indices.filter { checks[$0].outcome == .fail } let unsure = indices.filter { checks[$0].outcome == .unsure } - // Ties keep the server's order. - let passes = indices.filter { checks[$0].outcome == .pass } - .sorted { (checks[$0].margin ?? .infinity, $0) < (checks[$1].margin ?? .infinity, $1) } - .prefix(2) - return Array((fails + unsure + passes).prefix(limit)) + return Array((fails + unsure).prefix(limit)) } /// Whether a check's review line (`PlacementCheck.reviewThresholdFt`) explains its outcome, so @@ -104,20 +112,6 @@ public enum ResultReading { case .atLeast: return reviewThreshold > threshold && !(measured - error > reviewThreshold) } } - - /// How far `measured` clears `threshold` on the passing side, in units of `plusMinus` when it - /// is positive, else in the measurement's own units. Negative when it misses. Without a - /// comparison the side is unknown, so the distance to the limit either way. - public static func margin(measured: Double?, threshold: Double?, plusMinus: Double?, comparison: PlacementComparison?) -> Double? { - guard let measured, let threshold else { return nil } - let clearance = switch comparison { - case .atLeast: measured - threshold - case .atMost: threshold - measured - case nil: abs(measured - threshold) - } - guard let plusMinus, plusMinus > 0 else { return clearance } - return clearance / plusMinus - } } extension PlacementPolicy { @@ -175,9 +169,7 @@ extension PlacementResult { checks.map { check in ResultReading.Check( id: check.id, outcome: check.outcome, needsPerson: check.needsPerson, - viewCapturable: evidenceIndex(settling: check.id).map(capturable) ?? false, - margin: ResultReading.margin(measured: check.measuredFt, threshold: check.thresholdFt, - plusMinus: check.plusMinusFt, comparison: check.comparison) + viewCapturable: evidenceIndex(settling: check.id).map(capturable) ?? false ) } } diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Scene/ScanFolderCleanup.swift b/ios/HouseScanKit/Sources/HouseScanKit/Scene/ScanFolderCleanup.swift index 3335f8361..927e58c00 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Scene/ScanFolderCleanup.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Scene/ScanFolderCleanup.swift @@ -26,6 +26,13 @@ import Foundation /// can exist, and only those are deleted, whenever the deletion gets to run. Listing at deletion /// time instead let a cleanup that ran late delete a scan started after it: store A is made, /// Start over makes B, and A's cleanup then found B in the listing and deleted the scan in use. +/// +/// A folder whose bundle is still being built has no `scan.zip` yet, so it is listed as never +/// packaged. The packet is assembled in the folder first and zipped last, and Start over doesn't +/// wait for that write, so deleting on the listing alone took the folder from under it and the +/// scan never reached Saved scans. The app therefore deletes with `run(after:)`, waiting for the +/// bundle writes that were started before the listing; the recheck in `run` then keeps a folder +/// whose bundle landed meanwhile. public struct ScanFolderCleanup: Sendable { /// The bundle Share scan offers. A scan is completed when this file is a whole capture packet /// (`PacketArchiveCheck`); the file being there is not enough. @@ -132,7 +139,8 @@ public struct ScanFolderCleanup: Sendable { /// The deletion runs later than the listing, off the main actor, and a folder can change in /// between: a bundle write can start in a folder listed as never packaged, or finish in one. /// Each entry is read again and deleted only if it is still what it was when listed; one that - /// changed is left for the next cleanup. A change after this second read is not caught. + /// changed is left for the next cleanup. A change after this second read is not caught, so a + /// write still going when this runs can lose its folder: `run(after:)` waits for known writers. public func run() -> [(url: URL, error: any Error)] { let files = FileManager.default return obsolete.compactMap { url in @@ -146,4 +154,14 @@ public struct ScanFolderCleanup: Sendable { } } } + + /// `run()`, once `wait` returns. The app's `wait` is the scan bundle writes started before + /// this cleanup was listed (one chain: each write waits for the one before it), so a folder + /// listed while its packet was still being assembled is judged again only after that write + /// ends: a bundle that landed keeps the folder, and a write that failed before zipping leaves + /// it to be deleted. A `wait` that returns at once, as at launch with no write, is `run()`. + public func run(after wait: () async -> Void) async -> [(url: URL, error: any Error)] { + await wait() + return run() + } } diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift new file mode 100644 index 000000000..dc33d562d --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift @@ -0,0 +1,227 @@ +import Foundation + +/// The capture API's request and response bodies that House Scan sends and reads (`/v1/captures` +/// and below), written from the documented wire contract. Unknown response fields are ignored. +public enum CaptureAPI { + public struct Device: Codable, Sendable, Equatable { + public var model: String + public var systemVersion: String + public var appVersion: String + + public init(model: String, systemVersion: String, appVersion: String) { + self.model = model + self.systemVersion = systemVersion + self.appVersion = appVersion + } + } + + /// The create body. Expected counts are left out: they are optional, and a count that grows as + /// photos arrive would change the body the idempotency key is bound to. + public struct CreateRequest: Codable, Sendable, Equatable { + public var packetId: String + public var formatVersion = Packet04.formatVersion + public var tier: Packet04.Tier + public var flow = "guided" + public var device: Device + + public init(packetId: String, tier: Packet04.Tier, device: Device) { + self.packetId = packetId + self.tier = tier + self.device = device + } + } + + public struct Limits: Codable, Sendable, Equatable { + public var maxSinglePutBytes: Int? + public var urlTtlS: Int? + public var maxFiles: Int? + public var maxBatch: Int? + } + + public struct CreateResponse: Codable, Sendable, Equatable { + public var captureId: String + public var status: String + public var upload: Limits? + public var finalizeBy: String? + } + + public struct RegisterFile: Codable, Sendable, Equatable { + public var path: String + public var bytes: Int + public var sha256: String + public var md5: String + public var role: Packet04.Role + public var contentType: String + public var meta: Packet04.ImageRecord? + + public init(_ file: SealedFile) { + path = file.path + bytes = file.bytes + sha256 = file.sha256 + md5 = file.md5 + role = file.role + contentType = file.contentType + meta = file.meta + } + } + + public struct RegisterRequest: Codable, Sendable, Equatable { + public var files: [RegisterFile] + } + + public struct UploadTarget: Codable, Sendable, Equatable { + public var method: String + public var url: String? + public var headers: [String: String]? + public var expiresAt: String? + } + + public struct RegisteredFile: Codable, Sendable, Equatable { + public var path: String + public var state: String + public var upload: UploadTarget? + } + + public struct RegisterResponse: Codable, Sendable, Equatable { + public var files: [RegisteredFile] + } + + public struct CommitFile: Codable, Sendable, Equatable { + public var path: String + public var sha256: String + } + + public struct CommitRequest: Codable, Sendable, Equatable { + public var files: [CommitFile] + } + + public struct CommitResponse: Codable, Sendable, Equatable { + public var committed: [String] + public var notFound: [String] + public var mismatch: [String] + } + + public struct FinalizeResponse: Codable, Sendable, Equatable { + public var status: String + public var missing: [String] + public var runId: String + public var etaS: Int? + } + + public struct Event: Codable, Sendable, Equatable { + public var seq: Int + public var type: String + public var at: String + /// The fields the app reads. Others, which may carry text meant for the homeowner, are + /// not kept. + public var data: EventData? + } + + public struct EventData: Codable, Sendable, Equatable { + public var code: String? + public var kind: String? + public var stage: String? + public var status: String? + public var runId: String? + public var next: String? + } + + public struct EventsResponse: Codable, Sendable, Equatable { + public var status: String + public var next: Int + public var events: [Event] + } + + public struct ErrorItem: Codable, Sendable, Equatable { + public var code: String + public var pointer: String? + public var message: String? + } + + public struct ErrorBody: Codable, Sendable, Equatable { + public var errors: [ErrorItem] + } + + public static func errorCodes(_ body: Data) -> [String] { + (try? JSONDecoder().decode(ErrorBody.self, from: body))?.errors.map(\.code) ?? [] + } + + /// Statuses after which the capture changes no more. + public static let terminalStatuses: Set = ["complete", "manual_review", "failed", "expired", "needs_views"] + + /// Terminal statuses that can end a capture with no answer, so a result read carrying one and + /// a null outcome is final. The others come with the run's outcome. + public static let statusesWithoutOutcome: Set = ["failed", "expired"] +} + +/// An HTTP answer: status, body and the headers the uploader reads. +public struct HTTPReply: Sendable, Equatable { + public var status: Int + public var body: Data + public var retryAfter: Double? + + public init(status: Int, body: Data, retryAfter: Double? = nil) { + self.status = status + self.body = body + self.retryAfter = retryAfter + } +} + +/// How the uploader reaches the network. HTTP error statuses come back as replies; only transport +/// failures throw. An implementation must not follow redirects: the uploader checks each URL +/// before it sends a photo there, and a redirect would send it somewhere unchecked. A redirect +/// comes back as its own 3xx reply. +public protocol CaptureHTTP: Sendable { + func send(_ request: URLRequest) async throws -> HTTPReply + /// `file` as the whole body of `request`, streamed from disk. + func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply +} + +/// `CaptureHTTP` on one ordinary `URLSession`. +public struct URLSessionCaptureHTTP: CaptureHTTP { + public let session: URLSession + + public init(session: URLSession) { + self.session = session + } + + /// An ephemeral session: nothing cached or stored in cookies, and no credentials offered to + /// the signed storage URLs. + public static func ephemeral(timeout: Double = 60) -> URLSessionCaptureHTTP { + let configuration = URLSessionConfiguration.ephemeral + configuration.timeoutIntervalForRequest = timeout + configuration.httpCookieStorage = nil + configuration.urlCredentialStorage = nil + configuration.httpMaximumConnectionsPerHost = 4 + return URLSessionCaptureHTTP(session: URLSession(configuration: configuration)) + } + + public func send(_ request: URLRequest) async throws -> HTTPReply { + let (data, response) = try await session.data(for: request, delegate: NoRedirects.shared) + return Self.reply(data, response) + } + + public func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + let (data, response) = try await session.upload(for: request, fromFile: file, delegate: NoRedirects.shared) + return Self.reply(data, response) + } + + /// Refuses every redirect, so the task returns the 3xx response itself. URLSession follows + /// redirects by default, and App Transport Security alone would still allow one to another + /// https host, or to plain http on the local network, which the app's Info.plist permits. + final class NoRedirects: NSObject, URLSessionTaskDelegate, Sendable { + static let shared = NoRedirects() + + func urlSession( + _ session: URLSession, task: URLSessionTask, willPerformHTTPRedirection response: HTTPURLResponse, newRequest request: URLRequest + ) async -> URLRequest? { + nil + } + } + + static func reply(_ data: Data, _ response: URLResponse) -> HTTPReply { + let http = response as? HTTPURLResponse + return HTTPReply( + status: http?.statusCode ?? 0, body: data, retryAfter: http?.value(forHTTPHeaderField: "Retry-After").flatMap(Double.init)) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift new file mode 100644 index 000000000..364a96d01 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift @@ -0,0 +1,548 @@ +import Foundation +import simd + +/// A photo the scan kept, as the app stores it: the JPEG on disk and the facts of the ARFrame +/// whose pixels it holds, in that frame's own terms (raw world pose, the camera image's +/// intrinsics and size). +public struct KeptPhoto: Sendable { + public var t: Double + public var cameraToWorld: simd_float4x4 + /// [fx, fy, cx, cy] in pixels of the camera image (`cameraImageSize`). + public var cameraIntrinsics: SIMD4 + public var cameraImageSize: SIMD2 + /// Pixels of the stored JPEG. + public var width: Int + public var height: Int + public var tracking: PacketTracking + public var exposure: Packet04.Exposure? + public var jpeg: URL + /// The still's purpose ("meter_close"); nil for a keyframe. + public var purpose: String? + /// Reads the photo's ARKit depth. Called once, when the photo is kept, because the app writes a + /// retaken close-up's depth over the same files. + public var depth: @Sendable () -> Packet04Depth? + + public init( + t: Double, cameraToWorld: simd_float4x4, cameraIntrinsics: SIMD4, cameraImageSize: SIMD2, width: Int, height: Int, + tracking: PacketTracking, exposure: Packet04.Exposure?, jpeg: URL, purpose: String?, depth: @escaping @Sendable () -> Packet04Depth? = { nil } + ) { + self.t = t + self.cameraToWorld = cameraToWorld + self.cameraIntrinsics = cameraIntrinsics + self.cameraImageSize = cameraImageSize + self.width = width + self.height = height + self.tracking = tracking + self.exposure = exposure + self.jpeg = jpeg + self.purpose = purpose + self.depth = depth + } + + /// The observation in the stored JPEG's pixels: intrinsics scaled when the JPEG is not the + /// camera image's size. + public var observation: Packet04Observation { + let scale = SIMD2(Float(width), Float(height)) / cameraImageSize + let k = cameraIntrinsics + return Packet04Observation( + t: t, cameraToWorld: cameraToWorld, intrinsics: SIMD4(k.x * scale.x, k.y * scale.y, k.z * scale.x, k.w * scale.y), + width: width, height: height, tracking: tracking, exposure: exposure) + } +} + +/// The frame under the homeowner's tap, read when the tap happened. +public struct TapObservation: Sendable { + public var t: Double + public var cameraToWorld: simd_float4x4 + /// [fx, fy, cx, cy] in pixels of the camera image, which the JPEG stores unscaled. + public var intrinsics: SIMD4 + public var width: Int + public var height: Int + public var tracking: PacketTracking + /// The tap in the sensor image's pixels, (0, 0) its top-left corner. + public var pixel: SIMD2 + /// Encodes the frame's image; called off the main actor. + public var jpeg: @Sendable () -> Data? + + /// The same frame with the tap moved to where `world` (the raycast's hit) lands in its image, + /// so pixel, ray and hit agree in this one frame even when it is a frame or two newer than + /// the one the raycast used. Nil when the point is behind the camera or outside the image. + public func pointing(at world: SIMD3) -> TapObservation? { + let p = cameraToWorld.inverse * SIMD4(world, 1) + guard p.z < -0.05 else { return nil } + let u = Double(intrinsics.z + intrinsics.x * p.x / -p.z), v = Double(intrinsics.w - intrinsics.y * p.y / -p.z) + guard (0...Double(width)).contains(u), (0...Double(height)).contains(v) else { return nil } + var aimed = self + aimed.pixel = SIMD2(u, v) + return aimed + } + + public init( + t: Double, cameraToWorld: simd_float4x4, intrinsics: SIMD4, width: Int, height: Int, tracking: PacketTracking, + pixel: SIMD2, jpeg: @escaping @Sendable () -> Data? + ) { + self.t = t + self.cameraToWorld = cameraToWorld + self.intrinsics = intrinsics + self.width = width + self.height = height + self.tracking = tracking + self.pixel = pixel + self.jpeg = jpeg + } +} + +/// Where the coordinator reads the world's recorded streams: `CaptureRecorder` in the app. +public struct RecordingSource: Sendable { + /// Uptime and wall clock of the world's first recorded frame. + public var start: @Sendable () -> (uptime: Double, date: Date)? + /// Every row so far; called off the main actor. + public var rows: @Sendable () -> RecorderRows + + public init(start: @escaping @Sendable () -> (uptime: Double, date: Date)?, rows: @escaping @Sendable () -> RecorderRows) { + self.start = start + self.rows = rows + } +} + +/// The recorder's raw rows: trajectory `[t, state, reason, 16 pose values]`, intrinsics +/// `[t, fx, fy, cx, cy, w, h]`, and Core Motion's `[t, x, y, z]`, each sensor at its own times. +public struct RecorderRows: Sendable { + public var trajectory: [[Double]] + public var intrinsics: [[Double]] + public var accelerometer: [[Double]] + public var gyroscope: [[Double]] + + public init(trajectory: [[Double]], intrinsics: [[Double]], accelerometer: [[Double]], gyroscope: [[Double]]) { + self.trajectory = trajectory + self.intrinsics = intrinsics + self.accelerometer = accelerometer + self.gyroscope = gyroscope + } +} + +/// What the integration build does with a scan: one 0.4 packet per ARKit world, each kept photo +/// sealed into it on the phone as it is kept, the packet frozen when the scan is sent for +/// placement. Sending is separate: only a build allowed to send device data (`sends`) sends, and +/// only after the homeowner's yes for this scan, asked before it starts. From the yes on, every +/// sealed file goes up, and later photos go up as they are kept. A new scan asks again; a world +/// reset within the scan keeps its answer for the scan's next packet. +/// +/// Work for one world runs in order on that world's chain; a world reset or start over ends the +/// session, and nothing queued for it reaches the next one. +@MainActor +public final class CaptureSessionCoordinator { + public struct Environment: Sendable { + public var endpoint: URL + /// Whether device captures may go to `endpoint` at all (`CaptureIntegrationMode.send`). + public var sends: Bool + public var http: any CaptureHTTP + /// Each session writes to `/`. + public var capturesFolder: URL + public var sessionInfo: @Sendable (_ packetID: String, _ video: SIMD2) -> Packet04SessionInfo + public var device: CaptureAPI.Device + public var tier: Packet04.Tier + public var policy: CaptureUploader.Policy + public var log: @Sendable (String) -> Void + + public init( + endpoint: URL, sends: Bool, http: any CaptureHTTP, capturesFolder: URL, + sessionInfo: @escaping @Sendable (_ packetID: String, _ video: SIMD2) -> Packet04SessionInfo, + device: CaptureAPI.Device, tier: Packet04.Tier, policy: CaptureUploader.Policy = .init(), log: @escaping @Sendable (String) -> Void = { _ in } + ) { + self.endpoint = endpoint + self.sends = sends + self.http = http + self.capturesFolder = capturesFolder + self.sessionInfo = sessionInfo + self.device = device + self.tier = tier + self.policy = policy + self.log = log + } + } + + /// Checklist purposes the packet's `stills` take; any other still stays a keyframe only. + public static let stillPurposes: Set = ["meter_close", "meter_oblique"] + + /// The local code a capture ends with when the phone lost a photo or tap the scan accepted: + /// the packet can't be prepared complete. The upload, if one is running, ends + /// `.failed(step: "prepare", codes: [inputLostCode], status: 0)`; the session's + /// `preparationFailure` holds it either way. + /// Nonisolated, so status mapping outside the main actor can match it. + nonisolated public static let inputLostCode = "capture_input_lost" + + @MainActor + public final class Session { + /// This app's id for this session, new for every session even when two packets both call + /// their ARKit epoch "e1": a result is bound to it, so a late result can't land on a newer scan. + public let localID = UUID().uuidString + public let packetID: String + public let folder: URL + /// Nil until the homeowner says yes on a build that sends. + public fileprivate(set) var uploader: CaptureUploader? + public private(set) var producer: Packet04Producer? + let recording: RecordingSource + var chain: Task? + var sealing = false + var ended = false + /// The homeowner said no after a yes: this capture is never sent again. + var consentWithdrawn = false + var withdrawalFailure: ConsentWithdrawalError? + /// packet.json as frozen, once the scan was sent for placement. + var frozen: Data? + /// The server's result for this session's capture, decoded and bound to it. + public fileprivate(set) var result: CaptureResult.Record? + /// Set once a photo or tap the scan accepted couldn't be kept (`inputLostCode`). This + /// session then never seals, and never starts an upload. + public fileprivate(set) var preparationFailure: String? + + init(packetID: String, folder: URL, recording: RecordingSource) { + self.packetID = packetID + self.folder = folder + self.recording = recording + } + + /// Runs `work` after everything queued before it, unless the session has ended by then. + /// `cleanup` runs either way, after `work` when it ran. + func enqueue(_ work: @escaping @MainActor (Session) async -> Void, cleanup: (@MainActor @Sendable () -> Void)? = nil) { + let previous = chain + chain = Task { + await previous?.value + defer { cleanup?() } + guard !self.ended else { return } + await work(self) + } + } + + func producer(first photo: (t: Double, video: SIMD2), info: (String, SIMD2) -> Packet04SessionInfo) throws -> Packet04Producer { + if let producer { return producer } + // The packet starts at the world's first recorded frame; a photo can't be older. + let start = recording.start() ?? (photo.t, Date(timeIntervalSinceNow: photo.t - ProcessInfo.processInfo.systemUptime)) + let made = try Packet04Producer(folder: folder, info: info(packetID, photo.video), startedAtUptime: min(start.uptime, photo.t), startedAt: start.date) + producer = made + return made + } + } + + public let environment: Environment? + /// The homeowner's answer for this scan: nil before they were asked. + public private(set) var consent: Bool? + private var consentedAt: Date? + public private(set) var session: Session? + private var recording: RecordingSource? + /// The current session's status; nil when there is none or nothing is being sent. + public var onStatus: (@MainActor (CaptureUploadStatus?) -> Void)? + /// The current session's result, once the server has one. Never called for an ended session. + public var onResult: (@MainActor (CaptureResult.Record) -> Void)? + /// The packet's ARKit epoch: one packet is one world. + public static let epoch = "e1" + + public init(environment: Environment?) { + self.environment = environment + } + + /// Whether the homeowner should be asked now: a build that sends, a scan under way, and no + /// answer for it yet. + public var needsConsent: Bool { + environment?.sends == true && consent == nil && session != nil + } + + public enum SessionEndError: Error, Sendable, Equatable { + /// Sending stopped and this process won't resume the capture, but its ended marker + /// couldn't be written, so a relaunch could still find the saved yes. + case notRecorded(String) + } + + public enum ConsentWithdrawalError: Error, Sendable, Equatable { + /// Sending stopped in this process, but a relaunch may still find the old saved yes. + case notRecorded(String) + } + + /// A no after a yes stops this capture's upload for good: nothing kept later is sent, and a + /// second yes in the same scan does not start it again. What the server already has stays there. + /// Failure means sending stopped here but the withdrawal could not be saved for a relaunch. + @discardableResult + public func answerConsent(_ yes: Bool, at time: Date = Date()) -> Result { + consent = yes + consentedAt = yes ? time : nil + guard let session else { return .success(()) } + if let failure = session.withdrawalFailure { return .failure(failure) } + if yes { + startUploading(session) + } else if let uploader = session.uploader { + session.consentWithdrawn = true + session.uploader = nil + // The result distinguishes durable withdrawal from stopping only this live process. + switch uploader.withdrawConsent() { + case .marked: break + case .savedStateRemoved: + environment?.log("capture upload: the withdrawal marker could not be written; the saved upload was removed instead") + case .notRecorded(let error): + session.withdrawalFailure = .notRecorded(error) + environment?.log("capture upload: the withdrawal could not be saved (\(error)); sending stopped in this process, but a relaunch could still find the saved consent") + } + Task { await uploader.abandon(CaptureUploader.withdrawnReason) } + } + if let failure = session.withdrawalFailure { return .failure(failure) } + return .success(()) + } + + // MARK: Scan events + + /// The live source started recording a world. + public func begin(recording: RecordingSource) { + self.recording = recording + if environment != nil, session == nil { startSession() } + } + + /// The ARKit world was thrown away, or (`newScan`) the scan started over: this packet can't + /// be finished, and the next one belongs to `recording`'s new world. A new scan asks again. + /// Before this returns the old capture is ended in this process and its ended marker is + /// written; failure means the marker couldn't be, so only this process is sure not to resume it. + @discardableResult + public func newWorld(_ reason: String, recording: RecordingSource, newScan: Bool) -> Result { + let ended = endSession(reason) + if newScan { + consent = nil + consentedAt = nil + } + self.recording = recording + if environment != nil { startSession() } + return ended + } + + public func kept(_ photo: KeptPhoto) { + guard let session, !session.sealing, let environment else { return } + // The app writes a retaken close-up (JPEG and depth) over the same files, so both are taken + // now, the JPEG as an APFS clone, not when the queued work gets to them. + let depth = photo.depth() + let staged = environment.capturesFolder.appending(path: "staging/\(UUID().uuidString).jpg") + do { + try FileManager.default.createDirectory(at: staged.deletingLastPathComponent(), withIntermediateDirectories: true) + try FileManager.default.copyItem(at: photo.jpeg, to: staged) + } catch { + inputLost(session, "a kept photo could not be copied: \(error)") + return + } + // The copy is deleted even when a reset ends the session before this work runs. + session.enqueue({ session in + do { + let producer = try session.producer(first: (photo.t, photo.cameraImageSize), info: environment.sessionInfo) + let data = try await Task.detached(priority: .utility) { try Data(contentsOf: staged) }.value + var files: [SealedFile] = [] + if await producer.keyframeID(at: photo.t) == nil { + files += try await producer.sealKeyframe( + jpeg: data, observation: photo.observation, reason: photo.purpose == nil ? "auto" : "still", purpose: photo.purpose, depth: depth) + } + if let purpose = photo.purpose, Self.stillPurposes.contains(purpose), let keyframe = await producer.keyframeID(at: photo.t) { + files.append(try await producer.sealStill(purpose: purpose, keyframe: keyframe)) + } + await session.uploader?.add(files) + } catch { + self.inputLost(session, "a kept photo was not sealed: \(error)") + } + }, cleanup: { try? FileManager.default.removeItem(at: staged) }) + } + + /// The homeowner marked the meter: its frame becomes a keyframe and the tap goes on it. + public func meterTapped(_ tap: TapObservation, hit: Packet04.TapHit?) { + guard let session, !session.sealing, let environment else { return } + session.enqueue { session in + do { + let producer = try session.producer(first: (tap.t, SIMD2(Float(tap.width), Float(tap.height))), info: environment.sessionInfo) + guard let jpeg = await Task.detached(priority: .userInitiated, operation: { tap.jpeg() }).value else { + self.inputLost(session, "the meter tap's frame could not be encoded") + return + } + let observation = Packet04Observation( + t: tap.t, cameraToWorld: tap.cameraToWorld, intrinsics: tap.intrinsics, width: tap.width, height: tap.height, tracking: tap.tracking) + let files = try await producer.sealTap(id: "meter", label: "meter", jpeg: jpeg, observation: observation, pixel: tap.pixel, hit: hit) + await session.uploader?.add(files) + } catch { + self.inputLost(session, "the meter tap was not recorded: \(error)") + } + } + } + + /// The scan was sent for placement: freeze the packet with the streams recorded so far. + /// `acceptedCloseUpAt` is the frame time of the close-up the scan accepted, nil after a skip. + /// Later sends of the same scan (a retry, one more view) change nothing. + public func captureEnded(acceptedCloseUpAt: Double?) { + guard let session, !session.sealing, let environment else { return } + session.sealing = true + // A packet missing an accepted photo or tap is never frozen; its upload already ended. + guard session.preparationFailure == nil else { return } + session.enqueue { session in + // A photo or tap queued before this one may have been lost since: never freeze then. + guard session.preparationFailure == nil else { return } + guard let producer = session.producer else { + await session.uploader?.abandon("no photos were kept") + return + } + let recording = session.recording + let rows = await Task.detached(priority: .utility) { recording.rows() }.value + let poses = Packet04Streams.poseRows(trajectory: rows.trajectory, intrinsics: rows.intrinsics, tracking: PacketTracking.init(recorderState:reason:)) + do { + let finished = try await producer.finish( + poses: poses, accelerometer: Packet04Streams.motionRows(rows.accelerometer), + gyroscope: Packet04Streams.motionRows(rows.gyroscope), endedAtUptime: poses.last?.t ?? 0, acceptedCloseUpAt: acceptedCloseUpAt) + session.frozen = finished.packet + await session.uploader?.seal(packet: finished.packet, files: await producer.sealedFiles) + } catch { + environment.log("capture packet not finished: \(error)") + await session.uploader?.abandon("packet refused on the phone") + } + } + } + + /// Waits until the current session's queued work and uploads are idle. For tests and the + /// evidence run. + public func settle() async { + guard let session else { return } + while let chain = session.chain { + await chain.value + if session.chain == chain { break } + } + await session.uploader?.settled() + } + + /// After a relaunch: captures frozen before the app quit finish uploading, each only toward the + /// endpoint it was created on and with the yes it recorded for that endpoint. Returns them. + public func resumeSealedCaptures() -> [CaptureUploader] { + guard let environment, environment.sends, + let folders = try? FileManager.default.contentsOfDirectory(at: environment.capturesFolder, includingPropertiesForKeys: nil) + else { return [] } + var resumed: [CaptureUploader] = [] + for folder in folders { + guard let saved = try? CaptureUploadState.load(from: CaptureUploader.stateURL(in: folder)), saved.end == nil, saved.packet != nil, + let uploader = try? CaptureUploader.resume(folder: folder, base: environment.endpoint, http: environment.http, policy: environment.policy) + else { continue } + resumed.append(uploader) + Task { await uploader.kick() } + } + return resumed + } + + // MARK: Sessions + + private func startSession() { + guard let environment, let recording else { return } + let packetID = UUID().uuidString + let session = Session(packetID: packetID, folder: environment.capturesFolder.appending(path: packetID, directoryHint: .isDirectory), recording: recording) + self.session = session + environment.log("capture packet: new packet for this world") + if consent == true { startUploading(session) } + } + + /// Opens the capture on the server and queues everything sealed so far, then the frozen + /// packet if the scan was already sent. Photos kept later go up as they are sealed. + private func startUploading(_ session: Session) { + guard let environment, environment.sends, consent == true, let consentedAt, session.uploader == nil, !session.ended, + !session.consentWithdrawn, session.preparationFailure == nil else { return } + do { + let uploader = try CaptureUploader.start( + folder: session.folder, base: environment.endpoint, http: environment.http, + create: .init(packetId: session.packetID, tier: environment.tier, device: environment.device), consentedAt: consentedAt, + policy: environment.policy) + session.uploader = uploader + // Status and results from an ended session's uploader never reach the screen. + let publish: @Sendable (CaptureUploadStatus) -> Void = { [weak self, weak session] status in + Task { @MainActor in + guard let self, let session, self.session === session else { return } + self.onStatus?(status) + if status.resultAvailable, session.result == nil { await self.bindResult(of: session) } + } + } + let log = environment.log + Task { + await uploader.observe(publish, log: log) + await uploader.kick() + } + session.enqueue { session in + guard let producer = session.producer else { return } + let files = await producer.sealedFiles + if let frozen = session.frozen { + await uploader.seal(packet: frozen, files: files) + } else { + await uploader.add(files) + } + } + environment.log("capture upload: sending this world's packet") + } catch { + environment.log("capture upload not started: \(error)") + } + } + + /// Decodes the uploader's result and binds it to the session, capture and run that asked for + /// it. The analysis stays unverified: nothing in the response proves the server reconstructed + /// anything, so the spatial result is withheld (`CaptureResult.Record.placement`). + private func bindResult(of session: Session) async { + guard let uploader = session.uploader else { return } + let state = await uploader.snapshot + guard self.session === session, session.result == nil, let body = state.result, let captureID = state.captureID, + let runID = state.finalized?.runID else { return } + do { + let response = try JSONDecoder().decode(CaptureResult.Response.self, from: body) + let record = CaptureResult.Record( + response: response, + association: .init(sessionID: session.localID, captureID: captureID, runID: runID, epoch: Self.epoch)) + session.result = record + onResult?(record) + } catch { + environment?.log("capture result could not be read: \(error)") + } + } + + /// A photo or tap the scan accepted couldn't be kept, so `session`'s packet can't be complete. + /// Admission closes now, before the uploader's own turn, so its loop sends nothing more; the + /// upload then ends with `inputLostCode`. A session that already ended (a reset world's late + /// work) is left alone: its upload was abandoned, and the next world isn't this one. + private func inputLost(_ session: Session, _ detail: String) { + environment?.log("capture packet: \(detail); this capture can't be prepared") + guard !session.ended, session.preparationFailure == nil else { return } + session.preparationFailure = Self.inputLostCode + guard let uploader = session.uploader else { return } + uploader.stopSending() + Task { await uploader.failPreparation(Self.inputLostCode) } + } + + private func endSession(_ reason: String) -> Result { + guard let session else { return .success(()) } + self.session = nil + session.ended = true + onStatus?(nil) + guard let uploader = session.uploader else { + // A no that couldn't be recorded took the uploader away and left the old yes on disk; + // this end can't record anything either, so it says so. + if case .notRecorded(let error)? = session.withdrawalFailure { return .failure(.notRecorded(error)) } + return .success(()) + } + // Synchronous: closes admission, bars this process from resuming it, writes the marker a + // relaunch reads. The abandonment saved in the upload's own state follows on its turn. + let record = uploader.retire(reason) + Task { await uploader.abandon(reason) } + guard case .notRecorded(let error) = record else { return .success(()) } + environment?.log("capture upload: the end of this world's capture could not be recorded (\(error)); it won't resume in this process") + return .failure(.notRecorded(error)) + } +} + +extension PacketTracking { + /// The recorder's codes: state 0 normal, 1 limited, 2 not available; reason 0 none or + /// unknown, 1 initializing, 2 relocalizing, 3 excessive motion, 4 insufficient features. + public init(recorderState state: Int, reason: Int) { + let why: Reason? = switch reason { + case 1: .initializing + case 2: .relocalizing + case 3: .excessiveMotion + case 4: .insufficientFeatures + default: nil + } + self = switch state { + case 0: .normal + case 1: .limited(why) + default: .notAvailable + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadGate.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadGate.swift new file mode 100644 index 000000000..068ee46e2 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadGate.swift @@ -0,0 +1,72 @@ +import Foundation + +/// Whether this build and this homeowner allow a capture upload. No endpoint means nothing is +/// offered and nothing is sent; an endpoint without the homeowner's yes sends nothing either. +public enum CaptureUploadGate { + public enum Decision: Sendable, Equatable { + case off(String) + case on(URL) + } + + /// `endpoint` is the API base including `/v1`, from the build's Info.plist. An empty build + /// setting, an unexpanded `$(...)`, or anything but https (http only to this machine, for + /// tests) is off. + public static func decide(endpoint: String?, consented: Bool) -> Decision { + let text = endpoint?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard !text.isEmpty, !text.hasPrefix("$(") else { return .off("no capture endpoint in this build") } + guard let url = URL(string: text), let scheme = url.scheme?.lowercased(), let host = url.host(), url.user == nil, url.password == nil else { + return .off("capture endpoint is not a URL") + } + let local = host == "127.0.0.1" || host == "localhost" + guard scheme == "https" || (scheme == "http" && local) else { return .off("capture endpoint must be https") } + guard consented else { return .off("the homeowner has not agreed to send the capture") } + return .on(url) + } +} + +/// What the app does with its captures, from three build settings read once at launch. Only the +/// integration build records a 0.4 capture at all; only a build that also names an endpoint and +/// says device data may go there sends one, and then only after the homeowner's yes. +public enum CaptureIntegrationMode: Sendable, Equatable { + /// The client build, or no usable endpoint: nothing is recorded for the capture API or sent. + case off(String) + /// The capture is recorded on the phone and never sent: the endpoint may not receive device + /// data (an unauthenticated test server, unless the build says otherwise). + case recordOnly(URL) + /// Recorded, and sent after the homeowner's yes. + case send(URL) + + public var endpoint: URL? { + switch self { + case .off: nil + case .recordOnly(let url), .send(let url): url + } + } + + /// Where the scan's pictures come from. + public enum Source: Sendable, Equatable { + /// The phone's camera: sending needs the build's device-data switch. + case device + /// A recorded session played back (`-replay`). Its pictures can be anyone's, so it is sent + /// only to a receiver on this machine, and only when the run asks for that + /// (`sendToLocalReceiver`, the UI tests' switch); never to a remote endpoint. + case replay(sendToLocalReceiver: Bool) + } + + /// `integrationBuild` and `sendDeviceData` are Info.plist values that must read exactly "YES". + /// A launch argument can name `endpoint`, but can't turn a client build into an integration + /// build or allow device data. + public static func resolve(integrationBuild: String?, endpoint: String?, sendDeviceData: String?, source: Source = .device) -> CaptureIntegrationMode { + guard integrationBuild == "YES" else { return .off("not the integration build") } + switch CaptureUploadGate.decide(endpoint: endpoint, consented: true) { + case .off(let reason): return .off(reason) + case .on(let url): + switch source { + case .device: return sendDeviceData == "YES" ? .send(url) : .recordOnly(url) + case .replay(let local): + let loopback = url.host() == "127.0.0.1" || url.host() == "localhost" + return local && loopback ? .send(url) : .recordOnly(url) + } + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift new file mode 100644 index 000000000..f24ecce7a --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift @@ -0,0 +1,204 @@ +import Foundation + +/// One capture's upload, saved in its folder after every change so a relaunch resumes the same +/// capture with the same identity. The request bodies that idempotency keys are bound to (create +/// and finalize) are saved as the exact bytes first sent, never re-encoded. +public struct CaptureUploadState: Codable, Sendable, Equatable { + public static let fileName = "capture-upload.json" + /// Written into a capture's folder when the homeowner takes back their yes. `CaptureUploader.resume` + /// never resumes a folder that has it. + public static let withdrawnFileName = "consent-withdrawn" + + public static func withdrawnURL(in folder: URL) -> URL { folder.appending(path: withdrawnFileName) } + + /// Written into a capture's folder when its world ends before the capture was finished (a + /// reset, or starting over). It holds the reason. `CaptureUploader.resume` never resumes a folder + /// that has it. Not a withdrawal: the homeowner's yes wasn't taken back. + public static let endedFileName = "capture-ended" + + public static func endedURL(in folder: URL) -> URL { folder.appending(path: endedFileName) } + + /// Called with the uploader's folder registry locked so no initial save or reply can restore + /// a yes between these writes. Neither failing proves anything durable after a process exit. + static func recordWithdrawal(in folder: URL) -> CaptureUploader.WithdrawalRecord { + do { + try Data().write(to: withdrawnURL(in: folder), options: .atomic) + return .marked + } catch { + let markerError = CaptureUploader.describe(error) + do { + try FileManager.default.removeItem(at: CaptureUploader.stateURL(in: folder)) + return .savedStateRemoved + } catch let removal as NSError where removal.domain == NSCocoaErrorDomain && removal.code == NSFileNoSuchFileError { + return .savedStateRemoved + } catch { + return .notRecorded("marker: \(markerError); state removal: \(CaptureUploader.describe(error))") + } + } + } + + public enum FilePhase: Codable, Sendable, Equatable { + /// Sealed on the phone, not yet registered (or its URL must be fetched again). + case queued + case registered(url: String, headers: [String: String], expiresAt: Date?) + /// The storage PUT answered 2xx. Not yet received: only a commit says that. + case uploaded + /// Acknowledged by files:commit. + case committed + } + + public struct File: Codable, Sendable, Equatable { + public var sealed: SealedFile + public var phase: FilePhase = .queued + public var attempts = 0 + /// Register answers that left this file out, plus commit answers that didn't acknowledge + /// it. Nil in state saved before this was counted. + public var unacknowledged: Int? + /// Order in which the file was sealed, the tie-break after priority. + public var sequence: Int + + public init(sealed: SealedFile, phase: FilePhase = .queued, attempts: Int = 0, sequence: Int) { + self.sealed = sealed + self.phase = phase + self.attempts = attempts + self.sequence = sequence + } + } + + public enum End: Codable, Sendable, Equatable { + /// The ARKit world was reset or the scan started over: this packet will never be finished. + case abandoned(String) + /// The server refused something a retry can't change. + case failed(step: String, codes: [String], status: Int) + /// The capture reached a status after which it changes no more. + case finished(status: String) + } + + public struct Finalized: Codable, Sendable, Equatable { + public var status: String + public var missing: [String] + public var runID: String + + public init(status: String, missing: [String], runID: String) { + self.status = status + self.missing = missing + self.runID = runID + } + } + + /// Local identity of this upload. Answers that arrive for another attempt are dropped. + public var attemptID: String + public var packetID: String + /// The API base the capture was created on. A resume goes only there: its capture id and + /// signed URLs mean nothing to another server. + public var destination: String? + /// The homeowner's yes for this capture and this destination. A capture without one is + /// never resumed. + public var consent: Consent? + + public struct Consent: Codable, Sendable, Equatable { + public var grantedAt: Date + public var destination: String + + public init(grantedAt: Date, destination: String) { + self.grantedAt = grantedAt + self.destination = destination + } + } + /// The create body exactly as first sent. + public var createBody: Data + public var captureID: String? + public var maxBatch = 50 + public var files: [String: File] = [:] + public var nextSequence = 0 + /// packet.json exactly as frozen when the capture stopped. + public var packet: Data? + public var finalized: Finalized? + public var eventCursor = 0 + /// `retry_finalize` answers followed so far. + public var finalizeRetries = 0 + /// Result reads that found no answer yet. Saved so a relaunch continues the same limit; nil in + /// state saved before this was counted. + public var notReadyReads: Int? + public var backendStatus: String? + public var lastEvent: String? + public var result: Data? + public var end: End? + /// Wall-clock marks of each stage, for latency: no ids, URLs or photos. + public var marks: [String: Date] = [:] + + public init(attemptID: String, packetID: String, createBody: Data) { + self.attemptID = attemptID + self.packetID = packetID + self.createBody = createBody + } + + public var committedCount: Int { files.values.filter { $0.phase == .committed }.count } + public var committedBytes: Int { files.values.filter { $0.phase == .committed }.reduce(0) { $0 + $1.sealed.bytes } } + public var totalBytes: Int { files.values.reduce(0) { $0 + $1.sealed.bytes } } + + public func save(to url: URL) throws { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + try encoder.encode(self).write(to: url, options: .atomic) + } + + public static func load(from url: URL) throws -> CaptureUploadState { + try JSONDecoder().decode(CaptureUploadState.self, from: Data(contentsOf: url)) + } +} + +/// What the experience layer shows: separate facts, never one percentage. +public struct CaptureUploadStatus: Sendable, Equatable { + public enum Phase: Sendable, Equatable { + case creating, uploading, processing, finished, failed, abandoned + } + + public var phase: Phase + /// Files sealed on the phone for this capture. + public var retained: Int + /// Files the server acknowledged with a commit. + public var committed: Int + public var committedBytes: Int + public var totalBytes: Int + /// Whether packet.json is frozen (the capture stopped). + public var sealed: Bool + /// The capture's status as the server last reported it. + public var backendStatus: String? + public var lastEvent: String? + public var eventCursor: Int + public var resultAvailable: Bool + /// The server's id for the capture, once created: what an operator copies to find this scan. + public var captureID: String? + /// The run finalize started, once accepted. + public var runID: String? + /// Set while the uploader waits out a retryable failure. + public var retryingAt: Date? + public var detail: String? + + public init(_ state: CaptureUploadState, retryingAt: Date? = nil, detail: String? = nil) { + phase = switch state.end { + case .abandoned?: .abandoned + case .failed?: .failed + case .finished?: .finished + case nil: state.captureID == nil ? .creating : state.finalized == nil ? .uploading : .processing + } + retained = state.files.count + committed = state.committedCount + committedBytes = state.committedBytes + totalBytes = state.totalBytes + sealed = state.packet != nil + backendStatus = state.backendStatus + lastEvent = state.lastEvent + eventCursor = state.eventCursor + resultAvailable = state.result != nil + captureID = state.captureID + runID = state.finalized?.runID + self.retryingAt = retryingAt + if case .failed(let step, let codes, let status) = state.end { + self.detail = "\(step) \(status) \(codes.joined(separator: ","))" + } else { + self.detail = detail + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift new file mode 100644 index 000000000..a8ce5732b --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -0,0 +1,851 @@ +import Foundation +import Synchronization + +/// Sends one capture to the capture API while it is being taken: create once, then register, PUT +/// and commit each sealed file as it arrives, then finalize with the frozen packet (possibly +/// before the last commits), then follow the events to a result. +/// +/// Every answer is applied only while the attempt that asked is still current, so a reply that +/// arrives after a world reset or a new capture changes nothing. The saved state +/// (`CaptureUploadState`) is written after every change, which is what resumes a capture after a +/// relaunch with the same packet id, capture id and frozen request bodies. +public actor CaptureUploader { + public struct Policy: Sendable { + public var maxConcurrentPuts = 3 + /// Files per register and commit call: the contract's "about every 10 files". + public var batch = 10 + public var firstDelay = 1.0 + public var maxDelay = 30.0 + /// Seconds the server may hold an events poll. + public var eventsWait = 20 + /// A signed URL this close to expiry is fetched again before the PUT. + public var expiryMargin = 60.0 + /// PUTs of one file the storage refuses (not transient) before the upload stops, counted per + /// server capture. + public var maxRefusedPuts = 6 + /// `retry_finalize` answers followed before the upload stops, counted per server capture. + public var maxFinalizeRetries = 3 + /// Result reads that find the run's answer not readable yet before the upload stops. Each + /// waits the retry delay first, so the default allows about three minutes. No measurement + /// of how long a finished run's answer takes to become readable sets this number; it only + /// keeps an answer that never appears from holding the upload open. + public var maxNotReadyResults = 10 + /// Retries after a register answer leaves a file out or a commit doesn't acknowledge it, + /// counted per file per server capture. Each retry waits the retry delay; the upload stops + /// on the next unacknowledged answer, so a file is sent at most `maxUnacknowledged + 1` + /// times. No measurement sets this; it keeps a server that never takes a file from getting + /// the same photo forever. + public var maxUnacknowledged = 5 + /// Seconds between events polls that come back at once with nothing new and the capture + /// still running, for a server or proxy that doesn't hold the poll. Not measured. + public var minimumPollInterval = 2.0 + + public init() {} + + public func delay(afterFailure n: Int, retryAfter: Double?) -> Double { + if let retryAfter, retryAfter > 0 { return min(retryAfter, 120) } + return min(maxDelay, firstDelay * pow(2, Double(max(n, 1) - 1))) + } + } + + /// A failure a later try can fix. + struct Transient: Error { + var step: String + var detail: String + var retryAfter: Double? + } + + /// The homeowner took back their yes: nothing more is sent. + struct Withdrawn: Error {} + + /// The server refused something a retry can't change. + struct Refused: Error { + var step: String + var status: Int + var codes: [String] + } + + public nonisolated let folder: URL + private let base: URL + private let http: any CaptureHTTP + private let policy: Policy + private let now: @Sendable () -> Date + private let sleep: @Sendable (Double) async throws -> Void + private var state: CaptureUploadState + private var loop: Task? + private var failures = 0 + private var retryingAt: Date? + private var digestRetried: Set = [] + /// The registry covers public start/resume as well as coordinator discovery. An idle live + /// uploader still owns its folder, since it can receive more sealed files. Weak ownership + /// releases only after the uploader and its running task are gone. Withdrawn entries stay + /// for this process's lifetime, including when every disk write failed. + private struct FolderEntry { + weak var owner: CaptureUploader? + var withdrawn = false + /// A save failed here: the saved state is older than what this process sent, so this + /// process never resumes it. A relaunch may, under the consent it records. + var unsaved = false + /// Its world ended (`retire`): this process never resumes it, whatever reached the disk. + var retired = false + var stopped = false + var running: Task? + } + private static let folders = Mutex<[String: FolderEntry]>([:]) + private nonisolated let folderKey: String + + public enum OwnershipError: Error, Sendable, Equatable { + case alreadyOwned + case consentWithdrawn + /// Use resume for an existing saved upload; start must never overwrite its consent. + case savedUploadExists + case folderIdentityUnavailable + } + /// Set when a state change could not be saved. The upload ends here (`unsavedStateCode`) + /// rather than send a request the saved state doesn't know about. + private var saveFailed = false + private var observer: (@Sendable (CaptureUploadStatus) -> Void)? + private var log: (@Sendable (String) -> Void)? + + public nonisolated static func stateURL(in folder: URL) -> URL { folder.appending(path: CaptureUploadState.fileName) } + + /// Resolve symbolic links before asking for the filesystem's case-correct path. The + /// canonical-path resource alone preserves a final-component symlink on macOS. + /// Refuse an unknown identity instead of allowing a second registry key for one folder. + private static func canonicalFolder(_ folder: URL) throws -> URL { + let fresh = URL(fileURLWithPath: folder.path, isDirectory: true).standardizedFileURL.resolvingSymlinksInPath() + guard let path = try fresh.resourceValues(forKeys: [.canonicalPathKey]).canonicalPath else { + throw OwnershipError.folderIdentityUnavailable + } + return URL(fileURLWithPath: path, isDirectory: true) + } + + private init( + folder: URL, base: URL, http: any CaptureHTTP, state: CaptureUploadState, policy: Policy, + now: @escaping @Sendable () -> Date, sleep: @escaping @Sendable (Double) async throws -> Void + ) { + self.folder = folder + self.folderKey = folder.path + self.base = base + self.http = http + self.state = state + self.policy = policy + self.now = now + self.sleep = sleep + } + + /// A new capture. The create body is encoded once here and kept as those bytes for every retry. + /// `consentedAt` is when the homeowner said yes to sending this capture to `base`. + public static func start( + folder: URL, base: URL, http: any CaptureHTTP, create: CaptureAPI.CreateRequest, consentedAt: Date, policy: Policy = .init(), + now: @escaping @Sendable () -> Date = { Date() }, + sleep: @escaping @Sendable (Double) async throws -> Void = { try await Task.sleep(for: .seconds($0)) } + ) throws -> CaptureUploader { + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + let folder = try canonicalFolder(folder) + return try folders.withLock { entries in + entries = entries.filter { $0.value.owner != nil || $0.value.withdrawn || $0.value.unsaved || $0.value.retired } + guard entries[folder.path]?.withdrawn != true, + !FileManager.default.fileExists(atPath: CaptureUploadState.withdrawnURL(in: folder).path) + else { throw OwnershipError.consentWithdrawn } + guard entries[folder.path]?.owner == nil else { throw OwnershipError.alreadyOwned } + guard !FileManager.default.fileExists(atPath: stateURL(in: folder).path) else { throw OwnershipError.savedUploadExists } + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + var state = CaptureUploadState(attemptID: UUID().uuidString, packetID: create.packetId, createBody: try encoder.encode(create)) + state.destination = base.absoluteString + state.consent = .init(grantedAt: consentedAt, destination: base.absoluteString) + state.marks["started"] = now() + try state.save(to: stateURL(in: folder)) + let uploader = CaptureUploader(folder: folder, base: base, http: http, state: state, policy: policy, now: now, sleep: sleep) + entries[folder.path] = FolderEntry(owner: uploader) + return uploader + } + } + + /// Resumes a saved upload with its recorded consent and endpoint. Returns nil when absent, + /// ended or withdrawn; throws `alreadyOwned` while another live uploader owns the folder. + /// The new attempt id and ownership are saved together, before any request can start. + public static func resume( + folder: URL, base: URL, http: any CaptureHTTP, policy: Policy = .init(), + now: @escaping @Sendable () -> Date = { Date() }, + sleep: @escaping @Sendable (Double) async throws -> Void = { try await Task.sleep(for: .seconds($0)) } + ) throws -> CaptureUploader? { + guard FileManager.default.fileExists(atPath: folder.path) else { return nil } + let folder = try canonicalFolder(folder) + return try folders.withLock { entries in + entries = entries.filter { $0.value.owner != nil || $0.value.withdrawn || $0.value.unsaved || $0.value.retired } + let url = stateURL(in: folder) + guard entries[folder.path]?.withdrawn != true, entries[folder.path]?.unsaved != true, entries[folder.path]?.retired != true, + FileManager.default.fileExists(atPath: url.path), + !FileManager.default.fileExists(atPath: CaptureUploadState.withdrawnURL(in: folder).path), + !FileManager.default.fileExists(atPath: CaptureUploadState.endedURL(in: folder).path) + else { return nil } + guard entries[folder.path]?.owner == nil else { throw OwnershipError.alreadyOwned } + var state = try CaptureUploadState.load(from: url) + guard state.end == nil, state.destination == base.absoluteString, state.consent?.destination == base.absoluteString else { return nil } + state.attemptID = UUID().uuidString + try state.save(to: url) + let uploader = CaptureUploader(folder: folder, base: base, http: http, state: state, policy: policy, now: now, sleep: sleep) + entries[folder.path] = FolderEntry(owner: uploader) + return uploader + } + } + + public func observe(_ observer: @escaping @Sendable (CaptureUploadStatus) -> Void, log: (@Sendable (String) -> Void)? = nil) { + self.observer = observer + self.log = log + observer(status) + } + + public var status: CaptureUploadStatus { + CaptureUploadStatus(state, retryingAt: retryingAt, detail: saveFailed ? "upload state could not be saved" : nil) + } + public var snapshot: CaptureUploadState { state } + + // MARK: Input + + /// Sealed files, uploaded as soon as the capture exists. A path already known is ignored: + /// sealed files never change. + public func add(_ files: [SealedFile]) { + guard state.end == nil, state.packet == nil else { return } + var added = false + for file in files where state.files[file.path] == nil { + state.files[file.path] = .init(sealed: file, sequence: state.nextSequence) + state.nextSequence += 1 + added = true + } + guard added else { return } + mark("firstRetained") + persist() + kick() + } + + /// The capture stopped: `packet` is frozen as these bytes, and `files` is every file it lists. + /// Any the uploader has not seen yet (an `add` still on its way) is queued here, so the + /// server never waits for a listed file that nobody sends. + public func seal(packet: Data, files: [SealedFile]) { + guard state.end == nil, state.packet == nil else { return } + for file in files where state.files[file.path] == nil { + state.files[file.path] = .init(sealed: file, sequence: state.nextSequence) + state.nextSequence += 1 + } + state.packet = packet + mark("sealed") + persist() + kick() + } + + public static let withdrawnReason = "consent withdrawn" + + /// The code an upload ends with, at step `save` and status 0, when this phone couldn't save + /// the upload's state. It is a local storage problem: it says nothing about the photos or + /// the server, and no server answered. + public static let unsavedStateCode = "local_state_unsaved" + + /// The local end `failPreparation` writes: step `prepare`, status 0, no server answer. + static func isPreparationEnd(_ end: CaptureUploadState.End?) -> Bool { + guard case .failed(step: "prepare", _, status: 0)? = end else { return false } + return true + } + static let unsavedEnd = CaptureUploadState.End.failed(step: "save", codes: [unsavedStateCode], status: 0) + + /// How a withdrawal was recorded on disk. + public enum WithdrawalRecord: Sendable, Equatable { + /// The marker `resume` refuses was written. + case marked + /// The marker couldn't be written, so the saved state was removed instead. + case savedStateRemoved + /// Neither the marker nor the removal succeeded: a relaunch could still find the saved + /// yes until a later save records the upload as abandoned. + case notRecorded(String) + } + + /// Revokes the folder before returning and cancels its running task. A request already + /// admitted to a transport that ignores cancellation may finish; its reply cannot admit + /// another request or save resumable consent. `.notRecorded` means only the live process + /// is protected: after a crash the old saved yes may still exist. + @discardableResult + public nonisolated func withdrawConsent() -> WithdrawalRecord { + let (record, task) = Self.folders.withLock { entries in + entries[folderKey, default: FolderEntry()].withdrawn = true + let record = CaptureUploadState.recordWithdrawal(in: folder) + return (record, entries[folderKey]?.running) + } + // Cancellation handlers belong to the transport; never call them while holding our lock. + task?.cancel() + return record + } + + /// How a world's end was recorded on disk (`retire`). + public enum RetirementRecord: Sendable, Equatable { + /// The ended marker `resume` refuses was written. + case marked + /// It couldn't be written: this process won't resume the capture, but a relaunch could + /// still find its saved yes until a later save records the upload as abandoned. + case notRecorded(String) + } + + /// Ends this capture's world before returning: admission closes, this process never resumes + /// the folder, and the ended marker (`CaptureUploadState.endedURL`) is written, synchronously + /// and under the folder registry's lock, as `withdrawConsent` writes its own. The running task + /// is cancelled. The upload's saved end still comes from `abandon`, on the uploader's turn. + @discardableResult + public nonisolated func retire(_ reason: String) -> RetirementRecord { + let (record, task) = Self.folders.withLock { entries -> (RetirementRecord, Task?) in + entries[folderKey, default: FolderEntry()].stopped = true + entries[folderKey]?.retired = true + let record: RetirementRecord + do { + try Data(reason.utf8).write(to: CaptureUploadState.endedURL(in: folder), options: .atomic) + record = .marked + } catch { + record = .notRecorded(Self.describe(error)) + } + return (record, entries[folderKey]?.running) + } + task?.cancel() + return record + } + + /// Closes admission synchronously when a coordinator ends a world, before its queued actor + /// work can kick the upload. Ownership stays until the old uploader and its requests die. + nonisolated func stopSending() { + let task = Self.folders.withLock { entries in + entries[folderKey]?.stopped = true + return entries[folderKey]?.running + } + task?.cancel() + } + + /// Ends this upload for good: a world reset or starting over. Replies still in flight are + /// dropped when they arrive. + public func abandon(_ reason: String) { + stopSending() + // An end the app asks for replaces a save failure. A withdrawal also replaces an end this + // process reached but couldn't save, such as a result or a refusal, and the local end of + // a capture that lost an input (`failPreparation`): the homeowner's no is the reason it + // stopped. Otherwise an end stays: Start over doesn't rewrite a finished upload, and a + // reset after a loss keeps the loss. + let withdrawn = isWithdrawn + let replacedByWithdrawal = withdrawn && (saveFailed || Self.isPreparationEnd(state.end)) + guard state.end == nil || state.end == Self.unsavedEnd || replacedByWithdrawal else { return } + state.end = .abandoned(replacedByWithdrawal ? Self.withdrawnReason : reason) + state.attemptID = UUID().uuidString + loop?.cancel() + persist() + } + + /// Ends the upload because the phone lost a photo or tap the packet needs: step `prepare`, + /// status 0, `code`. Admission closes first, so nothing more is sent; a request already + /// admitted may finish, and its reply is dropped. A withdrawal still outranks it (`persist`), + /// and an end already reached stays. + public func failPreparation(_ code: String) { + stopSending() + guard state.end == nil else { return } + state.end = .failed(step: "prepare", codes: [code], status: 0) + state.attemptID = UUID().uuidString + loop?.cancel() + persist() + } + + /// Starts the loop if it is idle: after a relaunch, or when the network is back. + public func kick() { + guard loop == nil, state.end == nil, !saveFailed else { return } + Self.folders.withLock { entries in + guard let entry = entries[folderKey], entry.owner === self, !entry.withdrawn, !entry.stopped else { return } + let task = Task { await self.drive() } + loop = task + entries[folderKey]?.running = task + } + } + + /// Waits for the loop to go idle: nothing to send until more input, or ended. + public func settled() async { + while let running = loop { await running.value; if loop == running { loop = nil } } + } + + // MARK: Loop + + private func drive() async { + while !Task.isCancelled, state.end == nil, !saveFailed { + // A withdrawal sends nothing more; the save records the upload as abandoned. + if isWithdrawn { persist(); break } + let attempt = state.attemptID + do { + let progressed = try await step() + guard attempt == state.attemptID, state.end == nil else { break } + failures = 0 + if retryingAt != nil { retryingAt = nil; publish() } + if !progressed { break } + } catch let error as Transient { + guard attempt == state.attemptID else { break } + failures += 1 + let delay = policy.delay(afterFailure: failures, retryAfter: error.retryAfter) + retryingAt = now().addingTimeInterval(delay) + log?("capture-upload retry step=\(error.step) in=\(String(format: "%.1f", delay))s detail=\(error.detail)") + publish() + do { try await sleep(delay) } catch { break } + } catch is Withdrawn { + persist() + break + } catch let error as Refused { + guard attempt == state.attemptID else { break } + state.end = .failed(step: error.step, codes: error.codes, status: error.status) + log?("capture-upload failed step=\(error.step) status=\(error.status) codes=\(error.codes.joined(separator: ","))") + persist() + } catch { + break + } + } + loop = nil + Self.folders.withLock { $0[folderKey]?.running = nil } + } + + /// One unit of work. False when there is nothing to do until more input arrives. + private func step() async throws -> Bool { + if state.captureID == nil { try await create(); return true } + // The frozen packet goes as soon as the capture stops, before the files still uploading: + // the server answers awaiting_files and starts the run on the last commit. + if state.packet != nil, state.finalized == nil { try await finalize(); return true } + requeueExpiring() + let queued = ordered(where: { $0.phase == .queued }) + if !queued.isEmpty { try await register(Array(queued.prefix(min(policy.batch, state.maxBatch)))); return true } + // Each round of PUTs is committed before the next starts, so the first photos count as + // received after one round trip of uploads rather than a whole register batch. + let uploaded = ordered(where: { $0.phase == .uploaded }) + if !uploaded.isEmpty { try await commit(Array(uploaded.prefix(min(policy.batch, state.maxBatch)))); return true } + let registered = ordered(where: { if case .registered = $0.phase { true } else { false } }) + if !registered.isEmpty { try await put(Array(registered.prefix(policy.maxConcurrentPuts))); return true } + if state.finalized != nil, state.end == nil { + if state.result != nil || CaptureAPI.terminalStatuses.contains(state.backendStatus ?? "") { + try await fetchResult() + return true + } + try await pollEvents() + return true + } + return false + } + + private func ordered(where include: (CaptureUploadState.File) -> Bool) -> [CaptureUploadState.File] { + state.files.values.filter(include).sorted { ($0.sealed.priority, $0.sequence) < ($1.sealed.priority, $1.sequence) } + } + + // MARK: Calls + + private func create() async throws { + var request = jsonRequest("captures", method: "POST", body: state.createBody) + request.setValue(state.packetID, forHTTPHeaderField: "Idempotency-Key") + let (reply, attempt) = try await call("create", request) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let body = try? JSONDecoder().decode(CaptureAPI.CreateResponse.self, from: reply.body) else { + throw Refused(step: "create", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + state.captureID = body.captureId + state.backendStatus = body.status + if let batch = body.upload?.maxBatch, batch > 0 { state.maxBatch = batch } + mark("created") + persist() + } + + private func register(_ files: [CaptureUploadState.File]) async throws { + let body = try JSONEncoder().encode(CaptureAPI.RegisterRequest(files: files.map { .init($0.sealed) })) + let (reply, attempt) = try await call("register", jsonRequest("captures/\(captureID)/files", method: "POST", body: body)) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let answer = try? JSONDecoder().decode(CaptureAPI.RegisterResponse.self, from: reply.body) else { + if try captureGone(reply) { return } + throw Refused(step: "register", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + for file in answer.files where state.files[file.path] != nil { + if file.state == "committed" { + setPhase(file.path, .committed) + } else if let target = file.upload, target.method == "PUT", let url = target.url { + // Photos go only to an https URL, or http to this machine for tests, and only to a + // URL the PUT can use: a URL that doesn't parse would leave the file registered + // forever. This checks the URL as given; `CaptureHTTP` refuses redirects, so a + // storage server can't send the photo on to a URL this never checked. + if let problem = Self.uploadURLProblem(url) { throw Refused(step: "register", status: reply.status, codes: [problem]) } + setPhase(file.path, .registered(url: url, headers: target.headers ?? [:], expiresAt: target.expiresAt.flatMap(Self.parseDate))) + } else { + throw Refused(step: "register", status: reply.status, codes: ["upload_method_\(file.upload?.method ?? "none")"]) + } + } + let answered = Set(answer.files.map(\.path)) + try unacknowledged(files.map(\.sealed.path).filter { !answered.contains($0) }, step: "register", status: reply.status) + } + + /// Counts files the server didn't take, saves, and backs off (or stops once one has been left + /// out more than `maxUnacknowledged` times). Saves and returns when there are none. + private func unacknowledged(_ paths: [String], step: String, status: Int) throws { + for path in paths { + let count = (state.files[path]?.unacknowledged ?? 0) + 1 + state.files[path]?.unacknowledged = count + } + persist() + guard !paths.isEmpty else { return } + if paths.contains(where: { (state.files[$0]?.unacknowledged ?? 0) > policy.maxUnacknowledged }) { + throw Refused(step: step, status: status, codes: ["not_acknowledged"]) + } + throw Transient(step: step, detail: "\(paths.count) not acknowledged", retryAfter: nil) + } + + /// Why `text` can't receive a photo: nil when it is an https URL, or http to this machine. + static func uploadURLProblem(_ text: String) -> String? { + guard let url = URL(string: text), let scheme = url.scheme?.lowercased(), let host = url.host(), !host.isEmpty else { + return "upload_url_invalid" + } + // The same rule as the API's own scope under `.loopbackHTTP`: https anywhere, plain http only + // to 127.0.0.1, ::1 or localhost, so the two lists can't drift apart. + return CaptureAPIScope.allows(scheme: scheme, host: host.lowercased(), transport: .loopbackHTTP) ? nil : "upload_url_insecure" + } + + private func put(_ files: [CaptureUploadState.File]) async throws { + let attempt = state.attemptID + var transient: Transient? + var refused: Refused? + var index = 0 + while index < files.count { + // Each batch of PUTs is admitted only while the homeowner's yes stands. + if !canSend || Task.isCancelled { throw Withdrawn() } + let chunk = files[index..).self) { group in + for file in chunk { + guard case .registered(let url, let headers, _) = file.phase, Self.uploadURLProblem(url) == nil, let target = URL(string: url) else { continue } + var request = URLRequest(url: target) + request.httpMethod = "PUT" + // Exactly the headers the server signed; never the API's own credentials. + for (name, value) in headers { request.setValue(value, forHTTPHeaderField: name) } + let local = folder.appending(path: file.sealed.path) + let http = self.http + group.addTask { + if !self.canSend || Task.isCancelled { return (file.sealed.path, .failure(Withdrawn())) } + do { return (file.sealed.path, .success(try await http.upload(request, file: local))) } catch { return (file.sealed.path, .failure(error)) } + } + } + var out: [(String, Result)] = [] + for await result in group { out.append(result) } + return out + } + if !canSend || Task.isCancelled { throw Withdrawn() } + guard attempt == state.attemptID, state.end == nil else { return } + for (path, result) in results { + state.files[path]?.attempts += 1 + switch result { + case .success(let reply) where (200..<300).contains(reply.status): + setPhase(path, .uploaded) + case .success(let reply) where reply.status == 400 && !digestRetried.contains(path): + // BadDigest: the bytes on disk are sealed, so sending them once more is the retry. + digestRetried.insert(path) + case .success(let reply) where reply.status == 400: + // The same sealed bytes refused twice: the file on disk no longer matches its digest. + throw Refused(step: "put", status: 400, codes: ["bad_digest"]) + case .success(let reply) where Self.isTransient(reply.status): + transient = Transient(step: "put", detail: "status \(reply.status)", retryAfter: reply.retryAfter) + case .success(let reply): + // 403 expired signature or a spent token: a fresh URL, a bounded number of times. + guard (state.files[path]?.attempts ?? 0) < policy.maxRefusedPuts else { + throw Refused(step: "put", status: reply.status, codes: ["storage_refused"]) + } + setPhase(path, .queued) + case .failure(let error): + switch failure("put", error) { + case let error as Refused: refused = refused ?? error + case let error as Transient: transient = error + case let error: throw error + } + } + } + persist() + if let refused { throw refused } + } + if let transient { throw transient } + } + + private func commit(_ files: [CaptureUploadState.File]) async throws { + let body = try JSONEncoder().encode(CaptureAPI.CommitRequest(files: files.map { .init(path: $0.sealed.path, sha256: $0.sealed.sha256) })) + let (reply, attempt) = try await call("commit", jsonRequest("captures/\(captureID)/files:commit", method: "POST", body: body)) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let answer = try? JSONDecoder().decode(CaptureAPI.CommitResponse.self, from: reply.body) else { + if try captureGone(reply) { return } + throw Refused(step: "commit", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + let committed = Set(answer.committed) + for file in files { + // Only a commit acknowledges a file; notFound, mismatch or no mention sends it again. + setPhase(file.sealed.path, committed.contains(file.sealed.path) ? .committed : .queued) + } + if !committed.isEmpty { mark("firstCommit") } + if state.packet != nil, state.files.values.allSatisfy({ $0.phase == .committed }) { mark("lastCommit") } + try unacknowledged(files.map(\.sealed.path).filter { !committed.contains($0) }, step: "commit", status: reply.status) + } + + private func finalize() async throws { + guard let packet = state.packet else { return } + var request = jsonRequest("captures/\(captureID)/finalize", method: "POST", body: packet) + request.setValue(state.packetID, forHTTPHeaderField: "Idempotency-Key") + request.setValue(PacketFiles.sha256(packet), forHTTPHeaderField: "X-Packet-Sha256") + let (reply, attempt) = try await call("finalize", request) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let answer = try? JSONDecoder().decode(CaptureAPI.FinalizeResponse.self, from: reply.body) else { + if try captureGone(reply) { return } + throw Refused(step: "finalize", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + state.finalized = .init(status: answer.status, missing: answer.missing, runID: answer.runId) + state.backendStatus = answer.status + mark("finalizeAccepted") + persist() + } + + private func pollEvents() async throws { + var components = URLComponents(url: base.appending(path: "captures/\(captureID)/events"), resolvingAgainstBaseURL: false)! + components.queryItems = [.init(name: "after", value: String(state.eventCursor)), .init(name: "wait", value: String(policy.eventsWait))] + var request = URLRequest(url: components.url!) + request.setValue("application/json", forHTTPHeaderField: "Accept") + request.timeoutInterval = Double(policy.eventsWait) + 30 + let sent = now() + let (reply, attempt) = try await call("events", request) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let answer = try? JSONDecoder().decode(CaptureAPI.EventsResponse.self, from: reply.body) else { + if try captureGone(reply) { return } + throw Refused(step: "events", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + state.backendStatus = answer.status + state.eventCursor = max(state.eventCursor, answer.next) + for event in answer.events { + mark("firstEvent") + state.lastEvent = [event.type, event.data?.stage, event.data?.status, event.data?.kind, event.data?.code].compactMap { $0 }.joined(separator: " ") + // A verdict for another run is not this capture's result. + if event.type == "verdict_ready", event.data?.runId == nil || event.data?.runId == state.finalized?.runID { mark("verdictReady") } + if event.type == "failed", event.data?.next == "retry_finalize" { + // The server lost the finalize to a transient storage failure: send the same frozen + // packet again. + guard state.finalizeRetries < policy.maxFinalizeRetries else { + throw Refused(step: "finalize", status: 0, codes: [event.data?.code ?? "retry_finalize"]) + } + state.finalizeRetries += 1 + state.finalized = nil + } + } + if !answer.events.isEmpty { log?("capture-upload events status=\(answer.status) last=\(state.lastEvent ?? "")") } + persist() + // An answer that came back at once with nothing new means the server didn't hold the + // poll; waiting here keeps the loop from asking again immediately. + let waited = now().timeIntervalSince(sent) + if answer.events.isEmpty, !CaptureAPI.terminalStatuses.contains(answer.status), waited < policy.minimumPollInterval { + try await sleep(policy.minimumPollInterval - waited) + } + } + + private func fetchResult() async throws { + var request = URLRequest(url: base.appending(path: "captures/\(captureID)/result")) + request.setValue("application/json", forHTTPHeaderField: "Accept") + let (reply, attempt) = try await call("result", request) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status) else { + if try captureGone(reply) { return } + // The run reads as done before its answer is stored. + if reply.status == 404, CaptureAPI.errorCodes(reply.body).contains("result_not_ready") { try notReady(reply) } + throw Refused(step: "result", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + // The whole body must decode as the result the app reads, or a finished upload could hold + // an answer nothing can show. + let decoder = JSONDecoder() + guard let response = try? decoder.decode(CaptureResult.Response.self, from: reply.body), + let head = try? decoder.decode(ResultHead.self, from: reply.body) + else { + throw Refused(step: "result", status: reply.status, codes: ["result_unreadable"]) + } + if let runID = head.runId, runID != state.finalized?.runID { + throw Refused(step: "result", status: reply.status, codes: ["result_for_another_run"]) + } + let status = head.status ?? state.backendStatus ?? "unknown" + // Until the run has written its answer the API serves the capture's status with a null + // outcome, and that status can already be one the capture ends in. + if response.outcome == nil, !CaptureAPI.statusesWithoutOutcome.contains(status) { try notReady(reply) } + state.result = reply.body + state.end = .finished(status: status) + mark("result") + persist() + } + + /// The run and status as the server wrote them; `CaptureResult.Status` keeps no raw text. + private struct ResultHead: Decodable { + var runId: String? + var status: String? + } + + /// Always throws: a retry after the usual wait, or the end of the upload once + /// `maxNotReadyResults` reads have found no answer. + private func notReady(_ reply: HTTPReply) throws -> Never { + let notReadyReads = (state.notReadyReads ?? 0) + 1 + state.notReadyReads = notReadyReads + persist() + guard notReadyReads <= policy.maxNotReadyResults else { + throw Refused(step: "result", status: reply.status, codes: ["result_not_ready"]) + } + throw Transient(step: "result", detail: "result not ready (\(notReadyReads))", retryAfter: reply.retryAfter) + } + + // MARK: Plumbing + + private var captureID: String { state.captureID ?? "" } + + /// The server lost the capture (404: a redeploy with in-memory state). The same create body + /// opens it again, and every file goes up again under the new id. + private func captureGone(_ reply: HTTPReply) throws -> Bool { + guard reply.status == 404, CaptureAPI.errorCodes(reply.body).contains("capture_not_found") else { return false } + log?("capture-upload the server no longer has the capture; creating it again") + state.captureID = nil + state.finalized = nil + state.eventCursor = 0 + // A new server capture is a new run: none of the old one's retry limits carry over. + // Each still bounds the new run on its own. + state.notReadyReads = nil + state.finalizeRetries = 0 + digestRetried.removeAll() + for path in state.files.keys { + state.files[path]?.phase = .queued + state.files[path]?.attempts = 0 + state.files[path]?.unacknowledged = nil + } + persist() + return true + } + + private func requeueExpiring() { + let limit = now().addingTimeInterval(policy.expiryMargin) + for (path, file) in state.files { + if case .registered(_, _, let expires?) = file.phase, expires <= limit { state.files[path]?.phase = .queued } + } + } + + private func setPhase(_ path: String, _ phase: CaptureUploadState.FilePhase) { + state.files[path]?.phase = phase + } + + private func call(_ step: String, _ request: URLRequest) async throws -> (HTTPReply, String) { + // Every API request is admitted only while the homeowner's yes stands. One already sent + // finishes, but its reply can only save the upload as abandoned. + if !canSend || Task.isCancelled { throw Withdrawn() } + let attempt = state.attemptID + let reply: HTTPReply + do { + reply = try await http.send(request) + } catch { + throw failure(step, error) + } + if Self.isTransient(reply.status), attempt == state.attemptID { + throw Transient(step: step, detail: "status \(reply.status)", retryAfter: reply.retryAfter) + } + return (reply, attempt) + } + + private func jsonRequest(_ path: String, method: String, body: Data) -> URLRequest { + var request = URLRequest(url: base.appending(path: path)) + request.httpMethod = method + request.httpBody = body + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.setValue("application/json", forHTTPHeaderField: "Accept") + return request + } + + private func mark(_ name: String) { + guard state.marks[name] == nil else { return } + let at = now() + state.marks[name] = at + let since = state.marks["started"].map { at.timeIntervalSince($0) } ?? 0 + log?("capture-upload stage=\(name) t=\(String(format: "%.2f", since))s committed=\(state.committedCount)/\(state.files.count)") + } + + private nonisolated var isWithdrawn: Bool { Self.folders.withLock { $0[folderKey]?.withdrawn == true } } + + private nonisolated var canSend: Bool { + Self.folders.withLock { entries in + guard let entry = entries[folderKey] else { return false } + return entry.owner === self && !entry.withdrawn && !entry.stopped + } + } + + private func persist() { + do { + try Self.folders.withLock { entries in + guard entries[folderKey]?.owner === self else { throw OwnershipError.alreadyOwned } + let withdrawn = entries[folderKey]?.withdrawn == true + // A withdrawal outranks any end a late reply set (finished, failed). + if withdrawn, state.end != .abandoned(Self.withdrawnReason) { + state.end = .abandoned(Self.withdrawnReason) + // Replies still in flight belong to the old attempt and are dropped. + state.attemptID = UUID().uuidString + } + try state.save(to: Self.stateURL(in: folder)) + } + } catch { + saveFailed = true + // Admission closes before any suspended request resumes, as `stopSending` closes it: + // a credential fetch that returns after this sends nothing. One already handed to the + // transport may finish. + let running = Self.folders.withLock { entries -> Task? in + guard entries[folderKey]?.owner === self else { return nil } + entries[folderKey]?.unsaved = true + entries[folderKey]?.stopped = true + return entries[folderKey]?.running + } + // The saved state is now behind this one, so the upload ends in memory only: nothing + // claims it was saved, and nothing more is sent. An end already set stays: a + // withdrawal's, set before the write was tried, or a result or refusal. + // Replies in flight belong to the old attempt and are dropped. + if state.end == nil { state.end = Self.unsavedEnd } + state.attemptID = UUID().uuidString + log?("capture-upload stopped: the upload state could not be saved (\(Self.describe(error)))") + // Outside the lock: cancellation handlers belong to the transport. + running?.cancel() + } + publish() + } + + private func publish() { + observer?(status) + } + + /// 501 is the server saying it doesn't do what was asked (multipart), which waiting won't change. + static func isTransient(_ status: Int) -> Bool { status == 0 || status == 408 || status == 429 || ((500..<600).contains(status) && status != 501) } + + /// The error's domain and code only: a URLError's description can carry the signed URL. + /// What a transport error means for the upload. A withdrawal or cancellation that happened + /// meanwhile outranks the error, whatever it is: the request may have been cut short by that + /// very withdrawal (`ScopedCaptureHTTP` throws CancellationError once a credential wait is + /// cancelled), and retrying it would leave the upload waiting to retry instead of saved as + /// withdrawn. A refusal by the credential boundary won't change on retry: a URL outside the + /// API's scope, a malformed credential, one the provider couldn't supply, or a storage + /// request carrying Authorization. It ends the upload with the boundary's code and status 0, + /// since no server answered. Any other error, a network one included, is retried as before. + private func failure(_ step: String, _ error: any Error) -> any Error { + if !canSend || Task.isCancelled { + log?("capture-upload withdrawn step=\(step) detail=\(Self.describe(error))") + return Withdrawn() + } + guard let refusal = error as? ScopedCaptureHTTPError else { + return Transient(step: step, detail: Self.describe(error), retryAfter: nil) + } + return Refused(step: step, status: 0, codes: [refusal.code]) + } + + static func describe(_ error: any Error) -> String { + let ns = error as NSError + return "\(ns.domain) \(ns.code)" + } + + static func parseDate(_ text: String) -> Date? { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + if let date = formatter.date(from: text) { return date } + formatter.formatOptions = [.withInternetDateTime] + return formatter.date(from: text) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/ScopedCaptureHTTP.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/ScopedCaptureHTTP.swift new file mode 100644 index 000000000..51633c673 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/ScopedCaptureHTTP.swift @@ -0,0 +1,189 @@ +import Foundation + +/// The capture API's own requests, and nothing else, carry the credential the API issued. +/// +/// `CaptureUploader` sends two kinds of request through `CaptureHTTP`: `send` for the capture +/// API (create, register, commit, finalize, events, result) and `upload` for the storage URLs +/// the API signs. This decorator adds `Authorization: Bearer ` to an API request only +/// when the request's URL is inside the scope fixed at construction, and asks the credential +/// provider for the token only then. A storage upload never gets the credential, and the +/// provider isn't consulted for one, even when storage shares the API's origin. +/// +/// It refuses rather than falls back: a request outside the scope, a credential the provider +/// can't supply or one that isn't a well-formed token throws `ScopedCaptureHTTPError` and nothing +/// is sent. It adds no retry; the uploader's existing policy handles thrown errors. Redirects +/// stay refused by the wrapped transport (`URLSessionCaptureHTTP` refuses every one), so the +/// credential never reaches a redirect target. +/// +/// This is the client's side of the boundary only. It doesn't establish what the server +/// authorizes, attest result bytes or obtain the credential. +public struct ScopedCaptureHTTP: CaptureHTTP { + /// Supplies the API credential, refreshing it when the issuer requires. Called once per API + /// request that passes the scope check, never for a storage upload. + public typealias Credential = @Sendable () async throws -> String + + public let scope: CaptureAPIScope + private let credential: Credential + private let inner: any CaptureHTTP + + public init(scope: CaptureAPIScope, credential: @escaping Credential, inner: any CaptureHTTP) { + self.scope = scope + self.credential = credential + self.inner = inner + } + + public func send(_ request: URLRequest) async throws -> HTTPReply { + // The destination is checked before the credential is asked for. + guard let url = request.url else { throw ScopedCaptureHTTPError.destinationNotAllowed("no_url") } + if let reason = scope.problem(with: url) { throw ScopedCaptureHTTPError.destinationNotAllowed(reason) } + // A request cancelled before or during the credential fetch is never sent. A provider may + // wait without watching for cancellation and then return a token anyway; checking again + // after it returns keeps that token off a request the caller has given up on. This stops + // only new requests from this side: it doesn't withdraw anything the server already has. + try Task.checkCancellation() + let token: String + do { + token = try await credential() + } catch is CancellationError { + throw CancellationError() + } catch { + // Only the error's type: a provider's message may quote what it tried. + throw ScopedCaptureHTTPError.credentialUnavailable(String(describing: type(of: error))) + } + try Task.checkCancellation() + guard Self.isBearerToken(token) else { throw ScopedCaptureHTTPError.credentialMalformed } + var authorized = request + authorized.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") + return try await inner.send(authorized) + } + + /// A storage PUT goes out with exactly the headers the server signed. The uploader copies + /// those in and sets nothing else, and the storage URLs it receives carry their signature in + /// the query, so a signed set has no reason to hold `Authorization`. One that does is refused + /// instead of sent: forwarding it could hand a credential to storage, and stripping it could + /// break a signature in a way the server should hear about, not the bucket. + public func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + if request.value(forHTTPHeaderField: "Authorization") != nil { + throw ScopedCaptureHTTPError.storageRequestHasAuthorization + } + return try await inner.upload(request, file: file) + } + + /// RFC 6750's `b64token`: letters, digits and `-._~+/`, then optional `=` padding. Anything + /// else, a space, a quote or a line break included, could change the header or add another. + static func isBearerToken(_ token: String) -> Bool { + let body = token.utf8.reversed().drop { $0 == UInt8(ascii: "=") } + guard !body.isEmpty else { return false } + return body.allSatisfy { byte in + switch byte { + case UInt8(ascii: "a")...UInt8(ascii: "z"), UInt8(ascii: "A")...UInt8(ascii: "Z"), UInt8(ascii: "0")...UInt8(ascii: "9"): true + case UInt8(ascii: "-"), UInt8(ascii: "."), UInt8(ascii: "_"), UInt8(ascii: "~"), UInt8(ascii: "+"), UInt8(ascii: "/"): true + default: false + } + } + } +} + +/// Why `ScopedCaptureHTTP` refused a request. No case carries a credential, a URL or a header +/// value, so the description can be logged. +public enum ScopedCaptureHTTPError: Error, Equatable, CustomStringConvertible { + /// The URL is outside the configured API scope. The reason names the check that failed. + case destinationNotAllowed(String) + /// The credential provider threw. The value is the error's type name only. + case credentialUnavailable(String) + /// The provider returned an empty token, or one with characters a bearer token can't have. + case credentialMalformed + /// A storage upload already carries an `Authorization` header. + case storageRequestHasAuthorization + + /// The stable code an upload that hit this refusal ends with (`CaptureUploadState.End.failed`). + /// It names the kind of setup or credential problem, never a value. + public var code: String { + switch self { + case .destinationNotAllowed: "auth_destination_refused" + case .credentialUnavailable: "auth_credential_unavailable" + case .credentialMalformed: "auth_credential_malformed" + case .storageRequestHasAuthorization: "auth_storage_authorization" + } + } + + public var description: String { + switch self { + case .destinationNotAllowed(let reason): "the request is outside the capture API's scope (\(reason))" + case .credentialUnavailable(let type): "the capture API credential couldn't be obtained (\(type))" + case .credentialMalformed: "the capture API credential isn't a well-formed bearer token" + case .storageRequestHasAuthorization: "a storage upload carries an Authorization header" + } + } +} + +/// The capture API a credential is for: one scheme, host and port, and the paths under one +/// base path. Fixed when the uploader is configured, so a later settings change can't redirect +/// an in-flight capture's credential. +public struct CaptureAPIScope: Sendable, Equatable { + /// Plain http is allowed only by name, and only to this machine. + public enum Transport: Sendable, Equatable { + case httpsOnly + /// For tests and local development: http to `127.0.0.1`, `::1` or `localhost`. Never a + /// public host. + case loopbackHTTP + } + + public let scheme: String + public let host: String + public let port: Int + /// The base path without a trailing slash; empty for the origin's root. + public let basePath: String + public let transport: Transport + + /// The scope of `base`, the URL the uploader is given. Throws when `base` itself couldn't + /// receive a credential: not https (or loopback http under `.loopbackHTTP`), credentials in + /// the URL, a query or fragment, or a path with dot segments or encoded separators. + public init(base: URL, transport: Transport = .httpsOnly) throws { + guard let parts = URLComponents(url: base, resolvingAgainstBaseURL: false), + let scheme = parts.scheme?.lowercased(), let host = parts.host?.lowercased(), !host.isEmpty + else { throw ScopedCaptureHTTPError.destinationNotAllowed("base_invalid") } + guard parts.user == nil, parts.password == nil else { throw ScopedCaptureHTTPError.destinationNotAllowed("base_userinfo") } + guard parts.query == nil, parts.fragment == nil else { throw ScopedCaptureHTTPError.destinationNotAllowed("base_query") } + guard Self.allows(scheme: scheme, host: host, transport: transport) else { + throw ScopedCaptureHTTPError.destinationNotAllowed("base_insecure") + } + let path = parts.percentEncodedPath + guard Self.isPlainPath(path) else { throw ScopedCaptureHTTPError.destinationNotAllowed("base_path") } + self.scheme = scheme + self.host = host + self.port = parts.port ?? (scheme == "https" ? 443 : 80) + basePath = path.hasSuffix("/") ? String(path.dropLast()) : path + self.transport = transport + } + + /// Nil when `url` may receive the credential, else the check it fails. + public func problem(with url: URL) -> String? { + guard let parts = URLComponents(url: url, resolvingAgainstBaseURL: false), + let scheme = parts.scheme?.lowercased(), let host = parts.host?.lowercased() + else { return "url_invalid" } + if parts.user != nil || parts.password != nil { return "userinfo" } + if scheme != self.scheme { return "scheme" } + // Exact host: a suffix, a prefix or a trailing dot is another name. + if host != self.host { return "host" } + if (parts.port ?? (scheme == "https" ? 443 : 80)) != port { return "port" } + let path = parts.percentEncodedPath + if !Self.isPlainPath(path) { return "path_segments" } + guard path == basePath || path.hasPrefix(basePath + "/") || (basePath.isEmpty && path.isEmpty) else { return "path" } + return nil + } + + static func allows(scheme: String, host: String, transport: Transport) -> Bool { + if scheme == "https" { return true } + guard scheme == "http", transport == .loopbackHTTP else { return false } + return ["127.0.0.1", "::1", "[::1]", "localhost"].contains(host) + } + + /// No `.` or `..` segment, and no percent-encoded dot, slash or backslash that a server + /// might decode into one. + static func isPlainPath(_ path: String) -> Bool { + let lowered = path.lowercased() + if lowered.contains("%2e") || lowered.contains("%2f") || lowered.contains("%5c") || path.contains("\\") { return false } + return !path.split(separator: "/", omittingEmptySubsequences: false).contains { $0 == "." || $0 == ".." } + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureFolderOwnershipTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureFolderOwnershipTests.swift new file mode 100644 index 000000000..ea16aeffa --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureFolderOwnershipTests.swift @@ -0,0 +1,224 @@ +import Foundation +import HouseScanKit +import Synchronization +import Testing + +@Suite(.serialized) struct CaptureFolderOwnershipTests { + private func create(_ folder: URL, _ base: URL) throws -> CaptureUploader { + try CaptureUploader.start( + folder: folder, base: base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), + create: .init(packetId: "ownership-test", tier: .arkit, device: .init(model: "test", systemVersion: "test", appVersion: "test")), + consentedAt: Date()) + } + + @Test func concurrentStartsSaveOnlyTheOwnersAttempt() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "owners-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let owners = await withTaskGroup(of: CaptureUploader?.self, returning: [CaptureUploader].self) { group in + for _ in 0..<12 { + group.addTask { + do { return try create(root, server.base) } + catch { #expect(error as? CaptureUploader.OwnershipError == .alreadyOwned); return nil } + } + } + var owners: [CaptureUploader] = [] + for await owner in group { if let owner { owners.append(owner) } } + return owners + } + try #require(owners.count == 1) + let before = await owners[0].snapshot + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: root)) == before) + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned) { + try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + } + await owners[0].kick() + try #require(await CaptureUploaderTests.settles(owners[0])) + #expect(server.requests("POST captures").count == 1) + } + + @Test func concurrentResumesSaveOnlyTheOwnersAttempt() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "resume-owners-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + var first: CaptureUploader? = try create(root, server.base) + weak let released = first + first = nil + try #require(released == nil) + let owners = await withTaskGroup(of: CaptureUploader?.self, returning: [CaptureUploader].self) { group in + for _ in 0..<12 { + group.addTask { + do { return try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) } + catch { #expect(error as? CaptureUploader.OwnershipError == .alreadyOwned); return nil } + } + } + var owners: [CaptureUploader] = [] + for await owner in group { if let owner { owners.append(owner) } } + return owners + } + try #require(owners.count == 1) + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: root)) == (await owners[0].snapshot)) + await owners[0].kick() + try #require(await CaptureUploaderTests.settles(owners[0])) + #expect(server.requests("POST captures").count == 1) + } + + @Test func aliasesCannotAcquireTheSameFolder() throws { + let root = FileManager.default.temporaryDirectory.appending(path: "alias-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let folder = root.appending(path: "Capture") + let owner = try create(folder, server.base) + let link = root.appending(path: "link") + try FileManager.default.createSymbolicLink(at: link, withDestinationURL: folder) + var aliases = [link, root.appending(path: "Capture/../Capture")] + let lowercased = root.appending(path: "capture") + if FileManager.default.fileExists(atPath: lowercased.path) { aliases.append(lowercased) } + for alias in aliases { + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned, "alias: \(alias.path)") { try create(alias, server.base) } + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned, "alias: \(alias.path)") { + try CaptureUploader.resume(folder: alias, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + } + } + #expect(owner.folder.path == (try folder.resourceValues(forKeys: [.canonicalPathKey]).canonicalPath)) + #expect(server.state.withLock { $0.log.isEmpty }) + } + + @Test func idleOwnershipReleasesOnlyAfterTheUploaderDies() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "lifetime-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + var owner: CaptureUploader? = try create(root, server.base) + await owner?.kick() + try #require(await CaptureUploaderTests.settles(try #require(owner))) + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned) { + try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + } + weak let released = owner + owner = nil + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + #expect(throws: CaptureUploader.OwnershipError.savedUploadExists) { try create(root, server.base) } + let resumed = try #require(try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10))) + await resumed.kick() + try #require(await CaptureUploaderTests.settles(resumed)) + #expect(server.requests("POST captures").count == 1) + } + + @Test func anInFlightRequestKeepsOwnershipAfterTheCallerReleasesIt() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "inflight-owner-\(UUID().uuidString)") + let server = try LoopbackCaptureAPI() + defer { + server.state.withLock { $0.held = [] } + server.release("POST captures") + try? FileManager.default.removeItem(at: root) + } + server.state.withLock { $0.held = ["POST captures"] } + var owner: CaptureUploader? = try create(root, server.base) + await owner?.kick() + for _ in 0..<500 where server.state.withLock({ $0.parked["POST captures"]?.isEmpty ?? true }) { + try await Task.sleep(for: .milliseconds(10)) + } + try #require(server.state.withLock { !($0.parked["POST captures"]?.isEmpty ?? true) }) + weak let runningOwner = owner + owner = nil + #expect(runningOwner != nil) + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned) { + try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + } + server.state.withLock { $0.held = [] } + server.release("POST captures") + for _ in 0..<500 where runningOwner != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(runningOwner == nil) + let resumed = try #require(try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10))) + await resumed.kick() + try #require(await CaptureUploaderTests.settles(resumed)) + #expect(server.requests("POST captures").count == 1) + } + + /// Failure injection uses permissions only on this test's temporary folder. Restoring the + /// stale saved yes after stopping models the disk contents a failed withdrawal can leave. + @Test(arguments: [0, 1, 2]) func withdrawalBlocksReuseEvenWhenDiskStillHasConsent(failedWrites: Int) async throws { + var rig = try CaptureUploaderTests.Rig(http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10))) + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: rig.capture.folder.path) + rig.cleanUp() + } + rig.server.state.withLock { $0.held = ["POST captures/files"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + for _ in 0..<500 where rig.server.state.withLock({ $0.parked["POST captures/files"]?.isEmpty ?? true }) { + try await Task.sleep(for: .milliseconds(10)) + } + try #require(rig.server.state.withLock { !($0.parked["POST captures/files"]?.isEmpty ?? true) }) + let stateURL = CaptureUploader.stateURL(in: rig.capture.folder) + let staleConsent = try Data(contentsOf: stateURL) + let marker = CaptureUploadState.withdrawnURL(in: rig.capture.folder) + if failedWrites == 1 { + try FileManager.default.createDirectory(at: marker.appending(path: "blocked"), withIntermediateDirectories: true) + } else if failedWrites == 2 { + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: rig.capture.folder.path) + } + let result = rig.uploader.withdrawConsent() + switch failedWrites { + case 0: #expect(result == .marked) + case 1: #expect(result == .savedStateRemoved) + default: + guard case .notRecorded = result else { Issue.record("Both filesystem writes must fail"); return } + #expect(try Data(contentsOf: stateURL) == staleConsent) + } + rig.server.state.withLock { $0.held = [] } + rig.server.release("POST captures/files") + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(rig.server.requests("PUT upload").isEmpty) + #expect(rig.server.requests("POST captures/files:commit").isEmpty) + #expect(await rig.uploader.snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: rig.capture.folder.path) + if FileManager.default.fileExists(atPath: marker.path) { try FileManager.default.removeItem(at: marker) } + try staleConsent.write(to: stateURL, options: .atomic) + weak let released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + #expect(try CaptureUploader.resume(folder: rig.capture.folder, base: rig.server.base, http: rig.http) == nil) + #expect(throws: CaptureUploader.OwnershipError.consentWithdrawn) { try create(rig.capture.folder, rig.server.base) } + #expect(try Data(contentsOf: stateURL) == staleConsent) + } + + @Test(arguments: [false, true]) func withdrawalDuringPutsStopsTheNextBatchAndCommit(uncancellable: Bool) async throws { + let transport = URLSessionCaptureHTTP.ephemeral(timeout: 10) + let http: any CaptureHTTP = uncancellable ? UncancellableHTTP(transport) : transport + var policy = CaptureUploaderTests.Rig.fast + policy.maxConcurrentPuts = 2 + let rig = try CaptureUploaderTests.Rig(http: http, policy: policy) + defer { + rig.server.state.withLock { $0.held = [] } + rig.server.release("PUT upload") + rig.cleanUp() + } + rig.server.state.withLock { $0.held = ["PUT upload"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 3)) + for _ in 0..<500 where rig.server.state.withLock({ ($0.parked["PUT upload"]?.count ?? 0) < 2 }) { + try await Task.sleep(for: .milliseconds(10)) + } + try #require(rig.server.state.withLock { ($0.parked["PUT upload"]?.count ?? 0) == 2 }) + #expect(rig.uploader.withdrawConsent() == .marked) + await rig.uploader.kick() + rig.server.state.withLock { $0.held = [] } + rig.server.release("PUT upload") + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(rig.server.requests("PUT upload").count == 2) + #expect(rig.server.requests("POST captures/files:commit").isEmpty) + #expect(await rig.uploader.snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) + } + + @Test func withdrawalBeforeConcurrentKicksAdmitsNoRequests() async throws { + let rig = try CaptureUploaderTests.Rig() + defer { rig.cleanUp() } + #expect(rig.uploader.withdrawConsent() == .marked) + await withTaskGroup(of: Void.self) { group in + for _ in 0..<20 { group.addTask { await rig.uploader.kick() } } + } + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(rig.server.state.withLock { $0.log.isEmpty }) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift new file mode 100644 index 000000000..d10af08df --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift @@ -0,0 +1,260 @@ +import Foundation +import HouseScanKit +import simd +import Synchronization +import Testing + +/// A photo or tap the scan accepted but the phone couldn't keep makes the capture one that can't be +/// prepared: the running upload stops at once with a local code, nothing is finalized, and an +/// upload that hasn't started never starts. A withdrawal still outranks it. Frames the producer +/// leaves out on purpose (a repeated frame, photos after the scan was sent) are not losses. A +/// failure from a world that was reset never reaches the next one. +@Suite(.serialized) @MainActor struct CaptureLifecycleFailClosedTests { + let root = FileManager.default.temporaryDirectory.appending(path: "fail-closed-\(UUID().uuidString)") + var fixture: NativeCaptureFixture { NativeCaptureFixture(folder: root.appending(path: "store")) } + var captures: URL { root.appending(path: "Captures") } + + static let preparationFailed = CaptureUploadState.End.failed(step: "prepare", codes: ["capture_input_lost"], status: 0) + + func coordinator(_ server: LoopbackCaptureAPI, log: @escaping @Sendable (String) -> Void = { _ in }) -> CaptureSessionCoordinator { + CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment( + endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures, log: log)) + } + + /// A pose the producer refuses: scaled, so not rigid. + static func scaled(_ m: simd_float4x4) -> simd_float4x4 { + var out = m + out.columns.0 *= 2 + out.columns.1 *= 2 + out.columns.2 *= 2 + return out + } + + enum Loss: String, CaseIterable, Sendable { + /// The JPEG the scan kept is gone before it could be staged. + case stagingCopy + /// Staged, then refused by the producer when its turn came. + case queuedSeal + /// The tap's frame couldn't be encoded. + case tapFrame + /// The tap's seal was refused. + case tapSeal + } + + func lose(_ loss: Loss, in coordinator: CaptureSessionCoordinator) throws { + let t = fixture.start + 9 + switch loss { + case .stagingCopy: + var photo = try fixture.photo(at: t) + photo.jpeg = root.appending(path: "missing.jpg") + coordinator.kept(photo) + case .queuedSeal: + var photo = try fixture.photo(at: t) + photo.cameraToWorld = Self.scaled(photo.cameraToWorld) + coordinator.kept(photo) + case .tapFrame: + var (tap, hit) = try fixture.tap(at: t) + tap.jpeg = { nil } + coordinator.meterTapped(tap, hit: hit) + case .tapSeal: + var (tap, hit) = try fixture.tap(at: t) + tap.cameraToWorld = Self.scaled(tap.cameraToWorld) + coordinator.meterTapped(tap, hit: hit) + } + } + + /// The scan's photos so far, sent while it runs. + func startScan(_ coordinator: CaptureSessionCoordinator, consent: Bool?) async throws { + coordinator.begin(recording: fixture.recording) + if let consent { coordinator.answerConsent(consent) } + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + coordinator.kept(try fixture.photo(at: fixture.start + 4)) + await coordinator.settle() + } + + @Test(arguments: Loss.allCases) + func aLostInputStopsTheRunningUploadBeforeTheScanEnds(loss: Loss) async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + try await startScan(coordinator, consent: true) + let uploader = try #require(coordinator.session?.uploader) + #expect(await uploader.snapshot.end == nil) + + try lose(loss, in: coordinator) + await coordinator.settle() + // Ended by the loss itself, before the scan was sent. + #expect(await uploader.snapshot.end == Self.preparationFailed) + #expect(await uploader.status.retryingAt == nil) + #expect(coordinator.session?.preparationFailure == CaptureSessionCoordinator.inputLostCode) + + coordinator.kept(try fixture.photo(at: fixture.start + 10)) + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + await coordinator.settle() + #expect(await uploader.snapshot.end == Self.preparationFailed) + #expect(await uploader.snapshot.packet == nil) + #expect(server.requests("POST captures/finalize").isEmpty) + } + + /// Lost before the homeowner answered: a later yes sends nothing, so no capture is opened for + /// a packet that can't be complete. + @Test(arguments: Loss.allCases) + func aLostInputBeforeConsentOpensNoCapture(loss: Loss) async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + try await startScan(coordinator, consent: nil) + try lose(loss, in: coordinator) + await coordinator.settle() + coordinator.answerConsent(true) + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + await coordinator.settle() + #expect(coordinator.session?.uploader == nil) + #expect(coordinator.session?.preparationFailure == CaptureSessionCoordinator.inputLostCode) + #expect(server.state.withLock { $0.log.isEmpty }) + } + + /// The homeowner's no stands: a loss afterwards doesn't relabel the upload's end. + @Test func aWithdrawalOutranksALostInput() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + try await startScan(coordinator, consent: true) + let uploader = try #require(coordinator.session?.uploader) + guard case .success = coordinator.answerConsent(false) else { + Issue.record("the withdrawal wasn't recorded") + return + } + try lose(.queuedSeal, in: coordinator) + await coordinator.settle() + await uploader.settled() + #expect(await uploader.snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) + } + + /// The other order: the loss ends the upload first, then the homeowner says no. The recorded + /// withdrawal replaces the local end. A reset after a loss doesn't: it's not a withdrawal. + @Test func aWithdrawalAfterALossStillEndsTheUploadAsWithdrawn() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + try await startScan(coordinator, consent: true) + let uploader = try #require(coordinator.session?.uploader) + try lose(.queuedSeal, in: coordinator) + await coordinator.settle() + try #require(await uploader.snapshot.end == Self.preparationFailed) + guard case .success = coordinator.answerConsent(false) else { + Issue.record("the withdrawal wasn't recorded") + return + } + await uploader.settled() + for _ in 0..<500 { + if await uploader.snapshot.end != Self.preparationFailed { break } + try await Task.sleep(for: .milliseconds(10)) + } + #expect(await uploader.snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: uploader.folder)).end == .abandoned(CaptureUploader.withdrawnReason)) + } + + @Test func aResetAfterALossKeepsTheLossAsTheEnd() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + try await startScan(coordinator, consent: true) + let uploader = try #require(coordinator.session?.uploader) + try lose(.queuedSeal, in: coordinator) + await coordinator.settle() + coordinator.newWorld("world reset", recording: fixture.recording, newScan: false) + await uploader.settled() + try await Task.sleep(for: .milliseconds(50)) + #expect(await uploader.snapshot.end == Self.preparationFailed) + } + + /// A loss still queued when the scan is sent: the packet is never frozen, even with no upload + /// to refuse it. Nothing is settled between the lost input and the end of the capture. + @Test(arguments: [Loss.queuedSeal, Loss.tapSeal, Loss.tapFrame]) + func aLossQueuedBeforeTheEndKeepsThePacketUnfrozen(loss: Loss) async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + try await startScan(coordinator, consent: nil) + let folder = try #require(coordinator.session?.folder) + try lose(loss, in: coordinator) + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + await coordinator.settle() + #expect(coordinator.session?.preparationFailure == CaptureSessionCoordinator.inputLostCode) + #expect(!FileManager.default.fileExists(atPath: folder.appending(path: "packet.json").path)) + #expect(server.state.withLock { $0.log.isEmpty }) + } + + /// What the producer leaves out on purpose isn't a loss: the close-up's frame kept again, a + /// photo for a purpose that isn't a still, and photos after the scan was sent. + @Test func framesLeftOutOnPurposeStillFinish() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + try await startScan(coordinator, consent: true) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5)) + coordinator.kept(try fixture.photo(at: fixture.start + 6, purpose: "door_view")) + await coordinator.settle() + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + coordinator.kept(try fixture.photo(at: fixture.start + 11)) + await coordinator.settle() + #expect(await coordinator.session?.uploader?.snapshot.end == .finished(status: "manual_review")) + #expect(server.requests("POST captures/finalize").count == 1) + } + + /// A tap whose frame is still being encoded when the world is reset, and then fails, doesn't + /// touch the next world's session, upload or status. + @Test func aFailureFromAResetWorldDoesNotReachTheNextOne() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let lines = Mutex<[String]>([]) + let coordinator = coordinator(server) { line in lines.withLock { $0.append(line) } } + let statuses = Mutex<[CaptureUploadStatus?]>([]) + coordinator.onStatus = { status in statuses.withLock { $0.append(status) } } + try await startScan(coordinator, consent: true) + + let entered = Mutex(false) + let gate = DispatchSemaphore(value: 0) + var (tap, hit) = try fixture.tap(at: fixture.start + 9) + tap.jpeg = { + entered.withLock { $0 = true } + gate.wait() + return nil + } + coordinator.meterTapped(tap, hit: hit) + for _ in 0..<500 where !entered.withLock({ $0 }) { try await Task.sleep(for: .milliseconds(10)) } + try #require(entered.withLock { $0 }) + + coordinator.newWorld("world reset", recording: fixture.recording, newScan: false) + let next = try #require(coordinator.session) + let seen = statuses.withLock { $0.count } + gate.signal() + // The old world's work has run to its end once it logs the failed frame. + let failed = { lines.withLock { $0.contains { $0.contains("meter tap") } } } + for _ in 0..<500 where !failed() { try await Task.sleep(for: .milliseconds(10)) } + try #require(failed()) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + await coordinator.settle() + let nextUploader = try #require(next.uploader) + await nextUploader.settled() + + #expect(coordinator.session === next) + #expect(next.preparationFailure == nil) + #expect(await nextUploader.snapshot.end == nil) + #expect(statuses.withLock { $0.dropFirst(seen) }.allSatisfy { status in status?.phase != .failed }) + } +} + +/// The code is readable where the app maps an upload's end to a stage, outside the main actor. +struct InputLostCodeIsNonisolated { + static func matches(_ codes: [String]) -> Bool { codes.contains(CaptureSessionCoordinator.inputLostCode) } + + @Test func aNonisolatedMappingCanMatchTheCode() { + #expect(Self.matches(["capture_input_lost"])) + #expect(!Self.matches(["local_state_unsaved"])) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultSummaryTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultSummaryTests.swift new file mode 100644 index 000000000..e8c83cb24 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultSummaryTests.swift @@ -0,0 +1,142 @@ +import Foundation +import HouseScanKit +import simd +import Testing + +/// What the integration build shows of a capture-API result, from self-authored responses, and +/// the coordinator binding the result it fetched to its own scan. +@Suite struct CaptureResultSummaryTests { + static let world = CaptureResult.Association(sessionID: "session-A", captureID: "cap_A", runID: "run_A", epoch: "e1") + + static func response(_ json: String) throws -> CaptureResult.Response { + try JSONDecoder().decode(CaptureResult.Response.self, from: Data(json.utf8)) + } + + static func summary(_ json: String, association: CaptureResult.Association = world, analysis: CaptureResult.Analysis = .unverified) throws -> CaptureResultSummary { + CaptureResultSummary(.init(response: try response(json), association: association, analysis: analysis), world: world, meterAnchor: matrix_identity_float4x4) + } + + static let prompt = #"{"id":"vn1","kind":"wall_band","criteria":["C3"],"why":"unseen","promptId":"p1","prompt":{"title":"Step back and show the wall","body":"Include the ground under the meter."}}"# + + @Test func processingHasNoOutcomeAndNoMessage() throws { + let s = try Self.summary(#"{"runId":"run_A","status":"processing","viewsNeeded":[],"memberActions":[]}"#) + #expect(s.state == .working(.processing)) + #expect(s.message == nil && s.prompts.isEmpty && s.criteria == nil) + #expect(s.arUnavailable == .noOutcome(.processing)) + } + + /// A capture that failed or expired has no outcome, and it is over: it is not shown as + /// still working. + @Test(arguments: ["failed", "expired"]) func aCaptureThatEndedWithoutAnOutcomeIsNotWorking(status: String) throws { + let s = try Self.summary(#"{"runId":"run_A","status":"\#(status)","viewsNeeded":[],"memberActions":[]}"#) + #expect(s.state == .ended(CaptureResult.Status(rawValue: status))) + #expect(s.message == nil && s.prompts.isEmpty) + } + + @Test func manualReviewKeepsTheServersWordsAndNoInventedChecks() throws { + let s = try Self.summary(#""" + {"runId":"run_A","status":"manual_review","viewsNeeded":[],"memberActions":[], + "outcome":{"kind":"manual_review","profile":"C","message":"RECEIVER TEST: a person will look at this.","viewsNeeded":[],"reasons":[]}} + """#) + #expect(s.state == .decided(.manualReview)) + #expect(s.message == "RECEIVER TEST: a person will look at this.") + #expect(s.criteria == nil) + #expect(s.arUnavailable == .notEligible(.manualReview)) + } + + @Test func aMissingViewShowsItsPromptVerbatim() throws { + let s = try Self.summary(""" + {"runId":"run_A","status":"needs_views","viewsNeeded":[\(Self.prompt)],"memberActions":[], + "outcome":{"kind":"needs_more_photos","profile":"C","message":"One more view, please.","viewsNeeded":[\(Self.prompt)],"reasons":[]}} + """) + #expect(s.state == .decided(.needsMorePhotos)) + #expect(s.prompts.map(\.title) == ["Step back and show the wall"]) + #expect(s.prompts.map(\.body) == ["Include the ground under the meter."]) + } + + /// An eligible-shaped outcome from an unverified server stays without AR, and without the + /// criteria it didn't send. + @Test func eligibleWithoutCriteriaStaysUnverifiedAndOffAR() throws { + let s = try Self.summary(#""" + {"runId":"run_A","status":"complete","viewsNeeded":[],"memberActions":[], + "outcome":{"kind":"eligible","profile":"C","message":"A spot fits.","viewsNeeded":[],"reasons":[],"arkitEpoch":"e1", + "recommendedPlacement":{"wallId":"w1","startSM":1.2,"confidence":0.8, + "boxArkitWorld":{"pose":[1,0,0,0,0,1,0,0,0,0,1,0,0.5,0,-2,1],"size":[0.79,1.0,0.56]}}}} + """#) + #expect(s.state == .decided(.eligible)) + #expect(s.criteria == nil) + #expect(!s.analysisVerified) + #expect(s.arUnavailable == .analysisUnverified) + } + + @Test func anUnknownStatusStaysReadable() throws { + let s = try Self.summary(#"{"runId":"run_A","status":"paused_for_audit","viewsNeeded":[],"memberActions":[]}"#) + #expect(s.state == .working(.unknown("paused_for_audit"))) + #expect(s.arUnavailable == .noOutcome(.unknown("paused_for_audit"))) + } + + /// Even a verified result doesn't place a box for another session, capture or run. + @Test(arguments: [ + CaptureResult.Association(sessionID: "session-B", captureID: "cap_A", runID: "run_A", epoch: "e1"), + CaptureResult.Association(sessionID: "session-A", captureID: "cap_B", runID: "run_A", epoch: "e1"), + CaptureResult.Association(sessionID: "session-A", captureID: "cap_A", runID: "run_B", epoch: "e1"), + ]) + func aResultForAnotherScanPlacesNothing(_ association: CaptureResult.Association) throws { + let s = try Self.summary(#""" + {"runId":"run_A","status":"complete","viewsNeeded":[],"memberActions":[], + "outcome":{"kind":"eligible","profile":"C","message":"A spot fits.","viewsNeeded":[],"reasons":[],"arkitEpoch":"e1", + "recommendedPlacement":{"wallId":"w1","startSM":1.2,"confidence":0.8, + "boxArkitWorld":{"pose":[1,0,0,0,0,1,0,0,0,0,1,0,0.5,0,-2,1],"size":[0.79,1.0,0.56]}}}} + """#, association: association, analysis: .verified) + guard case .mismatch? = s.arUnavailable else { Issue.record("placed a box for another scan: \(String(describing: s.arUnavailable))"); return } + } +} + +/// The coordinator fetches the result through the real uploader and binds it to its own session. +@Suite(.serialized) @MainActor struct CaptureResultBindingTests { + @Test func theFetchedResultIsBoundToThisScanAndStaysUnverified() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "result-binding-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.resultMessage = "RECEIVER TEST: distinctive message 7f3" } + let coordinator = CaptureSessionCoordinator(environment: NativeCaptureFixture.environment( + endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: root.appending(path: "Captures"))) + var results: [CaptureResult.Record] = [] + coordinator.onResult = { results.append($0) } + try await NativeCaptureFixture(folder: root.appending(path: "store")).run(coordinator) + for _ in 0..<100 where results.isEmpty { try await Task.sleep(for: .milliseconds(20)) } + + let session = try #require(coordinator.session) + let record = try #require(results.first) + #expect(results.count == 1) + let state = await session.uploader!.snapshot + #expect(record.association == .init(sessionID: session.localID, captureID: state.captureID!, runID: state.finalized!.runID, epoch: "e1")) + #expect(record.response.outcome?.message == "RECEIVER TEST: distinctive message 7f3") + #expect(record.analysis == .unverified) + #expect(session.result == record) + } + + /// A result arriving for a session that has ended (a new scan started) is never published. + @Test func aResultForAnEndedSessionIsDropped() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "result-stale-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["GET captures/result"] } + let fixture = NativeCaptureFixture(folder: root.appending(path: "store")) + let coordinator = CaptureSessionCoordinator(environment: NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: root.appending(path: "Captures"))) + var results: [CaptureResult.Record] = [] + coordinator.onResult = { results.append($0) } + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 3)) + coordinator.captureEnded(acceptedCloseUpAt: nil) + for _ in 0..<500 where server.requests("GET captures/result").isEmpty { try await Task.sleep(for: .milliseconds(10)) } + let old = try #require(coordinator.session) + coordinator.newWorld("start over", recording: fixture.recording, newScan: true) + server.release("GET captures/result") + try await Task.sleep(for: .milliseconds(500)) + #expect(results.isEmpty) + #expect(old.result == nil) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultTests.swift new file mode 100644 index 000000000..ed3e9919a --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultTests.swift @@ -0,0 +1,320 @@ +import Foundation +import HouseScanKit +import Testing +import simd + +// Every body here is self-authored to the shape of the server's device result; none is a server +// example or a real capture. +@Suite struct CaptureResultTests { + typealias Association = CaptureResult.Association + + static let scan = Association(sessionID: "session-a", captureID: "cap-a", runID: "run-a", epoch: "e1") + + static let eligibleJSON = """ + {"runId": "run-a", "status": "complete", "viewsNeeded": [], "memberActions": [], + "outcome": {"kind": "eligible", "profile": "C", "message": "A battery fits on this wall.", + "viewsNeeded": [], "reasons": [], "arkitEpoch": "e1", + "frameToArkit": [1,0,0,0, 0,1,0,0, 0,0,1,0, 0,0,0,1], + "recommendedPlacement": {"wallId": "w1", "startSM": 1.1, "confidence": 0.7, "profile": "C", + "footprintM": {"w": 0.8, "h": 1.0, "d": 0.5}, + "boxSceneM": {"centerM": [1.5, 0.5, 0.3], "sizeM": [0.8, 1.0, 0.5], "yawDeg": 0}, + "boxArkitWorld": {"pose": [1,0,0,0, 0,1,0,0, 0,0,1,0, 1.5,0.5,0.3,1], "size": [0.8, 1.0, 0.5]}}, + "cableRoutePolylineArkit": null}, + "verdictUrl": null, "previewUrl": null, "reviewUrl": "/v1/captures/cap-a/review"} + """ + + static func decode(_ json: String) throws -> CaptureResult.Response { + try JSONDecoder().decode(CaptureResult.Response.self, from: Data(json.utf8)) + } + + static func eligible() throws -> CaptureResult.Response { try decode(eligibleJSON) } + + static func verified(_ response: CaptureResult.Response) -> CaptureResult.Record { + CaptureResult.Record(response: response, association: scan, analysis: .verified) + } + + // MARK: Decoding + + @Test func processingHasNoOutcome() throws { + let response = try Self.decode(""" + {"runId": null, "status": "processing", "viewsNeeded": [], "memberActions": [], "outcome": null} + """) + #expect(response.status == .processing) + #expect(response.runId == nil && response.outcome == nil && response.criteria == nil) + #expect(response.previewUrl == nil && response.reviewUrl == nil) + #expect(Self.verified(response).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + == .unavailable(.noOutcome(.processing))) + } + + @Test func eligibleWithoutCriteriaKeepsThemAbsent() throws { + let response = try Self.eligible() + #expect(response.outcome?.kind == .eligible) + #expect(response.criteria == nil) + #expect(response.reviewUrl == "/v1/captures/cap-a/review") + let box = try #require(response.outcome?.recommendedPlacement?.boxArkitWorld) + #expect(box.size == [0.8, 1.0, 0.5]) + guard case .box(let placed) = Self.verified(response).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) else { + Issue.record("expected a box") + return + } + #expect(placed.size == SIMD3(0.8, 1.0, 0.5)) + #expect(nearlyEqual(placed.anchorFromBox, translation(1.5, 0.5, 0.3))) + } + + @Test func eligibleWithCriteriaKeepsEachAsSent() throws { + let json = Self.eligibleJSON.replacingOccurrences( + of: "\"verdictUrl\": null", + with: """ + "criteria": [{"id": "c1", "outcome": "pass", "measuredFt": 4.2, "coverage": "observed"}, + {"id": "c2", "outcome": "unsure", "unsureCause": "unobserved"}, + {"id": "c3", "outcome": "waived"}], + "verdictUrl": null + """) + let criteria = try #require(try Self.decode(json).criteria) + #expect(criteria.map(\.outcome) == [.pass, .unsure, .unknown("waived")]) + #expect(criteria[0].measuredFt == 4.2 && criteria[0].coverage == "observed" && criteria[0].thresholdFt == nil) + #expect(criteria[1].unsureCause == "unobserved" && criteria[1].measuredFt == nil) + } + + @Test func manualReviewKeepsViewsAndPromptsVerbatim() throws { + let body = " Hold the phone at chest height,\nand include 3 ft left of the “meter”. " + let view = """ + {"id": "vn1", "kind": "past_end", "criteria": ["c2"], "why": "The left end was not seen.", + "promptId": "p-left-end", "prompt": {"title": "Show the left end", "body": \(try jsonString(body))}, + "side": "left", "wallId": "w1", "futureKey": {"nested": true}, + "referenceCrop": {"image": "k00002", "boxPx": [10, 20, 300, 400]}} + """ + let response = try Self.decode(""" + {"runId": "run-a", "status": "manual_review", "viewsNeeded": [\(view)], + "memberActions": ["retake_meter_closeups"], + "outcome": {"kind": "manual_review", "profile": "C", "message": "An installer will look at this scan.", + "viewsNeeded": [\(view)], "reasons": [], "arkitEpoch": "e1"}, + "criteria": [{"id": "c2", "outcome": "unsure"}], + "verdictUrl": null, "previewUrl": null, "reviewUrl": "/v1/captures/cap-a/review"} + """) + let outcome = try #require(response.outcome) + #expect(outcome.kind == .manualReview && outcome.reasons.isEmpty) + #expect(outcome.message == "An installer will look at this scan.") + #expect(response.viewsNeeded == outcome.viewsNeeded) + let needed = try #require(outcome.viewsNeeded.first) + #expect(needed.prompt.title == "Show the left end") + #expect(needed.prompt.body == body) + #expect(needed.kind == "past_end" && needed.wallId == "w1" && needed.band == nil) + #expect(needed.referenceCrop?.boxPx == [10, 20, 300, 400]) + #expect(response.memberActions == ["retake_meter_closeups"]) + #expect(Self.verified(response).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + == .unavailable(.notEligible(.manualReview))) + } + + @Test func unknownStatusAndKindDecodeAndWithholdAR() throws { + let unknownKind = try Self.decode(Self.eligibleJSON + .replacingOccurrences(of: "\"status\": \"complete\"", with: "\"status\": \"archived\"") + .replacingOccurrences(of: "\"kind\": \"eligible\"", with: "\"kind\": \"deferred\"")) + #expect(unknownKind.status == .unknown("archived")) + #expect(unknownKind.outcome?.kind == .unknown("deferred")) + #expect(unknownKind.outcome?.message == "A battery fits on this wall.") + #expect(Self.verified(unknownKind).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + == .unavailable(.notEligible(.unknown("deferred")))) + + var unknownStatus = try Self.eligible() + unknownStatus.status = .unknown("archived") + #expect(Self.verified(unknownStatus).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + == .unavailable(.statusNotComplete(.unknown("archived")))) + } + + @Test func missingStatusFailsToDecode() { + #expect(throws: DecodingError.self) { + try Self.decode(#"{"runId": "run-a", "viewsNeeded": [], "memberActions": [], "outcome": null}"#) + } + } + + // MARK: Gating + + @Test func eligibleResultIsUnverifiedByDefault() throws { + let record = CaptureResult.Record(response: try Self.eligible(), association: Self.scan) + #expect(record.analysis == .unverified) + #expect(record.placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) == .unavailable(.analysisUnverified)) + } + + enum Mismatch: CaseIterable, Sendable { + case worldSession, worldCapture, worldRun, worldEpoch, responseRun, responseRunMissing, outcomeEpoch + + var field: CaptureResult.IdentityField { + switch self { + case .worldSession: .session + case .worldCapture: .capture + case .worldRun, .responseRun, .responseRunMissing: .run + case .worldEpoch, .outcomeEpoch: .epoch + } + } + } + + @Test(arguments: Mismatch.allCases) + func everyIdentityMismatchWithholdsAR(_ mismatch: Mismatch) throws { + var response = try Self.eligible() + var world = Self.scan + switch mismatch { + case .worldSession: world.sessionID = "session-b" + case .worldCapture: world.captureID = "cap-b" + case .worldRun: world.runID = "run-b" + case .worldEpoch: world.epoch = "e2" + case .responseRun: response.runId = "run-b" + case .responseRunMissing: response.runId = nil + case .outcomeEpoch: response.outcome?.arkitEpoch = "e2" + } + let record = Self.verified(response) + #expect(record.placement(in: world, meterAnchor: matrix_identity_float4x4) == .unavailable(.mismatch(mismatch.field))) + #expect(record.response.outcome?.message == "A battery fits on this wall.") + } + + @Test func epochReusedByANewSessionIsNotAMatch() throws { + let newSession = Association(sessionID: "session-b", captureID: "cap-a", runID: "run-a", epoch: "e1") + #expect(newSession.epoch == Self.scan.epoch) + #expect(Self.verified(try Self.eligible()).placement(in: newSession, meterAnchor: matrix_identity_float4x4) + == .unavailable(.mismatch(.session))) + } + + enum BadBox: CaseIterable, Sendable { + case shortPose, twoSizes, nanPose, infiniteSize, zeroSize, scaledPose, shearedPose, mirroredPose, badBottomRow + case noBox, noEpoch, noPlacement + case floatOverflowSize, floatUnderflowSize, floatOverflowTranslation + + var expected: CaptureResult.Unavailable { + switch self { + case .shortPose: .malformedBox(.poseCount(15)) + case .twoSizes: .malformedBox(.sizeCount(2)) + case .nanPose, .infiniteSize, .floatOverflowSize, .floatOverflowTranslation: .malformedBox(.nonFinite) + case .zeroSize, .floatUnderflowSize: .malformedBox(.nonPositiveSize) + case .scaledPose, .shearedPose, .mirroredPose, .badBottomRow: .malformedBox(.notRigid) + case .noBox, .noEpoch: .notInARKitWorld + case .noPlacement: .noPlacement + } + } + + func apply(to outcome: inout CaptureResult.Outcome) { + switch self { + case .shortPose: outcome.recommendedPlacement?.boxArkitWorld?.pose.removeLast() + case .twoSizes: outcome.recommendedPlacement?.boxArkitWorld?.size = [0.8, 1.0] + case .nanPose: outcome.recommendedPlacement?.boxArkitWorld?.pose[13] = .nan + case .infiniteSize: outcome.recommendedPlacement?.boxArkitWorld?.size[1] = .infinity + case .zeroSize: outcome.recommendedPlacement?.boxArkitWorld?.size[2] = 0 + case .floatOverflowSize: outcome.recommendedPlacement?.boxArkitWorld?.size[0] = 1e40 + case .floatUnderflowSize: outcome.recommendedPlacement?.boxArkitWorld?.size[0] = 1e-50 + case .floatOverflowTranslation: outcome.recommendedPlacement?.boxArkitWorld?.pose[12] = 1e40 + case .scaledPose: + for i in 0..<11 { outcome.recommendedPlacement?.boxArkitWorld?.pose[i] *= 1.25 } + case .shearedPose: outcome.recommendedPlacement?.boxArkitWorld?.pose[4] = 0.3 + case .mirroredPose: outcome.recommendedPlacement?.boxArkitWorld?.pose[0] = -1 + case .badBottomRow: outcome.recommendedPlacement?.boxArkitWorld?.pose[3] = 0.5 + case .noBox: outcome.recommendedPlacement?.boxArkitWorld = nil + case .noEpoch: outcome.arkitEpoch = nil + case .noPlacement: outcome.recommendedPlacement = nil + } + } + } + + @Test(arguments: BadBox.allCases) + func malformedGeometryWithholdsARAndKeepsTheResult(_ bad: BadBox) throws { + var response = try Self.eligible() + var outcome = try #require(response.outcome) + bad.apply(to: &outcome) + response.outcome = outcome + let record = Self.verified(response) + #expect(record.placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) == .unavailable(bad.expected)) + #expect(record.response.outcome?.kind == .eligible) + #expect(record.response.outcome?.message == "A battery fits on this wall.") + } + + @Test func nonFiniteMeterAnchorWithholdsAR() throws { + var anchor = matrix_identity_float4x4 + anchor.columns.3.x = .nan + #expect(Self.verified(try Self.eligible()).placement(in: Self.scan, meterAnchor: anchor) == .unavailable(.invalidMeterAnchor)) + } + + // MARK: Transform + + @Test func boxIsExpressedInATranslatedRotatedMeterAnchor() throws { + let worldFromAnchor = pose(rotationY(degrees: 90), SIMD3(2, 0, -1)) + let anchorFromBox = pose(rotationY(degrees: 30), SIMD3(0.5, 0.4, 0.3)) + var response = try Self.eligible() + response.outcome?.recommendedPlacement?.boxArkitWorld?.pose = columnMajor(worldFromAnchor * anchorFromBox) + + let placement = Self.verified(response).placement(in: Self.scan, meterAnchor: float(worldFromAnchor)) + guard case .box(let placed) = placement else { + Issue.record("expected a box, got \(placement)") + return + } + #expect(nearlyEqual(placed.anchorFromBox, float(anchorFromBox))) + #expect(nearlyEqual(SIMD3(placed.anchorFromBox.columns.3.x, placed.anchorFromBox.columns.3.y, placed.anchorFromBox.columns.3.z), + SIMD3(0.5, 0.4, 0.3), 1e-5)) + #expect(placed.size == SIMD3(0.8, 1.0, 0.5)) + } + + /// The server builds the box's world pose from the scene box and the scene-to-ARKit similarity: + /// the similarity's scale is divided out of the rotation and multiplied into the size. The app + /// must take that size as it is, not scale it again by `frameToArkit`. + @Test func sizeCarriesTheServersScaleOnce() throws { + let scale = 1.25 + let frameToArkit = pose(rotationY(degrees: 40) * scale, SIMD3(1, 0, 2)) + let centerM = SIMD3(1.5, 0.5, 0.3), sizeM = SIMD3(0.8, 1.0, 0.56), yawDeg = 10.0 + + let rotation = rotation3(frameToArkit) * (1 / scale) * rotationY(degrees: yawDeg) + let center = frameToArkit * SIMD4(centerM, 1) + let serverPose = pose(rotation, SIMD3(center.x, center.y, center.z)) + let serverSize = sizeM * scale + + var response = try Self.eligible() + response.outcome?.recommendedPlacement?.boxArkitWorld?.pose = columnMajor(serverPose) + response.outcome?.recommendedPlacement?.boxArkitWorld?.size = [serverSize.x, serverSize.y, serverSize.z] + + let placement = Self.verified(response).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + guard case .box(let placed) = placement else { + Issue.record("expected a box, got \(placement)") + return + } + #expect(nearlyEqual(placed.size, SIMD3(1.0, 1.25, 0.7), 1e-6)) + #expect(nearlyEqual(placed.anchorFromBox, float(serverPose))) + for column in [placed.anchorFromBox.columns.0, placed.anchorFromBox.columns.1, placed.anchorFromBox.columns.2] { + #expect(abs(simd_length(SIMD3(column.x, column.y, column.z)) - 1) < 1e-5) + } + } +} + +private func jsonString(_ text: String) throws -> String { + String(decoding: try JSONEncoder().encode(text), as: UTF8.self) +} + +private func rotationY(degrees: Double) -> simd_double3x3 { + let a = degrees * .pi / 180 + return simd_double3x3(SIMD3(cos(a), 0, -sin(a)), SIMD3(0, 1, 0), SIMD3(sin(a), 0, cos(a))) +} + +private func rotation3(_ m: simd_double4x4) -> simd_double3x3 { + simd_double3x3( + SIMD3(m.columns.0.x, m.columns.0.y, m.columns.0.z), + SIMD3(m.columns.1.x, m.columns.1.y, m.columns.1.z), + SIMD3(m.columns.2.x, m.columns.2.y, m.columns.2.z)) +} + +private func pose(_ r: simd_double3x3, _ t: SIMD3) -> simd_double4x4 { + simd_double4x4(SIMD4(r.columns.0, 0), SIMD4(r.columns.1, 0), SIMD4(r.columns.2, 0), SIMD4(t, 1)) +} + +private func translation(_ x: Float, _ y: Float, _ z: Float) -> simd_float4x4 { + var m = matrix_identity_float4x4 + m.columns.3 = SIMD4(x, y, z, 1) + return m +} + +private func columnMajor(_ m: simd_double4x4) -> [Double] { + [m.columns.0, m.columns.1, m.columns.2, m.columns.3].flatMap { [$0.x, $0.y, $0.z, $0.w] } +} + +private func float(_ m: simd_double4x4) -> simd_float4x4 { + simd_float4x4(SIMD4(m.columns.0), SIMD4(m.columns.1), SIMD4(m.columns.2), SIMD4(m.columns.3)) +} + +private func nearlyEqual(_ a: simd_float4x4, _ b: simd_float4x4, _ tolerance: Float = 1e-5) -> Bool { + [(a.columns.0, b.columns.0), (a.columns.1, b.columns.1), (a.columns.2, b.columns.2), (a.columns.3, b.columns.3)] + .allSatisfy { simd_reduce_max(simd_abs($0 - $1)) <= tolerance } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift new file mode 100644 index 000000000..008d40758 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift @@ -0,0 +1,712 @@ +import Foundation +import HouseScanKit +import simd +import Synchronization +import Testing + +/// A scan shaped like the app's live capture, for the coordinator: 1920 x 1440 sensor JPEGs, the +/// camera 2 m from a wall at 1.5 m height walking +x, the recorder's raw rows (a 60 Hz trajectory +/// with each frame's intrinsics beside it, and Core Motion's accelerometer and gyro at about +/// 100 Hz on their own jittered clocks), the meter tap's frame and pixel, and the meter close-up +/// from its own frame. Pattern images, no house; the packet says `source.kind: synthetic`. +struct NativeCaptureFixture: Sendable { + static let width = 1920, height = 1440 + static let k = SIMD4(1445, 1445, 960, 720) + static let meter = SIMD3(0.4, 1.2, -2) + let start = 5000.0 + let end = 5012.0 + let folder: URL + + func pose(_ t: Double) -> simd_float4x4 { + var m = matrix_identity_float4x4 + m.columns.3 = SIMD4(Float(t - start) * 0.1, 1.5, 0, 1) + return m + } + + /// Where `world` lands in the sensor image of the frame at `t`. + func pixel(of world: SIMD3, at t: Double) -> SIMD2 { + let p = world - SIMD3(pose(t).columns.3.x, pose(t).columns.3.y, pose(t).columns.3.z) + let depth = -p.z + return SIMD2(Double(Self.k.z + Self.k.x * p.x / depth), Double(Self.k.w - Self.k.y * p.y / depth)) + } + + var recording: RecordingSource { + let fixture = self + return RecordingSource(start: { (fixture.start, Date(timeIntervalSince1970: 1_790_000_000)) }, rows: { fixture.rows }) + } + + var rows: RecorderRows { + var trajectory: [[Double]] = [], intrinsics: [[Double]] = [] + for i in 0...Int((end - start) * 60) { + let t = start + Double(i) / 60 + let m = pose(t) + let columns: [SIMD4] = [m.columns.0, m.columns.1, m.columns.2, m.columns.3] + let values: [Double] = columns.flatMap { (c: SIMD4) -> [Double] in [Double(c.x), Double(c.y), Double(c.z), Double(c.w)] } + trajectory.append([t, 0, 0] + values) + intrinsics.append([t, 1445, 1445, 960, 720, 1920, 1440]) + } + // Two sensors on their own clocks: neither shares a timestamp with the other. + var accel: [[Double]] = [], gyro: [[Double]] = [] + for i in 0.. URL { + let url = folder.appending(path: "\(name).jpg") + if !FileManager.default.fileExists(atPath: url.path) { + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + try makeJPEG(width: Self.width, height: Self.height, at: url) + } + return url + } + + func photo(at t: Double, purpose: String? = nil) throws -> KeptPhoto { + KeptPhoto( + t: t, cameraToWorld: pose(t), cameraIntrinsics: Self.k, cameraImageSize: SIMD2(1920, 1440), width: Self.width, height: Self.height, + tracking: .normal, exposure: .init(duration: 0.004, offset: 0, iso: 64, fNumber: 1.6), jpeg: try jpeg(named: "frame-\(t)"), purpose: purpose) + } + + /// The raycast ran on the frame at `t`; the frame the app records is two frames newer, as + /// ARKit's current frame can be, and the tap is re-aimed at the hit in that frame. + func tap(at t: Double) throws -> (TapObservation, Packet04.TapHit) { + let data = try Data(contentsOf: try jpeg(named: "tap-\(t)")) + let recorded = t + 2.0 / 60 + let camera = pose(recorded).columns.3 + let stale = TapObservation( + t: recorded, cameraToWorld: pose(recorded), intrinsics: Self.k, width: Self.width, height: Self.height, tracking: .normal, + pixel: pixel(of: Self.meter, at: t), jpeg: { data }) + let tap = try #require(stale.pointing(at: Self.meter)) + let hit = Packet04.TapHit( + position: [Self.meter.x, Self.meter.y, Self.meter.z].map(Double.init), target: "estimatedPlane", alignment: "vertical", + distance: Double(simd_distance(Self.meter, SIMD3(camera.x, camera.y, camera.z)))) + return (tap, hit) + } + + static func environment( + endpoint: URL, http: any CaptureHTTP, captures: URL, sends: Bool = true, eventsWait: Int = 0, log: @escaping @Sendable (String) -> Void = { _ in } + ) -> CaptureSessionCoordinator.Environment { + var policy = CaptureUploader.Policy() + policy.eventsWait = eventsWait + let device = CaptureAPI.Device(model: "iPhone15,4", systemVersion: "26.0", appVersion: "0.1.0 (1)") + return .init( + endpoint: endpoint, sends: sends, http: http, capturesFolder: captures, + sessionInfo: { packetID, video in + Packet04SessionInfo( + packetID: packetID, sessionID: packetID, source: .synthetic, appVersion: device.appVersion, deviceModel: device.model, + systemVersion: device.systemVersion, lidarAvailable: false, sceneDepthEnabled: false, meshReconstructionSupported: nil, + sceneReconstruction: nil, planeDetection: ["horizontal", "vertical"], videoWidth: Int(video.x), videoHeight: Int(video.y), + framesPerSecond: nil, timeZone: "America/Chicago") + }, + device: device, tier: .arkit, policy: policy, log: log) + } + + /// The whole scan through `coordinator`: the tap, the close-up, keyframes (one of them the + /// close-up's own frame again), then the capture ends. + @MainActor + func run(_ coordinator: CaptureSessionCoordinator, consent: Bool? = true, beforeEnd: @MainActor () async -> Void = {}) async throws { + coordinator.begin(recording: recording) + // The integration build asks before the scan starts. + if let consent { coordinator.answerConsent(consent) } + let (tap, hit) = try self.tap(at: start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.kept(try photo(at: start + 2.5, purpose: "meter_close")) + for i in 0..<5 { coordinator.kept(try photo(at: start + 3 + Double(i) * 1.5)) } + // The app keeps the close-up's frame twice when it is also a walk frame. + coordinator.kept(try photo(at: start + 2.5)) + await coordinator.settle() + await beforeEnd() + coordinator.captureEnded(acceptedCloseUpAt: start + 2.5) + coordinator.captureEnded(acceptedCloseUpAt: start + 2.5) + await coordinator.settle() + } +} + +@Suite(.serialized) @MainActor struct CaptureSessionCoordinatorTests { + let root = FileManager.default.temporaryDirectory.appending(path: "coordinator-\(UUID().uuidString)") + var fixture: NativeCaptureFixture { NativeCaptureFixture(folder: root.appending(path: "store")) } + var captures: URL { root.appending(path: "Captures") } + + @Test func nativeScanExportsAValidPacketAndUploadsItDuringTheScan() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures)) + let committedBeforeEnd = Mutex(0) + try await fixture.run(coordinator) { + let uploader = coordinator.session!.uploader! + committedBeforeEnd.withLock { $0 = 0 } + let count = await uploader.snapshot.committedCount + committedBeforeEnd.withLock { $0 = count } + } + let session = try #require(coordinator.session) + let state = await session.uploader!.snapshot + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: try #require(state.packet)) + + #expect(Packet04Check.problems(packet, folder: session.folder).isEmpty) + // Tap frame, close-up frame and five walk frames; the repeated close-up frame is not a new keyframe. + #expect(packet.keyframes.count == 7) + #expect(packet.keyframes.map(\.reason) == ["tap", "still", "auto", "auto", "auto", "auto", "auto"]) + #expect(packet.stills?.map(\.purpose) == ["meter_close"]) + #expect(packet.scaleReference.meterCloseUp.still == "meter_close") + let tap = try #require(packet.taps?.first) + #expect(tap.label == "meter" && tap.keyframe == "k00001" && tap.epoch == "e1") + // The ray through the tap pixel reaches the meter the raycast hit. + let toMeter = simd_normalize(SIMD3(tap.hit!.position[0] - tap.rayOrigin[0], tap.hit!.position[1] - tap.rayOrigin[1], tap.hit!.position[2] - tap.rayOrigin[2])) + #expect(simd_distance(toMeter, SIMD3(tap.rayDirection[0], tap.rayDirection[1], tap.rayDirection[2])) < 1e-5) + // Raw world poses, straight from the frames. + #expect(packet.keyframes[2].pose[12...14].map { $0 } == [0.3, 1.5, 0]) + // Both sensors' own times travel beside the proposed pairing. + #expect(packet.ext?["imuRawPairingStatus"]?.hasPrefix("proposal") == true) + #expect(Set(packet.files.filter { $0.role == .stream }.map(\.path)) + == ["streams/arkit_poses.csv.gz", "streams/imu_raw.csv.gz", "streams/accelerometer_raw.csv.gz", "streams/gyroscope_raw.csv.gz"]) + + #expect(committedBeforeEnd.withLock { $0 } == packet.files.count - 4, "every image was received before the scan ended") + #expect(state.end == .finished(status: "manual_review")) + #expect(server.requests("POST captures").count == 1) + #expect(server.requests("POST captures/finalize").count == 1) + + if let dir = ProcessInfo.processInfo.environment["HOUSESCAN_EXPORT_EVIDENCE_DIR"] { + let target = URL(fileURLWithPath: dir).appending(path: "native-export-local") + try? FileManager.default.removeItem(at: target) + try FileManager.default.createDirectory(at: target.deletingLastPathComponent(), withIntermediateDirectories: true) + try FileManager.default.copyItem(at: session.folder, to: target) + } + } + + @Test func onlyTheIntegrationBuildRecordsAndOnlyAnAllowedBuildSends() { + let url = "https://capture.example.com/v1" + #expect(CaptureIntegrationMode.resolve(integrationBuild: "NO", endpoint: url, sendDeviceData: "YES") == .off("not the integration build")) + #expect(CaptureIntegrationMode.resolve(integrationBuild: nil, endpoint: url, sendDeviceData: "YES") == .off("not the integration build")) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "$(HOUSESCAN_INTEGRATION_BUILD)", endpoint: url, sendDeviceData: "YES") == .off("not the integration build")) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: "", sendDeviceData: "YES") == .off("no capture endpoint in this build")) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: url, sendDeviceData: "NO") == .recordOnly(URL(string: url)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: url, sendDeviceData: nil) == .recordOnly(URL(string: url)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: url, sendDeviceData: "YES") == .send(URL(string: url)!)) + // A replay goes only to a receiver on this machine, only when asked, whatever the device switch says. + let local = "http://127.0.0.1:8765/v1" + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: local, sendDeviceData: "NO", source: .replay(sendToLocalReceiver: true)) == .send(URL(string: local)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: local, sendDeviceData: "YES", source: .replay(sendToLocalReceiver: false)) == .recordOnly(URL(string: local)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: url, sendDeviceData: "YES", source: .replay(sendToLocalReceiver: true)) == .recordOnly(URL(string: url)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "NO", endpoint: local, sendDeviceData: "YES", source: .replay(sendToLocalReceiver: true)) == .off("not the integration build")) + } + + /// A build that may not send device data records the capture on the phone and never sends it, + /// even with a yes. + @Test func aRecordOnlyBuildKeepsTheCaptureOnThePhone() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures, sends: false)) + try await fixture.run(coordinator) + #expect(coordinator.needsConsent == false) + coordinator.answerConsent(true) + await coordinator.settle() + let session = try #require(coordinator.session) + #expect(session.uploader == nil) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: Data(contentsOf: session.folder.appending(path: "packet.json"))) + #expect(Packet04Check.problems(packet, folder: session.folder).isEmpty) + #expect(server.state.withLock { $0.log.isEmpty }) + } + + /// No answer or a no sends nothing; the next scan asks again. A yes before the scan sends + /// during it, and a world reset inside the scan keeps that yes for the new world's packet. + @Test func consentIsPerScanAndOnlyAYesSends() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let environment = NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures) + + let unanswered = CaptureSessionCoordinator(environment: environment) + try await fixture.run(unanswered, consent: nil) + #expect(unanswered.needsConsent) + #expect(unanswered.session?.uploader == nil) + + let declining = CaptureSessionCoordinator(environment: environment) + try await fixture.run(declining, consent: false) + #expect(declining.session?.uploader == nil) + #expect(server.state.withLock { $0.log.isEmpty }) + declining.newWorld("start over", recording: fixture.recording, newScan: true) + #expect(declining.consent == nil && declining.needsConsent) + + let agreeing = CaptureSessionCoordinator(environment: environment) + agreeing.begin(recording: fixture.recording) + agreeing.answerConsent(true) + let firstWorld = try #require(agreeing.session) + agreeing.newWorld("world reset", recording: fixture.recording, newScan: false) + #expect(agreeing.consent == true) + let secondWorld = try #require(agreeing.session) + #expect(secondWorld.uploader != nil && secondWorld.packetID != firstWorld.packetID) + agreeing.newWorld("start over", recording: fixture.recording, newScan: true) + #expect(agreeing.consent == nil) + #expect(agreeing.session?.uploader == nil) + } + + /// After a relaunch a frozen capture resumes only with the yes it recorded and only toward the + /// endpoint it was created on. + @Test func resumeNeedsTheCapturesYesAndTheSameEndpoint() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let original = try LoopbackCaptureAPI() + let other = try LoopbackCaptureAPI() + // A capture frozen on the phone for `original` whose upload never started. + let folder = captures.appending(path: "frozen") + let capture = try SyntheticCapture04(folder: folder) + let images = try await capture.sealImages() + let (streams, packet) = try await capture.finish() + var saved = CaptureUploadState( + attemptID: "old-process", packetID: capture.producer.info.packetID, + createBody: try JSONEncoder().encode(CaptureAPI.CreateRequest(packetId: capture.producer.info.packetID, tier: .arkit, device: .init(model: "m", systemVersion: "s", appVersion: "a")))) + saved.destination = original.base.absoluteString + // First without a recorded yes: never resumed. + for (i, file) in (images + streams).enumerated() { saved.files[file.path] = .init(sealed: file, sequence: i) } + saved.packet = packet + try saved.save(to: CaptureUploader.stateURL(in: folder)) + let http = URLSessionCaptureHTTP.ephemeral(timeout: 10) + + #expect(CaptureSessionCoordinator(environment: NativeCaptureFixture.environment(endpoint: original.base, http: http, captures: captures)) + .resumeSealedCaptures().isEmpty) + saved.consent = .init(grantedAt: Date(), destination: original.base.absoluteString) + try saved.save(to: CaptureUploader.stateURL(in: folder)) + #expect(CaptureSessionCoordinator(environment: NativeCaptureFixture.environment(endpoint: other.base, http: http, captures: captures)) + .resumeSealedCaptures().isEmpty) + #expect(CaptureSessionCoordinator(environment: NativeCaptureFixture.environment(endpoint: original.base, http: http, captures: captures, sends: false)) + .resumeSealedCaptures().isEmpty) + try await Task.sleep(for: .milliseconds(200)) + #expect(original.state.withLock { $0.log.isEmpty } && other.state.withLock { $0.log.isEmpty }) + + let resumed = CaptureSessionCoordinator(environment: NativeCaptureFixture.environment(endpoint: original.base, http: http, captures: captures)) + .resumeSealedCaptures() + #expect(resumed.count == 1) + for uploader in resumed { await uploader.kick(); await uploader.settled() } + #expect(await resumed.first?.snapshot.end == .finished(status: "manual_review")) + #expect(other.state.withLock { $0.log.isEmpty }) + } + + /// A refused close-up and its accepted retake, written over the same file as the app does: + /// both stay sealed with their own pixels, and the scale reference names the accepted one, + /// with its pixels and pose. A skipped close-up names none. + @Test func onlyTheAcceptedCloseUpIsTheScaleReference() async throws { + defer { try? FileManager.default.removeItem(at: root) } + for accepted in [true, false] { + let coordinator = CaptureSessionCoordinator(environment: NativeCaptureFixture.environment( + endpoint: URL(string: "https://capture.invalid/v1")!, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures, sends: false)) + coordinator.begin(recording: fixture.recording) + let source = fixture.folder.appending(path: "meter_close.jpg") + try FileManager.default.createDirectory(at: fixture.folder, withIntermediateDirectories: true) + try? FileManager.default.removeItem(at: source) + try makeJPEG(width: NativeCaptureFixture.width, height: NativeCaptureFixture.height, at: source, shift: 7) + let refusedBytes = try Data(contentsOf: source) + // Its depth too lives in files the retake overwrites: 4 x 3 meters and confidence. + let depthFile = fixture.folder.appending(path: "meter_close.f32"), confidenceFile = fixture.folder.appending(path: "meter_close.conf.u8") + func writeDepth(_ meters: Float, _ confidence: UInt8) throws { + try PacketFiles.depthData(meters: [Float](repeating: meters, count: 12)).write(to: depthFile) + try Data(repeating: confidence, count: 12).write(to: confidenceFile) + } + try writeDepth(1.5, 2) + func closeUp(at t: Double) -> KeptPhoto { + KeptPhoto( + t: t, cameraToWorld: fixture.pose(t), cameraIntrinsics: NativeCaptureFixture.k, cameraImageSize: SIMD2(1920, 1440), + width: NativeCaptureFixture.width, height: NativeCaptureFixture.height, tracking: .normal, exposure: nil, jpeg: source, purpose: "meter_close", + depth: { + guard let map = try? Data(contentsOf: depthFile), let confidence = try? Data(contentsOf: confidenceFile) else { return nil } + return Packet04Depth(meters: PacketFiles.floats(littleEndian: map), confidence: [UInt8](confidence), width: 4, height: 3) + }) + } + coordinator.kept(closeUp(at: fixture.start + 2)) + // The retake overwrites the same files before the first shot's sealing has run. + try FileManager.default.removeItem(at: source) + try makeJPEG(width: NativeCaptureFixture.width, height: NativeCaptureFixture.height, at: source, shift: 101) + try writeDepth(0.9, 1) + let retakeBytes = try Data(contentsOf: source) + coordinator.kept(closeUp(at: fixture.start + 4)) + coordinator.captureEnded(acceptedCloseUpAt: accepted ? fixture.start + 4 : nil) + await coordinator.settle() + + let session = try #require(coordinator.session) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: Data(contentsOf: session.folder.appending(path: "packet.json"))) + #expect(packet.stills?.map(\.id) == ["meter_close", "meter_close-2"]) + #expect(try Data(contentsOf: session.folder.appending(path: "stills/meter_close.jpg")) == refusedBytes) + #expect(try Data(contentsOf: session.folder.appending(path: "stills/meter_close-2.jpg")) == retakeBytes) + // Each shot keeps its own depth and confidence. + #expect(try Data(contentsOf: session.folder.appending(path: "keyframes/k00001.depth.f32")) == PacketFiles.depthData(meters: [Float](repeating: 1.5, count: 12))) + #expect(try Data(contentsOf: session.folder.appending(path: "keyframes/k00001.confidence.u8")) == Data(repeating: 2, count: 12)) + #expect(try Data(contentsOf: session.folder.appending(path: "keyframes/k00002.depth.f32")) == PacketFiles.depthData(meters: [Float](repeating: 0.9, count: 12))) + #expect(Packet04Check.problems(packet, folder: session.folder).isEmpty) + if accepted { + #expect(packet.scaleReference.meterCloseUp.still == "meter_close-2") + let frame = try #require(packet.keyframes.first { $0.id == packet.stills?.last?.keyframe }) + #expect(frame.timestamp == fixture.start + 4 && frame.pose[12] == 0.4) + } else { + #expect(packet.scaleReference.meterCloseUp == .notCaptured) + } + } + } + + /// A world reset while the first world's photos are still going up: that packet is abandoned, + /// the new world gets its own packet id, and nothing kept after the reset joins the old one. + @Test func aWorldResetStartsANewPacketAndDropsTheOld() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/files"] } + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures)) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + let first = try #require(coordinator.session) + for _ in 0..<500 where server.requests("POST captures/files").isEmpty { try await Task.sleep(for: .milliseconds(10)) } + + coordinator.newWorld("world reset", recording: fixture.recording, newScan: false) + coordinator.kept(try fixture.photo(at: fixture.start + 4)) + server.release("POST captures/files") + server.state.withLock { $0.held = [] } + await coordinator.settle() + await first.uploader!.settled() + + let second = try #require(coordinator.session) + #expect(second.packetID != first.packetID) + let old = await first.uploader!.snapshot + #expect(old.end == .abandoned("world reset")) + #expect(old.files.values.allSatisfy { $0.phase == .queued }) + #expect(await second.uploader!.snapshot.files.keys.sorted() == ["keyframes/k00001.jpg"]) + #expect(await second.uploader!.snapshot.committedCount == 1) + let creates = server.requests("POST captures").compactMap { try? JSONDecoder().decode(CaptureAPI.CreateRequest.self, from: $0.body).packetId } + #expect(Set(creates) == [first.packetID, second.packetID]) + } + + /// A reset that ends the session before its kept photos are sealed still deletes their staged + /// copies, so no copy of a home photo outlives the packet it was kept for. + @Test func aResetDeletesPhotosStagedForTheEndedSession() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures, sends: false)) + let staging = captures.appending(path: "staging") + func staged() -> [String] { (try? FileManager.default.contentsOfDirectory(atPath: staging.path)) ?? [] } + + coordinator.begin(recording: fixture.recording) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + coordinator.kept(try fixture.photo(at: fixture.start + 3)) + #expect(staged().count == 2) + // No suspension between keeping and the reset, so the queued work finds the session ended. + coordinator.newWorld("start over", recording: fixture.recording, newScan: true) + coordinator.kept(try fixture.photo(at: fixture.start + 4)) + await coordinator.settle() + for _ in 0..<500 where !staged().isEmpty { try await Task.sleep(for: .milliseconds(10)) } + #expect(staged().isEmpty) + } + + /// A no after a yes stops sending: the upload is abandoned, nothing kept later goes up, the + /// packet is never finalized, and a second yes in the same scan does not start it again. + @Test func aNoAfterAYesStopsSending() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures)) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + await coordinator.settle() + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + await uploader.settled() + let putsBefore = server.requests("PUT upload").count + #expect(putsBefore == 1) + + coordinator.answerConsent(false) + coordinator.kept(try fixture.photo(at: fixture.start + 4)) + coordinator.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.captureEnded(acceptedCloseUpAt: nil) + await coordinator.settle() + await uploader.settled() + + #expect(await uploader.snapshot.end == .abandoned("consent withdrawn")) + #expect(coordinator.session === session && session.uploader == nil) + #expect(server.requests("PUT upload").count == putsBefore) + #expect(server.requests("POST captures").count == 1) + #expect(server.requests("POST captures/finalize").isEmpty) + } + + /// A withdrawal is saved before `answerConsent(false)` returns: a relaunch straight after it + /// does not resume the sealed capture, even though the uploader has not saved its end yet. + @Test func aWithdrawnCaptureIsNotResumedAfterARelaunch() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/finalize"] } + let environment = NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + for i in 0..<3 { coordinator.kept(try fixture.photo(at: fixture.start + 3 + Double(i) * 1.5)) } + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + let stateURL = CaptureUploader.stateURL(in: session.folder) + for _ in 0..<500 where (try? CaptureUploadState.load(from: stateURL))?.packet == nil { try await Task.sleep(for: .milliseconds(10)) } + #expect(try CaptureUploadState.load(from: stateURL).packet != nil) + + coordinator.answerConsent(false) + // The same main-actor turn: the abandon task has not run yet. + #expect(CaptureSessionCoordinator(environment: environment).resumeSealedCaptures().isEmpty) + #expect(try CaptureUploader.resume(folder: session.folder, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) == nil) + + server.release("POST captures/finalize") + server.state.withLock { $0.held = [] } + // The withdrawal cancelled the loop; the abandon task ends the upload in its own time. + for _ in 0..<500 where await uploader.snapshot.end == nil { try await Task.sleep(for: .milliseconds(10)) } + #expect(await uploader.snapshot.end == .abandoned("consent withdrawn")) + } + + /// A relaunch scan in the live process must not create a second sender for the sealed + /// folder. On the old code B survives A's withdrawal and sends the remaining files. + @Test func withdrawalCoversTheFolderDuringResumeDiscovery() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/finalize"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + for i in 0..<3 { coordinator.kept(try fixture.photo(at: fixture.start + 3 + Double(i) * 1.5)) } + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + let uploader = try #require(coordinator.session?.uploader) + try await waitUntilParked(server, "POST captures/finalize") + + let duplicates = CaptureSessionCoordinator(environment: environment).resumeSealedCaptures() + for duplicate in duplicates { await duplicate.kick() } + if !duplicates.isEmpty { + for _ in 0..<500 where server.requests("POST captures/finalize").count < 2 { + try await Task.sleep(for: .milliseconds(10)) + } + try #require(server.requests("POST captures/finalize").count == 2) + } + #expect(duplicates.isEmpty) + coordinator.answerConsent(false) + let puts = server.requests("PUT upload").count + let commits = server.requests("POST captures/files:commit").count + server.state.withLock { $0.held = [] } + server.release("POST captures/finalize") + try #require(await CaptureUploaderTests.settles(uploader)) + for duplicate in duplicates { try #require(await CaptureUploaderTests.settles(duplicate)) } + #expect(server.requests("PUT upload").count == puts) + #expect(server.requests("POST captures/files:commit").count == commits) + #expect(server.requests("GET captures/result").isEmpty) + #expect(CaptureSessionCoordinator(environment: environment).resumeSealedCaptures().isEmpty) + } + + @Test func droppingAnIdleCoordinatorReleasesFolderOwnership() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let environment = NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures) + var coordinator: CaptureSessionCoordinator? = CaptureSessionCoordinator(environment: environment) + coordinator?.begin(recording: fixture.recording) + coordinator?.answerConsent(true) + coordinator?.kept(try fixture.photo(at: fixture.start + 2)) + await coordinator?.settle() + let folder = try #require(coordinator?.session?.folder) + weak let oldSession = coordinator?.session + weak let oldUploader = coordinator?.session?.uploader + coordinator = nil + for _ in 0..<500 where oldSession != nil || oldUploader != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(oldSession == nil && oldUploader == nil) + let resumed = try #require(try CaptureUploader.resume(folder: folder, base: server.base, http: environment.http)) + await resumed.kick() + try #require(await CaptureUploaderTests.settles(resumed)) + #expect(server.requests("POST captures").count == 1) + } + + @Test func anUnsavedWithdrawalIsReturnedToTheCaller() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/files"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + try await waitUntilParked(server, "POST captures/files") + let stateURL = CaptureUploader.stateURL(in: session.folder) + let before = try Data(contentsOf: stateURL) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: session.folder.path) + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: session.folder.path) } + + let result = coordinator.answerConsent(false) + guard case .failure(.notRecorded(let detail)) = result else { + Issue.record("The caller must receive the persistence failure") + server.state.withLock { $0.held = [] } + server.release("POST captures/files") + return + } + #expect(detail.contains("marker:") && detail.contains("state removal:")) + #expect(try Data(contentsOf: stateURL) == before) + // Repeated answers must not turn the failed withdrawal into a reported success. + if case .success = coordinator.answerConsent(false) { Issue.record("Repeated no lost the failure") } + if case .success = coordinator.answerConsent(true) { Issue.record("Later yes lost the failure") } + #expect(session.uploader == nil) + #expect(try CaptureUploader.resume(folder: session.folder, base: server.base, http: environment.http) == nil) + #expect(CaptureSessionCoordinator(environment: environment).resumeSealedCaptures().isEmpty) + server.state.withLock { $0.held = [] } + server.release("POST captures/files") + try #require(await CaptureUploaderTests.settles(uploader)) + #expect(server.requests("PUT upload").isEmpty) + #expect(server.requests("POST captures/files:commit").isEmpty) + // All writes still fail, so the app cannot promise the persisted yes was removed. + #expect(try Data(contentsOf: stateURL) == before) + } + + /// Waits, suspending, until the loopback has parked an answer for `route`. + func waitUntilParked(_ server: LoopbackCaptureAPI, _ route: String) async throws { + for _ in 0..<1000 where server.state.withLock({ $0.parked[route]?.isEmpty ?? true }) { try await Task.sleep(for: .milliseconds(10)) } + #expect(server.state.withLock { !($0.parked[route]?.isEmpty ?? true) }) + } + + /// Blocks the main actor until `stateURL` holds something other than `before`, so nothing + /// queued on the main actor (the abandon task a withdrawal schedules) can run meanwhile. + func blockUntilSaved(_ stateURL: URL, differentFrom before: Data?) { + let deadline = Date().addingTimeInterval(5) + while Date() < deadline, (try? Data(contentsOf: stateURL)) == before { usleep(10_000) } + } + + /// A reply that lands after the homeowner withdrew, but before the uploader is abandoned, + /// can't save a capture a relaunch would resume, whether or not the withdrawal marker could + /// be written. The transport lets the reply arrive even though the request's task was + /// cancelled, as a background session does. The main actor is held from the withdrawal to the + /// checks, so the abandon task the withdrawal schedules can't run first. + @Test(arguments: [false, true]) func aReplyAfterWithdrawalCannotSaveAResumableCapture(markerBlocked: Bool) async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/files"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + let stateURL = CaptureUploader.stateURL(in: session.folder) + try await waitUntilParked(server, "POST captures/files") + if markerBlocked { + // A directory where the marker goes makes writing it fail. + try FileManager.default.createDirectory( + at: CaptureUploadState.withdrawnURL(in: session.folder).appending(path: "blocked"), withIntermediateDirectories: true) + } + + coordinator.answerConsent(false) + // No suspension from here to the checks. + let before = try? Data(contentsOf: stateURL) + #expect((before == nil) == markerBlocked) + server.release("POST captures/files") + // The uploader saves the register reply on its own executor. + blockUntilSaved(stateURL, differentFrom: before) + let saved = try CaptureUploadState.load(from: stateURL) + #expect(saved.end == .abandoned("consent withdrawn")) + if markerBlocked { + // The blocking directory sits at the marker's path and would stop a resume by itself; + // without it only the saved end can. + try FileManager.default.removeItem(at: CaptureUploadState.withdrawnURL(in: session.folder)) + } else { + #expect(try CaptureUploader.resume(folder: session.folder, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) == nil) + } + #expect(CaptureSessionCoordinator(environment: environment).resumeSealedCaptures().isEmpty) + #expect(server.requests("PUT upload").isEmpty) + + server.state.withLock { $0.held = [] } + await uploader.settled() + #expect(server.requests("PUT upload").isEmpty) + } + + /// A result that arrives after the withdrawal can't save the upload as finished: the + /// withdrawal outranks the end a late reply sets. + @Test func aResultAfterWithdrawalSavesTheUploadAsAbandoned() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["GET captures/result"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + for i in 0..<3 { coordinator.kept(try fixture.photo(at: fixture.start + 3 + Double(i) * 1.5)) } + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + let stateURL = CaptureUploader.stateURL(in: session.folder) + try await waitUntilParked(server, "GET captures/result") + + coordinator.answerConsent(false) + // No suspension from here to the check. + let before = try? Data(contentsOf: stateURL) + server.release("GET captures/result") + blockUntilSaved(stateURL, differentFrom: before) + #expect(try CaptureUploadState.load(from: stateURL).end == .abandoned("consent withdrawn")) + + server.state.withLock { $0.held = [] } + await uploader.settled() + #expect(await uploader.snapshot.end == .abandoned("consent withdrawn")) + } + + /// Withdrawing cancels the request the uploader is waiting on before the abandon task runs: + /// a held register request is given up at once instead of waiting for its answer, and no + /// photo is uploaded. + @Test func withdrawalCancelsTheRequestInFlight() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let logs = Mutex<[String]>([]) + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/files"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures, + log: { line in logs.withLock { $0.append(line) } }) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + let uploader = try #require(coordinator.session?.uploader) + try await waitUntilParked(server, "POST captures/files") + + coordinator.answerConsent(false) + // The main actor stays busy, so only the withdrawal itself can end the wait. + let deadline = Date().addingTimeInterval(5) + // The cut-short request ends as a withdrawal, not as a network failure to retry. + func gaveUp() -> Bool { logs.withLock { $0.contains { $0.contains("capture-upload withdrawn step=register") } } } + while Date() < deadline, !gaveUp() { usleep(10_000) } + #expect(gaveUp()) + + server.release("POST captures/files") + server.state.withLock { $0.held = [] } + await uploader.settled() + #expect(server.requests("PUT upload").isEmpty) + } + + /// Photos kept after the scan was sent don't change the frozen packet or start more uploads. + @Test func photosAfterTheScanWasSentAreNotAdded() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures)) + try await fixture.run(coordinator) + let before = await coordinator.session!.uploader!.snapshot + coordinator.kept(try fixture.photo(at: fixture.end - 0.5)) + await coordinator.settle() + let after = await coordinator.session!.uploader!.snapshot + #expect(after.packet == before.packet) + #expect(after.files.count == before.files.count) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionEndTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionEndTests.swift new file mode 100644 index 000000000..da6465217 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionEndTests.swift @@ -0,0 +1,117 @@ +import Foundation +import HouseScanKit +import Testing + +/// A world reset ends its capture for good before `newWorld` returns. The upload's own abandonment +/// still runs on the uploader's turn, so the ended record written synchronously here is what keeps +/// a relaunch from finding the old yes and resuming the capture the homeowner left. It is a +/// reset's record, not a withdrawal's. +@Suite(.serialized) @MainActor struct CaptureSessionEndTests { + let root = FileManager.default.temporaryDirectory.appending(path: "session-end-\(UUID().uuidString)") + var fixture: NativeCaptureFixture { NativeCaptureFixture(folder: root.appending(path: "store")) } + var captures: URL { root.appending(path: "Captures") } + static let endedFile = "capture-ended" + + func coordinator(_ server: LoopbackCaptureAPI) -> CaptureSessionCoordinator { + CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures)) + } + + func scan(_ coordinator: CaptureSessionCoordinator) async throws -> URL { + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + coordinator.kept(try fixture.photo(at: fixture.start + 4)) + await coordinator.settle() + let folder = try #require(coordinator.session?.folder) + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: folder)).end == nil) + return folder + } + + @Test func aResetRecordsTheEndBeforeNewWorldReturns() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + let folder = try await scan(coordinator) + + let ended = coordinator.newWorld("world reset", recording: fixture.recording, newScan: false) + // No suspension point since newWorld: only what it did synchronously is on disk. + guard case .success = ended else { + Issue.record("the reset's end wasn't recorded: \(ended)") + return + } + #expect(FileManager.default.fileExists(atPath: folder.appending(path: Self.endedFile).path)) + #expect(!FileManager.default.fileExists(atPath: CaptureUploadState.withdrawnURL(in: folder).path)) + await coordinator.settle() + } + + /// When the marker can't be written, newWorld says so, and this process still never resumes + /// the capture, even after its uploader is gone and with its saved state unended. + @Test func aResetThatCantBeRecordedSaysSoAndStillNeverResumesHere() async throws { + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + let folder = try await scan(coordinator) + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) + try? FileManager.default.removeItem(at: root) + } + weak let old = coordinator.session?.uploader + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) + + let ended = coordinator.newWorld("world reset", recording: fixture.recording, newScan: false) + guard case .failure(.notRecorded) = ended else { + Issue.record("expected the unwritable folder to leave the end unrecorded, got \(ended)") + return + } + #expect(!FileManager.default.fileExists(atPath: folder.appending(path: Self.endedFile).path)) + await coordinator.settle() + for _ in 0..<500 where old != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(old == nil) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: folder)).end == nil) + #expect(try CaptureUploader.resume(folder: folder, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) == nil) + } + + /// A no that couldn't be recorded (unwritable folder) leaves the old yes on disk. The reset + /// that follows has no uploader left to retire, and must report that failure, not success. + @Test func aResetAfterAnUnrecordedWithdrawalReportsTheFailure() async throws { + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + let folder = try await scan(coordinator) + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) + try? FileManager.default.removeItem(at: root) + } + let saved = try Data(contentsOf: CaptureUploader.stateURL(in: folder)) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) + guard case .failure(.notRecorded) = coordinator.answerConsent(false) else { + Issue.record("expected the withdrawal to go unrecorded in an unwritable folder") + return + } + let ended = coordinator.newWorld("world reset", recording: fixture.recording, newScan: false) + guard case .failure(.notRecorded) = ended else { + Issue.record("the reset reported \(ended) with the old yes still on disk") + return + } + await coordinator.settle() + #expect(try Data(contentsOf: CaptureUploader.stateURL(in: folder)) == saved) + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: folder)).end == nil) + } + + /// The record is what a relaunch reads: a folder carrying it isn't resumed, even with its + /// saved state still unended and nothing in this process remembering it. + @Test func aFolderWithTheEndedRecordIsNotResumed() async throws { + var rig = try CaptureUploaderTests.Rig() + defer { rig.cleanUp() } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await CaptureUploaderTests.settles(rig.uploader)) + weak let released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + try Data("world reset".utf8).write(to: rig.capture.folder.appending(path: Self.endedFile)) + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: rig.capture.folder)).end == nil) + + #expect(try CaptureUploader.resume(folder: rig.capture.folder, base: rig.server.base, http: rig.http, policy: CaptureUploaderTests.Rig.fast) == nil) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderRecoveryTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderRecoveryTests.swift new file mode 100644 index 000000000..d7b586417 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderRecoveryTests.swift @@ -0,0 +1,368 @@ +import Foundation +import HouseScanKit +import Synchronization +import Testing + +/// Two ways an upload could keep going on limits or state that no longer hold. A server that +/// lost the capture gets a new run, so the old run's retry limits must not end it early. A state +/// save that fails ends the upload in this process, so it never sends what the saved state +/// doesn't record. +/// +/// The loopback decides an answer when a request arrives and only delays sending it while its +/// route is held, so these tests change the server between one request and the next. +@Suite struct CaptureUploaderRecoveryTests { + typealias Rig = CaptureUploaderTests.Rig + + /// Waits until at least `count` requests on `route` are parked. Like `settles`, the budget is + /// this waiter's own turns, not wall time, and the answer is read after the last wait. + static func parked(_ server: LoopbackCaptureAPI, _ route: String, count: Int = 1, within seconds: Int = 20) async -> Bool { + let ready = { server.state.withLock { ($0.parked[route] ?? []).count >= count } } + for _ in 0..<(seconds * 100) { + if ready() { return true } + try? await Task.sleep(for: .milliseconds(10)) + } + return ready() + } + + /// Sends the answers parked on `route` and keeps holding it. + static func deliver(_ server: LoopbackCaptureAPI, _ route: String) { + let waiting = server.state.withLock { $0.parked.removeValue(forKey: route) ?? [] } + waiting.forEach { $0() } + } + + static func saved(_ rig: Rig) throws -> CaptureUploadState { + try CaptureUploadState.load(from: CaptureUploader.stateURL(in: rig.capture.folder)) + } + + static func routes(_ server: LoopbackCaptureAPI) -> [String] { server.state.withLock { $0.log.map(LoopbackCaptureAPI.route) } } + + // MARK: A recreated capture is a new run + + enum PutFault: String, CaseIterable, Sendable { + /// A second 400 BadDigest for a file whose one resend was spent. + case badDigest + /// A refused PUT past `maxRefusedPuts`. + case refused + } + + /// Every file of one sealed photo spends its PUT limits on the first server capture: a 400 + /// BadDigest (its one resend), a 403 refusal and a 200, three PUTs against `maxRefusedPuts` 3. + /// The server then loses the capture. The same fault once more for every file on the new + /// capture is retried, not the end of the upload, and the state a relaunch would load already + /// holds the new run's counts. + @Test(arguments: PutFault.allCases) + func aRecreatedCaptureGetsFreshPutLimits(fault: PutFault) async throws { + var policy = Rig.fast + policy.maxRefusedPuts = 3 + let rig = try Rig(policy: policy) + defer { rig.cleanUp() } + let files = try await rig.capture.sealImages(count: 1) + // One register answer and one round of PUTs covers every file, so each fault below lands + // once on each file. + let n = files.count + try #require(n <= policy.maxConcurrentPuts) + rig.server.state.withLock { + $0.badDigestPuts = n + $0.expireNextPuts = n + $0.held = ["PUT upload"] + } + await rig.uploader.add(files) + // Rounds one and two go through. A round starts only once the one before is answered, so + // a parked PUT past the 2n-th belongs to round three. + while true { + try #require(await Self.parked(rig.server, "PUT upload")) + if rig.server.requests("PUT upload").count > 2 * n { break } + Self.deliver(rig.server, "PUT upload") + } + try #require(await Self.parked(rig.server, "PUT upload", count: n)) + #expect(rig.server.requests("PUT upload").count == 3 * n) + + // Round three's 200s are decided. The server forgets the capture before the commit. + rig.server.state.withLock { s in + s.captures = [:] + s.byPacket = [:] + switch fault { + case .badDigest: s.badDigestPuts = n + case .refused: s.expireNextPuts = n + } + s.held = ["POST captures"] + } + Self.deliver(rig.server, "PUT upload") + try #require(await Self.parked(rig.server, "POST captures")) + let gone = try Self.saved(rig) + #expect(gone.captureID == nil) + #expect(gone.files.values.map(\.attempts) == Array(repeating: 0, count: n)) + #expect(gone.files.values.allSatisfy { $0.phase == .queued }) + rig.server.release("POST captures") + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(await rig.uploader.snapshot.end == nil) + + try await rig.finishAndSeal() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + #expect(rig.server.requests("POST captures").count == 2) + } + + /// The first server capture loses its finalize and follows `retry_finalize` once, the whole + /// limit. The server then loses the capture. A lost finalize on the new capture is sent again, + /// and the saved state reads 0 follows before the new capture exists. + @Test func aRecreatedCaptureGetsAFreshFinalizeLimit() async throws { + var policy = Rig.fast + policy.maxFinalizeRetries = 1 + let rig = try Rig(policy: policy) + defer { rig.cleanUp() } + rig.server.state.withLock { + $0.loseNextFinalize = true + $0.held = ["POST captures/finalize"] + } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await CaptureUploaderTests.settles(rig.uploader)) + try await rig.finishAndSeal() + try #require(await Self.parked(rig.server, "POST captures/finalize")) + Self.deliver(rig.server, "POST captures/finalize") + try #require(await Self.parked(rig.server, "POST captures/finalize")) + #expect(try Self.saved(rig).finalizeRetries == 1) + + // The second finalize is accepted. The server forgets the capture before the next poll. + rig.server.state.withLock { s in + s.captures = [:] + s.byPacket = [:] + s.loseNextFinalize = true + s.held = ["POST captures"] + } + Self.deliver(rig.server, "POST captures/finalize") + try #require(await Self.parked(rig.server, "POST captures")) + let gone = try Self.saved(rig) + #expect(gone.captureID == nil) + #expect(gone.finalizeRetries == 0) + rig.server.release("POST captures") + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + #expect(rig.server.requests("POST captures").count == 2) + #expect(rig.server.requests("POST captures/finalize").count == 4) + } + + // MARK: A state save that fails + + /// A failed save ends the upload in this process with the local code and status 0. The + /// observer hears it, nothing more is sent or retried, the saved file keeps its last good + /// bytes, and this process doesn't resume the folder from those older bytes. + @Test func aFailedSaveEndsTheUploadAndSendsNothingMore() async throws { + let sleeps = ScopedCaptureUploadTests.Sleeps() + var rig = try Rig(sleep: sleeps.record) + defer { rig.cleanUp() } + let folder = rig.capture.folder + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) } + let statuses = Mutex<[CaptureUploadStatus]>([]) + await rig.uploader.observe { status in statuses.withLock { $0.append(status) } } + rig.server.state.withLock { $0.held = ["POST captures"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 2)) + try #require(await Self.parked(rig.server, "POST captures")) + + let url = CaptureUploader.stateURL(in: folder) + let before = try Data(contentsOf: url) + // A folder that takes no new file: the atomic save can't write its replacement. + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) + rig.server.release("POST captures") + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + let unsaved = CaptureUploadState.End.failed(step: "save", codes: ["local_state_unsaved"], status: 0) + #expect(await rig.uploader.snapshot.end == unsaved) + let last = try #require(statuses.withLock { $0.last }) + #expect(last.phase == .failed) + #expect(last.detail == "save 0 local_state_unsaved") + #expect(last.retryingAt == nil) + #expect(try Data(contentsOf: url) == before) + + await rig.uploader.kick() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(Self.routes(rig.server) == ["POST captures"]) + #expect(sleeps.isEmpty) + + // Writable again, the folder still holds the older saved yes. Once the uploader is gone, + // this process doesn't pick that up and send the capture again. + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) + weak let released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + #expect(try CaptureUploader.resume(folder: folder, base: rig.server.base, http: rig.http, policy: Rig.fast, sleep: { _ in }) == nil) + #expect(try Data(contentsOf: url) == before) + #expect(Self.routes(rig.server) == ["POST captures"]) + } + + enum Order: String, CaseIterable, Sendable { + /// The withdrawal lands while the create is in flight; the reply's save then fails. + case withdrawalFirst + /// The save fails and ends the upload; the withdrawal comes after. + case saveFailureFirst + } + + /// A withdrawal outranks the save failure in either order. The folder is unwritable, so the + /// withdrawal itself can't be recorded (`notRecorded`) and no save succeeds. The coordinator + /// then abandons the upload as withdrawn, as it does on every withdrawal: the upload ends as + /// withdrawn, not as a local save failure, and sends nothing more. + @Test(arguments: Order.allCases) + func aWithdrawalOutranksAFailedSave(order: Order) async throws { + let sleeps = ScopedCaptureUploadTests.Sleeps() + let rig = try Rig(sleep: sleeps.record) + defer { rig.cleanUp() } + let folder = rig.capture.folder + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) } + rig.server.state.withLock { $0.held = ["POST captures"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 2)) + try #require(await Self.parked(rig.server, "POST captures")) + + let url = CaptureUploader.stateURL(in: folder) + let before = try Data(contentsOf: url) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) + if order == .saveFailureFirst { + rig.server.release("POST captures") + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(await rig.uploader.snapshot.end == .failed(step: "save", codes: ["local_state_unsaved"], status: 0)) + } + let record = rig.uploader.withdrawConsent() + guard case .notRecorded = record else { + Issue.record("expected the withdrawal to go unrecorded in an unwritable folder, got \(record)") + return + } + if order == .withdrawalFirst { rig.server.release("POST captures") } + try #require(await CaptureUploaderTests.settles(rig.uploader)) + await rig.uploader.abandon(CaptureUploader.withdrawnReason) + + #expect(await rig.uploader.snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) + #expect(await rig.uploader.status.phase == .abandoned) + #expect(try Data(contentsOf: url) == before) + await rig.uploader.kick() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(Self.routes(rig.server) == ["POST captures"]) + #expect(sleeps.isEmpty) + } + + /// Counts what reaches the network under `ScopedCaptureHTTP`. + final class CountingHTTP: CaptureHTTP { + let inner: any CaptureHTTP + let sends = Mutex(0) + init(_ inner: any CaptureHTTP) { self.inner = inner } + func send(_ request: URLRequest) async throws -> HTTPReply { + sends.withLock { $0 += 1 } + return try await inner.send(request) + } + func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + sends.withLock { $0 += 1 } + return try await inner.upload(request, file: file) + } + } + + /// A save that fails while a request waits for its credential stops that request. The create + /// waits on a provider that ignores cancellation. Meanwhile `add` saves into an unwritable + /// folder and fails. When the provider then returns a good token, nothing reaches the + /// network, nothing is retried, and the upload ends as the local save failure, not as a + /// withdrawal. + @Test func aFailedSaveStopsARequestWaitingForItsCredential() async throws { + let sleeps = ScopedCaptureUploadTests.Sleeps() + let gate = ScopedCaptureHTTPTests.Gate() + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.expectedBearer = ScopedCaptureUploadTests.token } + let network = CountingHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)) + let http = ScopedCaptureHTTP( + scope: try CaptureAPIScope(base: server.base, transport: .loopbackHTTP), + credential: { + await gate.wait() + return ScopedCaptureUploadTests.token + }, + inner: network) + let rig = try Rig(server: server, http: http, sleep: sleeps.record) + defer { rig.cleanUp() } + let folder = rig.capture.folder + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) } + let images = try await rig.capture.sealImages(count: 1) + await rig.uploader.kick() + await gate.entered() + + let url = CaptureUploader.stateURL(in: folder) + let before = try Data(contentsOf: url) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) + await rig.uploader.add(images) + gate.release() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + #expect(network.sends.withLock { $0 } == 0) + #expect(server.state.withLock { $0.log.isEmpty }) + #expect(await rig.uploader.snapshot.end == .failed(step: "save", codes: ["local_state_unsaved"], status: 0)) + #expect(await rig.uploader.status.phase == .failed) + #expect(await rig.uploader.status.retryingAt == nil) + #expect(sleeps.isEmpty) + #expect(try Data(contentsOf: url) == before) + await rig.uploader.kick() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(network.sends.withLock { $0 } == 0) + } + + enum UnsavedEnd: String, CaseIterable, Sendable { + /// The result arrives and ends the upload, and that save fails. + case finished + /// The server's answer ends the upload as refused, and that save fails. + case refused + } + + /// A withdrawal outranks an end this process reached but couldn't save. The end stays when + /// its save fails; the folder is unwritable, so the withdrawal goes unrecorded too. The + /// coordinator's `abandon(withdrawnReason)` must still end the upload as withdrawn. + /// + /// This holds at the library and coordinator boundary. The app doesn't offer Stop sending + /// once its screen shows a final state, so no button reaches this path today. + @Test(arguments: UnsavedEnd.allCases) + func aWithdrawalOutranksAnEndWhoseSaveFailed(end: UnsavedEnd) async throws { + let sleeps = ScopedCaptureUploadTests.Sleeps() + let rig = try Rig(sleep: sleeps.record) + defer { rig.cleanUp() } + let folder = rig.capture.folder + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) } + let route: String + switch end { + case .finished: + route = "GET captures/result" + rig.server.state.withLock { $0.held = [route] } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await CaptureUploaderTests.settles(rig.uploader)) + try await rig.finishAndSeal() + case .refused: + route = "POST captures/files" + // A register answer with a plain-http storage URL to another host, which ends the upload. + rig.server.state.withLock { + $0.held = [route] + $0.uploadURLOverride = "http://example.com/upload/x" + } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + } + try #require(await Self.parked(rig.server, route)) + + let url = CaptureUploader.stateURL(in: folder) + let before = try Data(contentsOf: url) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) + rig.server.release(route) + try #require(await CaptureUploaderTests.settles(rig.uploader)) + let reached = await rig.uploader.snapshot.end + switch end { + case .finished: #expect(reached == .finished(status: "manual_review")) + case .refused: #expect(reached == .failed(step: "register", codes: ["upload_url_insecure"], status: 200)) + } + #expect(try Data(contentsOf: url) == before) + let sent = Self.routes(rig.server) + + let record = rig.uploader.withdrawConsent() + guard case .notRecorded = record else { + Issue.record("expected the withdrawal to go unrecorded in an unwritable folder, got \(record)") + return + } + await rig.uploader.abandon(CaptureUploader.withdrawnReason) + #expect(await rig.uploader.snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) + #expect(await rig.uploader.status.phase == .abandoned) + #expect(try Data(contentsOf: url) == before) + await rig.uploader.kick() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(Self.routes(rig.server) == sent) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderSettlesTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderSettlesTests.swift new file mode 100644 index 000000000..85ced8153 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderSettlesTests.swift @@ -0,0 +1,61 @@ +import Foundation +import HouseScanKit +import Testing + +/// `CaptureUploaderTests.settles` is a hang safeguard, not a timing requirement, so a test +/// process that can't run its tasks for a while must not read as an uploader that never settles. +/// Hosted run 37219668935 stalled every test task for about 42 s after start (trivial tests +/// reported 28–43 s); four ScopedCaptureUploadTests then failed `settles` at its 20 s budget +/// although their uploads end at the first request. +/// +/// These tests block every cooperative thread (sleeping, not spinning) for longer than a 1 s +/// budget while a real uploader settles. That stalls the whole test process, so they run only +/// when asked: `HSK_STARVE_COOPERATIVE_POOL=1 swift test --filter CaptureUploaderSettlesTests`. +@Suite(.enabled(if: ProcessInfo.processInfo.environment["HSK_STARVE_COOPERATIVE_POOL"] == "1"), .serialized) +struct CaptureUploaderSettlesTests { + typealias Rig = CaptureUploaderTests.Rig + + /// Occupies every cooperative thread for `seconds`. Twice the processor count, so the threads + /// a waiter frees by suspending are taken too. + static func starve(for seconds: Double) { + for _ in 0..<(2 * ProcessInfo.processInfo.activeProcessorCount) { + Task.detached { usleep(useconds_t(seconds * 1_000_000)) } + } + } + + @Test func anIdleUploaderSettlesThroughAStall() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + Self.starve(for: 1.5) + #expect(await CaptureUploaderTests.settles(rig.uploader, within: 1)) + } + + /// The failing hosted case: a malformed credential ends the upload at once, with a stall + /// longer than the budget in between. + @Test func aRefusedCredentialSettlesThroughAStall() async throws { + let sleeps = ScopedCaptureUploadTests.Sleeps() + let rig = try ScopedCaptureUploadTests.rig(provider: ScopedCaptureHTTPTests.Provider([""]), sleeps: sleeps) + defer { rig.cleanUp() } + await rig.uploader.kick() + Self.starve(for: 1.5) + #expect(await CaptureUploaderTests.settles(rig.uploader, within: 1)) + #expect(await rig.uploader.snapshot.end == .failed(step: "create", codes: ["auth_credential_malformed"], status: 0)) + #expect(sleeps.isEmpty) + } +} + +/// The safeguard still ends: an uploader held at its first request never settles, and `settles` +/// says so after at least its budget, every turn being a wait of 20 ms or more. +@Suite struct CaptureUploaderSettlesBoundTests { + @Test func aHeldUploaderDoesNotSettle() async throws { + let rig = try CaptureUploaderTests.Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.held = ["POST captures"] } + await rig.uploader.kick() + let started = ContinuousClock.now + #expect(await !CaptureUploaderTests.settles(rig.uploader, within: 1)) + #expect(ContinuousClock.now - started >= .seconds(1)) + rig.server.release("POST captures") + #expect(await CaptureUploaderTests.settles(rig.uploader)) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift new file mode 100644 index 000000000..224cecef5 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift @@ -0,0 +1,634 @@ +import Foundation +import HouseScanKit +import Synchronization +import Testing + +/// The uploader against `LoopbackCaptureAPI` over real HTTP, through the same `URLSession` +/// transport the app uses. Retries wait 10 ms instead of seconds. +@Suite(.serialized) struct CaptureUploaderTests { + struct Rig { + let server: LoopbackCaptureAPI + let root: URL + let capture: SyntheticCapture04 + private var ownedUploader: CaptureUploader? + var uploader: CaptureUploader { + guard let ownedUploader else { preconditionFailure("The test released this uploader to simulate process exit") } + return ownedUploader + } + + mutating func releaseUploader() { ownedUploader = nil } + let http: any CaptureHTTP + + init( + packetID: String = UUID().uuidString, appVersion: String = "test (1)", server: LoopbackCaptureAPI? = nil, + http: any CaptureHTTP = URLSessionCaptureHTTP.ephemeral(timeout: 10), policy: CaptureUploader.Policy = Rig.fast, + sleep: @escaping @Sendable (Double) async throws -> Void = { _ in try await Task.sleep(for: .milliseconds(10)) }, + now: @escaping @Sendable () -> Date = { Date() } + ) throws { + self.http = http + self.server = try server ?? LoopbackCaptureAPI() + root = FileManager.default.temporaryDirectory.appending(path: "uploader-\(UUID().uuidString)") + capture = try SyntheticCapture04(folder: root.appending(path: "packet"), packetID: packetID) + ownedUploader = try CaptureUploader.start( + folder: capture.folder, base: self.server.base, http: http, + create: .init(packetId: packetID, tier: .arkit, device: .init(model: "iPhone15,4", systemVersion: "26.0", appVersion: appVersion)), + consentedAt: Date(), policy: policy, now: now, sleep: sleep) + } + + static var fast: CaptureUploader.Policy { + var policy = CaptureUploader.Policy() + policy.eventsWait = 0 + return policy + } + + func finishAndSeal() async throws { + let (streams, packet) = try await capture.finish() + await uploader.seal(packet: packet, files: streams) + } + + func cleanUp() { try? FileManager.default.removeItem(at: root) } + } + + /// Photos go up during the capture; the frozen packet finalizes early, the streams follow, and + /// the capture ends with a result for its own run. + @Test func joinedPathCommitsPhotosBeforeTheCaptureEnds() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + let images = try await rig.capture.sealImages() + await rig.uploader.add(images) + await rig.uploader.settled() + + let during = await rig.uploader.snapshot + #expect(during.committedCount == images.count) + #expect(during.packet == nil) + #expect(rig.server.requests("POST captures/finalize").isEmpty) + + try await rig.finishAndSeal() + await rig.uploader.settled() + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "manual_review")) + #expect(done.finalized?.status == "awaiting_files") + #expect(done.finalized?.missing.sorted() == [ + "streams/accelerometer_raw.csv.gz", "streams/arkit_poses.csv.gz", "streams/gyroscope_raw.csv.gz", "streams/imu_raw.csv.gz", + ]) + #expect(done.committedCount == images.count + 4) + #expect(done.result != nil) + let marks = done.marks + #expect(try #require(marks["firstCommit"]) < #require(marks["sealed"])) + #expect(try #require(marks["sealed"]) <= #require(marks["lastCommit"])) + + let puts = rig.server.requests("PUT upload") + #expect(puts.count == images.count + 4) + #expect(puts.allSatisfy { $0.headers["authorization"] == nil && $0.headers["content-md5"] != nil }) + #expect(rig.server.requests("POST captures").count == 1) + } + + /// A create and a finalize whose answers are lost are sent again as the same bytes, and the + /// server keeps one capture. + @Test func lostAnswersAreReplayedWithTheFrozenBodies() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.dropNext = ["POST captures", "POST captures/finalize"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 2)) + await rig.uploader.settled() + try await rig.finishAndSeal() + await rig.uploader.settled() + + let creates = rig.server.requests("POST captures") + let finals = rig.server.requests("POST captures/finalize") + #expect(creates.count == 2) + #expect(Set(creates.map(\.body)).count == 1) + #expect(finals.count == 2) + #expect(Set(finals.map(\.body)).count == 1) + #expect(rig.server.state.withLock { $0.captures.count } == 1) + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + } + + /// A relaunch resumes the saved capture: same packet and capture ids, no second create, and the + /// old process's attempt id is replaced. + @Test func relaunchResumesTheSameCapture() async throws { + var rig = try Rig() + defer { rig.cleanUp() } + await rig.uploader.add(try await rig.capture.sealImages(count: 2)) + await rig.uploader.settled() + let before = await rig.uploader.snapshot + + // The old process releases every uploader before the new one resumes its saved state. + weak let released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + let resumed = try #require(try CaptureUploader.resume( + folder: rig.capture.folder, base: rig.server.base, http: rig.http, policy: Rig.fast, sleep: { _ in })) + let after = await resumed.snapshot + #expect(after.packetID == before.packetID) + #expect(after.captureID == before.captureID) + #expect(after.createBody == before.createBody) + #expect(after.attemptID != before.attemptID) + + let (streams, packet) = try await rig.capture.finish() + await resumed.seal(packet: packet, files: streams) + await resumed.settled() + #expect(await resumed.snapshot.end == .finished(status: "manual_review")) + #expect(rig.server.requests("POST captures").count == 1) + } + + /// The same packet id with a different create body is the server's idempotency conflict; the + /// uploader stops instead of retrying it. + @Test func aChangedCreateBodyIsRefusedNotRetried() async throws { + let packetID = UUID().uuidString + let first = try Rig(packetID: packetID) + defer { first.cleanUp() } + await first.uploader.kick() + await first.uploader.settled() + #expect(await first.uploader.snapshot.captureID != nil) + + let second = try Rig(packetID: packetID, appVersion: "test (2)", server: first.server) + defer { second.cleanUp() } + await second.uploader.kick() + await second.uploader.settled() + #expect(await second.uploader.snapshot.end == .failed(step: "create", codes: ["idempotency_conflict"], status: 409)) + #expect(first.server.requests("POST captures").count == 2) + } + + /// A storage URL answered 403 (expired) is registered again and the file sent to the new URL. + @Test func anExpiredURLIsFetchedAgain() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.expireNextPuts = 1 } + let images = try await rig.capture.sealImages(count: 1) + await rig.uploader.add(images) + await rig.uploader.settled() + let snapshot = await rig.uploader.snapshot + #expect(snapshot.committedCount == images.count) + #expect(rig.server.requests("PUT upload").count == images.count + 1) + let registered = rig.server.requests("POST captures/files").flatMap { request -> [String] in + let body = try? JSONDecoder().decode(CaptureAPI.RegisterRequest.self, from: request.body) + return body?.files.map(\.path) ?? [] + } + #expect(registered.count == images.count + 1) + } + + /// Only paths the commit names as committed advance; notFound and mismatch go up again. + @Test func mixedCommitAnswersAdvanceOnlyCommittedFiles() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + let images = try await rig.capture.sealImages(count: 2) + // Priority order: the still, then k00001, then k00002. + let paths = ["stills/meter_close.jpg", "keyframes/k00001.jpg", "keyframes/k00002.jpg"] + #expect(Set(images.map(\.path)) == Set(paths)) + rig.server.state.withLock { $0.commitOverride = (committed: [paths[0]], notFound: [paths[1]], mismatch: [paths[2]]) } + let seen = Mutex<[Int]>([]) + await rig.uploader.observe { status in seen.withLock { $0.append(status.committed) } } + await rig.uploader.add(images) + await rig.uploader.settled() + + #expect(await rig.uploader.snapshot.committedCount == 3) + // After the first commit exactly one file counted as received. + #expect(seen.withLock { $0.first { $0 > 0 } } == 1) + let puts = rig.server.requests("PUT upload") + #expect(puts.count == 5) + } + + /// A world reset while a register answer is in flight: the late answer changes nothing, and the + /// next capture is a new packet that never reuses the old one. + @Test func resetDuringAPendingAnswerDropsTheLateReply() async throws { + // A background session's transfer outlives the task that asked for it, so the reply + // still arrives after the reset; only the attempt check keeps it out. + let rig = try Rig(http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10))) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.held = ["POST captures/files"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 2)) + for _ in 0..<500 where rig.server.requests("POST captures/files").isEmpty { try await Task.sleep(for: .milliseconds(10)) } + #expect(rig.server.requests("POST captures/files").count == 1) + + await rig.uploader.abandon("world reset") + rig.server.release("POST captures/files") + try await Task.sleep(for: .milliseconds(300)) + await rig.uploader.settled() + + let after = await rig.uploader.snapshot + #expect(after.end == .abandoned("world reset")) + #expect(after.files.values.allSatisfy { $0.phase == .queued }) + #expect(rig.server.requests("PUT upload").isEmpty) + // Nothing more is sent for the abandoned capture. + let late = try await rig.capture.producer.sealKeyframe( + jpeg: try SyntheticCapture04.jpeg(in: rig.root), observation: rig.capture.observation(SyntheticCapture04.start + 1.9), reason: "motion") + await rig.uploader.add(late) + await rig.uploader.settled() + #expect(rig.server.requests("POST captures/files").count == 1) + + let next = try Rig(server: rig.server) + defer { next.cleanUp() } + await next.uploader.add(try await next.capture.sealImages(count: 1)) + await next.uploader.settled() + #expect(await next.uploader.snapshot.captureID != after.captureID) + #expect(await next.uploader.snapshot.committedCount == 2) + } + + /// A finalize the server accepted and then lost (`failed`, next `retry_finalize`) is sent + /// again as the same bytes, and the run then completes. + @Test func aLostFinalizeIsSentAgainOnRetryFinalize() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.loseNextFinalize = true } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + let finals = rig.server.requests("POST captures/finalize") + #expect(finals.count == 2) + #expect(Set(finals.map(\.body)).count == 1) + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + #expect(await rig.uploader.snapshot.finalizeRetries == 1) + } + + /// The run is done but its answer is not readable yet (404 `result_not_ready`): the uploader + /// reads again after a wait instead of failing the capture. + @Test func aResultNotReadableYetIsReadAgain() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.notReadyResults = 2 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "manual_review")) + #expect(try Self.outcomeKind(done.result) == .manualReview) + #expect(rig.server.requests("GET captures/result").count == 3) + } + + /// The API answers a result read with the capture's status and a null outcome until the run + /// has written its answer. A terminal status with that body is not the answer. + @Test func aResultWithoutItsOutcomeIsReadAgain() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.outcomelessResults = 2 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "manual_review")) + #expect(try Self.outcomeKind(done.result) == .manualReview) + #expect(rig.server.requests("GET captures/result").count == 3) + } + + /// An answer that never becomes readable ends the upload visibly after the policy's reads, + /// rather than waiting forever. + @Test func aResultThatNeverArrivesEndsTheUpload() async throws { + var policy = Rig.fast + policy.maxNotReadyResults = 3 + let rig = try Rig(policy: policy) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.outcomelessResults = 100 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .failed(step: "result", codes: ["result_not_ready"], status: 200)) + #expect(done.result == nil) + #expect(rig.server.requests("GET captures/result").count == 4) + } + + /// A 200 whose body is not a result is refused by name instead of being kept as the answer. + @Test func anUnreadableResultStopsTheUpload() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.unreadableResults = 1 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .failed(step: "result", codes: ["result_unreadable"], status: 200)) + #expect(done.result == nil) + #expect(rig.server.requests("GET captures/result").count == 1) + } + + /// An outcome missing fields the app needs is refused by name instead of finishing the upload + /// with an answer nothing can show. + @Test func aMalformedOutcomeStopsTheUpload() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.malformedOutcomeResults = 1 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .failed(step: "result", codes: ["result_unreadable"], status: 200)) + #expect(done.result == nil) + #expect(rig.server.requests("GET captures/result").count == 1) + } + + /// Whether `uploader` goes idle within about `seconds`. A loop that never goes idle fails the + /// test here instead of hanging the run; the process ending stops it. + /// + /// A hang safeguard, not a timing requirement, so the budget is this waiter's own turns, 20 ms + /// apart, like the suite's `for _ in 0..<500 where` waits, not wall time. A stalled test process + /// holds the uploader and this waiter up together, and a wall-clock deadline expired on time + /// neither got to run (CaptureUploaderSettlesTests). The answer is read once more after the + /// last wait, which the old loop skipped. + static func settles(_ uploader: CaptureUploader, within seconds: Int = 20) async -> Bool { + let done = Mutex(false) + Task.detached { await uploader.settled(); done.withLock { $0 = true } } + for _ in 0..<(seconds * 50) { + if done.withLock({ $0 }) { return true } + try? await Task.sleep(for: .milliseconds(20)) + } + return done.withLock { $0 } + } + + /// A storage URL that is not https (or http to this machine) is refused when it is + /// registered, before any bytes go to it; one that doesn't parse is refused the same way + /// instead of leaving the file registered and the loop spinning. + @Test(arguments: [ + ("http://storage.example.com/upload/x", "upload_url_insecure"), ("", "upload_url_invalid"), + // Near matches of a loopback name are public hosts. + ("http://[::2]:9/upload/x", "upload_url_insecure"), ("http://localhost.example.com/upload/x", "upload_url_insecure"), + ("http://127.0.0.1.example.com/upload/x", "upload_url_insecure"), + ]) + func aStorageURLThatIsNotSafeIsRefusedBeforeAnyPut(url: String, code: String) async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.uploadURLOverride = url } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "register", codes: [code], status: 200)) + #expect(rig.server.requests("POST captures/files").count == 1) + #expect(rig.server.requests("PUT upload").isEmpty) + } + + /// Plain http to this machine over IPv6 is allowed, as ScopedCaptureHTTP's `.loopbackHTTP` + /// allows it: the file is registered for its PUT instead of refused. Nothing listens on `::1` + /// here, so the PUT itself fails as a network error and the upload waits to retry. + @Test func aLoopbackStorageURLOverIPv6IsNotRefused() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + let url = "http://[::1]:\(rig.server.port)/v1/uploads/x" + rig.server.state.withLock { $0.uploadURLOverride = url } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + let registered = { () async -> Bool in + await rig.uploader.snapshot.files.values.contains { if case .registered(url, _, _) = $0.phase { true } else { false } } + } + for _ in 0..<500 { + if await rig.uploader.snapshot.end != nil { break } + if await registered() { break } + try await Task.sleep(for: .milliseconds(10)) + } + #expect(await rig.uploader.snapshot.end == nil) + #expect(await registered()) + await rig.uploader.abandon("test done") + } + + /// Storage that answers a PUT with a redirect doesn't get the photo sent on: the real + /// `URLSession` transport refuses every redirect, so the photo reaches only the URL that passed + /// the check, and the 307 comes back as a storage refusal. The redirect goes to a second local + /// server, which must see nothing. + @Test func aStorageRedirectIsNotFollowed() async throws { + let elsewhere = try LoopbackCaptureAPI() + var policy = Rig.fast + policy.maxRefusedPuts = 2 + let rig = try Rig(policy: policy) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.redirects["PUT upload"] = "http://127.0.0.1:\(elsewhere.port)/upload/elsewhere" } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "put", codes: ["storage_refused"], status: 307)) + #expect(!rig.server.requests("PUT upload").isEmpty) + #expect(elsewhere.state.withLock { $0.log.isEmpty }) + } + + /// The API's own requests aren't redirected either: a 307 on create comes back as the reply, + /// which the uploader refuses by its status, and the second server sees nothing. + @Test func anAPIRedirectIsNotFollowed() async throws { + let elsewhere = try LoopbackCaptureAPI() + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.redirects["POST captures"] = "\(elsewhere.base.absoluteString)/captures" } + await rig.uploader.kick() + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "create", codes: [], status: 307)) + #expect(rig.server.requests("POST captures").count == 1) + #expect(elsewhere.state.withLock { $0.log.isEmpty }) + } + + /// Commits that never acknowledge a file back off between tries and stop after the policy's + /// limit, instead of sending the same photo again and again. + @Test func aCommitThatNeverAcknowledgesBacksOffAndStops() async throws { + var policy = Rig.fast + policy.maxUnacknowledged = 3 + let sleeps = Mutex<[Double]>([]) + let rig = try Rig(policy: policy, sleep: { seconds in sleeps.withLock { $0.append(seconds) } }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.commitNeverAcknowledges = true } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "commit", codes: ["not_acknowledged"], status: 200)) + // The images are the close-up still and its keyframe. + #expect(rig.server.requests("POST captures/files:commit").count == 4) + #expect(sleeps.withLock { $0.count } == 3) + } + + /// Register answers that leave a file out back off and stop the same way. + @Test func aRegisterThatOmitsFilesBacksOffAndStops() async throws { + var policy = Rig.fast + policy.maxUnacknowledged = 3 + let sleeps = Mutex<[Double]>([]) + let rig = try Rig(policy: policy, sleep: { seconds in sleeps.withLock { $0.append(seconds) } }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.registerOmitsFiles = true } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "register", codes: ["not_acknowledged"], status: 200)) + #expect(rig.server.requests("POST captures/files").count == 4) + #expect(rig.server.requests("PUT upload").isEmpty) + #expect(sleeps.withLock { $0.count } == 3) + } + + /// The limit on reads that find no answer is saved with the upload, so relaunching doesn't + /// start it again: across two processes the reads still stop at `maxNotReadyResults + 1`. + @Test func theNotReadyLimitSurvivesARelaunch() async throws { + var policy = Rig.fast + policy.maxNotReadyResults = 3 + let waits = Mutex(0) + // The first process quits during its second wait. + var rig = try Rig(policy: policy, sleep: { _ in + let n = waits.withLock { $0 += 1; return $0 } + if n >= 2 { throw CancellationError() } + }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.outcomelessResults = 100 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + try #require(await Self.settles(rig.uploader)) + #expect(rig.server.requests("GET captures/result").count == 2) + #expect(await rig.uploader.snapshot.end == nil) + + // The old process releases every uploader before the new one resumes its saved state. + weak let released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + let resumed = try #require(try CaptureUploader.resume( + folder: rig.capture.folder, base: rig.server.base, http: rig.http, policy: policy, sleep: { _ in })) + await resumed.kick() + try #require(await Self.settles(resumed)) + #expect(await resumed.snapshot.end == .failed(step: "result", codes: ["result_not_ready"], status: 200)) + #expect(rig.server.requests("GET captures/result").count == 4) + } + + /// A server that answers an events poll at once with nothing new is not polled again at once: + /// the uploader waits out `minimumPollInterval` first. The clock moves only when the uploader + /// sleeps, so every empty answer arrives at once and each wait is the whole interval. The + /// terminal answer is fetched without a wait. + @Test func emptyEventsAnsweredAtOnceArePaced() async throws { + let clock = Mutex(Date(timeIntervalSince1970: 1_000_000)) + let sleeps = Mutex<[Double]>([]) + let rig = try Rig( + sleep: { seconds in sleeps.withLock { $0.append(seconds) }; clock.withLock { $0 += seconds } }, + now: { clock.withLock { $0 } }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.processingPolls = 3 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + let interval = CaptureUploader.Policy().minimumPollInterval + #expect(sleeps.withLock { $0 } == [interval, interval, interval]) + #expect(rig.server.requests("GET captures/events").count == 4) + } + + /// An empty answer the server held for the whole interval needs no further wait. + @Test func emptyEventsHeldPastTheIntervalAreNotPaced() async throws { + // Every reading of the clock is 3 s after the last, longer than the 2 s interval. + let clock = Mutex(Date(timeIntervalSince1970: 1_000_000)) + let sleeps = Mutex<[Double]>([]) + let rig = try Rig( + sleep: { seconds in sleeps.withLock { $0.append(seconds) } }, + now: { clock.withLock { $0 += 3; return $0 } }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.processingPolls = 3 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + #expect(sleeps.withLock { $0 }.isEmpty) + #expect(rig.server.requests("GET captures/events").count == 4) + } + + /// A capture the server lost and the uploader opened again is a new run, so it gets a fresh + /// not-ready limit: the old run's reads don't end it. + @Test func aRecreatedCaptureStartsANewNotReadyLimit() async throws { + var policy = Rig.fast + policy.maxNotReadyResults = 3 + let waits = Mutex(0) + // The first process quits during its third wait, after three not-ready reads. + var rig = try Rig(policy: policy, sleep: { _ in + let n = waits.withLock { $0 += 1; return $0 } + if n >= 3 { throw CancellationError() } + }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.outcomelessResults = 100 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + try #require(await Self.settles(rig.uploader)) + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: rig.capture.folder)).notReadyReads == 3) + + // The server forgets every capture (a redeploy with in-memory state). The next run has + // one not-ready read before its answer. + rig.server.state.withLock { $0.captures = [:]; $0.byPacket = [:]; $0.outcomelessResults = 1 } + // The old process releases every uploader before the new one resumes its saved state. + weak let released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + let resumed = try #require(try CaptureUploader.resume( + folder: rig.capture.folder, base: rig.server.base, http: rig.http, policy: policy, sleep: { _ in })) + await resumed.kick() + try #require(await Self.settles(resumed)) + #expect(await resumed.snapshot.end == .finished(status: "manual_review")) + #expect(rig.server.requests("POST captures").count == 2) + } + + /// `failed` ends a capture without an answer, so its outcomeless result is final at once. + @Test func aFailedCaptureEndsWithoutWaitingForAnOutcome() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.endStatus = "failed" } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "failed")) + #expect(try Self.outcomeKind(done.result) == nil) + #expect(rig.server.requests("GET captures/result").count == 1) + } + + static func outcomeKind(_ body: Data?) throws -> CaptureResult.OutcomeKind? { + try JSONDecoder().decode(CaptureResult.Response.self, from: #require(body)).outcome?.kind + } + + /// Storage that keeps refusing the sealed bytes' digest stops the upload after one retry + /// instead of cycling through register and PUT. + @Test func aDigestRefusedTwiceStopsTheUpload() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.badDigestPuts = 100 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + await rig.uploader.settled() + #expect(await rig.uploader.snapshot.end == .failed(step: "put", codes: ["bad_digest"], status: 400)) + #expect(rig.server.requests("PUT upload").count <= 4) + } + + /// A file listed in the frozen packet that reached `seal` before its own `add` is still sent. + @Test func sealQueuesListedFilesItHasNotSeen() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + let images = try await rig.capture.sealImages(count: 2) + await rig.uploader.add(Array(images.prefix(1))) + await rig.uploader.settled() + let (streams, packet) = try await rig.capture.finish() + await rig.uploader.seal(packet: packet, files: images + streams) + await rig.uploader.add(Array(images.suffix(1))) + await rig.uploader.settled() + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "manual_review")) + #expect(done.committedCount == images.count + streams.count) + } + + @Test func noEndpointOrNoConsentMeansNoUpload() { + #expect(CaptureUploadGate.decide(endpoint: nil, consented: true) == .off("no capture endpoint in this build")) + #expect(CaptureUploadGate.decide(endpoint: "", consented: true) == .off("no capture endpoint in this build")) + #expect(CaptureUploadGate.decide(endpoint: "$(HOUSESCAN_CAPTURE_API_URL)", consented: true) == .off("no capture endpoint in this build")) + #expect(CaptureUploadGate.decide(endpoint: "http://example.com/v1", consented: true) == .off("capture endpoint must be https")) + #expect(CaptureUploadGate.decide(endpoint: "https://user:secret@example.com/v1", consented: true) == .off("capture endpoint is not a URL")) + #expect(CaptureUploadGate.decide(endpoint: "https://example.com/v1", consented: false) == .off("the homeowner has not agreed to send the capture")) + #expect(CaptureUploadGate.decide(endpoint: "https://example.com/v1", consented: true) == .on(URL(string: "https://example.com/v1")!)) + } +} + +/// Runs each request in a detached task, so cancelling the caller doesn't cancel the request: +/// the way a background URLSession behaves. +struct UncancellableHTTP: CaptureHTTP { + let inner: any CaptureHTTP + init(_ inner: any CaptureHTTP) { self.inner = inner } + + func send(_ request: URLRequest) async throws -> HTTPReply { + try await Task.detached { try await inner.send(request) }.value + } + + func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + try await Task.detached { try await inner.upload(request, file: file) }.value + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CountedWait.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CountedWait.swift new file mode 100644 index 000000000..db0012ea9 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CountedWait.swift @@ -0,0 +1,47 @@ +import Testing + +/// Waits until `condition` holds, as a hang safeguard and not a timing requirement: the photo +/// processing suites' `until` helpers call this so a capture that never reaches a state fails its +/// test instead of hanging the run. +/// +/// The budget is this waiter's own turns, `seconds * 50` waits of 20 ms or more, not wall time, the +/// semantics `CaptureUploaderTests.settles` took in 6c1f68ea. A stalled test process holds the code +/// under test and this waiter up together, and a wall-clock deadline can expire before either gets +/// to run: hosted run 37219668935 stalled every task for about 42 s, and in run 37229126189 a 10 s +/// wall-clock wait here gave up on an upload that takes about 4 s locally. `condition` is read once +/// more after the last wait, so a state reached during that wait still counts. It runs in the +/// caller's isolation (`#isolation`), so a main-actor suite's condition is read on the main actor +/// and never sent across. +func countedWait( + _ seconds: Double, isolation: isolated (any Actor)? = #isolation, _ condition: () -> Bool +) async throws -> Bool { + for _ in 0..= .milliseconds(200)) + #expect(reads == 11) + } + + @Test func aConditionThatHoldsOnlyAtTheFinalReadCounts() async throws { + var reads = 0 + #expect(try await countedWait(0.2) { reads += 1; return reads == 11 }) + } + + @Test func aConditionThatAlreadyHoldsReturnsWithoutWaiting() async throws { + var reads = 0 + #expect(try await countedWait(30) { reads += 1; return true }) + #expect(reads == 1) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/FrameCalibrationRowTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/FrameCalibrationRowTests.swift new file mode 100644 index 000000000..a5df48783 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/FrameCalibrationRowTests.swift @@ -0,0 +1,68 @@ +import HouseScanKit +import simd +import Testing + +@Suite struct FrameCalibrationRowTests { + /// ARKit's intrinsics, column-major: fx and fy on the diagonal, cx and cy in the last column. + static func intrinsics(fx: Float, fy: Float, cx: Float, cy: Float) -> simd_float3x3 { + simd_float3x3(SIMD3(fx, 0, 0), SIMD3(0, fy, 0), SIMD3(cx, cy, 1)) + } + + /// A trajectory row as the recorder writes it: t, tracking state and reason, then the pose's + /// 16 values column by column. + static func trajectoryRow(_ t: Double, x: Float) -> [Double] { + var m = matrix_identity_float4x4 + m.columns.3 = SIMD4(x, 1.5, 0, 1) + let columns = [m.columns.0, m.columns.1, m.columns.2, m.columns.3] + return [t, 0, 0] + columns.flatMap { [Double($0.x), Double($0.y), Double($0.z), Double($0.w)] } + } + + @Test func theRowIsTheFramesOwnValuesInTheirOriginalUnits() { + // Every value distinct, so a swapped column or axis would show. + let k = Self.intrinsics(fx: 1445.25, fy: 1446.5, cx: 961.75, cy: 719.125) + #expect(FrameCalibrationRow.row(t: 12.345, intrinsics: k, imageWidth: 1920, imageHeight: 1440) + == [12.345, 1445.25, 1446.5, 961.75, 719.125, 1920, 1440]) + // Off-diagonal entries (skew, the bottom row) don't leak into the row. + var skewed = k + skewed.columns.1.x = 3 + skewed.columns.0.z = 7 + #expect(FrameCalibrationRow.row(t: 1, intrinsics: skewed, imageWidth: 1920, imageHeight: 1440) + == FrameCalibrationRow.row(t: 1, intrinsics: k, imageWidth: 1920, imageHeight: 1440)) + } + + /// Calibration and resolution change from frame to frame independently; each pose gets the + /// calibration of the frame with its own t, through #192's join. + @Test func eachPoseJoinsItsOwnFramesCalibration() throws { + let frames: [(t: Double, k: SIMD4, size: SIMD2)] = [ + (100.000, SIMD4(1445, 1445, 960, 720), SIMD2(1920, 1440)), + (100.017, SIMD4(1450, 1449, 958, 722), SIMD2(1920, 1440)), + (100.033, SIMD4(1450, 1449, 958, 722), SIMD2(1280, 960)), + (100.050, SIMD4(963, 964, 640, 480), SIMD2(1280, 960)), + ] + let trajectory = frames.enumerated().map { Self.trajectoryRow($1.t, x: Float($0)) } + let intrinsics = frames.map { + FrameCalibrationRow.row( + t: $0.t, intrinsics: Self.intrinsics(fx: $0.k.x, fy: $0.k.y, cx: $0.k.z, cy: $0.k.w), imageWidth: $0.size.x, imageHeight: $0.size.y) + } + // Rows out of order, as nothing promises they arrive sorted. + let poses = Packet04Streams.poseRows(trajectory: trajectory, intrinsics: intrinsics.reversed(), tracking: PacketTracking.init(recorderState:reason:)) + #expect(poses.map(\.t) == frames.map(\.t)) + #expect(poses.map(\.intrinsics) == frames.map(\.k)) + #expect(poses.map { $0.cameraToWorld.columns.3.x } == [0, 1, 2, 3]) + #expect(intrinsics.map { [$0[5], $0[6]] } == frames.map { [$0.size.x, $0.size.y] }) + } + + /// A pose whose frame has no calibration row is left out, and a row at another t, however + /// close, isn't borrowed for it. + @Test func aPoseWithoutItsOwnRowIsOmittedNotGuessed() { + let k = Self.intrinsics(fx: 1445, fy: 1445, cx: 960, cy: 720) + let trajectory = [100.0, 100.5, 101.0].enumerated().map { Self.trajectoryRow($1, x: Float($0)) } + let intrinsics = [ + FrameCalibrationRow.row(t: 100.0, intrinsics: k, imageWidth: 1920, imageHeight: 1440), + // 100.5 has no row; 101.0's row is a hair off its pose's t. + FrameCalibrationRow.row(t: 101.0.nextUp, intrinsics: k, imageWidth: 1920, imageHeight: 1440), + ] + let poses = Packet04Streams.poseRows(trajectory: trajectory, intrinsics: intrinsics, tracking: PacketTracking.init(recorderState:reason:)) + #expect(poses.map(\.t) == [100.0]) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/KeptScansTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/KeptScansTests.swift index 521549436..9b21146c7 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/KeptScansTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/KeptScansTests.swift @@ -219,6 +219,255 @@ import Testing #expect(try Set(files.contentsOfDirectory(atPath: root.path)) == ["intact1", "intact2", "rewritten", "startsWriting", "finishesWriting"]) } + // MARK: A bundle still being written when Start over lists its folder (B44) + // + // The async tests below carry a five-minute time limit, only so that a gate never released + // fails instead of hanging; the gates and assertions decide pass or fail. In the concurrent + // package suite these tests reported up to 82.5 s (hosted run 37161498639, which had no + // limit), so a one-minute limit was too tight to rely on. + + /// A scan folder whose bundle is being written: the photos, stamp and raw streams the packet + /// is built from and the half-assembled `packet/`, but no `scan.zip` yet. + static func packing(_ name: String, in root: URL) throws -> URL { + try scan(name, in: root) + let folder = root.appending(path: name) + let files = FileManager.default + try Data("{}".utf8).write(to: folder.appending(path: "scan-stamp.json")) + try files.createDirectory(at: folder.appending(path: "streams-raw"), withIntermediateDirectories: true) + try Data([1, 2, 3]).write(to: folder.appending(path: "streams-raw/trajectory.bin")) + try files.createDirectory(at: folder.appending(path: "packet/photos"), withIntermediateDirectories: true) + try Data("photo".utf8).write(to: folder.appending(path: "packet/photos/p00001.jpg")) + return folder + } + + /// What a bundle write leaves when it ends. + enum WriteEnd: Sendable, CaseIterable { + /// The zip written to the end and `packet/` removed, as `zipPacket` does. + case bundled + /// It failed before zipping: `packet/` is left and there is no zip. + case failedBeforeZipping + /// A zip cut short is left. + case partialZip + + func apply(to folder: URL) throws { + let bundle = folder.appending(path: ScanFolderCleanup.bundleName) + switch self { + case .bundled: + try KeptScansTests.packet.write(to: bundle) + try FileManager.default.removeItem(at: folder.appending(path: "packet")) + case .failedBeforeZipping: + break + case .partialZip: + try KeptScansTests.packet.prefix(KeptScansTests.packet.count / 2).write(to: bundle) + } + } + } + + /// A bundle write the test holds open, then ends as `end` says; like the app's, it starts + /// only after the write before it (`after`). + struct HeldWrite { + let task: Task + private let release: AsyncStream.Continuation + + init(_ folder: URL, ending end: WriteEnd, after earlier: Task? = nil) { + let (gate, release) = AsyncStream.makeStream() + self.release = release + task = Task { + await earlier?.value + for await _ in gate { break } + try? end.apply(to: folder) + } + } + + func finish() { release.finish() } + } + + /// Deletes with `run(after:)`, waiting for `writes` as the app does, and says through + /// `arrived` when it has reached that wait. Returns the folders it failed to delete. + static func deleting(_ cleanup: ScanFolderCleanup, after writes: Task?, arrived: AsyncStream.Continuation) -> Task<[URL], Never> { + Task { + await cleanup.run(after: { + arrived.yield() + await writes?.value + }).map(\.url) + } + } + + /// Start over lists the previous scan's folder while its bundle is still being assembled: + /// with no zip yet it is listed as never packaged. The deletion waits for the write; a zip + /// written to the end keeps the folder, one cut short keeps it without a place (as any + /// partial bundle), and a write that failed before zipping leaves it to be deleted. + @Test(.timeLimit(.minutes(5)), arguments: WriteEnd.allCases) + func aBundleStillBeingWrittenIsJudgedOnlyAfterItsWrite(end: WriteEnd) async throws { + let root = try Self.makeRoot() + defer { try? FileManager.default.removeItem(at: root) } + let folder = try Self.packing("previous", in: root) + try Self.scan("new", in: root) + let write = HeldWrite(folder, ending: end) + + let cleanup = ScanFolderCleanup(root: root, keeping: "new") + // What B44 deleted: the folder being written is listed with the never-packaged ones. + #expect(Self.names(cleanup.obsolete) == ["previous"]) + let (arrived, arrive) = AsyncStream.makeStream() + let deleting = Self.deleting(cleanup, after: write.task, arrived: arrive) + var arrivals = arrived.makeAsyncIterator() + _ = await arrivals.next() + // The cleanup is waiting for the write, which is still assembling the packet. + #expect(FileManager.default.fileExists(atPath: folder.appending(path: "packet/photos/p00001.jpg").path)) + + write.finish() + #expect(await deleting.value.isEmpty) + let next = ScanFolderCleanup(root: root, keeping: "next") + switch end { + case .bundled: + #expect(FileManager.default.fileExists(atPath: folder.appending(path: ScanFolderCleanup.bundleName).path)) + #expect(Self.names(next.keptCompleted) == ["previous"]) + case .failedBeforeZipping: + #expect(!FileManager.default.fileExists(atPath: folder.path)) + case .partialZip: + #expect(FileManager.default.fileExists(atPath: folder.path)) + #expect(Self.names(next.incompleteBundles) == ["previous"]) + #expect(!Self.names(next.obsolete).contains("previous")) + } + } + + /// At launch there is no write to wait for: the deletion is `run()`'s, at once. + @Test func withNoWriteTheDeletionRunsAtOnce() async throws { + let root = try Self.makeRoot() + defer { try? FileManager.default.removeItem(at: root) } + try Self.scan("abandoned", in: root) + try Self.scan("run1", in: root, bundledMinutesAgo: 90) + try Self.scan("run2", in: root, bundledMinutesAgo: 30) + try Self.scan("run3", in: root, bundledMinutesAgo: 5) + try Self.scan("now", in: root) + + let none: Task? = nil + let failed = await ScanFolderCleanup(root: root, keeping: "now").run(after: { await none?.value }) + #expect(failed.isEmpty) + #expect(try Set(FileManager.default.contentsOfDirectory(atPath: root.path)) == ["now", "run2", "run3"]) + } + + /// The ends of named bundle writes, and the cleanups waiting for them. Ending a write + /// releases the cleanups waiting for it and records which it released, under the lock, so a + /// test reads which cleanup a write let go instead of inferring it from scheduling. + final class WriteEnds: @unchecked Sendable { + private let lock = NSLock() + private var ended: Set = [] + private var waiting: [(write: String, cleanup: String, resume: CheckedContinuation)] = [] + private var released: [String: [String]] = [:] + + /// Waits until `write` ends. `registered` is told once `cleanup` is on the list, so a + /// write ended after that releases it. + func wait(for write: String, as cleanup: String, registered: AsyncStream.Continuation) async { + await withCheckedContinuation { (resume: CheckedContinuation) in + lock.lock() + let already = ended.contains(write) + if already { released[write, default: []].append(cleanup) } else { waiting.append((write, cleanup, resume)) } + lock.unlock() + registered.yield() + if already { resume.resume() } + } + } + + func end(_ write: String) { + lock.lock() + ended.insert(write) + let ready = waiting.filter { $0.write == write } + waiting.removeAll { $0.write == write } + released[write, default: []] += ready.map(\.cleanup) + lock.unlock() + for waiter in ready { waiter.resume.resume() } + } + + func released(by write: String) -> [String] { + lock.lock() + defer { lock.unlock() } + return released[write] ?? [] + } + } + + /// Start over twice while the first scan's bundle is written, as the app chains them: the + /// second scan's write starts only after the first, so each cleanup waits for the newest + /// write when it listed (the chain head). The first write's end releases only the first + /// cleanup; the second cleanup waits for the second write, which is held before zipping with + /// its inputs in place, and finds them intact. The first scan's bundle is kept, the second's + /// only if its write ran (one queued for a scan already started over is skipped), and the + /// newest folders are never listed. + @Test(.timeLimit(.minutes(5)), arguments: [true, false]) + func rapidStartOversWaitForTheWholeChain(secondScanWrites: Bool) async throws { + let root = try Self.makeRoot() + defer { try? FileManager.default.removeItem(at: root) } + let files = FileManager.default + let ends = WriteEnds() + let first = try Self.packing("first", in: root) + let (firstGate, releaseFirst) = AsyncStream.makeStream() + let firstWrite = Task { + for await _ in firstGate { break } + try? WriteEnd.bundled.apply(to: first) + ends.end("first") + } + + // Start over: the second scan's folder is made and the first cleanup lists. + try Self.scan("second", in: root) + let firstCleanup = ScanFolderCleanup(root: root, keeping: "second") + #expect(Self.names(firstCleanup.obsolete) == ["first"]) + // The second scan captures its packet inputs, and its write queues behind the first. + let second = try Self.packing("second", in: root) + let secondInputs = second.appending(path: "packet/photos/p00001.jpg") + let (secondGate, releaseSecond) = AsyncStream.makeStream() + let secondWrite = Task { () -> Bool? in + await firstWrite.value + guard secondScanWrites else { + ends.end("second") + return nil + } + for await _ in secondGate { break } + // Its inputs must still be there; this write never recreates them. + let intact = FileManager.default.fileExists(atPath: secondInputs.path) + if intact { try? WriteEnd.bundled.apply(to: second) } + ends.end("second") + return intact + } + + // Start over again: the second cleanup lists both earlier folders. + try Self.scan("third", in: root) + let secondCleanup = ScanFolderCleanup(root: root, keeping: "third") + #expect(Self.names(secondCleanup.obsolete) == ["first", "second"]) + try Self.scan("fourth", in: root) + + let (registered, register) = AsyncStream.makeStream() + let deletingFirst = Task { + await firstCleanup.run(after: { await ends.wait(for: "first", as: "firstCleanup", registered: register) }).map(\.url) + } + let deletingSecond = Task { + await secondCleanup.run(after: { await ends.wait(for: "second", as: "secondCleanup", registered: register) }).map(\.url) + } + var registrations = registered.makeAsyncIterator() + _ = await registrations.next() + _ = await registrations.next() + #expect(files.fileExists(atPath: first.appending(path: "packet").path)) + + releaseFirst.finish() + await firstWrite.value + #expect(ends.released(by: "first") == ["firstCleanup"]) + #expect(await deletingFirst.value.isEmpty) + #expect(files.fileExists(atPath: first.appending(path: ScanFolderCleanup.bundleName).path)) + if secondScanWrites { + // The first write has ended and its cleanup has run; the second cleanup still waits. + #expect(files.fileExists(atPath: secondInputs.path)) + } + + releaseSecond.finish() + let wrote = await secondWrite.value + #expect(wrote == (secondScanWrites ? true : nil)) + #expect(ends.released(by: "second") == ["secondCleanup"]) + #expect(await deletingSecond.value.isEmpty) + var expected: Set = ["first", "third", "fourth"] + if secondScanWrites { expected.insert("second") } + #expect(try Set(files.contentsOfDirectory(atPath: root.path)) == expected) + #expect(files.fileExists(atPath: second.appending(path: ScanFolderCleanup.bundleName).path) == secondScanWrites) + } + /// Bundles saved at the same moment keep the order by name, newest name first, as before. @Test func equalSaveTimesFallBackToNameOrder() throws { let root = try Self.makeRoot() diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift new file mode 100644 index 000000000..4a85db542 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift @@ -0,0 +1,403 @@ +import CryptoKit +import Foundation +import Network +import Synchronization +import HouseScanKit + +/// A local stand-in for the capture API over real HTTP on 127.0.0.1, so the tests drive the +/// actual `URLSession` transport. It implements the documented behaviour the uploader relies on: +/// idempotent create and finalize bound to the exact body, register with signed PUT URLs, +/// Content-MD5 checked on the PUT, commit as the only acknowledgement, early finalize with +/// `awaiting_files`, and events leading to a result. Faults are injected per test. +/// +/// Written for these tests from the public description of the API; it is not the server's code. +final class LoopbackCaptureAPI: Sendable { + struct Request: Sendable { + var method: String + var path: String + var query: [String: String] + var headers: [String: String] + var body: Data + } + + struct Registration { + var sha256: String + var bytes: Int + var md5: String + var contentType: String + } + + struct Capture { + var id: String + var packetID: String + var createBody: Data + var registered: [String: Registration] = [:] + var stored: [String: Data] = [:] + var committed: Set = [] + var packetSHA: String? + var listed: [String] = [] + var runID: String? + var events: [(type: String, data: [String: String])] = [] + var status = "uploading" + /// Accepted a finalize, then lost it to a storage failure: no run until it is sent again. + var finalizeLost = false + } + + struct State { + var captures: [String: Capture] = [:] + var byPacket: [String: String] = [:] + var tokens: [String: (capture: String, path: String)] = [:] + var log: [Request] = [] + var nextID = 1 + /// Close the connection without answering the next request whose route matches. + var dropNext: Set = [] + /// Answer the next N storage PUTs 403, as an expired signature. + var expireNextPuts = 0 + /// Answer the next N storage PUTs 400 BadDigest. + var badDigestPuts = 0 + /// Accept the next finalize, then report it lost with `failed` / `retry_finalize`. + var loseNextFinalize = false + /// The homeowner message the result carries. + var resultMessage = "The loopback receiver's test result." + /// Answer the next N result reads 404 `result_not_ready`: the run is done but its answer + /// is not readable yet. + var notReadyResults = 0 + /// Answer the next N result reads 200 with the capture's status and run but a null + /// outcome, the body the API serves before the run has written its answer. + var outcomelessResults = 0 + /// Answer the next N result reads 200 with a body that is not JSON. + var unreadableResults = 0 + /// Answer the next N result reads 200 with an outcome that has a kind but none of the + /// other fields an outcome must carry. + var malformedOutcomeResults = 0 + /// The status a capture ends in once every listed file is committed. Only `manual_review` + /// ends with an outcome; `failed` ends with a `failed` event and none. + var endStatus = "manual_review" + /// Every register answer gives this URL for its PUTs instead of the loopback's own. + var uploadURLOverride: String? + /// Register answers leave out every file, as if the server never saw the request's list. + var registerOmitsFiles = false + /// Commits never acknowledge a file: every path comes back in `notFound`. + var commitNeverAcknowledges = false + /// Once every listed file is committed, the next N event polls answer at once with status + /// `processing` and no events, as a server that doesn't hold the poll would; then the run ends. + var processingPolls = 0 + /// Routes ("PUT upload", "POST captures", ...) answered `307` with the given `Location`, as + /// a server sending the request somewhere else would. + var redirects: [String: String] = [:] + /// Answer the next commit with this split instead of the truth. + var commitOverride: ((committed: [String], notFound: [String], mismatch: [String]))? + /// Opt-in: every API request must carry `Authorization: Bearer `, or it gets 401 + /// `unauthorized`. Nil keeps the default, which refuses any Authorization on an API request. + /// A signed PUT is refused with any Authorization either way. + var expectedBearer: String? + /// Extra headers every register answer adds to a file's signed headers, as a server that + /// signs one more header would. + var extraSignedHeaders: [String: String] = [:] + /// Routes whose answers wait until `release` is called. + var held: Set = [] + var parked: [String: [@Sendable () -> Void]] = [:] + } + + let state = Mutex(State()) + private let listener: NWListener + private let queue = DispatchQueue(label: "loopback-capture-api") + var port: UInt16 { listener.port?.rawValue ?? 0 } + + init() throws { + let parameters = NWParameters.tcp + parameters.acceptLocalOnly = true + parameters.requiredLocalEndpoint = .hostPort(host: "127.0.0.1", port: .any) + listener = try NWListener(using: parameters) + let ready = DispatchSemaphore(value: 0) + listener.stateUpdateHandler = { if case .ready = $0 { ready.signal() } } + listener.newConnectionHandler = { [queue] connection in + connection.start(queue: queue) + Self.receive(connection, buffer: Data()) { request in self.handle(request, connection) } + } + listener.start(queue: queue) + guard ready.wait(timeout: .now() + 5) == .success, listener.port != nil else { throw URLError(.cannotConnectToHost) } + } + + deinit { listener.cancel() } + + var base: URL { URL(string: "http://127.0.0.1:\(port)/v1")! } + + func requests(_ route: String) -> [Request] { state.withLock { $0.log.filter { Self.route($0) == route } } } + + func release(_ route: String) { + let waiting = state.withLock { s -> [@Sendable () -> Void] in + s.held.remove(route) + return s.parked.removeValue(forKey: route) ?? [] + } + waiting.forEach { $0() } + } + + /// "POST /v1/captures/{id}/files" style names, ids replaced. + static func route(_ r: Request) -> String { + let parts = r.path.split(separator: "/").map(String.init) + if parts.first == "upload" { return "\(r.method) upload" } + guard parts.count >= 2 else { return "\(r.method) \(r.path)" } + if parts.count == 2 { return "\(r.method) captures" } + return "\(r.method) captures/\(parts[2...].joined(separator: "/"))" + } + + // MARK: HTTP over the connection + + private static func receive(_ connection: NWConnection, buffer: Data, handler: @escaping @Sendable (Request) -> Void) { + connection.receive(minimumIncompleteLength: 1, maximumLength: 1 << 20) { data, _, done, error in + var buffer = buffer + if let data { buffer.append(data) } + if let request = parse(buffer) { return handler(request) } + if done || error != nil { return connection.cancel() } + receive(connection, buffer: buffer, handler: handler) + } + } + + private static func parse(_ data: Data) -> Request? { + guard let end = data.range(of: Data("\r\n\r\n".utf8)) else { return nil } + let head = String(decoding: data[..= 2 else { return nil } + var headers: [String: String] = [:] + for field in head.dropFirst() { + guard let colon = field.firstIndex(of: ":") else { continue } + headers[field[..= length else { return nil } + let target = URLComponents(string: String(line[1])) + var query: [String: String] = [:] + for item in target?.queryItems ?? [] { query[item.name] = item.value } + var path = target?.path ?? "" + if path.hasPrefix("/v1/") { path.removeFirst(4) } else if path.hasPrefix("/") { path.removeFirst() } + return Request(method: String(line[0]), path: path, query: query, headers: headers, body: Data(body.prefix(length))) + } + + private func handle(_ request: Request, _ connection: NWConnection) { + let route = Self.route(request) + let (drop, hold, redirect) = state.withLock { s -> (Bool, Bool, String?) in + s.log.append(request) + return (s.dropNext.remove(route) != nil, s.held.contains(route), s.redirects[route]) + } + let (status, body) = redirect == nil ? answer(request) : (307, Data()) + let location = redirect.map { "Location: \($0)\r\n" } ?? "" + let send: @Sendable () -> Void = { + if drop { return connection.cancel() } + let head = "HTTP/1.1 \(status) X\r\n\(location)Content-Type: application/json\r\nContent-Length: \(body.count)\r\nConnection: close\r\n\r\n" + connection.send(content: Data(head.utf8) + body, completion: .contentProcessed { _ in connection.cancel() }) + } + if hold { + state.withLock { $0.parked[route, default: []].append(send) } + } else { + send() + } + } + + // MARK: API behaviour + + private func json(_ object: Any) -> Data { try! JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]) } + private func error(_ status: Int, _ code: String) -> (Int, Data) { (status, json(["errors": [["code": code, "pointer": "/", "message": code]]])) } + + private func answer(_ r: Request) -> (Int, Data) { + state.withLock { s in + let parts = r.path.split(separator: "/").map(String.init) + if parts.first == "upload" { return put(r, token: parts.last ?? "", &s) } + if let bearer = s.expectedBearer { + if r.headers["authorization"] != "Bearer \(bearer)" { return error(401, "unauthorized") } + } else if r.headers["authorization"] != nil { + return error(400, "unexpected_authorization") + } + if r.method == "POST", parts == ["captures"] { return create(r, &s) } + guard parts.count >= 2, var capture = s.captures[parts[1]] else { return error(404, "capture_not_found") } + defer { s.captures[capture.id] = capture } + switch (r.method, parts.dropFirst(2).joined(separator: "/")) { + case ("POST", "files"): return register(r, &capture, &s) + case ("POST", "files:commit"): return commit(r, &capture, &s) + case ("POST", "finalize"): return finalize(r, &capture, &s) + case ("GET", "events"): return events(r, &capture, &s) + case ("GET", "result"): + if s.notReadyResults > 0 { + s.notReadyResults -= 1 + return error(404, "result_not_ready") + } + if s.unreadableResults > 0 { + s.unreadableResults -= 1 + return (200, Data("not a result".utf8)) + } + if s.malformedOutcomeResults > 0 { + s.malformedOutcomeResults -= 1 + return (200, json(["runId": capture.runID ?? "", "status": capture.status, "viewsNeeded": [], "memberActions": [], + "outcome": ["kind": "manual_review"]])) + } + let withheld = s.outcomelessResults > 0 + if withheld { s.outcomelessResults -= 1 } + let outcome: Any = capture.status == "manual_review" && !withheld + ? ["kind": "manual_review", "profile": "C", "message": s.resultMessage, "viewsNeeded": [], "reasons": []] as [String: Any] + : NSNull() + return (200, json(["runId": capture.runID ?? "", "status": capture.status, "viewsNeeded": [], "memberActions": [], "outcome": outcome])) + default: return error(404, "route") + } + } + } + + private func create(_ r: Request, _ s: inout State) -> (Int, Data) { + guard let body = try? JSONSerialization.jsonObject(with: r.body) as? [String: Any], let packetID = body["packetId"] as? String else { + return error(422, "schema") + } + guard let key = r.headers["idempotency-key"] else { return error(422, "idempotency_key_missing") } + guard key == packetID else { return error(422, "idempotency_key_mismatch") } + guard body["formatVersion"] as? String == "0.4" else { return error(422, "format_version_unsupported") } + let status: Int + let capture: Capture + if let existing = s.byPacket[packetID].flatMap({ s.captures[$0] }) { + guard existing.createBody == r.body else { return error(409, "idempotency_conflict") } + (status, capture) = (200, existing) + } else { + capture = Capture(id: "cap_\(s.nextID)", packetID: packetID, createBody: r.body) + s.nextID += 1 + s.captures[capture.id] = capture + s.byPacket[packetID] = capture.id + status = 201 + } + return (status, json([ + "captureId": capture.id, "status": "uploading", "eventsUrl": "/v1/captures/\(capture.id)/events", + "finalizeBy": "2099-01-01T00:00:00Z", "upload": ["maxSinglePutBytes": 16_777_216, "maxBatch": 50, "urlTtlS": 3600, "maxFiles": 2000, "partBytes": 8_388_608], + ])) + } + + private func register(_ r: Request, _ c: inout Capture, _ s: inout State) -> (Int, Data) { + guard let body = try? JSONSerialization.jsonObject(with: r.body) as? [String: Any], let files = body["files"] as? [[String: Any]], + (1...50).contains(files.count) else { return error(422, "batch_too_large") } + var out: [[String: Any]] = [] + for file in files { + guard let path = file["path"] as? String, Packet04.isRelPath(path), let sha = file["sha256"] as? String, sha.count == 64, + let bytes = file["bytes"] as? Int, let md5 = file["md5"] as? String, Data(base64Encoded: md5)?.count == 16, + let role = file["role"] as? String, Packet04.Role(rawValue: role) != nil, let type = file["contentType"] as? String + else { return error(422, "schema") } + let meta = file["meta"] as? [String: Any] + let image = role == "keyframe" || role == "still" + if image != (meta != nil) || (meta.map { $0.count != 1 } ?? false) { return error(422, "meta_mismatch") } + if let previous = c.registered[path], previous.sha256 != sha { return error(409, "sha256_conflict") } + if c.packetSHA != nil, !c.listed.contains(path) { return error(409, "not_in_manifest") } + c.registered[path] = Registration(sha256: sha, bytes: bytes, md5: md5, contentType: type) + if c.committed.contains(path) { + out.append(["path": path, "state": "committed"]) + continue + } + if s.registerOmitsFiles { continue } + let token = UUID().uuidString + s.tokens[token] = (c.id, path) + out.append(["path": path, "state": "pending", "upload": [ + "method": "PUT", "url": s.uploadURLOverride ?? "http://127.0.0.1:\(port)/upload/\(token)", + "headers": ["Content-Type": type, "Content-MD5": md5].merging(s.extraSignedHeaders) { _, extra in extra }, + "expiresAt": "2099-01-01T00:00:00Z", + ]]) + } + return (200, json(["files": out])) + } + + private func put(_ r: Request, token: String, _ s: inout State) -> (Int, Data) { + guard r.method == "PUT", let (captureID, path) = s.tokens[token], var c = s.captures[captureID], let reg = c.registered[path] else { + return (403, Data("token_invalid".utf8)) + } + if r.headers["authorization"] != nil { return (400, Data("authorization sent to storage".utf8)) } + if s.badDigestPuts > 0 { + s.badDigestPuts -= 1 + return (400, Data("BadDigest".utf8)) + } + if s.expireNextPuts > 0 { + s.expireNextPuts -= 1 + s.tokens[token] = nil + return (403, Data("AccessDeniedRequest has expired".utf8)) + } + guard r.headers["content-md5"] == reg.md5, Data(Insecure.MD5.hash(data: r.body)).base64EncodedString() == reg.md5, + r.headers["content-type"] == reg.contentType + else { return (400, Data("BadDigest".utf8)) } + c.stored[path] = r.body + s.captures[captureID] = c + return (200, Data()) + } + + private func commit(_ r: Request, _ c: inout Capture, _ s: inout State) -> (Int, Data) { + guard let body = try? JSONSerialization.jsonObject(with: r.body) as? [String: Any], let files = body["files"] as? [[String: Any]] else { + return error(422, "schema") + } + var committed: [String] = [], notFound: [String] = [], mismatch: [String] = [] + for file in files { + guard let path = file["path"] as? String, let sha = file["sha256"] as? String else { return error(422, "schema") } + guard let data = c.stored[path] else { notFound.append(path); continue } + if PacketFiles.sha256(data) != sha || c.registered[path]?.sha256 != sha { mismatch.append(path); continue } + committed.append(path) + } + if let forced = s.commitOverride { + s.commitOverride = nil + (committed, notFound, mismatch) = forced + } + if s.commitNeverAcknowledges { + notFound += committed + mismatch + committed = [] + mismatch = [] + } + c.committed.formUnion(committed) + if !committed.isEmpty { c.events.append(("files_committed", ["committed": String(c.committed.count)])) } + startIfComplete(&c, endStatus: s.endStatus) + return (200, json(["committed": committed, "notFound": notFound, "mismatch": mismatch])) + } + + private func finalize(_ r: Request, _ c: inout Capture, _ s: inout State) -> (Int, Data) { + guard r.headers["idempotency-key"] == c.packetID else { return error(422, "idempotency_key_mismatch") } + let sha = PacketFiles.sha256(r.body) + if let declared = r.headers["x-packet-sha256"], declared != sha { return error(422, "packet_sha_mismatch") } + if let previous = c.packetSHA, previous != sha { return error(409, "packet_sha_conflict") } + guard let packet = try? JSONDecoder().decode(Packet04.Packet.self, from: r.body) else { return error(422, "schema") } + guard packet.packetId == c.packetID else { return error(422, "packet_id_mismatch") } + let problems = Packet04Check.problems(packet, folder: nil) + if !problems.isEmpty { return error(422, "schema") } + for entry in packet.files { + if let reg = c.registered[entry.path], reg.sha256 != entry.sha256 || reg.bytes != entry.bytes { return error(422, "sha256_mismatch") } + } + if c.packetSHA == nil { + c.packetSHA = sha + c.listed = packet.files.map(\.path) + c.runID = "run_" + sha.prefix(16) + c.status = "awaiting_files" + if s.loseNextFinalize { + s.loseNextFinalize = false + c.finalizeLost = true + c.events.append(("failed", ["code": "storage_unavailable", "next": "retry_finalize"])) + } + startIfComplete(&c, endStatus: s.endStatus) + } else if c.finalizeLost { + c.finalizeLost = false + startIfComplete(&c, endStatus: s.endStatus) + } + let missing = c.listed.filter { !c.committed.contains($0) } + return (202, json(["status": missing.isEmpty ? "processing" : "awaiting_files", "missing": missing, "runId": c.runID!, "etaS": 1])) + } + + private func startIfComplete(_ c: inout Capture, endStatus: String) { + guard c.packetSHA != nil, !c.finalizeLost, c.status == "awaiting_files", c.listed.allSatisfy(c.committed.contains) else { return } + c.status = endStatus + c.events.append(("stage", ["stage": "ingest", "status": "ok", "runId": c.runID!])) + if endStatus == "manual_review" { + c.events.append(("verdict_ready", ["kind": "manual_review", "runId": c.runID!])) + } else { + c.events.append(("failed", ["code": "loopback_run_failed", "runId": c.runID!])) + } + } + + private func events(_ r: Request, _ c: inout Capture, _ s: inout State) -> (Int, Data) { + if c.status == "manual_review" || c.status == "failed", s.processingPolls > 0, c.runID != nil { + s.processingPolls -= 1 + let after = Int(r.query["after"] ?? "0") ?? 0 + return (200, json(["status": "processing", "next": after, "events": [] as [Any]])) + } + let after = Int(r.query["after"] ?? "0") ?? 0 + let list = c.events.enumerated().filter { $0.offset + 1 > after }.map { index, event in + ["seq": index + 1, "type": event.type, "at": "2026-09-27T00:00:00Z", "data": event.data] as [String: Any] + } + return (200, json(["status": c.status, "next": c.events.count, "events": list])) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/MeterTapObservationTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/MeterTapObservationTests.swift new file mode 100644 index 000000000..d014a3c58 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/MeterTapObservationTests.swift @@ -0,0 +1,239 @@ +import Foundation +import HouseScanKit +import simd +import Testing + +/// Two ARFrames that differ in pose, intrinsics and image size, and a meter hit both can see. +enum TwoFrames { + static let hit = SIMD3(0.4, 1.2, -2) + + static let a = MeterTapFrame( + t: 10, cameraToWorld: pose(x: 0, y: 1.5, z: 0, yawDegrees: 0), intrinsics: SIMD4(1445, 1445, 960, 720), width: 1920, height: 1440, + tracking: .normal) + static let b = MeterTapFrame( + t: 10.033, cameraToWorld: pose(x: 0.3, y: 1.4, z: 0.1, yawDegrees: 10), intrinsics: SIMD4(1001, 1003, 641, 479), width: 1280, + height: 960, tracking: .normal) + + static func pose(x: Float, y: Float, z: Float, yawDegrees: Float) -> simd_float4x4 { + let r = yawDegrees * .pi / 180 + return simd_float4x4( + SIMD4(cos(r), 0, -sin(r), 0), SIMD4(0, 1, 0, 0), SIMD4(sin(r), 0, cos(r), 0), SIMD4(x, y, z, 1)) + } + + static func columns(_ m: simd_float4x4) -> [Double] { + [m.columns.0, m.columns.1, m.columns.2, m.columns.3].flatMap { [Double($0.x), Double($0.y), Double($0.z), Double($0.w)] } + } + + /// How far `point` lies from the ray, meters. + static func miss(_ ray: (origin: [Double], direction: [Double]), _ point: SIMD3) -> Double { + let o = SIMD3(ray.origin[0], ray.origin[1], ray.origin[2]), d = simd_normalize(SIMD3(ray.direction[0], ray.direction[1], ray.direction[2])) + let v = SIMD3(point) - o + return simd_length(v - simd_dot(v, d) * d) + } + + static func ray(_ frame: MeterTapFrame, pixel: SIMD2) -> (origin: [Double], direction: [Double]) { + let k = frame.intrinsics + return Packet04Check.ray(pose: columns(frame.cameraToWorld), intrinsics: [k.x, k.y, k.z, k.w].map(Double.init), pixel: [pixel.x, pixel.y]) + } +} + +@Suite struct MeterTapFrameTests { + func observation(_ frame: MeterTapFrame) throws -> TapObservation { + try frame.observation(hit: TwoFrames.hit, jpeg: { Data([0xFF]) }).get() + } + + /// The tap carries one frame's time, pose, intrinsics and image size, and its pixel is the hit + /// seen through that frame's camera: the ray rebuilt the way the packet rebuilds it reaches the + /// hit. The same pixel with the other frame's camera, or this frame's pose with the other's + /// intrinsics, misses it by centimeters, so pairing frames would fail this test. + @Test func thePixelAndRayComeFromTheSnapshotsOwnCamera() throws { + let a = try observation(TwoFrames.a), b = try observation(TwoFrames.b) + #expect(a.t == TwoFrames.a.t && a.cameraToWorld == TwoFrames.a.cameraToWorld && a.intrinsics == TwoFrames.a.intrinsics) + #expect(a.width == 1920 && a.height == 1440) + #expect(TwoFrames.miss(TwoFrames.ray(TwoFrames.a, pixel: a.pixel), TwoFrames.hit) < 1e-4) + #expect(TwoFrames.miss(TwoFrames.ray(TwoFrames.b, pixel: b.pixel), TwoFrames.hit) < 1e-4) + #expect(simd_distance(a.pixel, b.pixel) > 50, "the two frames should see the hit at different pixels") + // Mixed: B's camera with A's pixel, and A's pose with B's intrinsics. + #expect(TwoFrames.miss(TwoFrames.ray(TwoFrames.b, pixel: a.pixel), TwoFrames.hit) > 0.05) + var mixed = TwoFrames.a + mixed.intrinsics = TwoFrames.b.intrinsics + #expect(TwoFrames.miss(TwoFrames.ray(mixed, pixel: a.pixel), TwoFrames.hit) > 0.05) + } + + @Test func aFrameWithoutNormalTrackingGivesNoTap() { + for tracking in [PacketTracking.limited(.relocalizing), .limited(nil), .notAvailable] { + var frame = TwoFrames.a + frame.tracking = tracking + #expect(frame.observation(hit: TwoFrames.hit, jpeg: { Data() }).failureValue == .trackingLimited) + } + } + + @Test func aHitBehindTheCameraOrOutsideTheImageGivesNoTap() { + // Behind: the camera looks along -z from z = 0. + #expect(TwoFrames.a.observation(hit: SIMD3(0.4, 1.2, 2), jpeg: { Data() }).failureValue == .hitOutsideImage) + // In front, far to the side: past the image's right edge. + #expect(TwoFrames.a.observation(hit: SIMD3(5, 1.2, -2), jpeg: { Data() }).failureValue == .hitOutsideImage) + } + + @Test func theHitRecordsItsDistanceFromTheSnapshotsCamera() { + let hit = TwoFrames.a.tapHit(TwoFrames.hit, estimatedPlane: true) + #expect(hit.target == "estimatedPlane" && hit.alignment == "vertical") + #expect(abs((hit.distance ?? 0) - Double(simd_distance(TwoFrames.hit, SIMD3(0, 1.5, 0)))) < 1e-6) + #expect(TwoFrames.a.tapHit(TwoFrames.hit, estimatedPlane: false).target == "existingPlaneGeometry") + } +} + +extension Result { + var failureValue: Failure? { + if case .failure(let failure) = self { return failure } + return nil + } +} + +/// The meter tap through the controller and the real coordinator, against the in-process fixture. +@Suite(.serialized) @MainActor struct MeterTapControllerTests { + let root = FileManager.default.temporaryDirectory.appending(path: "meter-tap-\(UUID().uuidString)") + var fixture: NativeCaptureFixture { NativeCaptureFixture(folder: root.appending(path: "store")) } + + func controller(_ http: FixtureCaptureHTTP) -> PhotoProcessingController { + var environment = NativeCaptureFixture.environment(endpoint: FixtureCaptureHTTP.base, http: http, captures: root.appending(path: "Captures")) + environment.policy.firstDelay = 0.01 + environment.policy.maxDelay = 0.02 + return PhotoProcessingController(setup: .ready(environment, standIn: true)) + } + + func begin(_ controller: PhotoProcessingController, scan: String = "scan-1", recording: String = "rec-1") -> ScanContext { + let context = ScanContext(scanID: scan, profile: controller.profile, recordingSessionID: recording) + controller.beginScan(context, recording: fixture.recording) + return context + } + + /// The fixture's frame one second in, as the snapshot would read it, with its image encoded. + func snapshotTap() throws -> (TapObservation, Packet04.TapHit) { + let t = fixture.start + 1 + let frame = MeterTapFrame( + t: t, cameraToWorld: fixture.pose(t), intrinsics: NativeCaptureFixture.k, width: NativeCaptureFixture.width, + height: NativeCaptureFixture.height, tracking: .normal) + let jpeg = try Data(contentsOf: try fixture.jpeg(named: "tap-\(t)")) + return (try frame.observation(hit: NativeCaptureFixture.meter, jpeg: { jpeg }).get(), frame.tapHit(NativeCaptureFixture.meter, estimatedPlane: true)) + } + + /// A hang safeguard counted in the waiter's own turns (`countedWait`), not wall time. + func until(_ seconds: Double = 30, _ condition: () -> Bool) async throws -> Bool { + try await countedWait(seconds, condition) + } + + /// The forwarded tap is sealed with a ray, rebuilt from its keyframe's pose and intrinsics, + /// that reaches the hit within the server's replay tolerance; the packet passes its checks, and + /// every request went to the in-process fixture. + @Test func aForwardedTapIsSealedWithARayThroughTheHit() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .candidate) + let controller = controller(http) + let scan = begin(controller) + controller.answerConsent(true) + let (tap, hit) = try snapshotTap() + controller.meterTapped(tap, hit: hit, scan: scan) + #expect(controller.meterTap == .forwarded) + controller.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + for i in 0..<5 { controller.kept(try fixture.photo(at: fixture.start + 3 + Double(i) * 1.5)) } + await controller.settle() + controller.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + try #require(try await until { controller.status?.isFinal == true }, "\(String(describing: controller.status))") + let folder = try #require(controller.captureFolder) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: Data(contentsOf: folder.appending(path: "packet.json"))) + #expect(Packet04Check.problems(packet, folder: folder).isEmpty) + let sealed = try #require(packet.taps?.first) + #expect(sealed.label == "meter" && sealed.time == tap.t) + let toHit = simd_normalize(SIMD3(NativeCaptureFixture.meter) - SIMD3(sealed.rayOrigin[0], sealed.rayOrigin[1], sealed.rayOrigin[2])) + let error = zip([toHit.x, toHit.y, toHit.z], sealed.rayDirection).map { abs($0 - $1) }.max() ?? .infinity + #expect(error <= Packet04Check.tapReplayTolerance, "ray misses the hit by \(error)") + #expect(http.routes.allSatisfy { $0.hasPrefix("POST captures") || $0.hasPrefix("GET captures") || $0 == "PUT upload" }) + controller.endScan(recording: fixture.recording) + } + + /// After a yes, a failed tap ends the packet as not prepared at once, and nothing more of it + /// is sealed or finalized. + @Test(arguments: MeterTapFailure.allCases) + func aFailedTapAfterAYesEndsThePacketNotPrepared(reason: MeterTapFailure) async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .candidate) + let controller = controller(http) + let scan = begin(controller) + controller.answerConsent(true) + controller.meterTapFailed(reason, scan: scan) + #expect(controller.meterTap == .failed(reason)) + #expect(controller.status?.stage == .ended(.notPrepared)) + controller.kept(try fixture.photo(at: fixture.start + 3)) + controller.captureEnded(acceptedCloseUpAt: nil) + try await Task.sleep(for: .milliseconds(500)) + #expect(controller.status?.stage == .ended(.notPrepared)) + #expect(!http.routes.contains("POST captures/finalize")) + controller.endScan(recording: fixture.recording) + } + + /// Before an answer, a failed tap is kept: a later yes ends the packet as not prepared, and a + /// no keeps its own end. + @Test func aFailedTapBeforeAnAnswerIsKeptForIt() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let yes = controller(FixtureCaptureHTTP(answer: .candidate)) + let scan = begin(yes) + yes.meterTapFailed(.encodeFailed, scan: scan) + #expect(yes.status?.stage == .capturing(sent: 0)) + yes.answerConsent(true) + #expect(yes.status?.stage == .ended(.notPrepared)) + // The yes never reached the coordinator: no upload started, so no consented upload state + // was saved for a packet that can't be sent (a relaunch could otherwise find it). + let saved = FileManager.default.enumerator(at: root.appending(path: "Captures"), includingPropertiesForKeys: nil)? + .compactMap { $0 as? URL }.filter { $0.lastPathComponent == CaptureUploader.stateURL(in: root).lastPathComponent } ?? [] + #expect(saved.isEmpty, "upload state saved after a yes on a packet without its tap: \(saved)") + yes.endScan(recording: fixture.recording) + + let no = controller(FixtureCaptureHTTP(answer: .candidate)) + let other = begin(no, scan: "scan-2") + no.meterTapFailed(.noFrame, scan: other) + no.answerConsent(false) + no.captureEnded(acceptedCloseUpAt: nil) + #expect(no.status?.stage == .ended(.consentNotGiven)) + no.endScan(recording: fixture.recording) + } + + /// A tap result that arrives after a world reset or Start over belongs to a packet that no + /// longer exists: it is neither forwarded nor allowed to end the new packet. + @Test func aLateTapAfterAWorldResetOrStartOverIsDropped() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let controller = controller(FixtureCaptureHTTP(answer: .candidate)) + let first = begin(controller) + controller.answerConsent(true) + let (tap, hit) = try snapshotTap() + + let reset = first.inNewWorld(recordingSessionID: "rec-2") + controller.worldReset(reset, recording: fixture.recording) + controller.meterTapFailed(.encodeFailed, scan: first) + controller.meterTapped(tap, hit: hit, scan: first) + #expect(controller.meterTap == .none) + #expect(controller.status?.stage == .capturing(sent: 0)) + + controller.endScan(recording: fixture.recording) + let next = begin(controller, scan: "scan-2", recording: "rec-3") + controller.answerConsent(true) + controller.meterTapFailed(.trackingLimited, scan: reset) + controller.meterTapped(tap, hit: hit, scan: first) + #expect(controller.meterTap == .none) + #expect(controller.status?.stage == .capturing(sent: 0)) + #expect(controller.context == next) + controller.endScan(recording: fixture.recording) + } + + /// One tap per packet: a second result for the same packet changes nothing. + @Test func onlyThePacketsFirstTapResultCounts() throws { + defer { try? FileManager.default.removeItem(at: root) } + let controller = controller(FixtureCaptureHTTP(answer: .candidate)) + let scan = begin(controller) + let (tap, hit) = try snapshotTap() + controller.meterTapped(tap, hit: hit, scan: scan) + controller.meterTapFailed(.encodeFailed, scan: scan) + #expect(controller.meterTap == .forwarded) + controller.endScan(recording: fixture.recording) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/Packet04Tests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/Packet04Tests.swift new file mode 100644 index 000000000..909bb67b2 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/Packet04Tests.swift @@ -0,0 +1,277 @@ +import Foundation +import HouseScanKit +import simd +import Testing + +/// A synthetic ARKit capture for the 0.4 tests: a camera walking along +x at 1.5 m height, +/// 96 x 72 JPEGs with intrinsics that fit them, 60 Hz poses and 100 Hz IMU with the gyro 1 ms +/// behind the accelerometer. `source` is synthetic, so no packet from it is evidence. +struct SyntheticCapture04 { + static let start = 1000.0 + let folder: URL + let producer: Packet04Producer + + init(folder: URL, packetID: String = UUID().uuidString) throws { + self.folder = folder + producer = try Packet04Producer(folder: folder, info: Self.info(packetID: packetID), startedAtUptime: Self.start, startedAt: Date(timeIntervalSince1970: 1_790_000_000)) + } + + static func info(packetID: String) -> Packet04SessionInfo { + Packet04SessionInfo( + packetID: packetID, sessionID: "S-1", source: .synthetic, appVersion: "test (1)", deviceModel: "iPhone15,4", systemVersion: "26.0", + lidarAvailable: false, sceneDepthEnabled: false, meshReconstructionSupported: false, sceneReconstruction: "none", + planeDetection: ["horizontal", "vertical"], videoWidth: 1920, videoHeight: 1440, framesPerSecond: 60, timeZone: "America/Chicago") + } + + static func pose(_ t: Double) -> simd_float4x4 { + var m = matrix_identity_float4x4 + m.columns.3 = SIMD4(Float(t - start) * 0.5, 1.5, -2, 1) + return m + } + + static let intrinsics = SIMD4(80, 80, 48, 36) + + func observation(_ t: Double) -> Packet04Observation { + Packet04Observation(t: t, cameraToWorld: Self.pose(t), intrinsics: Self.intrinsics, width: 96, height: 72, tracking: .normal) + } + + static func jpeg(in folder: URL) throws -> Data { + let url = folder.appending(path: "source-\(UUID().uuidString).jpg") + try makeJPEG(width: 96, height: 72, at: url) + defer { try? FileManager.default.removeItem(at: url) } + return try Data(contentsOf: url) + } + + /// Seals `count` keyframes and the head-on meter still on the first one; returns every file. + func sealImages(count: Int = 3) async throws -> [SealedFile] { + let scratch = folder.deletingLastPathComponent() + var files: [SealedFile] = [] + for i in 0.. [Packet04Streams.PoseRow] { + stride(from: start, through: end, by: 1.0 / 60).map { .init(t: $0, tracking: .normal, cameraToWorld: pose($0), intrinsics: SIMD4(1440, 1440, 960, 720)) } + } + + static func imu(until end: Double = start + 2) -> (accel: [Packet04Streams.MotionRow], gyro: [Packet04Streams.MotionRow]) { + let times = Array(stride(from: start, through: end, by: 0.01)) + return (times.map { .init(t: $0, value: SIMD3(0, -1, 0)) }, times.map { .init(t: $0 + 0.001, value: SIMD3(0.01, 0, 0)) }) + } + + func finish() async throws -> (streams: [SealedFile], packet: Data) { + let imu = Self.imu() + return try await producer.finish(poses: Self.poses(), accelerometer: imu.accel, gyroscope: imu.gyro, endedAtUptime: Self.start + 2, acceptedCloseUpAt: Self.start + 0.2) + } +} + +func temporaryPacketFolder(_ name: String) -> URL { + FileManager.default.temporaryDirectory.appending(path: "\(name)-\(UUID().uuidString)/packet") +} + +@Suite struct Packet04Tests { + @Test func finishedPacketPassesItsChecksAndListsEverySealedFile() async throws { + let folder = temporaryPacketFolder("p04-normal") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + let images = try await capture.sealImages() + let (streams, data) = try await capture.finish() + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: data) + + #expect(Packet04Check.problems(packet, folder: folder).isEmpty) + #expect(Set(packet.files.map(\.path)) == Set((images + streams).map(\.path))) + #expect(packet.formatVersion == "0.4") + #expect(packet.session.tier == .arkit) + #expect(packet.epochs == [.init(id: "e1", startTime: SyntheticCapture04.start, reason: "sessionStart")]) + #expect(packet.keyframes.allSatisfy { $0.epoch == "e1" }) + #expect(packet.scaleReference.meterCloseUp == .init(captured: true, still: "meter_close", arkitDistanceM: nil, side: nil)) + #expect(packet.scaleReference.ext.obliqueCloseUp == .notCaptured) + #expect(packet.tracking == [.init(time: SyntheticCapture04.start, state: "normal", epoch: "e1")]) + // The exact bytes finish returned are what is on disk. + #expect(try Data(contentsOf: folder.appending(path: "packet.json")) == data) + } + + @Test func keyframeKeepsTheRawWorldPoseAndItsOwnIntrinsics() async throws { + let folder = temporaryPacketFolder("p04-pose") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + let t = SyntheticCapture04.start + 0.7 + let files = try await capture.producer.sealKeyframe(jpeg: try SyntheticCapture04.jpeg(in: folder.deletingLastPathComponent()), observation: capture.observation(t), reason: "motion") + guard case .keyframe(let record)? = files.first?.meta else { Issue.record("no keyframe record"); return } + #expect(record.pose == [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0.35, 1.5, -2, 1]) + #expect(record.intrinsics == [80, 80, 48, 36]) + #expect(record.timestamp == t) + #expect(files.first?.path == "keyframes/k00001.jpg") + #expect(files.first?.role == .keyframe) + } + + @Test func sealedDigestsMatchTheBytesOnDisk() async throws { + let folder = temporaryPacketFolder("p04-digest") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + for file in try await capture.sealImages() { + let data = try Data(contentsOf: folder.appending(path: file.path)) + #expect(file.bytes == data.count) + #expect(file.sha256 == PacketFiles.sha256(data)) + #expect(Data(base64Encoded: file.md5)?.count == 16) + } + } + + @Test func refusesARotatedImageAndIntrinsicsOfAnotherSize() async throws { + let folder = temporaryPacketFolder("p04-refuse") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + var portrait = capture.observation(SyntheticCapture04.start + 1) + (portrait.width, portrait.height) = (72, 96) + await #expect(throws: Packet04Error.self) { + _ = try await capture.producer.sealKeyframe(jpeg: try SyntheticCapture04.jpeg(in: folder.deletingLastPathComponent()), observation: portrait, reason: "motion") + } + var wrong = capture.observation(SyntheticCapture04.start + 1) + wrong.intrinsics = SIMD4(1440, 1440, 960, 720) + await #expect(throws: Packet04Error.self) { + _ = try await capture.producer.sealKeyframe(jpeg: try SyntheticCapture04.jpeg(in: folder.deletingLastPathComponent()), observation: wrong, reason: "motion") + } + } + + @Test func nothingIsSealedAfterFinishAndFinishReturnsTheSameBytes() async throws { + let folder = temporaryPacketFolder("p04-frozen") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + _ = try await capture.sealImages() + let first = try await capture.finish() + let second = try await capture.finish() + #expect(first.packet == second.packet) + await #expect(throws: Packet04Error.sealedAfterFinish("keyframe")) { + _ = try await capture.producer.sealKeyframe(jpeg: try SyntheticCapture04.jpeg(in: folder.deletingLastPathComponent()), observation: capture.observation(SyntheticCapture04.start + 1.9), reason: "motion") + } + } + + @Test func tapRayReplaysFromItsKeyframe() async throws { + let folder = temporaryPacketFolder("p04-tap") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + _ = try await capture.sealImages(count: 1) + try await capture.producer.addTap(id: "tap1", label: "meter", keyframe: "k00001", pixel: SIMD2(48, 36), time: SyntheticCapture04.start + 0.2, hit: nil) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: try await capture.finish().packet) + let tap = try #require(packet.taps?.first) + // The principal point looks straight down -z from the camera. + #expect(tap.rayOrigin == [0.1, 1.5, -2]) + #expect(tap.rayDirection == [0, 0, -1]) + + var broken = packet + broken.taps?[0].rayDirection = [0, 0.01, -1] + #expect(Packet04Check.problems(broken, folder: nil).contains { $0.hasPrefix("tap tap1 ray replays with error") }) + } + + /// A tap ray must be two 3-vectors of finite numbers. Joined and compared pairwise, a missing + /// or misshaped vector, or a NaN, could otherwise pass the replay check unseen. + @Test(arguments: [ + ([0.1, 1.5, -2], []), + ([0.1, 1.5], [-2, 0, 0, -1]), + ([0.1, 1.5, -2], [0, 0, -1, 0]), + ([0.1, 1.5, -2], [0, Double.nan, -1]), + ([0.1, .infinity, -2], [0, 0, -1]), + ] as [([Double], [Double])]) + func aTapRayThatIsNotTwoFinite3VectorsIsRefused(origin: [Double], direction: [Double]) async throws { + let folder = temporaryPacketFolder("p04-tap-shape") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + _ = try await capture.sealImages(count: 1) + try await capture.producer.addTap(id: "tap1", label: "meter", keyframe: "k00001", pixel: SIMD2(48, 36), time: SyntheticCapture04.start + 0.2, hit: nil) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: try await capture.finish().packet) + #expect(!Packet04Check.problems(packet, folder: nil).contains { $0.hasPrefix("tap tap1") }) + + var broken = packet + broken.taps?[0].rayOrigin = origin + broken.taps?[0].rayDirection = direction + #expect(Packet04Check.problems(broken, folder: nil).contains { $0.hasPrefix("tap tap1") }) + } + + @Test func checksCatchAMalformedPacket() async throws { + let folder = temporaryPacketFolder("p04-malformed") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + _ = try await capture.sealImages() + let good = try JSONDecoder().decode(Packet04.Packet.self, from: try await capture.finish().packet) + + var unlisted = good + unlisted.files.removeAll { $0.path == "keyframes/k00002.jpg" } + #expect(Packet04Check.problems(unlisted, folder: folder) == ["keyframe k00002 image keyframes/k00002.jpg is not in files"]) + + var badPath = good + badPath.files[0].path = "../escape.jpg" + #expect(Packet04Check.problems(badPath, folder: nil).contains("file path ../escape.jpg is not a RelPath")) + + var badStill = good + badStill.stills?[0].keyframe = "k00009" + #expect(Packet04Check.problems(badStill, folder: nil).contains("still meter_close names unknown keyframe k00009")) + + try Data("changed".utf8).write(to: folder.appending(path: "keyframes/k00003.jpg")) + #expect(Packet04Check.problems(good, folder: folder).contains { $0.hasPrefix("file keyframes/k00003.jpg is 7 bytes") }) + } + + @Test func imuRowsPairOnlyCloseGyroSamplesAndReportTheMeasuredRate() throws { + let accel = (0..<100).map { Packet04Streams.MotionRow(t: 10 + Double($0) * 0.01, value: SIMD3(0, -1, 0)) } + // A gyro sample 1 ms after each accelerometer sample, except the 50th's, so the nearest to + // that accelerometer sample is 9 ms away: it is left out, not given a far reading. + var gyro = accel.map { Packet04Streams.MotionRow(t: $0.t + 0.001, value: SIMD3(0.5, 0, 0)) } + gyro.remove(at: 50) + let out = try Packet04Streams.imuRaw(accelerometer: accel, gyroscope: gyro) + let lines = String(decoding: out.csv, as: UTF8.self).split(separator: "\n") + #expect(lines.first == "t,ax,ay,az,gx,gy,gz") + #expect(lines.count == 1 + 99) + #expect(!lines.contains { $0.hasPrefix("10.5,") }) + #expect(abs(out.rateHz - 100) < 0.01) + #expect(out.pairingNote["imuRawRowsKept"] == "99 of 100") + } + + @Test func poseRowsJoinEachFrameToItsOwnIntrinsics() { + func trajectory(_ t: Double, x: Double) -> [Double] { [t, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, x, 1.5, -2, 1] } + let rows = Packet04Streams.poseRows( + trajectory: [trajectory(1, x: 0.1), trajectory(2, x: 0.2), trajectory(3, x: 0.3)], + // Frame 2 wrote no intrinsics row: it is left out, not given frame 1's or frame 3's. + intrinsics: [[1, 1440, 1441, 960, 720, 1920, 1440], [3, 1500, 1501, 961, 721, 1920, 1440]], + tracking: { state, _ in state == 0 ? .normal : .notAvailable }) + #expect(rows.map(\.t) == [1, 3]) + #expect(rows[1].intrinsics == SIMD4(1500, 1501, 961, 721)) + #expect(rows[1].cameraToWorld.columns.3 == SIMD4(0.3, 1.5, -2, 1)) + #expect(rows[0].tracking == .normal) + } + + @Test func aSlowIMUIsRefusedNotRelabelled() { + let accel = (0..<20).map { Packet04Streams.MotionRow(t: Double($0) * 0.05, value: SIMD3(0, -1, 0)) } + #expect(throws: Packet04Error.streamTooSlow(stream: "imuRaw", measuredHz: 1 / 0.05, minimumHz: 50)) { + _ = try Packet04Streams.imuRaw(accelerometer: accel, gyroscope: accel) + } + } + + @Test func gzipStreamsDecompressWithTheSystemGzip() throws { + let text = Data((0..<2000).map { "\($0),0.5,-1\n" }.joined().utf8) + let url = FileManager.default.temporaryDirectory.appending(path: "p04-\(UUID().uuidString).csv.gz") + defer { try? FileManager.default.removeItem(at: url) } + try Gzip.compress(text).write(to: url) + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/gzip") + process.arguments = ["-dc", url.path] + let pipe = Pipe() + process.standardOutput = pipe + try process.run() + let out = pipe.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + #expect(process.terminationStatus == 0) + #expect(out == text) + } + + @Test func idAndPathPatternsMatchTheServer() { + #expect(Packet04.isStorageID(UUID().uuidString)) + #expect(!Packet04.isStorageID("short")) + #expect(!Packet04.isStorageID("has.dot-in-it")) + #expect(Packet04.isSegmentID("k00001") && Packet04.isSegmentID("meter_close")) + #expect(!Packet04.isSegmentID("-lead") && !Packet04.isSegmentID("a.b")) + #expect(Packet04.isRelPath("keyframes/k00001.depth.f32") && Packet04.isRelPath("streams/imu_raw.csv.gz")) + #expect(!Packet04.isRelPath("a/../b") && !Packet04.isRelPath("a//b") && !Packet04.isRelPath("foo..bar") && !Packet04.isRelPath(".hidden")) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift index 0904fda17..e171be1ab 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift @@ -6,20 +6,22 @@ import ImageIO import simd import Testing -/// The capture packet's manifest schema, version 1.1, byte for byte from t3/packet at d5439cf -/// (packet/manifest.schema.json). `vendoredManifestSchemaIsTheRecordedRevision` fails if the copy -/// is edited by hand. There is no live copy to compare with: t3/packet retired 1.1 at 6a12700 in -/// favour of the server team's packet 0.4, which the writer moves to next. +/// The capture packet's manifest schema, version 1.1: t3/packet at d5439cf +/// (packet/manifest.schema.json, sha256 44c9c9be…), plus one additive change made here, the +/// optional `attrs.inferred` on a wall end and `t`'s description (B-12, packet/README.md). +/// `vendoredManifestSchemaIsTheRecordedRevision` fails if the copy changes without this record. +/// There is no live copy to compare with: t3/packet retired 1.1 at 6a12700 in favour of the server +/// team's packet 0.4, which the writer moves to next. enum PacketSchema { static let name = "manifest.schema.json" - static let sha256 = "44c9c9beb1a95b85d069d9abc52c193008a50aa75e1dff52f5d47431f10c5c75" + static let sha256 = "54d2a0d104162801a35599c5300acff09b2c2061aa8e88291f2621c04398bc27" static func validator() throws -> JSONSchemaValidator { try JSONSchemaValidator(schema: SceneSchemas.data(name)) } } /// A JPEG of `width` x `height` with a colour pattern, written by ImageIO. `orientation` sets the /// EXIF orientation tag. -func makeJPEG(width: Int, height: Int, at url: URL, orientation: Int? = nil) throws { +func makeJPEG(width: Int, height: Int, at url: URL, orientation: Int? = nil, shift: Int = 0) throws { let space = CGColorSpaceCreateDeviceRGB() let context = try #require(CGContext( data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 4 * width, space: space, @@ -28,7 +30,7 @@ func makeJPEG(width: Int, height: Int, at url: URL, orientation: Int? = nil) thr for y in 0..] = [SIMD3(-2, -0.5, 0), SIMD3(-1.25, 0.5, 0)] let drive: [SIMD3] = [SIMD3(2, -1.5, 0.5), SIMD3(2, -1.5, 4)] #expect(marks[3].points == window) @@ -362,7 +367,9 @@ struct SyntheticPacket { #expect(Set(planes[1].keys) == ["id", "alignment", "pose", "extent_m"]) let marks = try #require(json["marks"] as? [[String: Any]]) #expect(Set(marks[0].keys) == ["id", "kind", "points", "t", "photo_ids"]) - #expect(Set(marks[1].keys) == ["id", "kind", "points", "t", "side", "end_kind"]) + // The left end is inferred: the flag, no mark time. The right end is the homeowner's. + #expect(Set(marks[1].keys) == ["id", "kind", "points", "side", "end_kind", "attrs"]) + #expect(Set(marks[2].keys) == ["id", "kind", "points", "t", "side", "end_kind"]) #expect((marks[3]["attrs"] as? [String: Bool]) == ["operable": true]) let guidance = try #require(json["guidance"] as? [[String: Any]]) #expect(Set(guidance[1].keys) == ["id", "kind", "origin", "message", "band", "span_m", "t_shown", "t_resolved", "outcome"]) @@ -767,7 +774,7 @@ private func photoJPEG() throws -> URL { let fence = try PacketMark.from(.fence(foot: [SIMD3(-1, 0, 3), SIMD3(1, 0, 3)]), id: "f", wall: wall, frame: frame) #expect(fence.kind == .fence && fence.points == [SIMD3(-1, -1.5, 3), SIMD3(1, -1.5, 3)]) #expect(throws: PacketError.self) { try PacketMark.from(.driveway(edge: [SIMD3(0, 0, 1)]), id: "x", wall: wall, frame: frame) } - let end = PacketMark.wallEnd(id: "e", side: .right, endKind: .limit, s: 2.5, wall: wall, frame: frame) + let end = PacketMark.wallEnd(id: "e", side: .right, endKind: .limit, s: 2.5, wall: wall, frame: frame, stamp: .marked(at: nil)) #expect(end.points == [SIMD3(2.5, 0, 0)] && end.side == .right && end.endKind == .limit) #expect(PacketMark.meter(id: "m").points == [.zero]) } @@ -796,7 +803,7 @@ private func photoJPEG() throws -> URL { @Suite struct PacketSchemaTests { @Test func vendoredManifestSchemaIsTheRecordedRevision() throws { let digest = SHA256.hash(data: try SceneSchemas.data(PacketSchema.name)).map { String(format: "%02x", $0) }.joined() - #expect(digest == PacketSchema.sha256, "Schemas/\(PacketSchema.name) is not the copy taken from t3/packet d5439cf") + #expect(digest == PacketSchema.sha256, "Schemas/\(PacketSchema.name) is not t3/packet d5439cf plus attrs.inferred") } /// The schema is not a rubber stamp: a manifest missing required fields or breaking a pattern diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift new file mode 100644 index 000000000..ddd8d64cb --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift @@ -0,0 +1,536 @@ +import Foundation +import HouseScanKit +import Synchronization +import Testing + +@Suite struct ScanContextTests { + let profile = ProcessingProfile(backend: .photoProcessing, answers: .standIn, origin: FixtureCaptureHTTP.base) + + @Test func aWorldResetKeepsTheScanAndItsProfileButStartsANewSpatialSession() { + let first = ScanContext(scanID: "scan-a", profile: profile, recordingSessionID: "rec-1") + let second = first.inNewWorld(recordingSessionID: "rec-2") + #expect(second.scanID == first.scanID) + #expect(second.profile == first.profile) + #expect(first.spatial == SpatialSession(recordingSessionID: "rec-1", worldEpoch: 0)) + #expect(second.spatial == SpatialSession(recordingSessionID: "rec-2", worldEpoch: 1)) + #expect(second.inNewWorld(recordingSessionID: "rec-3").spatial.worldEpoch == 2) + } + + @Test func theStoredChoiceReadsBackAndAnUnknownOneReadsAsTheDefault() throws { + let suite = "processing-backend-\(UUID().uuidString)" + let defaults = try #require(UserDefaults(suiteName: suite)) + defer { defaults.removePersistentDomain(forName: suite) } + #expect(ProcessingBackendPreference.read(defaults) == .legacy) + ProcessingBackendPreference.write(.photoProcessing, to: defaults) + #expect(ProcessingBackendPreference.read(defaults) == .photoProcessing) + defaults.set("onDevice", forKey: ProcessingBackendPreference.key) + #expect(ProcessingBackendPreference.read(defaults) == .legacy) + } +} + +@Suite struct PhotoCaptureBindingTests { + static let world = SpatialSession(recordingSessionID: "rec-1", worldEpoch: 0) + let binding = PhotoCaptureBinding( + spatial: world, origin: FixtureCaptureHTTP.base, captureSessionID: "session-1", packetID: "packet-1", epoch: "e1") + let matching = PhotoCaptureObservation( + spatial: world, origin: FixtureCaptureHTTP.base, captureSessionID: "session-1", packetID: "packet-1", captureID: "cap_1", + runID: "run_1") + + func record( + sessionID: String = "session-1", captureID: String = "cap_1", runID: String = "run_1", responseRun: String = "run_1", epoch: String = "e1" + ) throws -> CaptureResult.Record { + let body = """ + {"runId": "\(responseRun)", "status": "manual_review", "viewsNeeded": [], "memberActions": [], + "outcome": {"kind": "manual_review", "profile": "p", "message": "m", "viewsNeeded": [], "reasons": []}} + """ + let response = try JSONDecoder().decode(CaptureResult.Response.self, from: Data(body.utf8)) + return CaptureResult.Record(response: response, association: .init(sessionID: sessionID, captureID: captureID, runID: runID, epoch: epoch)) + } + + @Test func theRecordingsOwnAnswerIsAccepted() throws { + #expect(binding.mismatch(try record(), observed: matching) == nil) + } + + @Test func anAnswerAssociatedWithAnotherWorldIsRefusedWhateverElseMatches() throws { + var observed = matching + observed.spatial = SpatialSession(recordingSessionID: "rec-2", worldEpoch: 1) + #expect(binding.mismatch(try record(), observed: observed) == .world) + observed.spatial = SpatialSession(recordingSessionID: "rec-1", worldEpoch: 1) + #expect(binding.mismatch(try record(), observed: observed) == .world) + observed.spatial = nil + #expect(binding.mismatch(try record(), observed: observed) == .world) + } + + @Test func anAnswerWhoseRunOrLocalAssociationDiffersIsRefused() throws { + var observed = matching + observed.origin = URL(string: "https://elsewhere.invalid/v1") + #expect(binding.mismatch(try record(), observed: observed) == .origin) + observed = matching + observed.captureSessionID = "session-2" + #expect(binding.mismatch(try record(), observed: observed) == .session) + #expect(binding.mismatch(try record(sessionID: "session-2"), observed: matching) == .session) + observed = matching + observed.packetID = "packet-2" + #expect(binding.mismatch(try record(), observed: observed) == .packet) + #expect(binding.mismatch(try record(captureID: "cap_2"), observed: matching) == .capture) + observed = matching + observed.captureID = nil + #expect(binding.mismatch(try record(), observed: observed) == .capture) + #expect(binding.mismatch(try record(runID: "run_2"), observed: matching) == .run) + #expect(binding.mismatch(try record(responseRun: "run_2"), observed: matching) == .run) + #expect(binding.mismatch(try record(epoch: "e2"), observed: matching) == .epoch) + } + + @Test func aRefusalReadsAsTheStageItEndsIn() { + typealias Stage = PhotoProcessingStatus.Stage + #expect(Stage.refusal(step: "create", codes: ["schema"], status: 422) == .ended(.refused(step: "create"))) + #expect(Stage.refusal(step: "put", codes: ["storage_refused"], status: 403) == .ended(.refused(step: "put"))) + #expect(Stage.refusal(step: "result", codes: ["result_not_ready"], status: 404) == .ended(.answerNotReady)) + #expect(Stage.refusal(step: "result", codes: ["result_unreadable"], status: 200) == .ended(.answerUnreadable)) + #expect(Stage.refusal(step: "result", codes: ["result_for_another_run"], status: 200) == .ended(.answerMismatch(.run))) + #expect(Stage.refusal(step: "result", codes: [], status: 500) == .ended(.refused(step: "result"))) + // The boundary's codes at status 0, where no server answered; the same text from a server isn't the phone's setup. + for problem in PhotoSetupProblem.allCases { + #expect(Stage.refusal(step: "create", codes: [problem.code], status: 0) == .ended(.setupRefused(step: "create", problem))) + #expect(Stage.refusal(step: "create", codes: [problem.code], status: 401) == .ended(.refused(step: "create"))) + } + // The uploader's own end when this phone couldn't save its state: status 0, no server. The + // same code with a server status isn't the phone's failure. + #expect(Stage.refusal(step: "save", codes: [CaptureUploader.unsavedStateCode], status: 0) == .ended(.uploadStateUnsaved)) + #expect(Stage.refusal(step: "save", codes: [CaptureUploader.unsavedStateCode], status: 500) == .ended(.refused(step: "save"))) + // Status 0 alone doesn't make it this failure: the code at another step isn't the uploader's own end. + #expect(Stage.refusal(step: "put", codes: [CaptureUploader.unsavedStateCode], status: 0) == .ended(.refused(step: "put"))) + // The coordinator's end for a photo or tap the phone lost: not prepared, never the service's + // refusal. With a server status, or at another step, the code isn't that end. + #expect(Stage.refusal(step: "prepare", codes: [CaptureSessionCoordinator.inputLostCode], status: 0) == .ended(.notPrepared)) + #expect(Stage.refusal(step: "prepare", codes: [CaptureSessionCoordinator.inputLostCode], status: 500) == .ended(.refused(step: "prepare"))) + #expect(Stage.refusal(step: "put", codes: [CaptureSessionCoordinator.inputLostCode], status: 0) == .ended(.refused(step: "put"))) + } +} + +/// `FixtureCaptureHTTP` with every request to one route held until `release`, to deliver an +/// answer after the scan it was for has gone. +final class HeldResultHTTP: CaptureHTTP, Sendable { + private struct Held { + var open = false + var waiting: [CheckedContinuation] = [] + } + + let inner: FixtureCaptureHTTP + private let held = Mutex(Held()) + + init(_ inner: FixtureCaptureHTTP) { + self.inner = inner + } + + var waiting: Int { held.withLock { $0.waiting.count } } + + func send(_ request: URLRequest) async throws -> HTTPReply { + if request.httpMethod == "GET", request.url?.lastPathComponent == "result" { + await withCheckedContinuation { (continuation: CheckedContinuation) in + let open = held.withLock { state -> Bool in + if state.open { return true } + state.waiting.append(continuation) + return false + } + if open { continuation.resume() } + } + } + return try await inner.send(request) + } + + func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + try await inner.upload(request, file: file) + } + + func release() { + let waiting = held.withLock { state -> [CheckedContinuation] in + state.open = true + defer { state.waiting = [] } + return state.waiting + } + waiting.forEach { $0.resume() } + } +} + +@Suite(.serialized) @MainActor struct PhotoProcessingControllerTests { + let root = FileManager.default.temporaryDirectory.appending(path: "photo-processing-\(UUID().uuidString)") + var fixture: NativeCaptureFixture { NativeCaptureFixture(folder: root.appending(path: "store")) } + + func controller(_ http: any CaptureHTTP, maxConcurrentPuts: Int? = nil) -> PhotoProcessingController { + var environment = NativeCaptureFixture.environment(endpoint: FixtureCaptureHTTP.base, http: http, captures: root.appending(path: "Captures")) + // Retries and unreadable answers come round in milliseconds, not the app's seconds. + environment.policy.firstDelay = 0.01 + environment.policy.maxDelay = 0.02 + if let maxConcurrentPuts { environment.policy.maxConcurrentPuts = maxConcurrentPuts } + return PhotoProcessingController(setup: .ready(environment, standIn: true)) + } + + func context(_ controller: PhotoProcessingController, scan: String = "scan-1", recording: String = "rec-1") -> ScanContext { + ScanContext(scanID: scan, profile: controller.profile, recordingSessionID: recording) + } + + /// The scan from the meter tap to the send: the tap, the close-up and five walk photos. + func capture(_ controller: PhotoProcessingController, consent: Bool?) async throws { + if let consent { controller.answerConsent(consent) } + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + if let scan = controller.context { controller.meterTapped(tap, hit: hit, scan: scan) } + controller.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + for i in 0..<5 { controller.kept(try fixture.photo(at: fixture.start + 3 + Double(i) * 1.5)) } + await controller.settle() + controller.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + } + + /// A hang safeguard counted in the waiter's own turns (`countedWait`), not wall time. + func until(_ seconds: Double = 30, _ condition: () -> Bool) async throws -> Bool { + try await countedWait(seconds, condition) + } + + @Test func theFixtureWritesTheCoordinatorsEpoch() { + #expect(FixtureCaptureHTTP.arkitEpoch == CaptureSessionCoordinator.epoch) + } + + @Test(arguments: [ + FixtureCaptureHTTP.Answer.candidate, .needsViews, .manualReview, .notEligible, .failed, .expired, .notReady, .unreadable, .wrongRun, + ]) + func eachAnswerReachesTheScanAsItsTypedStage(answer: FixtureCaptureHTTP.Answer) async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: answer) + let controller = controller(http) + controller.beginScan(context(controller), recording: fixture.recording) + try await capture(controller, consent: true) + try #require(try await until { controller.status?.isFinal == true }, "no final stage: \(String(describing: controller.status))") + let stage = try #require(controller.status?.stage) + let message = answer.message ?? "" + switch answer { + case .candidate: + // Proposed, never confirmed: AR stays withheld because nobody verified the analysis. + #expect(stage == .answered(.candidate(.init(message: message, previewMade: false, ar: .withheld(.analysisUnverified))))) + case .needsViews: + guard case .answered(.needsMorePhotos(let more)) = stage else { + Issue.record("expected more photos, got \(stage)") + return + } + #expect(more.prompts.map(\.title) == ["Fixture: show the ground below the meter"]) + #expect(more.message == message) + let parent = try #require(more.parent) + #expect(parent.viewIDs == ["view_fixture_1"]) + #expect(parent.profile == controller.profile) + #expect(parent.captureID.hasPrefix("cap_fixture_")) + #expect(parent.runID.hasPrefix("run_fixture_")) + #expect(parent.folder == controller.captureFolder) + case .manualReview: + #expect(stage == .answered(.installerReview(message: message))) + case .notEligible: + #expect(stage == .answered(.noCandidate(message: message))) + case .failed: + #expect(stage == .ended(.processingFailed)) + case .expired: + #expect(stage == .ended(.expired)) + case .notReady: + #expect(stage == .ended(.answerNotReady)) + case .unreadable: + #expect(stage == .ended(.answerUnreadable)) + case .wrongRun: + #expect(stage == .ended(.answerMismatch(.run))) + case .refuseCreate, .hold: + Issue.record("not an argument of this test") + } + #expect(controller.status?.consent == .granted) + #expect(controller.status?.standIn == true) + #expect(http.routes.contains("POST captures/finalize")) + controller.endScan(recording: fixture.recording) + } + + @Test func aRefusedCaptureEndsWithTheStepTheServiceRefused() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .refuseCreate) + let controller = controller(http) + controller.beginScan(context(controller), recording: fixture.recording) + try await capture(controller, consent: true) + try #require(try await until { controller.status?.isFinal == true }) + #expect(controller.status?.stage == .ended(.refused(step: "create"))) + #expect(!http.routes.contains("POST captures/files")) + controller.endScan(recording: fixture.recording) + } + + /// The phone couldn't finish saving a photo kept before the send. This drives the controller + /// directly: it proves what happens once the engine's wait gives up, not the engine's 10 s wait. + /// A storage upload is held in flight when the capture is marked unprepared. The scan ends as + /// not prepared; the held request, already admitted, may finish; after it the uploader starts + /// no new request and retries nothing, a photo kept later isn't forwarded, and the capture is + /// never sealed or finalized, nor does any later report change the stage. + @Test func aCaptureMissingAPhotoIsNeverSealedAndItsUploaderStops() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let inner = FixtureCaptureHTTP(answer: .candidate) + let http = HoldFirstPut(inner) + // Released even if an expectation below fails, so the held upload can't strand the test. + defer { http.release() } + // One storage upload at a time, so the held one is the only request admitted when the + // capture ends; with the default of three, others could land after the snapshot below. + let controller = controller(http, maxConcurrentPuts: 1) + controller.beginScan(context(controller), recording: fixture.recording) + controller.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + if let scan = controller.context { controller.meterTapped(tap, hit: hit, scan: scan) } + controller.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + controller.kept(try fixture.photo(at: fixture.start + 3)) + try #require(try await until(10) { http.isHolding }, "no storage upload was held") + let before = inner.routes + controller.capturePreparationFailed() + #expect(controller.status?.stage == .ended(.notPrepared)) + http.release() + controller.kept(try fixture.photo(at: fixture.start + 4)) + controller.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + // Longer than the uploader waits between retries and polls with this test's policy. + try await Task.sleep(for: .seconds(2.5)) + let after = Array(inner.routes.dropFirst(before.count)) + #expect(after.count <= 1 && after.allSatisfy { $0 == "PUT upload" }, "requests after the capture ended: \(after)") + #expect(!inner.routes.contains("POST captures/finalize")) + #expect(controller.status?.stage == .ended(.notPrepared)) + controller.endScan(recording: fixture.recording) + } + + /// A photo the scan accepted is lost before the phone could stage it, while the upload runs: + /// the coordinator stops the upload with its lost-input end, and the scan ends as not prepared, + /// not as the service refusing it. The capture is never sealed or finalized. + @Test func aPhotoThePhoneLostWhileSendingEndsTheScanNotPrepared() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .candidate) + let controller = controller(http) + defer { controller.endScan(recording: fixture.recording) } + let scan = context(controller) + controller.beginScan(scan, recording: fixture.recording) + controller.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + controller.meterTapped(tap, hit: hit, scan: scan) + controller.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + await controller.settle() + // A photo the scan accepted whose JPEG is gone before the coordinator could stage it. + var lost = try fixture.photo(at: fixture.start + 4) + lost.jpeg = root.appending(path: "missing.jpg") + controller.kept(lost) + try #require(try await until { controller.status?.isFinal == true }, "no final stage: \(String(describing: controller.status))") + #expect(controller.status?.stage == .ended(.notPrepared)) + #expect(controller.status?.consent == .granted) + // Ended before the send: the capture is never sealed or finalized. + controller.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + await controller.settle() + #expect(controller.status?.stage == .ended(.notPrepared)) + #expect(!http.routes.contains("POST captures/finalize")) + } + + /// While the service processes, the phone can no longer save the upload's state (its folder + /// turned read-only): the real uploader ends and the scan says sending stopped on this phone, + /// not that the service refused it, and nothing more is sent. A withdrawal that couldn't be + /// saved keeps its own stage instead (`stoppingPartWayStopsSendingAndSaysWhetherThePhoneSavedIt`). + @Test func anUploadStateThePhoneCantSaveEndsTheScanAsALocalFailure() async throws { + let http = FixtureCaptureHTTP(answer: .hold) + let controller = controller(http) + // Before anything can throw: the session ends and the folders go even if a wait times out. + var lockedFolder: URL? + defer { + if let lockedFolder { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: lockedFolder.path) } + controller.endScan(recording: fixture.recording) + try? FileManager.default.removeItem(at: root) + } + controller.beginScan(context(controller), recording: fixture.recording) + try await capture(controller, consent: true) + try #require(try await until { controller.status?.stage == .processing }) + let folder = try #require(controller.captureFolder) + lockedFolder = folder + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) + try #require(try await until { controller.status?.isFinal == true }, "no final stage: \(String(describing: controller.status))") + #expect(controller.status?.stage == .ended(.uploadStateUnsaved)) + #expect(controller.status?.consent == .granted) + let sent = http.routes.count + try await Task.sleep(for: .seconds(2.5)) + #expect(http.routes.count == sent) + } + + @Test(arguments: [false, nil] as [Bool?]) + func withoutAYesNothingIsSent(consent: Bool?) async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .candidate) + let controller = controller(http) + controller.beginScan(context(controller), recording: fixture.recording) + #expect(controller.status?.consent == .asking) + try await capture(controller, consent: consent) + #expect(controller.status?.stage == .ended(.consentNotGiven)) + let answered: PhotoProcessingStatus.Consent = consent == false ? .declined : .asking + #expect(controller.status?.consent == answered) + try await Task.sleep(for: .milliseconds(300)) + #expect(http.routes.isEmpty) + controller.endScan(recording: fixture.recording) + } + + @Test(arguments: [true, false]) + func stoppingPartWayStopsSendingAndSaysWhetherThePhoneSavedIt(saved: Bool) async throws { + let http = FixtureCaptureHTTP(answer: .hold) + let controller = controller(http) + controller.beginScan(context(controller), recording: fixture.recording) + try await capture(controller, consent: true) + try #require(try await until { controller.status?.stage == .processing }) + let folder = try #require(controller.captureFolder) + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) + try? FileManager.default.removeItem(at: root) + } + // Neither the withdrawal marker nor the saved state's removal can be written. + if !saved { try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) } + controller.stopSending() + #expect(controller.status?.stage == .ended(.withdrawn(recorded: saved))) + #expect(controller.status?.consent == .withdrawn(recorded: saved)) + let sent = http.routes.count + // Longer than the uploader waits between polls of a service that answers at once. + try await Task.sleep(for: .seconds(2.5)) + #expect(http.routes.count == sent) + // A second yes doesn't revive the capture. + controller.answerConsent(true) + #expect(controller.status?.stage == .ended(.withdrawn(recorded: saved))) + controller.endScan(recording: fixture.recording) + } + + @Test func aLateAnswerAfterStartOverNeverReachesTheNextScan() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = HeldResultHTTP(FixtureCaptureHTTP(answer: .candidate)) + let controller = controller(http) + controller.beginScan(context(controller, scan: "scan-1"), recording: fixture.recording) + try await capture(controller, consent: true) + try #require(try await until { http.waiting > 0 }, "the result was never asked for") + controller.endScan(recording: fixture.recording) + #expect(controller.status == nil) + let next = context(controller, scan: "scan-2", recording: "rec-2") + controller.beginScan(next, recording: fixture.recording) + http.release() + try await Task.sleep(for: .milliseconds(500)) + #expect(controller.context == next) + #expect(controller.status == PhotoProcessingStatus(stage: .capturing(sent: 0), consent: .asking, standIn: true)) + #expect(http.inner.routes.filter { $0 == "POST captures" }.count == 1) + controller.endScan(recording: fixture.recording) + } + + @Test func aWorldResetSendsTheNextPacketUnderTheSameYesAndBindsItsAnswerToTheNewWorld() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .manualReview) + let controller = controller(http) + let first = context(controller) + controller.beginScan(first, recording: fixture.recording) + controller.answerConsent(true) + controller.kept(try fixture.photo(at: fixture.start + 3)) + await controller.settle() + let firstFolder = controller.captureFolder + let next = first.inNewWorld(recordingSessionID: "rec-2") + controller.worldReset(next, recording: fixture.recording) + #expect(controller.status?.consent == .granted) + #expect(controller.status?.stage == .capturing(sent: 0)) + #expect(controller.captureFolder != firstFolder) + try await capture(controller, consent: nil) + try #require(try await until { controller.status?.isFinal == true }) + #expect(controller.status?.stage == .answered(.installerReview(message: FixtureCaptureHTTP.Answer.manualReview.message ?? ""))) + #expect(controller.context?.spatial == next.spatial) + #expect(controller.context?.scanID == first.scanID) + #expect(http.routes.filter { $0 == "POST captures" }.count == 2) + controller.endScan(recording: fixture.recording) + } + + @Test func theScansUploaderOwnsItsFolderAgainstAnyOtherStartOrResume() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .hold) + let controller = controller(http) + controller.beginScan(context(controller), recording: fixture.recording) + controller.answerConsent(true) + controller.kept(try fixture.photo(at: fixture.start + 3)) + await controller.settle() + let folder = try #require(controller.captureFolder) + let create = CaptureAPI.CreateRequest(packetId: "other", tier: .arkit, device: .init(model: "m", systemVersion: "s", appVersion: "a")) + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned) { + _ = try CaptureUploader.start(folder: folder, base: FixtureCaptureHTTP.base, http: http, create: create, consentedAt: Date()) + } + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned) { + _ = try CaptureUploader.resume(folder: folder, base: FixtureCaptureHTTP.base, http: http) + } + controller.endScan(recording: fixture.recording) + } + + @Test func aBuildWithoutSetupSaysSoAndSendsNothing() throws { + let controller = PhotoProcessingController(setup: .notSetUp("live sending is off in this build")) + #expect(controller.profile.answers == .notSetUp("live sending is off in this build")) + #expect(controller.profile.origin == nil) + controller.beginScan(context(controller), recording: fixture.recording) + let ended = PhotoProcessingStatus(stage: .ended(.notSetUp("live sending is off in this build")), consent: .notNeeded, standIn: false) + #expect(controller.status == ended) + controller.answerConsent(true) + controller.captureEnded(acceptedCloseUpAt: nil) + #expect(controller.status == ended) + #expect(controller.captureFolder == nil) + } + + @Test func anEndpointThatMayNotReceiveTheScanIsNotSetUp() { + var environment = NativeCaptureFixture.environment( + endpoint: FixtureCaptureHTTP.base, http: FixtureCaptureHTTP(answer: .candidate), captures: root.appending(path: "Captures")) + environment.sends = false + let controller = PhotoProcessingController(setup: .ready(environment, standIn: true)) + guard case .notSetUp = controller.profile.answers else { + Issue.record("a record-only endpoint was offered as set up") + return + } + } + + @Test func aScanWhoseProfileIsNotThisBuildsSendsNothing() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .candidate) + let controller = controller(http) + let elsewhere = ProcessingProfile(backend: .photoProcessing, answers: .service, origin: URL(string: "https://elsewhere.invalid/v1")) + controller.beginScan(ScanContext(scanID: "scan-1", profile: elsewhere, recordingSessionID: "rec-1"), recording: fixture.recording) + #expect(controller.status?.stage == .ended(.notSetUp("the scan's processing setup is not this build's"))) + try await capture(controller, consent: true) + try await Task.sleep(for: .milliseconds(300)) + #expect(http.routes.isEmpty) + controller.endScan(recording: fixture.recording) + } +} + +/// The capture fixture with its first storage upload held until `release()`: a request already +/// admitted, still waiting for its answer. +private final class HoldFirstPut: CaptureHTTP, @unchecked Sendable { + private let inner: FixtureCaptureHTTP + private let lock = NSLock() + private var held = false + private var released = false + private var waiting: CheckedContinuation? + + init(_ inner: FixtureCaptureHTTP) { self.inner = inner } + + var isHolding: Bool { lock.withLock { waiting != nil } } + + func release() { + let next = lock.withLock { () -> CheckedContinuation? in + released = true + defer { waiting = nil } + return waiting + } + next?.resume() + } + + func send(_ request: URLRequest) async throws -> HTTPReply { try await inner.send(request) } + + func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + let hold = lock.withLock { () -> Bool in + guard !held, !released else { return false } + held = true + return true + } + if hold { + await withCheckedContinuation { continuation in + let resumeNow = lock.withLock { () -> Bool in + if released { return true } + waiting = continuation + return false + } + if resumeNow { continuation.resume() } + } + } + return try await inner.upload(request, file: file) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/ResultCardActionsTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/ResultCardActionsTests.swift index 9a1cd9a53..36f547ecd 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/ResultCardActionsTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/ResultCardActionsTests.swift @@ -12,7 +12,7 @@ import Testing } @Test func withTheCameraUpEachAnswerKeepsItsButton() { - #expect(Self.primary(.fits) == .showAR(clean: true)) + #expect(Self.primary(.candidate) == .showAR(clean: true)) #expect(Self.primary(.oneMoreLook, hasViewToTake: true) == .takeView) #expect(Self.primary(.installer, spotIsClean: false) == .showAR(clean: false)) #expect(Self.primary(.installer) == .showAR(clean: true)) @@ -21,7 +21,7 @@ import Testing /// No spot, nothing to show in AR; no view to take, no "Show me". @Test func nothingToShowMeansNoButton() { - #expect(Self.primary(.fits, hasSpot: false) == nil) + #expect(Self.primary(.candidate, hasSpot: false) == nil) #expect(Self.primary(.installer, hasSpot: false) == nil) #expect(Self.primary(.oneMoreLook, hasViewToTake: false) == nil) } @@ -31,7 +31,7 @@ import Testing /// button already was. Starting over starts a new camera, so it stays. @Test func aFailedCameraOffersNothingThatNeedsIt() { #expect(Self.primary(.oneMoreLook, hasViewToTake: true, sourceAvailable: false) == nil) - #expect(Self.primary(.fits, sourceAvailable: false) == nil) + #expect(Self.primary(.candidate, sourceAvailable: false) == nil) #expect(Self.primary(.installer, spotIsClean: false, sourceAvailable: false) == nil) #expect(Self.primary(.notHere, hasSpot: false, sourceAvailable: false) == .startOver) } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/ResultReadingTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/ResultReadingTests.swift index 3f525a0ef..d6d6b2496 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/ResultReadingTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/ResultReadingTests.swift @@ -112,8 +112,9 @@ import simd #expect(result.answer { _ in true } == .oneMoreLook) } - /// A manual_review held back only by unapproved rules, with every check passing, fits. - @Test func allPassUnderUnapprovedRulesFits() throws { + /// A manual_review held back only by unapproved rules, with every check passing, is a + /// candidate like a pass: the rules' approval was never what made a spot a confirmed fit. + @Test func allPassUnderUnapprovedRulesIsACandidate() throws { var object = try Self.sampleObject() object["checks"] = [ Self.check("wall_backing", "pass"), @@ -123,7 +124,7 @@ import simd let result = try Self.decode(object) #expect(result.decision == .manualReview && !result.policy.autoApprove && result.spot != nil) #expect(result.spotIsClean) - #expect(result.answer { _ in true } == .fits) + #expect(result.answer { _ in true } == .candidate) } /// The same answer with `checks: []`, which the schema allows: no evidence is not a fit. @@ -208,7 +209,7 @@ import simd /// ios/HouseScanUITests/Fixtures/results, the answers the screenshots and UI tests show /// through `-uiDemoResultFile`. Each must stay in the schema and read as its name says. @Test(arguments: [ - ("pass", ResultReading.Answer.fits), ("reject-nearest", .notHere), + ("pass", ResultReading.Answer.candidate), ("reject-nearest", .notHere), ("unsure-view", .oneMoreLook), ("no-clean-spot", .installer), ("no-spot-no-nearest", .notHere), ("review-band", .installer), ("view-not-offered", .oneMoreLook), ]) @@ -236,9 +237,9 @@ import simd ResultReading.answer(decision: decision, policyApproved: approved, hasSpot: spot, checks: checks) } - #expect(answer(.pass, true, true, [pass]) == .fits) + #expect(answer(.pass, true, true, [pass]) == .candidate) // Only the rules' approval holds it back. - #expect(answer(.manualReview, false, true, [pass]) == .fits) + #expect(answer(.manualReview, false, true, [pass]) == .candidate) #expect(answer(.manualReview, false, false, [pass]) == .installer) #expect(answer(.manualReview, true, true, [pass, byView, byPerson]) == .oneMoreLook) #expect(answer(.manualReview, true, true, [pass, byPerson]) == .installer) @@ -255,28 +256,107 @@ import simd #expect(unexplained.needsPerson) } + // MARK: No confirmed fit (B17) + + /// The reading has no answer that says a battery fits: a passing spot rests on space the scan + /// can't show was confirmed, so the most it can be is a candidate. + @Test func noAnswerSaysABatteryFits() { + #expect(ResultReading.Answer(rawValue: "fits") == nil) + typealias Check = ResultReading.Check + let outcomes: [PlacementOutcome] = [.pass, .unsure, .fail] + // Each check set once with a check this app doesn't know, so a new server check can't + // open a shortcut to a stronger answer. + var sets: [[Check]] = [[]] + for id in ["wall_backing", "a_check_this_app_does_not_know", ResultReading.meterWorkingSpaceCheckID] { + sets = sets.flatMap { set in [set] + outcomes.map { set + [Check(id: id, outcome: $0, viewCapturable: true)] } } + } + for decision in [PlacementDecision.pass, .manualReview, .reject] { + for approved in [true, false] { + for hasSpot in [true, false] { + for checks in sets { + let answer = ResultReading.answer(decision: decision, policyApproved: approved, hasSpot: hasSpot, checks: checks) + let described = "\(decision) approved=\(approved) spot=\(hasSpot) checks=\(checks.map { "\($0.id)=\($0.outcome)" })" + // A reject without a spot stays negative whatever its checks say. + if decision == .reject, !hasSpot { #expect(answer == .notHere, "\(described)") } + guard answer == .candidate else { continue } + // A candidate has a spot and at least one check, every one passing. + #expect(hasSpot && !checks.isEmpty && checks.allSatisfy { $0.outcome == .pass } && decision != .reject, + "\(described)") + } + } + } + } + } + + /// A pass that contradicts itself (no spot, no checks, or a check that didn't pass) goes to a + /// person rather than reading as a candidate, and its checks stay as sent. + @Test func anInconsistentPassGoesToAnInstaller() { + typealias Check = ResultReading.Check + let answer = { (spot: Bool, checks: [Check]) in + ResultReading.answer(decision: .pass, policyApproved: true, hasSpot: spot, checks: checks) + } + #expect(answer(false, []) == .installer) + #expect(answer(true, []) == .installer) + #expect(answer(false, [Check(id: "a", outcome: .pass)]) == .installer) + #expect(answer(true, [Check(id: "a", outcome: .pass), Check(id: "b", outcome: .unsure, viewCapturable: true)]) == .installer) + #expect(answer(true, [Check(id: "a", outcome: .pass), Check(id: "b", outcome: .fail)]) == .installer) + #expect(answer(true, [Check(id: "a", outcome: .pass)]) == .candidate) + } + + /// An installer's review can still carry a spot, and the reading keeps it: only the words + /// over it change. + @Test func anInstallerReviewKeepsItsSpot() throws { + let result = try Self.sample(adding: Self.check(ResultReading.meterWorkingSpaceCheckID, "unsure", measured: -0.5, threshold: 0)) + #expect(result.answer { _ in true } == .installer) + #expect(result.spot != nil) + } + + /// Reading an answer changes nothing in it. A passing answer bound to the scene sent stays + /// bound, decodes to the same result before and after it is read, and keeps the server's + /// decision and every check's outcome: the candidate is how the app words it, not a new + /// answer. + @Test func readingAPassLeavesTheAnswerAsTheServerSentIt() throws { + let scene = Data(#"{"note":"the bytes this phone sent"}"#.utf8) + var object = try #require(try JSONSerialization.jsonObject(with: ResultBindingTests.answer(declaring: PacketFiles.sha256(scene))) as? [String: Any]) + object["decision"] = "pass" + object["checks"] = try #require(object["checks"] as? [[String: Any]]).map { check in + var check = check + check["outcome"] = "pass" + check.removeValue(forKey: "unsure_cause") + return check + } + object["missing_evidence"] = [Any]() + let data = try JSONSerialization.data(withJSONObject: object) + #expect(try SceneSchemas.result().validate(data) == []) + try ResultBinding.check(answer: data, submittedScene: scene) + + let result = try PlacementResult.decode(data) + let sent = data + #expect(result.answer { _ in true } == .candidate) + #expect(ResultReading.cardLines(result.readingChecks { _ in true }).isEmpty) + #expect(data == sent) + #expect(try PlacementResult.decode(data) == result) + #expect(result.decision == .pass) + #expect(result.checks.allSatisfy { $0.outcome == .pass }) + try ResultBinding.check(answer: data, submittedScene: scene) + } + // MARK: The card's check lines - @Test func cardLinesPutFailuresFirstThenUnsureThenTheClosestPasses() { + /// The card leads with what failed and what is unsure. A passing check is never on it: the + /// card read a pass as space confirmed clear, which a candidate can't claim (B17). Every check + /// stays in Details. + @Test func cardLinesPutFailuresFirstThenUnsureAndNoPasses() { typealias Check = ResultReading.Check let checks = [ - Check(id: "wide", outcome: .pass, margin: 9), + Check(id: "pass", outcome: .pass), Check(id: "unsure", outcome: .unsure), - Check(id: "close", outcome: .pass, margin: 0.5), Check(id: "fail", outcome: .fail), - Check(id: "unmeasured", outcome: .pass), - Check(id: "closer", outcome: .pass, margin: 0.2), + Check(id: "unsure2", outcome: .unsure), + Check(id: "fail2", outcome: .fail), ] - #expect(ResultReading.cardLines(checks) == [3, 1, 5]) - let passes = checks.filter { $0.outcome == .pass } - #expect(ResultReading.cardLines(passes) == [3, 1]) // closer, close: two passes at most - } - - @Test func marginIsInUnitsOfTheErrorWhenThereIsOne() throws { - #expect(abs(try #require(ResultReading.margin(measured: 3.6, threshold: 3, plusMinus: 0.3, comparison: .atLeast)) - 2) < 1e-9) - #expect(ResultReading.margin(measured: 18, threshold: 20, plusMinus: nil, comparison: .atMost) == 2) - #expect(ResultReading.margin(measured: 2.5, threshold: 3, plusMinus: 0, comparison: .atLeast) == -0.5) - #expect(ResultReading.margin(measured: 2.5, threshold: 3, plusMinus: nil, comparison: nil) == 0.5) - #expect(ResultReading.margin(measured: nil, threshold: 3, plusMinus: 0.3, comparison: .atLeast) == nil) + #expect(ResultReading.cardLines(checks) == [2, 4, 1]) + #expect(ResultReading.cardLines(checks, limit: 5) == [2, 4, 1, 3]) + #expect(ResultReading.cardLines(checks.filter { $0.outcome == .pass }).isEmpty) } } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/Schemas/manifest.schema.json b/ios/HouseScanKit/Tests/HouseScanKitTests/Schemas/manifest.schema.json index dd235455b..deebb59b0 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/Schemas/manifest.schema.json +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/Schemas/manifest.schema.json @@ -535,6 +535,7 @@ } }, "t": { + "description": "When the homeowner made the mark. Absent on a wall end the app inferred (attrs.inferred); an absent t never means the homeowner marked it.", "$ref": "#/$defs/time" }, "photo_ids": { @@ -561,6 +562,10 @@ "properties": { "operable": { "type": "boolean" + }, + "inferred": { + "description": "On a wall end only, and only true: the app placed this end from the walk (where it reached, or past what a past_end request showed), not the homeowner. Such an end has no t. Its point and end_kind are as for a marked end.", + "type": "boolean" } } } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureHTTPTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureHTTPTests.swift new file mode 100644 index 000000000..9c612a72c --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureHTTPTests.swift @@ -0,0 +1,351 @@ +import Foundation +import HouseScanKit +import Synchronization +import Testing + +/// The capture API credential reaches the configured API and nothing else. Two kinds of test: +/// a recording transport shows exactly which request would leave and with what headers (no +/// socket), and `LoopbackPlacementServer` runs the real `URLSessionCaptureHTTP` over HTTP on +/// 127.0.0.1 for what only a socket shows: the header on the wire and a refused redirect. +@Suite struct ScopedCaptureHTTPTests { + static let base = URL(string: "https://api.example.com/v1")! + static let token = "eyJhbGciOiJIUzI1NiJ9.e30.c2lnbmF0dXJl" + + /// Records what would be sent. Not a socket. + final class Recorder: CaptureHTTP, Sendable { + let sent = Mutex<[(kind: String, request: URLRequest)]>([]) + + func send(_ request: URLRequest) async throws -> HTTPReply { + sent.withLock { $0.append(("send", request)) } + return HTTPReply(status: 200, body: Data("{}".utf8)) + } + + func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + sent.withLock { $0.append(("upload", request)) } + return HTTPReply(status: 200, body: Data()) + } + } + + /// A provider that counts its calls and returns `tokens` in order, the last repeating. + final class Provider: Sendable { + let calls = Mutex(0) + let tokens: [String] + init(_ tokens: [String] = [ScopedCaptureHTTPTests.token]) { self.tokens = tokens } + + var credential: ScopedCaptureHTTP.Credential { + { [self] in + let index = calls.withLock { count in + defer { count += 1 } + return count + } + return tokens[min(index, tokens.count - 1)] + } + } + } + + static func scoped(_ provider: Provider, inner: any CaptureHTTP, base: URL = base, transport: CaptureAPIScope.Transport = .httpsOnly) throws + -> ScopedCaptureHTTP + { + ScopedCaptureHTTP(scope: try CaptureAPIScope(base: base, transport: transport), credential: provider.credential, inner: inner) + } + + static func request(_ url: String, method: String = "POST") -> URLRequest { + var request = URLRequest(url: URL(string: url)!) + request.httpMethod = method + request.httpBody = Data(#"{"packetId":"p1"}"#.utf8) + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.setValue("p1", forHTTPHeaderField: "Idempotency-Key") + return request + } + + // MARK: API requests (request inspection, no socket) + + @Test func anAPIRequestCarriesTheCredentialAndIsOtherwiseUnchanged() async throws { + let recorder = Recorder() + let provider = Provider() + let http = try Self.scoped(provider, inner: recorder) + _ = try await http.send(Self.request("https://api.example.com/v1/captures")) + let sent = try #require(recorder.sent.withLock { $0.first }) + #expect(sent.kind == "send") + #expect(sent.request.value(forHTTPHeaderField: "Authorization") == "Bearer \(Self.token)") + #expect(sent.request.httpMethod == "POST") + #expect(sent.request.httpBody == Data(#"{"packetId":"p1"}"#.utf8)) + #expect(sent.request.value(forHTTPHeaderField: "Content-Type") == "application/json") + #expect(sent.request.value(forHTTPHeaderField: "Idempotency-Key") == "p1") + #expect(provider.calls.withLock { $0 } == 1) + } + + /// The base path itself, its subpaths, a case-different host and an explicit default port + /// are all the same API. + @Test(arguments: [ + "https://api.example.com/v1", "https://api.example.com/v1/captures/c1/events?wait=25", + "https://API.example.com/v1/captures", "https://api.example.com:443/v1/captures", + ]) + func everyRequestInsideTheScopeIsAllowed(url: String) async throws { + let recorder = Recorder() + _ = try await Self.scoped(Provider(), inner: recorder).send(Self.request(url)) + #expect(recorder.sent.withLock { $0.count } == 1) + } + + /// Each of these is refused before the provider is asked for anything, and nothing is sent. + @Test(arguments: [ + ("https://api.example.com.evil.test/v1/captures", "host"), + ("https://evilapi.example.com/v1/captures", "host"), + ("https://example.com/v1/captures", "host"), + ("https://api.example.com./v1/captures", "host"), + ("https://api.example.com:8443/v1/captures", "port"), + ("http://api.example.com/v1/captures", "scheme"), + ("https://user:pass@api.example.com/v1/captures", "userinfo"), + ("https://api.example.com/v10/captures", "path"), + ("https://api.example.com/other", "path"), + ("https://api.example.com/", "path"), + ("https://api.example.com/v1/../admin", "path_segments"), + ("https://api.example.com/v1/%2e%2e/admin", "path_segments"), + ("https://api.example.com/v1/captures%2F..%2Fadmin", "path_segments"), + ]) + func aRequestOutsideTheScopeIsRefusedBeforeTheCredentialIsRead(url: String, reason: String) async throws { + let recorder = Recorder() + let provider = Provider() + let http = try Self.scoped(provider, inner: recorder) + await #expect(throws: ScopedCaptureHTTPError.destinationNotAllowed(reason)) { + try await http.send(Self.request(url)) + } + #expect(provider.calls.withLock { $0 } == 0) + #expect(recorder.sent.withLock { $0.isEmpty }) + } + + /// The provider is asked for every API request, so a token it refreshes reaches the next + /// request, at the same fixed scope. + @Test func aRefreshedCredentialIsUsedForTheNextRequest() async throws { + let recorder = Recorder() + let provider = Provider(["first-token", "second-token"]) + let http = try Self.scoped(provider, inner: recorder) + _ = try await http.send(Self.request("https://api.example.com/v1/captures")) + _ = try await http.send(Self.request("https://api.example.com/v1/captures/c1/files")) + let headers = recorder.sent.withLock { $0.map { $0.request.value(forHTTPHeaderField: "Authorization") } } + #expect(headers == ["Bearer first-token", "Bearer second-token"]) + await #expect(throws: ScopedCaptureHTTPError.destinationNotAllowed("host")) { + try await http.send(Self.request("https://other.example.com/v1/captures")) + } + #expect(provider.calls.withLock { $0 } == 2) + } + + /// An empty token, padding alone, or one with a space, quote or line break is refused, and + /// the error names no part of it. + @Test(arguments: ["", "==", "two words", "line\r\nX-Injected: 1", "quote\"d", "tab\tbed", "café"]) + func aMalformedCredentialIsRefusedWithoutItsValue(token: String) async throws { + let recorder = Recorder() + let http = try Self.scoped(Provider([token]), inner: recorder) + let thrown = await #expect(throws: ScopedCaptureHTTPError.self) { + try await http.send(Self.request("https://api.example.com/v1/captures")) + } + #expect(thrown == .credentialMalformed) + #expect(recorder.sent.withLock { $0.isEmpty }) + if !token.isEmpty { #expect(!String(describing: thrown).contains(token)) } + } + + @Test(arguments: ["abc", "a.b-c_d~e+f/g", "dGVzdA==", ScopedCaptureHTTPTests.token]) + func wellFormedBearerTokensAreSent(token: String) async throws { + let recorder = Recorder() + _ = try await Self.scoped(Provider([token]), inner: recorder).send(Self.request("https://api.example.com/v1/captures")) + #expect(recorder.sent.withLock { $0.first?.request.value(forHTTPHeaderField: "Authorization") } == "Bearer \(token)") + } + + /// A provider that can't supply a credential stops the request. Nothing goes out + /// unauthenticated, and the error carries the type, not the provider's message. + @Test func anUnavailableCredentialSendsNothing() async throws { + struct Expired: Error, CustomStringConvertible { var description: String { "token sk-live-secret expired" } } + let recorder = Recorder() + let http = ScopedCaptureHTTP(scope: try CaptureAPIScope(base: Self.base), credential: { throw Expired() }, inner: recorder) + let thrown = await #expect(throws: ScopedCaptureHTTPError.self) { + try await http.send(Self.request("https://api.example.com/v1/captures")) + } + #expect(thrown == .credentialUnavailable("Expired")) + #expect(!String(describing: thrown).contains("sk-live-secret")) + #expect(recorder.sent.withLock { $0.isEmpty }) + } + + /// Holds the credential provider until the test releases it, ignoring cancellation while it + /// waits, as a provider that doesn't watch for it would. + final class Gate: Sendable { + private let waiting = Mutex?>(nil) + private let enteredStream: AsyncStream + private let enteredContinuation: AsyncStream.Continuation + + init() { + (enteredStream, enteredContinuation) = AsyncStream.makeStream() + } + + /// Waits for `release`; reports to `entered` once it is waiting. + func wait() async { + await withCheckedContinuation { continuation in + waiting.withLock { $0 = continuation } + enteredContinuation.yield() + } + } + + func entered() async { + for await _ in enteredStream { return } + } + + func release() { + waiting.withLock { + $0?.resume() + $0 = nil + } + } + } + + /// Cancelled while the provider is waiting, the request isn't sent, even though the + /// provider then returns a well-formed token. + @Test func aRequestCancelledDuringTheCredentialFetchIsNeverSent() async throws { + let recorder = Recorder() + let gate = Gate() + let http = ScopedCaptureHTTP( + scope: try CaptureAPIScope(base: Self.base), + credential: { + await gate.wait() + return ScopedCaptureHTTPTests.token + }, + inner: recorder) + let request = Self.request("https://api.example.com/v1/captures") + let task = Task { try await http.send(request) } + await gate.entered() + task.cancel() + gate.release() + let result = await task.result + #expect(throws: CancellationError.self) { try result.get() } + #expect(recorder.sent.withLock { $0.isEmpty }) + } + + /// A request already cancelled doesn't reach the provider at all. + @Test func aRequestCancelledBeforeTheCredentialFetchNeverReachesTheProvider() async throws { + let recorder = Recorder() + let provider = Provider() + let http = try Self.scoped(provider, inner: recorder) + let request = Self.request("https://api.example.com/v1/captures") + let task = Task { () async throws -> HTTPReply in + withUnsafeCurrentTask { $0?.cancel() } + return try await http.send(request) + } + let result = await task.result + #expect(throws: CancellationError.self) { try result.get() } + #expect(provider.calls.withLock { $0 } == 0) + #expect(recorder.sent.withLock { $0.isEmpty }) + } + + /// An Authorization the caller already set is replaced, never sent alongside or instead. + @Test func anAPIRequestCarriesOnlyTheIssuedCredential() async throws { + let recorder = Recorder() + var request = Self.request("https://api.example.com/v1/captures") + request.setValue("Bearer stale", forHTTPHeaderField: "Authorization") + _ = try await Self.scoped(Provider(), inner: recorder).send(request) + #expect(recorder.sent.withLock { $0.first?.request.value(forHTTPHeaderField: "Authorization") } == "Bearer \(Self.token)") + } + + // MARK: Storage uploads (request inspection, no socket) + + /// A signed PUT goes out exactly as built, without asking the provider, even to the API's + /// own origin. + @Test(arguments: ["https://storage.example.com/bucket/k0.jpg?X-Goog-Signature=abc", "https://api.example.com/v1/upload/token-1"]) + func aSignedPutIsSentUnchangedAndWithoutTheCredential(url: String) async throws { + let recorder = Recorder() + let provider = Provider() + var request = URLRequest(url: URL(string: url)!) + request.httpMethod = "PUT" + request.setValue("image/jpeg", forHTTPHeaderField: "Content-Type") + request.setValue("rL0Y20zC+Fzt72VPzMSk2A==", forHTTPHeaderField: "Content-MD5") + _ = try await Self.scoped(provider, inner: recorder).upload(request, file: URL(fileURLWithPath: "/dev/null")) + let sent = try #require(recorder.sent.withLock { $0.first }) + #expect(sent.kind == "upload") + #expect(sent.request == request) + #expect(sent.request.value(forHTTPHeaderField: "Authorization") == nil) + #expect(provider.calls.withLock { $0 } == 0) + } + + /// The chosen policy: storage requests carry their signature in the URL, so an Authorization + /// header on one is refused, not forwarded and not stripped. + @Test func aStorageRequestCarryingAuthorizationIsRefused() async throws { + let recorder = Recorder() + let provider = Provider() + var request = URLRequest(url: URL(string: "https://storage.example.com/bucket/k0.jpg?sig=abc")!) + request.httpMethod = "PUT" + request.setValue("Bearer leaked", forHTTPHeaderField: "Authorization") + let http = try Self.scoped(provider, inner: recorder) + await #expect(throws: ScopedCaptureHTTPError.storageRequestHasAuthorization) { + try await http.upload(request, file: URL(fileURLWithPath: "/dev/null")) + } + #expect(recorder.sent.withLock { $0.isEmpty }) + #expect(provider.calls.withLock { $0 } == 0) + } + + // MARK: The scope itself + + @Test(arguments: [ + ("http://api.example.com/v1", "base_insecure"), ("https://u:p@api.example.com/v1", "base_userinfo"), + ("https://api.example.com/v1?x=1", "base_query"), ("https://api.example.com/v1/../x", "base_path"), + ("ftp://api.example.com/v1", "base_insecure"), + ]) + func aBaseThatCouldNotReceiveACredentialIsRefused(base: String, reason: String) { + #expect(throws: ScopedCaptureHTTPError.destinationNotAllowed(reason)) { try CaptureAPIScope(base: URL(string: base)!) } + } + + /// Plain http is allowed only by name and only to this machine. + @Test func loopbackHTTPIsAnExplicitLocalOnlyChoice() throws { + #expect(throws: ScopedCaptureHTTPError.destinationNotAllowed("base_insecure")) { + try CaptureAPIScope(base: URL(string: "http://127.0.0.1:8080/v1")!) + } + #expect(throws: ScopedCaptureHTTPError.destinationNotAllowed("base_insecure")) { + try CaptureAPIScope(base: URL(string: "http://api.example.com/v1")!, transport: .loopbackHTTP) + } + let scope = try CaptureAPIScope(base: URL(string: "http://127.0.0.1:8080/v1")!, transport: .loopbackHTTP) + #expect(scope.problem(with: URL(string: "http://127.0.0.1:8080/v1/captures")!) == nil) + #expect(scope.problem(with: URL(string: "http://127.0.0.1:8081/v1/captures")!) == "port") + #expect(scope.problem(with: URL(string: "http://localhost:8080/v1/captures")!) == "host") + } + + // MARK: Over a real socket + + /// The real transport on 127.0.0.1: an API request arrives with the credential, a storage + /// PUT to the same origin arrives without it, and a redirect from either is not followed, + /// so the second listener receives nothing. + @Test func overHTTPTheCredentialReachesOnlyTheAPIAndNoRedirectTarget() async throws { + let elsewhere = try LoopbackPlacementServer { _ in .init(status: 200, body: Data("{}".utf8)) } + let target = elsewhere.base.absoluteString + let api = try LoopbackPlacementServer { request in + if request.path.hasSuffix("/redirect") || request.path.hasSuffix("/upload/redirect") { + return .init(status: 307, headers: ["Location": "\(target)\(request.path)"], body: Data()) + } + return .init(status: 200, body: Data("{}".utf8)) + } + let provider = Provider() + let http = try Self.scoped( + provider, inner: URLSessionCaptureHTTP.ephemeral(timeout: 10), + base: api.base.appending(path: "v1"), transport: .loopbackHTTP) + + let sent = try await http.send(Self.request("\(api.base.absoluteString)/v1/captures")) + #expect(sent.status == 200) + let redirected = try await http.send(Self.request("\(api.base.absoluteString)/v1/redirect")) + #expect(redirected.status == 307) + + let file = FileManager.default.temporaryDirectory.appending(path: "scoped-\(UUID().uuidString).bin") + try Data("photo".utf8).write(to: file) + defer { try? FileManager.default.removeItem(at: file) } + for path in ["upload/token-1", "upload/redirect"] { + var put = URLRequest(url: URL(string: "\(api.base.absoluteString)/v1/\(path)")!) + put.httpMethod = "PUT" + put.setValue("application/octet-stream", forHTTPHeaderField: "Content-Type") + _ = try await http.upload(put, file: file) + } + + let seen = api.requests + #expect(seen.map(\.path) == ["/v1/captures", "/v1/redirect", "/v1/upload/token-1", "/v1/upload/redirect"]) + #expect(seen[0].headers["authorization"] == "Bearer \(Self.token)") + #expect(seen[1].headers["authorization"] == "Bearer \(Self.token)") + #expect(seen[2].headers["authorization"] == nil) + #expect(seen[3].headers["authorization"] == nil) + #expect(seen[2].body == Data("photo".utf8)) + #expect(elsewhere.requests.isEmpty) + #expect(provider.calls.withLock { $0 } == 2) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureUploadTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureUploadTests.swift new file mode 100644 index 000000000..ecaeac30a --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureUploadTests.swift @@ -0,0 +1,211 @@ +import Foundation +import HouseScanKit +import Synchronization +import Testing + +/// The whole uploader through `ScopedCaptureHTTP`, over real HTTP to the loopback capture API +/// with an expected bearer (`LoopbackCaptureAPI.State.expectedBearer`). The credential reaches +/// every API request and no signed PUT. A refusal at the credential boundary ends the upload at +/// once with its code, instead of being retried as a network failure would be. +@Suite struct ScopedCaptureUploadTests { + typealias Rig = CaptureUploaderTests.Rig + typealias Provider = ScopedCaptureHTTPTests.Provider + static let token = "loopback-capture-token" + + /// Every backoff the uploader asked for. Empty means it never waited to retry. + final class Sleeps: Sendable { + private let values = Mutex<[Double]>([]) + var isEmpty: Bool { values.withLock { $0.isEmpty } } + var record: @Sendable (Double) async throws -> Void { { [self] seconds in values.withLock { $0.append(seconds) } } } + } + + /// A loopback API expecting `token`, and an uploader on it whose transport is scoped to + /// `scopeBase` (the loopback's own base unless a test points it elsewhere). + static func rig( + provider: Provider, scopeBase: ((LoopbackCaptureAPI) -> URL)? = nil, sleeps: Sleeps, + configure: (inout LoopbackCaptureAPI.State) -> Void = { _ in } + ) throws -> Rig { + let server = try LoopbackCaptureAPI() + server.state.withLock { + $0.expectedBearer = token + configure(&$0) + } + let scope = try CaptureAPIScope(base: scopeBase?(server) ?? server.base, transport: .loopbackHTTP) + let http = ScopedCaptureHTTP(scope: scope, credential: provider.credential, inner: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + return try Rig(server: server, http: http, sleep: sleeps.record) + } + + @Test func aWholeCaptureUploadsWithTheCredentialOnEveryAPIRequestAndNoPut() async throws { + let sleeps = Sleeps() + let provider = Provider([Self.token]) + let rig = try Self.rig(provider: provider, sleeps: sleeps) + defer { rig.cleanUp() } + let images = try await rig.capture.sealImages() + await rig.uploader.add(images) + try #require(await CaptureUploaderTests.settles(rig.uploader)) + try await rig.finishAndSeal() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "manual_review")) + #expect(done.result != nil) + let log = rig.server.state.withLock { $0.log } + let puts = log.filter { LoopbackCaptureAPI.route($0) == "PUT upload" } + let api = log.filter { LoopbackCaptureAPI.route($0) != "PUT upload" } + #expect(puts.count >= images.count + 4) + #expect(!api.isEmpty) + #expect(api.allSatisfy { $0.headers["authorization"] == "Bearer \(Self.token)" }) + #expect(puts.allSatisfy { $0.headers["authorization"] == nil }) + // At least one fetch per API request the uploader made. Not equality: URLSession may + // resend a request on a reused connection, which the server logs twice. + #expect(provider.calls.withLock { $0 } > 0) + } + + /// Each refusal ends the upload at its step with the boundary's code and status 0, sends the + /// server nothing, and sleeps for no retry. + @Test func aDestinationOutsideTheScopeEndsTheUploadWithoutARetry() async throws { + let sleeps = Sleeps() + let provider = Provider([Self.token]) + let rig = try Self.rig(provider: provider, scopeBase: { URL(string: "http://127.0.0.1:\(Int($0.port) + 1)/v1")! }, sleeps: sleeps) + defer { rig.cleanUp() } + await rig.uploader.kick() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "create", codes: ["auth_destination_refused"], status: 0)) + #expect(rig.server.state.withLock { $0.log.isEmpty }) + #expect(provider.calls.withLock { $0 } == 0) + #expect(sleeps.isEmpty) + } + + @Test(arguments: [("", "auth_credential_malformed"), ("has space", "auth_credential_malformed")]) + func aMalformedCredentialEndsTheUploadWithoutARetry(token: String, code: String) async throws { + let sleeps = Sleeps() + let provider = Provider([token]) + let rig = try Self.rig(provider: provider, sleeps: sleeps) + defer { rig.cleanUp() } + await rig.uploader.kick() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "create", codes: [code], status: 0)) + #expect(rig.server.state.withLock { $0.log.isEmpty }) + #expect(provider.calls.withLock { $0 } == 1) + #expect(sleeps.isEmpty) + } + + /// A provider that can't supply a credential ends the upload; nothing is sent without one. + /// Retrying a credential refresh belongs in the provider, by its own policy. + @Test func anUnavailableCredentialEndsTheUploadWithoutARetry() async throws { + struct Offline: Error {} + let sleeps = Sleeps() + let calls = Mutex(0) + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.expectedBearer = Self.token } + let http = ScopedCaptureHTTP( + scope: try CaptureAPIScope(base: server.base, transport: .loopbackHTTP), + credential: { + calls.withLock { $0 += 1 } + throw Offline() + }, + inner: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + let rig = try Rig(server: server, http: http, sleep: sleeps.record) + defer { rig.cleanUp() } + await rig.uploader.kick() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "create", codes: ["auth_credential_unavailable"], status: 0)) + #expect(server.state.withLock { $0.log.isEmpty }) + #expect(calls.withLock { $0 } == 1) + #expect(sleeps.isEmpty) + } + + /// Signed headers that include Authorization end the upload at the PUT, with no PUT sent + /// and no retry. The API requests before it carried the credential as usual. + @Test func signedHeadersCarryingAuthorizationEndTheUploadAtThePut() async throws { + let sleeps = Sleeps() + let rig = try Self.rig(provider: Provider([Self.token]), sleeps: sleeps) { + $0.extraSignedHeaders = ["Authorization": "Bearer storage-credential"] + } + defer { rig.cleanUp() } + await rig.uploader.add(try await rig.capture.sealImages(count: 2)) + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "put", codes: ["auth_storage_authorization"], status: 0)) + #expect(rig.server.requests("PUT upload").isEmpty) + #expect(rig.server.requests("POST captures").first?.headers["authorization"] == "Bearer \(Self.token)") + #expect(sleeps.isEmpty) + } + + /// A withdrawal that lands while the credential is being fetched outranks the refusal that + /// follows: the upload stops as withdrawn, not as a credential failure. + @Test func aWithdrawalOutranksACredentialRefusal() async throws { + struct Offline: Error {} + let sleeps = Sleeps() + let uploader = Mutex(nil) + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.expectedBearer = Self.token } + let http = ScopedCaptureHTTP( + scope: try CaptureAPIScope(base: server.base, transport: .loopbackHTTP), + credential: { + _ = uploader.withLock { $0 }?.withdrawConsent() + throw Offline() + }, + inner: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + let rig = try Rig(server: server, http: http, sleep: sleeps.record) + defer { rig.cleanUp() } + uploader.withLock { $0 = rig.uploader } + await rig.uploader.kick() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + let end = await rig.uploader.snapshot.end + #expect(end != .failed(step: "create", codes: ["auth_credential_unavailable"], status: 0)) + #expect(server.state.withLock { $0.log.isEmpty }) + #expect(sleeps.isEmpty) + } + + /// A withdrawal that lands while a provider is waiting, one that ignores cancellation and + /// then returns a good token, ends the upload as withdrawn: nothing is sent, no retry is + /// scheduled, and the end is saved. The wrapper throws CancellationError there, which the + /// uploader must not read as a network failure to retry. + @Test func aWithdrawalWhileTheProviderWaitsEndsTheUploadAsWithdrawn() async throws { + let sleeps = Sleeps() + let gate = ScopedCaptureHTTPTests.Gate() + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.expectedBearer = Self.token } + let http = ScopedCaptureHTTP( + scope: try CaptureAPIScope(base: server.base, transport: .loopbackHTTP), + credential: { + await gate.wait() + return Self.token + }, + inner: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + let rig = try Rig(server: server, http: http, sleep: sleeps.record) + defer { rig.cleanUp() } + await rig.uploader.kick() + await gate.entered() + #expect(rig.uploader.withdrawConsent() == .marked) + gate.release() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + let snapshot = await rig.uploader.snapshot + #expect(snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) + #expect(await rig.uploader.status.retryingAt == nil) + #expect(server.state.withLock { $0.log.isEmpty }) + #expect(sleeps.isEmpty) + let saved = try CaptureUploadState.load(from: CaptureUploader.stateURL(in: rig.capture.folder)) + #expect(saved.end == .abandoned(CaptureUploader.withdrawnReason)) + } + + /// With the default loopback (no expected bearer), the old assertion stands: an API request + /// that carries Authorization is refused. + @Test func theDefaultLoopbackStillRefusesAnyAPIAuthorization() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + var request = URLRequest(url: rig.server.base.appending(path: "captures")) + request.httpMethod = "POST" + request.setValue("Bearer x", forHTTPHeaderField: "Authorization") + request.httpBody = Data("{}".utf8) + let reply = try await URLSessionCaptureHTTP.ephemeral(timeout: 10).send(request) + #expect(reply.status == 400) + #expect(CaptureAPI.errorCodes(reply.body) == ["unexpected_authorization"]) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedPhotoProcessingTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedPhotoProcessingTests.swift new file mode 100644 index 000000000..7e1012fd8 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedPhotoProcessingTests.swift @@ -0,0 +1,151 @@ +import Foundation +import HouseScanKit +import Synchronization +import Testing + +/// `FixtureCaptureHTTP` whose register answers sign each storage upload with an Authorization +/// header, as a misconfigured service might. +final class AuthorizingStorageHTTP: CaptureHTTP, Sendable { + let inner: FixtureCaptureHTTP + + init(_ inner: FixtureCaptureHTTP) { + self.inner = inner + } + + func send(_ request: URLRequest) async throws -> HTTPReply { + let reply = try await inner.send(request) + guard request.httpMethod == "POST", request.url?.lastPathComponent == "files", + var body = try JSONSerialization.jsonObject(with: reply.body) as? [String: Any], + let files = body["files"] as? [[String: Any]] else { return reply } + body["files"] = files.map { file -> [String: Any] in + guard var upload = file["upload"] as? [String: Any], var headers = upload["headers"] as? [String: String] else { return file } + headers["Authorization"] = "Bearer storage" + upload["headers"] = headers + var signed = file + signed["upload"] = upload + return signed + } + return HTTPReply(status: reply.status, body: try JSONSerialization.data(withJSONObject: body), retryAfter: reply.retryAfter) + } + + func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + try await inner.upload(request, file: file) + } +} + +/// The scoped credential boundary's refusals as photo processing sees them: the real +/// `ScopedCaptureHTTP` and `CaptureUploader` end the upload with their code at status 0, and the +/// scan ends in the setup state, never as the homeowner's scan being refused. +@Suite(.serialized) @MainActor struct ScopedPhotoProcessingTests { + struct NoCredential: Error {} + + let root = FileManager.default.temporaryDirectory.appending(path: "scoped-photo-\(UUID().uuidString)") + + func controller(_ http: any CaptureHTTP) -> PhotoProcessingController { + var policy = CaptureUploader.Policy() + policy.eventsWait = 0 + policy.firstDelay = 0.01 + policy.maxDelay = 0.02 + let environment = CaptureSessionCoordinator.Environment( + endpoint: FixtureCaptureHTTP.base, sends: true, http: http, capturesFolder: root.appending(path: "Captures"), + sessionInfo: { packetID, video in SyntheticCapture.sessionInfo(packetID: packetID, video: video, appVersion: "test") }, + device: .init(model: "synthetic", systemVersion: "synthetic", appVersion: "test"), tier: .arkit, policy: policy) + return PhotoProcessingController(setup: .ready(environment, standIn: true)) + } + + func scoped(base: URL = FixtureCaptureHTTP.base, credential: @escaping ScopedCaptureHTTP.Credential, inner: any CaptureHTTP) throws + -> ScopedCaptureHTTP + { + ScopedCaptureHTTP(scope: try CaptureAPIScope(base: base), credential: credential, inner: inner) + } + + /// A hang safeguard counted in the waiter's own turns (`countedWait`), not wall time. + func until(_ seconds: Double = 30, _ condition: () -> Bool) async throws -> Bool { + try await countedWait(seconds, condition) + } + + func run(_ controller: PhotoProcessingController) async throws { + let capture = SyntheticCapture.standard + controller.beginScan(ScanContext(scanID: "scan", profile: controller.profile, recordingSessionID: "rec"), recording: capture.recording) + controller.answerConsent(true) + for photo in try capture.photos(in: root.appending(path: "photos")) { controller.kept(photo) } + controller.captureEnded(acceptedCloseUpAt: capture.acceptedCloseUpAt) + } + + @Test(arguments: [PhotoSetupProblem.destinationRefused, .credentialUnavailable, .credentialMalformed, .storageAuthorization]) + func eachBoundaryRefusalEndsInTheSetupState(problem: PhotoSetupProblem) async throws { + defer { try? FileManager.default.removeItem(at: root) } + let fixture = FixtureCaptureHTTP(answer: .candidate) + let http: ScopedCaptureHTTP + switch problem { + case .destinationRefused: + let elsewhere = try #require(URL(string: "https://elsewhere.invalid/v1")) + http = try scoped(base: elsewhere, credential: { "token" }, inner: fixture) + case .credentialUnavailable: + http = try scoped(credential: { throw NoCredential() }, inner: fixture) + case .credentialMalformed: + http = try scoped(credential: { "not a token" }, inner: fixture) + case .storageAuthorization: + http = try scoped(credential: { "token" }, inner: AuthorizingStorageHTTP(fixture)) + } + let controller = controller(http) + try await run(controller) + try #require(try await until { controller.status?.isFinal == true }, "\(String(describing: controller.status))") + let step = problem == .storageAuthorization ? "put" : "create" + #expect(controller.status?.stage == .ended(.setupRefused(step: step, problem))) + #expect(controller.status?.consent == .granted) + // Refused on the phone: nothing reached the API, or past register for the storage case. + if problem == .storageAuthorization { + #expect(!fixture.routes.contains("PUT upload") && !fixture.routes.contains("POST captures/finalize")) + } else { + #expect(fixture.routes.isEmpty) + } + controller.endScan(recording: SyntheticCapture.standard.recording) + } + + /// A credential that stops being available once the service has the whole capture, so the + /// refusal comes at the result read, after the service may have processed the scan. The scan + /// ends in the same setup state, at the result step. + @Test func aRefusalAtTheResultStepIsTheSameSetupState() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let fixture = FixtureCaptureHTTP(answer: .candidate) + // The fixture lists a route once the request got past the boundary, so the events poll + // that finds the run finished gets its credential, and the result read after it doesn't. + let http = try scoped(credential: { + if fixture.routes.contains("GET captures/events") { throw NoCredential() } + return "token" + }, inner: fixture) + let controller = controller(http) + try await run(controller) + try #require(try await until { controller.status?.isFinal == true }, "\(String(describing: controller.status))") + #expect(controller.status?.stage == .ended(.setupRefused(step: "result", .credentialUnavailable))) + #expect(fixture.routes.contains("POST captures/finalize")) + #expect(!fixture.routes.contains("GET captures/result")) + controller.endScan(recording: SyntheticCapture.standard.recording) + } + + /// The homeowner stops sending while the credential is being fetched, and the provider then + /// fails: the scan ends withdrawn, never in the setup state. + @Test func aWithdrawalOutranksARefusalThatFollowsIt() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let fixture = FixtureCaptureHTTP(answer: .candidate) + let asked = Mutex(0) + let http = try scoped(credential: { + asked.withLock { $0 += 1 } + // Waits without watching for cancellation, then fails. + await Task.detached { try? await Task.sleep(for: .milliseconds(400)) }.value + throw NoCredential() + }, inner: fixture) + let controller = controller(http) + controller.beginScan( + ScanContext(scanID: "scan", profile: controller.profile, recordingSessionID: "rec"), recording: SyntheticCapture.standard.recording) + controller.answerConsent(true) + try #require(try await until(10) { asked.withLock { $0 } > 0 }, "the credential was never asked for") + controller.stopSending() + #expect(controller.status?.stage == .ended(.withdrawn(recorded: true))) + try await Task.sleep(for: .seconds(1)) + #expect(controller.status?.stage == .ended(.withdrawn(recorded: true))) + #expect(fixture.routes.isEmpty) + controller.endScan(recording: SyntheticCapture.standard.recording) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift index d479ca93b..be8adf4a4 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift @@ -243,6 +243,40 @@ import Testing #expect(left.band == .ground && left.span == -5 ... -3) let right = try #require(GapPlanner().plan(for: item(#"{"kind":"past_end","side":"right","message":"m"}"#), leftEnd: -3, rightEnd: 4)) #expect(right.span == 4...6) + // One owner of the 2 m: the item's request is the planner's past-end request from that end. + #expect(left == GapPlanner().pastEndPlan(side: .left, end: -3)) + #expect(right == GapPlanner().pastEndPlan(side: .right, end: 4)) + } + + /// "It turns a corner" on a past_end request: the request can't follow the corner, so it is + /// skipped, and the request planned from the homeowner's corner is recorded with it + /// (`ScanEngine.skipCurrentGap(deferring:)`). The next answer's repeat of that side is not + /// raised; a past_end from a different end on that side, or another request, still is. Ends + /// at -3 and 4 m; the corner nearer or farther than the cleared end, on either side. + @Test(arguments: [ + (WalkSide.left, Float(-2), Float(-1)), (.left, -4, -5), + (.right, 3, 2), (.right, 5, 6), + ]) + func aCornerIsNotAskedForAgainFromThatEnd(side: WalkSide, corner: Float, laterEnd: Float) throws { + let planner = GapPlanner() + let pastEnd = try item(#"{"kind":"past_end","side":"\#(side.rawValue)","message":"m"}"#) + let ground = try item(#"{"kind":"band","band":"ground","span_ft":[3.0,5.5],"message":"m"}"#) + func ends(_ s: Float) -> (left: Float, right: Float) { side == .left ? (s, 4) : (-3, s) } + let cleared = ends(side == .left ? -3 : 4) + let raised = try #require(planner.plan(for: pastEnd, leftEnd: cleared.left, rightEnd: cleared.right)) + let deferred = planner.pastEndPlan(side: side, end: corner) + func next(at end: (left: Float, right: Float), skipped: [GapPlan]) -> [PlacementMissingEvidence] { + planner.serverRequests( + in: [pastEnd, ground], leftEnd: end.left, rightEnd: end.right, limitEnds: [], + asked: [raised], skipped: skipped, limit: 5 + ).map(\.item) + } + // The skipped request alone misses the repeat: the corner moved the end. + #expect(next(at: ends(corner), skipped: [raised]) == [pastEnd, ground]) + // With the corner's request recorded, the repeat is not raised; the ground request is. + #expect(next(at: ends(corner), skipped: [raised, deferred]) == [ground]) + // The end moved on from the corner later: walking past it is a new view. + #expect(next(at: ends(laterEnd), skipped: [raised, deferred]) == [pastEnd, ground]) } /// Issue #39: the answer asks for two views and the homeowner can't get to the first. The diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/SyntheticCaptureTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/SyntheticCaptureTests.swift new file mode 100644 index 000000000..ef14c31ce --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/SyntheticCaptureTests.swift @@ -0,0 +1,123 @@ +import Foundation +import HouseScanKit +import Testing + +@Suite struct CaptureFixtureLaunchTests { + @Test(arguments: [true, false]) + func aReleaseBuildNeverUsesTheFixture(onReplay: Bool) { + for synthetic in [true, false] { + let launch = CaptureFixtureLaunch.resolve(debugBuild: false, onReplay: onReplay, answer: .candidate, syntheticCapture: synthetic) + #expect(launch == .off("a release build has no capture fixture")) + } + } + + @Test func theLiveCameraNeverUsesTheFixture() { + for synthetic in [true, false] { + let launch = CaptureFixtureLaunch.resolve(debugBuild: true, onReplay: false, answer: .candidate, syntheticCapture: synthetic) + #expect(launch == .off("the capture fixture runs only on a replay, never the live camera")) + } + } + + @Test func theSyntheticCaptureNeedsTheFixtureAndNothingIsOnByDefault() { + #expect(CaptureFixtureLaunch.resolve(debugBuild: true, onReplay: true, answer: nil, syntheticCapture: true) + == .off("the synthetic capture needs the capture fixture")) + #expect(CaptureFixtureLaunch.resolve(debugBuild: true, onReplay: true, answer: nil, syntheticCapture: false) + == .off("sending captures is off in this build")) + } + + @Test func aDebugReplayUsesWhatItAskedFor() { + #expect(CaptureFixtureLaunch.resolve(debugBuild: true, onReplay: true, answer: .hold, syntheticCapture: false) == .on(.hold, syntheticCapture: false)) + #expect(CaptureFixtureLaunch.resolve(debugBuild: true, onReplay: true, answer: .candidate, syntheticCapture: true) + == .on(.candidate, syntheticCapture: true)) + } +} + +@Suite(.serialized) @MainActor struct SyntheticCaptureTests { + let root = FileManager.default.temporaryDirectory.appending(path: "synthetic-capture-\(UUID().uuidString)") + + @Test func itIsTheRecordingOf192sSyntheticFixture() { + let fixture = NativeCaptureFixture(folder: root) + let capture = SyntheticCapture.standard + #expect(capture.start == fixture.start && capture.end == fixture.end) + let ours = capture.rows, theirs = fixture.rows + #expect(ours.trajectory == theirs.trajectory) + #expect(ours.intrinsics == theirs.intrinsics) + #expect(ours.accelerometer == theirs.accelerometer) + #expect(ours.gyroscope == theirs.gyroscope) + #expect(capture.recording.start()?.uptime == capture.start) + #expect(capture.recording.start()?.date == SyntheticCapture.startedAt) + } + + @Test func twoRunsMakeTheSameRowsAndPhotos() throws { + defer { try? FileManager.default.removeItem(at: root) } + let first = try SyntheticCapture().photos(in: root.appending(path: "a")) + let second = try SyntheticCapture().photos(in: root.appending(path: "b")) + #expect(first.count == 6) + for (a, b) in zip(first, second) { + #expect(a.t == b.t && a.purpose == b.purpose && a.cameraToWorld == b.cameraToWorld) + #expect(try Data(contentsOf: a.jpeg) == Data(contentsOf: b.jpeg)) + } + #expect(Set(try first.map { try Data(contentsOf: $0.jpeg) }).count == 6, "two photos share an image") + #expect(SyntheticCapture().rows.accelerometer == SyntheticCapture.standard.rows.accelerometer) + } + + func controller(_ http: FixtureCaptureHTTP) -> PhotoProcessingController { + var policy = CaptureUploader.Policy() + policy.eventsWait = 0 + policy.firstDelay = 0.01 + policy.maxDelay = 0.02 + let environment = CaptureSessionCoordinator.Environment( + endpoint: FixtureCaptureHTTP.base, sends: true, http: http, capturesFolder: root.appending(path: "Captures"), + sessionInfo: { packetID, video in SyntheticCapture.sessionInfo(packetID: packetID, video: video, appVersion: "test") }, + device: .init(model: "synthetic", systemVersion: "synthetic", appVersion: "test"), tier: .arkit, policy: policy) + return PhotoProcessingController(setup: .ready(environment, standIn: true)) + } + + /// A hang safeguard counted in the waiter's own turns (`countedWait`), not wall time. + func until(_ condition: () -> Bool) async throws -> Bool { + try await countedWait(30, condition) + } + + /// The capture through the real coordinator and uploader: a packet the phone's checks pass, + /// labelled synthetic, and the fixture's answer as the scan's typed stage. + @Test func itsPacketPassesThePhonesChecksSaysSyntheticAndGetsAnAnswer() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .candidate) + let controller = controller(http) + let capture = SyntheticCapture.standard + controller.beginScan(ScanContext(scanID: "scan", profile: controller.profile, recordingSessionID: "rec"), recording: capture.recording) + controller.answerConsent(true) + for photo in try capture.photos(in: root.appending(path: "photos")) { controller.kept(photo) } + controller.captureEnded(acceptedCloseUpAt: capture.acceptedCloseUpAt) + try #require(try await until { controller.status?.isFinal == true }, "\(String(describing: controller.status))") + #expect(controller.status?.stage == .answered(.candidate(.init( + message: FixtureCaptureHTTP.Answer.candidate.message ?? "", previewMade: false, ar: .withheld(.analysisUnverified))))) + let folder = try #require(controller.captureFolder) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: Data(contentsOf: folder.appending(path: "packet.json"))) + #expect(packet.source.kind == .synthetic) + #expect(Packet04Check.problems(packet, folder: folder).isEmpty) + #expect(packet.stills?.map(\.purpose) == ["meter_close"]) + #expect(packet.scaleReference.meterCloseUp.still == "meter_close") + controller.endScan(recording: capture.recording) + } + + /// The same capture without its motion, as a replay is: the phone's checks still refuse it. + @Test func withoutMotionThePacketIsStillRefusedOnThePhone() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .candidate) + let controller = controller(http) + let capture = SyntheticCapture.standard + let still = RecordingSource(start: { (capture.start, SyntheticCapture.startedAt) }, rows: { + let rows = capture.rows + return RecorderRows(trajectory: rows.trajectory, intrinsics: rows.intrinsics, accelerometer: [], gyroscope: []) + }) + controller.beginScan(ScanContext(scanID: "scan", profile: controller.profile, recordingSessionID: "rec"), recording: still) + controller.answerConsent(true) + for photo in try capture.photos(in: root.appending(path: "photos")) { controller.kept(photo) } + controller.captureEnded(acceptedCloseUpAt: capture.acceptedCloseUpAt) + try #require(try await until { controller.status?.isFinal == true }) + #expect(controller.status?.stage == .ended(.notPrepared)) + #expect(!http.routes.contains("POST captures/finalize")) + controller.endScan(recording: capture.recording) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/WallEndProvenanceTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/WallEndProvenanceTests.swift new file mode 100644 index 000000000..2dc72dae7 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/WallEndProvenanceTests.swift @@ -0,0 +1,202 @@ +import Foundation +import HouseScanKit +import simd +import Testing + +// B-12: a wall end the homeowner marked and one the app inferred from the walk keep the same +// place and kind; only the packet's `t` and `attrs.inferred` tell them apart. The wall is z = 0 +// facing +z with the meter 1.5 m up, synthetic values chosen for the tests. + +@Suite struct WallEndProvenanceTests { + static let wall = SceneWall(meter: SIMD3(0, 1.5, 0), outward: SIMD3(0, 0, 1), groundY: 0) + + static func end(_ stamp: WallEndStamp, kind: PacketMark.EndKind = .unexplored) throws -> PacketMark { + let frame = try #require(MeterFrame(wall: wall)) + return .wallEnd(id: "wall_end_right", side: .right, endKind: kind, s: 2.5, wall: wall, frame: frame, stamp: stamp) + } + + static func json(_ mark: PacketMark) throws -> [String: Any] { + try #require(JSONSerialization.jsonObject(with: JSONEncoder().encode(mark)) as? [String: Any]) + } + + // MARK: Stamps + + @Test func aHomeownerMarkKeepsItsTimeAndMeetsTheRequest() { + let stamp = WallEndStamp.marked(at: 12.5) + #expect(stamp.source == .homeowner && stamp.markedAt == 12.5 && !stamp.isInferred) + #expect(stamp.markEndOutcome == .met) + } + + @Test func anInferredEndHasNoTimeAndNeverMeetsAMarkRequest() { + let stamp = WallEndStamp.inferred + #expect(stamp.source == .inferred && stamp.markedAt == nil && stamp.isInferred) + #expect(stamp.markEndOutcome == .superseded) + } + + /// A homeowner mark made before the packet's clock started has no time but is still the + /// homeowner's: absence of a time is never read as either source. + @Test func aMarkWithoutAClockIsStillTheHomeowners() { + let stamp = WallEndStamp.marked(at: nil) + #expect(stamp.source == .homeowner && stamp.markedAt == nil) + #expect(stamp.markEndOutcome == .met) + #expect(stamp != .inferred) + } + + // MARK: Past-end requests + + @Test(arguments: [ + // A mark during the request stands, met or not: it may be nearer than the cleared end. + (true, true, PastEndSettlement.keepMarked), + (true, false, PastEndSettlement.keepMarked), + // Met by views with nobody marking: the end moves on, inferred. + (false, true, PastEndSettlement.moveOn), + // "I can't get there": the cleared end comes back. + (false, false, PastEndSettlement.restore), + ]) + func pastEndSettlement(marked: Bool, met: Bool, expected: PastEndSettlement) { + #expect(PastEndSettlement.when(endMarkedDuringRequest: marked, met: met) == expected) + } + + // MARK: Packet encoding + + @Test func aMarkedEndCarriesItsTimeAndNoFlag() throws { + let object = try Self.json(try Self.end(.marked(at: 12.5))) + #expect(object["t"] as? Double == 12.5) + #expect(object["attrs"] == nil) + } + + @Test func anInferredEndCarriesTheFlagAndNoTime() throws { + let object = try Self.json(try Self.end(.inferred)) + #expect(object["t"] == nil) + #expect((object["attrs"] as? [String: Any])?["inferred"] as? Bool == true) + } + + /// Provenance changes nothing about where the end is or what it is. + @Test(arguments: [PacketMark.EndKind.unexplored, .limit]) + func provenanceKeepsPlaceAndKind(kind: PacketMark.EndKind) throws { + let marked = try Self.end(.marked(at: 3), kind: kind) + let inferred = try Self.end(.inferred, kind: kind) + #expect(marked.points == inferred.points && marked.points == [SIMD3(2.5, 0, 0)]) + #expect(marked.side == inferred.side && marked.endKind == kind && inferred.endKind == kind) + #expect(marked.kind == .wallEnd && inferred.kind == .wallEnd) + } + + @Test func bothRoundTripThroughJSON() throws { + for mark in [try Self.end(.marked(at: 7)), try Self.end(.inferred), try Self.end(.marked(at: nil))] { + let decoded = try JSONDecoder().decode(PacketMark.self, from: JSONEncoder().encode(mark)) + #expect(decoded == mark) + } + } + + /// An older packet's wall end, without attrs, decodes as not inferred, its time kept. + @Test func anEndWithoutAttrsIsNotInferred() throws { + let json = #"{"id": "e", "kind": "wall_end", "points": [[2.5, 0, 0]], "t": 4, "side": "right", "end_kind": "limit"}"# + let mark = try JSONDecoder().decode(PacketMark.self, from: Data(json.utf8)) + #expect(!mark.inferred && mark.t == 4) + } + + /// The flag sits beside an opening's `operable` without disturbing it. + @Test func operableStaysAlone() throws { + let frame = try #require(MeterFrame(wall: Self.wall)) + let window = PacketMark.opening(.window, id: "w", span: 0 ... 1, bottom: 1, top: 2, operable: true, wall: Self.wall, frame: frame) + let attrs = try #require(try Self.json(window)["attrs"] as? [String: Any]) + #expect(attrs.keys.sorted() == ["operable"]) + } + + /// The written synthetic packet carries an inferred left end and a marked right end, and its + /// manifest validates (`syntheticPacketIsComplete`). Here: the schema reads the flag as a + /// boolean, so a manifest with any other value fails at that mark. + @Test func theSchemaRejectsANonBooleanFlag() throws { + let folder = URL(fileURLWithPath: NSTemporaryDirectory()).appending(path: "hs-provenance-\(UUID())") + let jpegs = URL(fileURLWithPath: NSTemporaryDirectory()).appending(path: "hs-provenance-jpegs-\(UUID())") + try FileManager.default.createDirectory(at: jpegs, withIntermediateDirectories: true) + defer { + try? FileManager.default.removeItem(at: folder) + try? FileManager.default.removeItem(at: jpegs) + } + _ = try SyntheticPacket().write(to: folder, jpegs: jpegs) + let data = try Data(contentsOf: folder.appendingPathComponent("manifest.json")) + let validator = try PacketSchema.validator() + #expect(try validator.validate(data) == []) + var manifest = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any]) + var marks = try #require(manifest["marks"] as? [[String: Any]]) + #expect((marks[1]["attrs"] as? [String: Any])?["inferred"] as? Bool == true && marks[1]["t"] == nil) + marks[1]["attrs"] = ["inferred": "yes"] + manifest["marks"] = marks + let errors = try validator.validate(JSONSerialization.data(withJSONObject: manifest)) + #expect(errors.contains { $0.contains("marks") && $0.contains("inferred") }, "\(errors)") + } + + /// The writer's own check: an inferred end with a time, or the flag on another kind of mark, + /// is refused before anything is written. + @Test func theWriterRefusesAnInferredEndWithATime() throws { + // Times after the synthetic capture's start, so only the rule under test can refuse. + let t = SyntheticPacket.start + 3 + var mark = try Self.end(.inferred) + mark.t = t + let folder = URL(fileURLWithPath: NSTemporaryDirectory()).appending(path: "hs-provenance-\(UUID())") + defer { try? FileManager.default.removeItem(at: folder) } + var writer = try PacketWriter(folder: folder, session: try SyntheticPacket().session) + #expect(throws: PacketError.invalidMark(id: "wall_end_right", reason: "an inferred wall end has no mark time: nobody marked it")) { + try writer.setMarks([mark]) + } + var meter = PacketMark.meter(id: "m") + meter.inferred = true + #expect(throws: PacketError.invalidMark(id: "m", reason: "only a wall end can be inferred")) { try writer.setMarks([meter]) } + #expect(throws: Never.self) { try writer.setMarks([try Self.end(.inferred), try Self.end(.marked(at: t)).renamed("wall_end_left")]) } + } +} + +private extension PacketMark { + func renamed(_ id: String) -> PacketMark { + var mark = self + mark.id = id + return mark + } +} + +// MARK: Shorter bounds + +/// An end the homeowner marks nearer than the end a past-end request cleared bounds the export: +/// nothing seen past it is reported, on either band, and the other side is untouched. +@Suite struct NearerEndBoundsTests { + @Test func aNearerEndClipsWhatIsReportedAndLeavesTheOtherSide() throws { + var map = CoverageMap(wall: standardWall()) + for (i, c) in stride(from: Float(-3), through: 3, by: 0.25).enumerated() { + map.observe(wallCamera(s: c), trackingNormal: true, time: Double(i)) + map.observe(groundCamera(s: c), trackingNormal: true, time: Double(i) + 0.5) + } + map.setEnd(.left, at: -2.5) + map.setEnd(.right, at: 2.5) + let before = try #require(map.wallSeenSpans().map(\.span.upperBound).max()) + #expect(before > 2 && before <= 2.5 + 1e-4) + #expect((map.coveredIntervals(.ground).map(\.upperBound).max() ?? 0) > 2) + // The wall really stops at 1.5 m. + map.setEnd(.right, at: 1.5) + #expect(map.rightEnd == 1.5 && map.leftEnd == -2.5) + for span in map.wallSeenSpans() + map.groundDepthSpans() { + #expect(span.span.upperBound <= 1.5 + 1e-4, "\(span.span) runs past the nearer end") + } + #expect(map.coveredIntervals(.wall).allSatisfy { $0.upperBound <= 1.5 + 1e-4 }) + #expect(map.coveredIntervals(.ground).allSatisfy { $0.upperBound <= 1.5 + 1e-4 }) + // The left side is untouched. + #expect((map.wallSeenSpans().map(\.span.lowerBound).min() ?? 0) < -2) + } + + /// A nearer end keeps the walk's minimum (`WallFrame.minWallLength`), measured from the other + /// end, or from the meter while that side has none, and never moves the other end. + @Test func aNearerEndKeepsTheMinimumWallLength() { + var map = CoverageMap(wall: standardWall()) + map.setEnd(.left, at: -1) + let least = WallFrame.minWallLength + #expect(map.endWouldLeaveTooLittle(.right, at: -1 + least - 0.01)) + #expect(!map.endWouldLeaveTooLittle(.right, at: -1 + least + 0.01)) + // No right end: measured from the meter. + #expect(map.endWouldLeaveTooLittle(.left, at: -(least - 0.01))) + #expect(!map.endWouldLeaveTooLittle(.left, at: -(least + 0.01))) + map.setEnd(.right, at: 0.2) + #expect(map.endWouldLeaveTooLittle(.left, at: -0.1), "a 0.3 m wall") + #expect(!map.endWouldLeaveTooLittle(.left, at: 0.2 - least - 0.01)) + #expect(map.leftEnd == -1 && map.rightEnd == 0.2, "asking moved an end") + } +} diff --git a/ios/HouseScanUITests/EndAimCircleAssertions.swift b/ios/HouseScanUITests/EndAimCircleAssertions.swift new file mode 100644 index 000000000..01c9a1e0a --- /dev/null +++ b/ios/HouseScanUITests/EndAimCircleAssertions.swift @@ -0,0 +1,41 @@ +import XCTest + +/// The circle "Wall ends here" marks at (`endAimCircle`, identifier `aim.circle`), read from the +/// production layout: the app's own view, as VoiceOver sees it, not a test-only overlay. +extension XCTestCase { + /// Checks the circle is the 56 pt reticle drawn in the middle of the window, where the ray + /// "Wall ends here" uses goes through (`ScanEngine.circleEnd`: the middle of the sensor image, + /// which the camera view fills and centres), is wholly on screen, and is under neither the + /// card nor any of `covers`. + /// + /// The card ends at its last part plus the scrim's padding under it (`InstructionCard`): + /// unfolded, the reply (`reply`), 12 pt above the scrim's edge; folded around the circle, + /// Details, 8 pt above it, since the reply has moved to the actions. + @MainActor + func assertEndCircleOpen( + _ app: XCUIApplication, folded: Bool, reply: String, covers: [String], _ context: String, + file: StaticString = #filePath, line: UInt = #line + ) { + func element(_ identifier: String) -> XCUIElement { app.descendants(matching: .any)[identifier].firstMatch } + let window = app.windows.firstMatch.frame + let circle = element("aim.circle") + XCTAssertTrue(circle.waitForExistence(timeout: 5), "\(context): no aiming circle", file: file, line: line) + let frame = circle.frame + XCTAssertEqual(frame.width, 56, accuracy: 1, "\(context): the aiming circle is \(frame.size), not the 56 pt reticle", file: file, line: line) + XCTAssertEqual(frame.height, 56, accuracy: 1, "\(context): the aiming circle is \(frame.size), not the 56 pt reticle", file: file, line: line) + XCTAssertEqual(frame.midX, window.midX, accuracy: 1, "\(context): the circle \(frame) must be the middle of the camera", file: file, line: line) + XCTAssertEqual(frame.midY, window.midY, accuracy: 1, "\(context): the circle \(frame) must be the middle of the camera", file: file, line: line) + XCTAssertTrue(window.contains(frame), "\(context): the circle \(frame) is not wholly on screen", file: file, line: line) + + let parts = (folded ? ["instruction", "instruction.details"] : ["instruction", reply]).map(element).filter(\.exists) + let padding: CGFloat = folded ? 8 : 12 + if let last = parts.map(\.frame.maxY).max() { + XCTAssertGreaterThanOrEqual(frame.minY, last + padding, "\(context): the circle \(frame) is under the card, which ends at \(last + padding)", file: file, line: line) + } + for identifier in covers { + let cover = element(identifier) + guard cover.exists else { continue } + XCTAssertFalse(frame.intersects(cover.frame), "\(context): the circle \(frame) is under \(identifier) \(cover.frame)", file: file, line: line) + } + } +} diff --git a/ios/HouseScanUITests/GapFollowUpFoldUITests.swift b/ios/HouseScanUITests/GapFollowUpFoldUITests.swift new file mode 100644 index 000000000..6be0228b9 --- /dev/null +++ b/ios/HouseScanUITests/GapFollowUpFoldUITests.swift @@ -0,0 +1,130 @@ +import XCTest + +/// A request the finished check sent back ("One more view to finish"), read from the production +/// layout in demo mode. At the largest text size the card folds (`GapRequestScreen.followUpFolds`): +/// the side to aim at leads, the follow-up, the stretch and the check's words are under Details, +/// "I can't get there" sits among the actions, wholly on screen and usable once its lock passes, +/// and the camera between them stays open. At the default size nothing changes. +final class GapFollowUpFoldUITests: XCTestCase { + private static let largestText = ["-UIPreferredContentSizeCategoryName", "UICTContentSizeCategoryAccessibilityXXXL"] + private static let followUp = ["-uiDemo", "-uiDemoFreeze", "-practiceMeter", "NO", "-uiDemoPhase", "gapRequest", "-uiDemoFollowUp"] + /// `CameraChromeLayout.minCameraWindow`: room for the largest aim ring (64 pt radius, 108 + /// percent at its pulse) with a margin, the minimum every folded aiming card keeps. + private static let minCameraWindow: CGFloat = 180 + /// `InstructionCard`'s `replyLock` (600 ms), with room for a slow runner. + private static let lockWait: TimeInterval = 3 + + override func setUp() { + continueAfterFailure = false + } + + @MainActor + private func launch(_ extra: [String]) -> XCUIApplication { + let app = XCUIApplication() + app.launchArguments = Self.followUp + extra + app.launch() + XCTAssertTrue(app.descendants(matching: .any)["screen.gapRequest"].waitForExistence(timeout: 15)) + return app + } + + @MainActor + private func element(_ app: XCUIApplication, _ identifier: String) -> XCUIElement { + app.descendants(matching: .any)[identifier].firstMatch + } + + @MainActor + private func waitUntilEnabled(_ element: XCUIElement) -> Bool { + let enabled = XCTNSPredicateExpectation(predicate: NSPredicate(format: "isEnabled == true"), object: element) + return XCTWaiter().wait(for: [enabled], timeout: Self.lockWait) == .completed + } + + @MainActor + private func attach(_ app: XCUIApplication, _ name: String) { + let shot = XCTAttachment(screenshot: app.screenshot()) + shot.name = name + shot.lifetime = .keepAlways + add(shot) + } + + /// At the default size the card keeps every word and its reply, as before: the follow-up over + /// the stretch's two ends, no Details, and "I can't get there" as the card's trailing pill, + /// under the words, enabled once its lock passes. + @MainActor + func testAtTheDefaultSizeTheCardKeepsItsWordsAndReply() throws { + let app = launch([]) + defer { app.terminate() } + let window = app.windows.firstMatch.frame + let card = element(app, "instruction") + XCTAssertTrue(card.waitForExistence(timeout: 5)) + XCTAssertTrue(card.label.contains("One more view to finish"), card.label) + XCTAssertTrue(card.label.contains("4 ft to 7 ft right of your meter"), card.label) + XCTAssertFalse(element(app, "instruction.details").exists, "the default size folded the card") + let reply = app.buttons["action.skipGap"] + XCTAssertTrue(reply.waitForExistence(timeout: 5)) + XCTAssertTrue(waitUntilEnabled(reply), "I can't get there stayed locked") + XCTAssertTrue(reply.isHittable) + XCTAssertTrue(window.contains(reply.frame), "I can't get there \(reply.frame) isn't wholly on screen") + XCTAssertGreaterThanOrEqual(reply.frame.minY, card.frame.maxY, "the reply must sit under the card's words") + // The card's pill hugs its words at the card's trailing edge; the moved reply is full width. + XCTAssertLessThan(reply.frame.width, window.width * 0.7, "the reply \(reply.frame) left the card") + XCTAssertGreaterThan(reply.frame.midX, window.midX, "the reply \(reply.frame) isn't the card's trailing pill") + XCTAssertTrue(element(app, "action.showResult").exists) + attach(app, "gapRequest-followUp-default") + } + + /// At the largest text size the card folds: a short lead and Details, "I can't get there" + /// wholly on screen below the card and enabled once its lock passes, and at least + /// `minCameraWindow` of camera between them. + @MainActor + func testAtTheLargestSizeTheCardFoldsAndTheReplyStaysOnScreen() throws { + let app = launch(Self.largestText) + defer { app.terminate() } + let window = app.windows.firstMatch.frame + let card = element(app, "instruction") + XCTAssertTrue(card.waitForExistence(timeout: 5)) + XCTAssertTrue(card.label.contains("Show the ground right of your meter"), card.label) + XCTAssertFalse(card.label.contains("One more view to finish"), "the follow-up should be under Details: \(card.label)") + XCTAssertFalse(card.label.contains("4 ft to 7 ft"), "the stretch should be under Details: \(card.label)") + let details = element(app, "instruction.details") + XCTAssertTrue(details.exists, "the folded card has no Details") + // The card's scrim ends 8 pt under Details (`assertEndCircleOpen`). + let cardBottom = max(card.frame.maxY, details.frame.maxY) + 8 + let reply = app.buttons["action.skipGap"] + XCTAssertTrue(reply.waitForExistence(timeout: 5)) + XCTAssertTrue(waitUntilEnabled(reply), "I can't get there stayed locked") + XCTAssertTrue(reply.isHittable, "I can't get there can't be tapped where it is") + XCTAssertTrue(window.contains(reply.frame), "I can't get there \(reply.frame) isn't wholly on screen in \(window)") + XCTAssertGreaterThanOrEqual(reply.frame.minY, cardBottom, "I can't get there \(reply.frame) is under the card, which ends at \(cardBottom)") + XCTAssertGreaterThanOrEqual( + reply.frame.minY - cardBottom, Self.minCameraWindow, "only \(reply.frame.minY - cardBottom) pt of camera between the card and the actions") + XCTAssertEqual(reply.label, "I can't get there") + attach(app, "gapRequest-followUp-AX5-folded") + } + + /// Details holds every word the unfolded card had: the follow-up, the stretch's two ends and + /// the check's own words. "I can't get there" stays reachable below the open card. + @MainActor + func testAtTheLargestSizeDetailsHoldsEveryWord() throws { + let app = launch(Self.largestText) + defer { app.terminate() } + let details = element(app, "instruction.details") + XCTAssertTrue(details.waitForExistence(timeout: 5)) + details.tap() + let words = element(app, "instruction.detail") + XCTAssertTrue(words.waitForExistence(timeout: 5), "Details didn't open") + for text in ["One more view to finish", "Show the ground from 4 ft to 7 ft right of your meter", "A second look at the ground"] { + XCTAssertTrue(words.label.contains(text), "Details lacks \"\(text)\": \(words.label)") + } + attach(app, "gapRequest-followUp-AX5-details") + let reply = app.buttons["action.skipGap"] + XCTAssertTrue(reply.exists) + // Scroll until the whole button is in view, not just until it is hittable: a sliver at the + // bottom edge is already hittable (local run ax5-fold-07be16e4-ui stopped there). + let window = app.windows.firstMatch.frame + for _ in 0..<6 where !(reply.isHittable && window.contains(reply.frame)) { app.swipeUp(velocity: .slow) } + XCTAssertTrue(reply.isHittable, "I can't get there can't be reached with Details open") + XCTAssertTrue(window.contains(reply.frame), "I can't get there \(reply.frame) isn't wholly on screen with Details open") + XCTAssertGreaterThanOrEqual(reply.frame.minY, words.frame.maxY, "I can't get there is under the open card") + attach(app, "gapRequest-followUp-AX5-details-reply") + } +} diff --git a/ios/HouseScanUITests/PastEndRecoveryUITests.swift b/ios/HouseScanUITests/PastEndRecoveryUITests.swift new file mode 100644 index 000000000..14d2250ab --- /dev/null +++ b/ios/HouseScanUITests/PastEndRecoveryUITests.swift @@ -0,0 +1,545 @@ +import CryptoKit +import Foundation +import XCTest + +/// B-12: a past_end request (the walk stopped at an end, and the check asks to walk on past it) +/// offers the walk's own "Wall ends here" for that end, with the circle, the refusal when the +/// circle isn't on the wall, and the end question; "I can't get there" stays the honest decline. +/// +/// Two kinds of test, which prove different things. The frozen demo (`-uiDemoGap pastEnd`) shows +/// the screen's states for the captures and checks what is offered, at the default and the +/// largest text size. The replay runs the real engine with the test answering its uploads +/// (`-answersFromGate`), and checks what marking again or declining does to the scene the next +/// upload sends and to the packet's marks and guidance. There the autopilot marks the end with a +/// tap where the replay shows the place (`-autopilotMarkPastEnd`), through the same `EndAim` check +/// as the circle, but no camera is aimed: aiming a phone at a real wall is not tested here. +final class PastEndRecoveryUITests: XCTestCase { + private static let largestText = ["-UIPreferredContentSizeCategoryName", "UICTContentSizeCategoryAccessibilityXXXL"] + + override func setUp() { + continueAfterFailure = false + } + + // MARK: The screen (demo engine) + + @MainActor + func testAPastEndRequestOffersWallEndsHere() throws { + try offersWallEndsHere(textSize: [], name: "gapRequest-pastEnd") + } + + @MainActor + func testAPastEndRequestOffersWallEndsHereAtLargestTextSize() throws { + try offersWallEndsHere(textSize: Self.largestText, name: "gapRequest-pastEnd-AX5") + } + + @MainActor + func testAPastEndRequestOnTheLeftKeepsTheCircleOpenAtLargestTextSize() throws { + let app = launch(["-uiDemoGap", "pastEndLeft"] + Self.largestText) + let card = label(app, "instruction") + XCTAssertTrue(card.contains("Keep walking left"), "the folded lead names the side: \(card)") + let mark = app.buttons["action.markEnd"] + XCTAssertTrue(mark.waitForExistence(timeout: 5)) + scrollIntoView(mark, in: app) + XCTAssertTrue(mark.isHittable, "Wall ends here can't be reached") + assertEndCircleOpen(app, folded: true, reply: "action.skipGap", covers: Self.endCovers, "left past-end, AX5") + snap(app, "gapRequest-pastEndLeft-AX5") + } + + @MainActor + private func offersWallEndsHere(textSize: [String], name: String) throws { + let folded = !textSize.isEmpty + let app = launch(["-uiDemoGap", "pastEnd"] + textSize) + let card = label(app, "instruction") + if folded { + // Folded around the circle: walking on leads; the request's words, the eyebrow and + // the way to mark the end are under Details. + XCTAssertTrue(card.contains("Keep walking right"), "card reads: \(card)") + XCTAssertFalse(card.contains("If the wall stops sooner"), "the how-to words must fold under Details: \(card)") + XCTAssertFalse(card.contains("One more view to finish"), "the eyebrow must fold under Details: \(card)") + } else { + XCTAssertTrue(card.contains("One more view to finish"), "card reads: \(card)") + XCTAssertTrue(card.contains("Keep walking past the right end"), "the request's own words: \(card)") + XCTAssertTrue(card.contains("If the wall stops sooner, aim where it stops and tap Wall ends here."), "card reads: \(card)") + } + XCTAssertTrue(element(app, "action.skipGap").exists, "\"I can't get there\" stays the decline") + let mark = app.buttons["action.markEnd"] + XCTAssertTrue(mark.waitForExistence(timeout: 5), "a past-end request must offer Wall ends here") + XCTAssertEqual(mark.label, "Wall ends here") + scrollIntoView(mark, in: app) + XCTAssertTrue(mark.isHittable, "Wall ends here can't be reached") + // Where "Wall ends here" can be pressed, the circle it marks at is in view. + assertEndCircleOpen(app, folded: folded, reply: "action.skipGap", covers: Self.endCovers, name) + snap(app, name) + guard folded else { return } + + // Details opens every word of the unfolded card, and closes back to the open circle. + scrollToTop(app) + let details = app.buttons["instruction.details"] + XCTAssertTrue(details.waitForExistence(timeout: 5), "the folded card needs Details") + tapWhenReady(details) + let detail = element(app, "instruction.detail") + XCTAssertTrue(detail.waitForExistence(timeout: 5), "Details must open the folded words") + for words in ["One more view to finish.", "Show the ground", "Keep walking past the right end", "If the wall stops sooner, aim where it stops and tap Wall ends here."] { + XCTAssertTrue(detail.label.contains(words), "Details must hold \"\(words)\": \(detail.label)") + } + snap(app, "\(name)-details") + tapWhenReady(details) + XCTAssertTrue(waitForAbsence(detail), "Details must close again") + scrollIntoView(mark, in: app) + XCTAssertTrue(mark.isHittable) + assertEndCircleOpen(app, folded: true, reply: "action.skipGap", covers: Self.endCovers, "\(name), Details closed") + // The decline moved from the card to the actions, and stays within reach. + let decline = app.buttons["action.skipGap"] + scrollIntoView(decline, in: app) + XCTAssertTrue(decline.isHittable, "I can't get there can't be reached") + XCTAssertEqual(decline.label, "I can't get there") + XCTAssertGreaterThan(decline.frame.minY, mark.frame.maxY - 0.5, "the decline sits under Wall ends here") + } + + /// Coaching riding along leads the folded card in a few words, and its whole note opens + /// Details: under the lead, the dark note's lines reached the circle. + @MainActor + func testCoachingOnAFoldedPastEndCardKeepsTheCircleOpen() throws { + let app = launch(["-uiDemoGap", "pastEnd", "-uiDemoCoaching", "tooDark"] + Self.largestText) + let card = label(app, "instruction") + XCTAssertTrue(card.contains("Try your flashlight"), "card reads: \(card)") + XCTAssertFalse(card.contains("Try your phone's flashlight"), "the coaching's note must fold under Details: \(card)") + let mark = app.buttons["action.markEnd"] + scrollIntoView(mark, in: app) + XCTAssertTrue(mark.isHittable) + assertEndCircleOpen(app, folded: true, reply: "action.skipGap", covers: Self.endCovers, "coached past-end, AX5") + snap(app, "gapRequest-pastEnd-tooDark-AX5") + scrollToTop(app) + tapWhenReady(app.buttons["instruction.details"]) + let detail = element(app, "instruction.detail") + XCTAssertTrue(detail.waitForExistence(timeout: 5)) + for words in ["It's dark here. Try your phone's flashlight", "Keep walking past the right end", "If the wall stops sooner"] { + XCTAssertTrue(detail.label.contains(words), "Details must hold \"\(words)\": \(detail.label)") + } + } + + /// The circle on the other side of the meter, on either side: the folded card leads with + /// which end to face, and the circle stays open. + @MainActor + func testTheOtherSideRefusalFoldsOnBothSides() throws { + for (gap, asked, landed) in [("pastEnd", "right", "left"), ("pastEndLeft", "left", "right")] { + let app = launch(["-uiDemoGap", gap, "-uiDemoEndMarkOtherSide"] + Self.largestText) + let card = label(app, "instruction") + XCTAssertTrue(card.contains("Face the \(asked) end"), "\(gap): card reads: \(card)") + let mark = app.buttons["action.markEnd"] + scrollIntoView(mark, in: app) + XCTAssertTrue(mark.isHittable) + assertEndCircleOpen(app, folded: true, reply: "action.skipGap", covers: Self.endCovers, "\(gap) other side, AX5") + snap(app, "gapRequest-\(gap)-otherSide-AX5") + scrollToTop(app) + tapWhenReady(app.buttons["instruction.details"]) + let detail = element(app, "instruction.detail") + XCTAssertTrue(detail.waitForExistence(timeout: 5)) + XCTAssertTrue(detail.label.contains("That's the \(landed) side of your meter. Turn to the \(asked) end"), "\(gap): Details reads: \(detail.label)") + app.terminate() + } + } + + /// What may cover the circle: the actions and the card's reply. The card itself is checked by + /// its bottom edge (`assertEndCircleOpen`). + private static let endCovers = ["action.markEnd", "action.skipGap", "action.showResult", "action.endCorner", "action.endBlocked", "action.endEnds"] + + /// The circle off the wall: the card says why and what to do, the decline stays, and pressing + /// again with the circle still off the wall refuses again rather than marking anything. + @MainActor + func testARefusedEndOnARequestSaysWhatToDo() throws { + try refusal(textSize: [], name: "gapRequest-pastEnd-refused") + } + + @MainActor + func testARefusedEndOnARequestSaysWhatToDoAtLargestTextSize() throws { + try refusal(textSize: Self.largestText, name: "gapRequest-pastEnd-refused-AX5") + } + + @MainActor + private func refusal(textSize: [String], name: String) throws { + let folded = !textSize.isEmpty + let app = launch(["-uiDemoGap", "pastEnd", "-uiDemoEndMarkRefusal"] + textSize) + // Folded, the correction leads and the reason goes under Details. + let says = folded ? "Aim at the wall" : "The circle isn't on the wall" + var card = label(app, "instruction") + XCTAssertTrue(card.contains(says), "card reads: \(card)") + if folded { + XCTAssertFalse(card.contains("The circle isn't on the wall"), "the reason must fold under Details: \(card)") + } else { + XCTAssertTrue(card.contains("Aim it at the wall where it stops or turns"), "card reads: \(card)") + } + XCTAssertTrue(element(app, "action.skipGap").exists) + let mark = app.buttons["action.markEnd"] + scrollIntoView(mark, in: app) + XCTAssertTrue(mark.isHittable) + assertEndCircleOpen(app, folded: folded, reply: "action.skipGap", covers: Self.endCovers, name) + snap(app, name) + tapWhenReady(mark) + card = label(app, "instruction") + XCTAssertTrue(card.contains(says), "a second press marked something: \(card)") + XCTAssertFalse(element(app, "action.endBlocked").exists, "no end was marked, so nothing to ask") + guard folded else { return } + scrollToTop(app) + tapWhenReady(app.buttons["instruction.details"]) + let detail = element(app, "instruction.detail") + XCTAssertTrue(detail.waitForExistence(timeout: 5)) + XCTAssertTrue(detail.label.contains("The circle isn't on the wall. Aim it at the wall where it stops or turns, then tap Wall ends here."), "Details reads: \(detail.label)") + } + + /// "Wall ends here" asks the walk's end question; it replaces the request and its decline + /// until answered, and "Something blocks it" settles the request, which goes on to the check. + @MainActor + func testMarkingTheEndAsksWhatIsThereAndSettlesTheRequest() throws { + try question(textSize: [], name: "gapRequest-pastEnd-question") + } + + @MainActor + func testMarkingTheEndAsksWhatIsThereAtLargestTextSize() throws { + try question(textSize: Self.largestText, name: "gapRequest-pastEnd-question-AX5") + } + + @MainActor + private func question(textSize: [String], name: String) throws { + let app = launch(["-uiDemoGap", "pastEnd"] + textSize) + let mark = app.buttons["action.markEnd"] + scrollIntoView(mark, in: app) + tapWhenReady(mark) + let asked = app.descendants(matching: .any) + .matching(NSPredicate(format: "identifier == 'instruction' AND label CONTAINS %@", "What's at the right end?")).firstMatch + XCTAssertTrue(asked.waitForExistence(timeout: 5), "marking the end must ask what is there") + XCTAssertFalse(element(app, "action.skipGap").exists, "the question replaces the decline until answered") + XCTAssertFalse(element(app, "action.markEnd").exists) + for answer in ["action.endCorner", "action.endBlocked", "action.endEnds"] { + XCTAssertTrue(app.buttons[answer].exists, "\(answer) missing") + } + // In their order, and the question needs no circle. + let tops = ["action.endCorner", "action.endBlocked", "action.endEnds"].map { app.buttons[$0].frame.minY } + XCTAssertEqual(tops, tops.sorted(), "the answers moved out of order: \(tops)") + XCTAssertFalse(element(app, "aim.circle").exists, "the question shows no circle") + snap(app, name) + let blocked = app.buttons["action.endBlocked"] + scrollIntoView(blocked, in: app) + tapWhenReady(blocked) + XCTAssertTrue(element(app, "screen.uploading").waitForExistence(timeout: 10), "the answer must settle the request") + } + + /// "It turns a corner" can't settle a past-end request, which can't follow the corner: no + /// check mark, no "Got it, thanks", straight on to the check, as "I can't get there" goes. + @MainActor + func testACornerAtThePastEndGoesOnWithoutSettlingIt() throws { + let app = launch(["-uiDemoGap", "pastEnd"]) + let mark = app.buttons["action.markEnd"] + scrollIntoView(mark, in: app) + tapWhenReady(mark) + let corner = app.buttons["action.endCorner"] + XCTAssertTrue(corner.waitForExistence(timeout: 5)) + scrollIntoView(corner, in: app) + tapWhenReady(corner) + XCTAssertFalse(label(app, "instruction").contains("Got it, thanks"), "a corner settled the request") + XCTAssertTrue(element(app, "screen.uploading").waitForExistence(timeout: 10), "the corner must move on to the check") + } + + /// Only a past-end request offers the end: the phone's own request and the server's other + /// kinds keep their own controls. + @MainActor + func testOtherRequestsDoNotOfferTheEnd() throws { + for gap in [[], ["-uiDemoGap", "groundOut"], ["-uiDemoGap", "walkOut"], ["-uiDemoGap", "overhead"]] { + let app = launch(gap) + XCTAssertFalse(app.buttons["action.markEnd"].exists, "\(gap) offers Wall ends here") + XCTAssertFalse(label(app, "instruction").contains("Wall ends here"), "\(gap) mentions Wall ends here") + app.terminate() + } + } + + // MARK: The real engine (replay, answer gate) + + /// The check asks to walk past the left end. The autopilot marks it again nearer than before + /// and answers "Something blocks it". The next upload's scene has the shorter left bound as a + /// limit; the request's guidance is met and the walk's own "mark the end" stays as it was; the + /// packet's left end is the homeowner's mark, with its time and no inferred flag. + @MainActor + func testMarkingThePastEndAgainShortensTheWallOnTheRealEngine() throws { + let run = try runReplay(markPastEnd: true) + let mark = try XCTUnwrap(run.pastEndMark, "the autopilot left no past-end-mark.json") + let marked = try XCTUnwrap(mark["s"] as? Double), cleared = try XCTUnwrap(mark["cleared_s"] as? Double) + XCTAssertEqual(mark["side"] as? String, "left") + XCTAssertGreaterThan(marked, cleared, "the end was meant to come nearer than the cleared one") + + let first = try XCTUnwrap(run.scenes.first), last = try XCTUnwrap(run.scenes.last) + XCTAssertGreaterThanOrEqual(run.scenes.count, 2) + XCTAssertEqual(try Self.endKind(last, "left"), "limit") + XCTAssertEqual(try Self.leftEndFeet(last), marked * Self.feetPerMeter, accuracy: 0.2, "the scene's left bound is the mark") + XCTAssertGreaterThan(try Self.leftEndFeet(last), try Self.leftEndFeet(first) + 1, "the left bound came nearer") + XCTAssertEqual(try Self.endKind(last, "right"), try Self.endKind(first, "right"), "the right side changed") + // Wall seen past the nearer end is not reported. Ground past it is, now that it is a limit + // (server README, "Ends and corners"): that is ground seen, not wall claimed. + XCTAssertTrue(try Self.observedLeftEdges(last, band: "wall").allSatisfy { $0 >= marked * Self.feetPerMeter - 0.05 }, "wall reported past the nearer end") + + let packet = try XCTUnwrap(run.packet, "no packet-marks-guidance.json") + let left = try XCTUnwrap(Self.wallEnd(packet, "left")) + XCTAssertNotNil(left["t"], "a homeowner's mark keeps its time") + XCTAssertNil((left["attrs"] as? [String: Any])?["inferred"], "a homeowner's mark is not inferred") + XCTAssertEqual(left["end_kind"] as? String, "limit") + let pastEnd = Self.guidance(packet, kind: "gap_past_end") + XCTAssertEqual(pastEnd.last?["outcome"] as? String, "met", "\(pastEnd)") + attach(run) + } + + /// The check asks to walk past the left end; the replay can't show it, and the autopilot says + /// "I can't get there". The next upload's scene has the left end back where it was, with its + /// kind, and claims nothing past it; the request is cannot_reach, and the packet's left end + /// keeps the source and time it had. + @MainActor + func testDecliningThePastEndRestoresTheEndOnTheRealEngine() throws { + let run = try runReplay(markPastEnd: false) + XCTAssertNil(run.pastEndMark) + XCTAssertGreaterThanOrEqual(run.scenes.count, 2) + let first = try XCTUnwrap(run.scenes.first), last = try XCTUnwrap(run.scenes.last) + XCTAssertEqual(try Self.leftEndFeet(last), try Self.leftEndFeet(first), accuracy: 0.01, "the left end moved") + XCTAssertEqual(try Self.endKind(last, "left"), try Self.endKind(first, "left")) + let bound = try Self.leftEndFeet(last) + // Past an unexplored end nothing is reported; past a limit only ground is (the autopilot's + // fallback marks a limit when no frame shows the end). + let pastBand: String? = try Self.endKind(last, "left") == "limit" ? "wall" : nil + XCTAssertTrue(try Self.observedLeftEdges(last, band: pastBand).allSatisfy { $0 >= bound - 0.05 }, "coverage claimed past the restored end") + + let packet = try XCTUnwrap(run.packet, "no packet-marks-guidance.json") + let left = try XCTUnwrap(Self.wallEnd(packet, "left")) + let firstPacketLeft = try XCTUnwrap(run.firstPacketLeftEnd, "the first upload's packet had no left end") + XCTAssertEqual(left["t"] as? Double, firstPacketLeft["t"] as? Double, "the restored end lost or changed its time") + XCTAssertEqual((left["attrs"] as? [String: Any])?["inferred"] as? Bool, (firstPacketLeft["attrs"] as? [String: Any])?["inferred"] as? Bool) + let pastEnd = Self.guidance(packet, kind: "gap_past_end") + XCTAssertEqual(pastEnd.last?["outcome"] as? String, "cannot_reach", "\(pastEnd)") + attach(run) + } + + /// The check asks to walk past the left end. The autopilot marks it again nearer and answers + /// "It turns a corner", which the request can't follow. The next answer asks again to walk + /// past the left end, now from the corner, and also for ground on the right. The repeat is not + /// raised; the ground request is. The corner stays the homeowner's unexplored end, nothing past + /// it is reported, and the request closed cannot_reach, not met. + @MainActor + func testACornerAtThePastEndIsNotAskedForAgainOnTheRealEngine() throws { + let run = try runReplay(markPastEnd: true, corner: true) + let mark = try XCTUnwrap(run.pastEndMark, "the autopilot left no past-end-mark.json") + let marked = try XCTUnwrap(mark["s"] as? Double), cleared = try XCTUnwrap(mark["cleared_s"] as? Double) + XCTAssertEqual(mark["side"] as? String, "left") + XCTAssertGreaterThan(marked, cleared, "the end was meant to come nearer than the cleared one") + + // The first upload, one after the corner, one after the ground request. + XCTAssertGreaterThanOrEqual(run.scenes.count, 3, run.log) + let last = try XCTUnwrap(run.scenes.last) + XCTAssertEqual(try Self.endKind(last, "left"), "unexplored") + XCTAssertEqual(try Self.leftEndFeet(last), marked * Self.feetPerMeter, accuracy: 0.2, "the scene's left bound is the corner") + XCTAssertTrue(try Self.observedLeftEdges(last, band: nil).allSatisfy { $0 >= marked * Self.feetPerMeter - 0.05 }, "coverage claimed past the corner") + + let packet = try XCTUnwrap(run.packet, "no packet-marks-guidance.json") + let left = try XCTUnwrap(Self.wallEnd(packet, "left")) + XCTAssertNotNil(left["t"], "the corner is the homeowner's mark, with its time") + XCTAssertNil((left["attrs"] as? [String: Any])?["inferred"], "the corner is not inferred") + XCTAssertEqual(left["end_kind"] as? String, "unexplored") + let pastEnd = Self.guidance(packet, kind: "gap_past_end") + XCTAssertEqual(pastEnd.count, 1, "the repeated past_end was raised again: \(pastEnd)") + XCTAssertEqual(pastEnd.first?["outcome"] as? String, "cannot_reach", "\(pastEnd)") + // The phone's own requests are gap_band too; the server's are the ones in its answers. + let serverBand = Self.guidance(packet, kind: "gap_band").filter { $0["origin"] as? String == "server" } + XCTAssertEqual(serverBand.count, 1, "the ground request in the same answer was not raised: \(serverBand)") + attach(run) + } + + private struct ReplayRun { + var scenes: [[String: Any]] + var packet: [String: Any]? + var firstPacketLeftEnd: [String: Any]? + var pastEndMark: [String: Any]? + var log: String + } + + /// Plays the synthetic wall with the test as the placement server. The first answer is the + /// bundled sample, whose missing evidence asks to walk past the left end; later answers drop + /// that item, so the request is raised once. Returns every scene uploaded, in order, and the + /// packet projection and mark record the autopilot leaves at the result. + /// + /// With `corner`, the autopilot answers "It turns a corner", and the second answer asks again + /// to walk past the left end together with the sample's ground request on the right. + @MainActor + private func runReplay(markPastEnd: Bool, corner: Bool = false) throws -> ReplayRun { + let sample = String(decoding: try Data(contentsOf: UploadRecoveryUITests.sampleResult), as: UTF8.self) + let zeros = String(repeating: "0", count: 64) + XCTAssertTrue(sample.contains("\"input_sha256\": \"\(zeros)\"")) + // The first answer asks only to walk past the left end, so that request is the one raised; + // the sample's ground request ahead of it would take the first upload. Later answers ask + // for nothing, so the request is raised once. + var answer = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(sample.utf8)) as? [String: Any]) + let missing = (answer["missing_evidence"] as? [[String: Any]]) ?? [] + let pastEnd = missing.filter { $0["kind"] as? String == "past_end" && $0["side"] as? String == "left" } + XCTAssertEqual(pastEnd.count, 1, "the sample no longer asks to walk past the left end") + answer["missing_evidence"] = pastEnd + let onlyPastEnd = String(decoding: try JSONSerialization.data(withJSONObject: answer, options: [.sortedKeys]), as: UTF8.self) + let ground = missing.filter { $0["kind"] as? String == "band" && $0["band"] as? String == "ground" } + XCTAssertEqual(ground.count, 1, "the sample no longer asks for ground") + answer["missing_evidence"] = pastEnd + ground + let pastEndAgain = String(decoding: try JSONSerialization.data(withJSONObject: answer, options: [.sortedKeys]), as: UTF8.self) + answer["missing_evidence"] = [[String: Any]]() + let nothingMissing = String(decoding: try JSONSerialization.data(withJSONObject: answer, options: [.sortedKeys]), as: UTF8.self) + XCTAssertTrue(onlyPastEnd.contains("\"input_sha256\":\"\(zeros)\""), "the binding placeholder moved") + + let files = FileManager.default + let gate = URL(fileURLWithPath: NSTemporaryDirectory()).appending(path: "housescan-gate-\(UUID().uuidString)", directoryHint: .isDirectory) + try files.createDirectory(at: gate, withIntermediateDirectories: true) + defer { try? files.removeItem(at: gate) } + // Every screen up to the result is let through; the result's stays shut, so the autopilot + // stops there with its files written. + for phase in ["onboarding", "findMeter", "meterCloseUp", "wallWalk", "markFeatures", "gapRequest", "uploading", "spotConfirm"] { + try Data().write(to: gate.appending(path: phase)) + } + let server = GateServer(gate: gate) { body, index in + let template = switch index { + case 0: onlyPastEnd + case 1 where corner: pastEndAgain + default: nothingMissing + } + return Data(template.replacingOccurrences(of: zeros, with: UploadRecoveryUITests.sha256(body)).utf8) + } + defer { server.stop() } + + let app = XCUIApplication() + app.launchArguments = [ + "-replay", FullFlowUITests.fixture, "-autopilot", "-autopilotHold", "1.0", "-autopilotGate", gate.path, + "-practiceMeter", "NO", "-serverURL", "http://placement.invalid", "-answersFromGate", + ] + (markPastEnd ? ["-autopilotMarkPastEnd"] : []) + (corner ? ["-autopilotPastEndCorner"] : []) + app.launch() + defer { app.terminate() } + let any = app.descendants(matching: .any) + XCTAssertTrue(any["screen.result"].waitForExistence(timeout: 600), "the flow never reached the result") + let sceneFile = gate.appending(path: "scene.json") + let deadline = Date().addingTimeInterval(30) + while !files.fileExists(atPath: sceneFile.path), Date() < deadline { Thread.sleep(forTimeInterval: 0.5) } + let requests = server.requests.filter { $0.target == "POST /v1/placements" } + let scenes = try requests.map { try XCTUnwrap(JSONSerialization.jsonObject(with: $0.body) as? [String: Any]) } + func json(_ name: String) -> [String: Any]? { + (try? Data(contentsOf: gate.appending(path: name))).flatMap { try? JSONSerialization.jsonObject(with: $0) as? [String: Any] } + } + // The packet of the upload that raised the past-end request, which the autopilot leaves as + // `packet-first.json` when the request shows: its left end is the walk's own. + let first = json("packet-first.json").flatMap { Self.wallEnd($0, "left") } + return ReplayRun( + scenes: scenes, packet: json("packet-marks-guidance.json"), firstPacketLeftEnd: first, + pastEndMark: json("past-end-mark.json"), log: requests.map(\.target).joined(separator: "\n")) + } + + // MARK: Reading scenes and packets + + private static let feetPerMeter = 3.280839895 + + /// The left end of the first wall's baseline, in feet along it from the meter (negative left). + private static func leftEndFeet(_ scene: [String: Any]) throws -> Double { + let walls = try XCTUnwrap(scene["walls"] as? [[String: Any]]) + let baseline = try XCTUnwrap(walls.first?["baseline"] as? [[Double]]) + let meter = try XCTUnwrap((scene["meter"] as? [String: Any])?["pos"] as? [Double]) + let first = try XCTUnwrap(baseline.first), last = try XCTUnwrap(baseline.last) + let length = hypot(last[0] - first[0], last[1] - first[1]) + return ((first[0] - meter[0]) * (last[0] - first[0]) + (first[1] - meter[2]) * (last[1] - first[1])) / length + } + + private static func endKind(_ scene: [String: Any], _ side: String) throws -> String? { + let coverage = try XCTUnwrap(scene["coverage"] as? [String: Any]) + return (coverage["ends"] as? [String: [String: String]])?[side]?["kind"] + } + + /// The lower edge of every observed span of `band` (every band when nil), in feet. + private static func observedLeftEdges(_ scene: [String: Any], band: String?) throws -> [Double] { + let coverage = try XCTUnwrap(scene["coverage"] as? [String: Any]) + let observed = ((coverage["observed"] as? [[String: Any]]) ?? []).filter { band == nil || $0["band"] as? String == band } + return observed.compactMap { ($0["span_ft"] as? [Double]).map { min($0[0], $0[1]) } } + } + + private static func wallEnd(_ packet: [String: Any], _ side: String) -> [String: Any]? { + ((packet["marks"] as? [[String: Any]]) ?? []).first { $0["kind"] as? String == "wall_end" && $0["side"] as? String == side } + } + + private static func guidance(_ packet: [String: Any], kind: String) -> [[String: Any]] { + ((packet["guidance"] as? [[String: Any]]) ?? []).filter { $0["kind"] as? String == kind } + } + + // MARK: Helpers + + @MainActor + private func attach(_ run: ReplayRun) { + let text = "uploads: \(run.scenes.count)\npast-end mark: \(run.pastEndMark ?? [:])\npacket: \(run.packet ?? [:])" + let attachment = XCTAttachment(string: text) + attachment.name = "pastEnd-replay" + attachment.lifetime = .keepAlways + add(attachment) + } + + @MainActor + private func launch(_ arguments: [String]) -> XCUIApplication { + let app = XCUIApplication() + app.launchArguments = ["-practiceMeter", "NO", "-uiDemo", "-uiDemoFreeze", "-uiDemoPhase", "gapRequest"] + arguments + app.launch() + XCTAssertTrue(element(app, "screen.gapRequest").waitForExistence(timeout: 15), "screen.gapRequest never appeared") + XCTAssertTrue(element(app, "instruction").waitForExistence(timeout: 5)) + return app + } + + /// Scrolls in measured steps until the whole element is in the window: at the largest text + /// sizes the actions sit below the card, reached by scrolling. + @MainActor + private func scrollIntoView(_ target: XCUIElement, in app: XCUIApplication) { + let window = app.windows.firstMatch.frame + let scroll = app.scrollViews.firstMatch + guard scroll.exists else { return } + let start = scroll.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.6)) + for _ in 0..<12 where !window.contains(target.frame) { + let frame = target.frame + let shift: CGFloat = frame.maxY > window.maxY + ? -min(frame.maxY - window.maxY + 24, window.height / 3) + : min(window.minY - frame.minY + 24, window.height / 3) + start.press(forDuration: 0.05, thenDragTo: start.withOffset(CGVector(dx: 0, dy: shift))) + } + } + + /// Drags the screen down until its top shows. + @MainActor + private func scrollToTop(_ app: XCUIApplication) { + let scroll = app.scrollViews.firstMatch + guard scroll.exists else { return } + let start = scroll.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.3)) + for _ in 0..<4 { + start.press(forDuration: 0.05, thenDragTo: start.withOffset(CGVector(dx: 0, dy: app.windows.firstMatch.frame.height * 0.4))) + } + } + + @MainActor + private func waitForAbsence(_ target: XCUIElement) -> Bool { + let gone = XCTNSPredicateExpectation(predicate: NSPredicate(format: "exists == false"), object: target) + return XCTWaiter().wait(for: [gone], timeout: 5) == .completed + } + + @MainActor + private func snap(_ app: XCUIApplication, _ name: String) { + let shot = XCTAttachment(screenshot: app.screenshot()) + shot.name = name + shot.lifetime = .keepAlways + add(shot) + } + + @MainActor + private func element(_ app: XCUIApplication, _ identifier: String) -> XCUIElement { + app.descendants(matching: .any)[identifier].firstMatch + } + + @MainActor + private func label(_ app: XCUIApplication, _ identifier: String) -> String { + ElementRead.snapshot(element(app, identifier))?.label ?? "" + } + + @MainActor + private func tapWhenReady(_ target: XCUIElement) { + let ready = XCTNSPredicateExpectation(predicate: NSPredicate(format: "isEnabled == true AND isHittable == true"), object: target) + XCTAssertEqual(XCTWaiter().wait(for: [ready], timeout: 10), .completed, "\(target) never took taps") + target.tap() + } +} diff --git a/ios/HouseScanUITests/PhotoProcessingUITests.swift b/ios/HouseScanUITests/PhotoProcessingUITests.swift new file mode 100644 index 000000000..3729028a7 --- /dev/null +++ b/ios/HouseScanUITests/PhotoProcessingUITests.swift @@ -0,0 +1,454 @@ +import XCTest + +/// Photo processing (beta) in the running app. +/// +/// The real-engine tests run the synthetic replay with the autopilot and the DEBUG capture fixture +/// (`-photoProcessingFixture`), which answers the capture API inside the app; its routes land in +/// `photo-transport.log` in the gate folder. A replay records no motion and only about 2 poses a +/// second, so its 0.4 packet fails the phone's own checks: these runs exercise the choice, consent, +/// sending during the walk and the end the phone reaches, not a service answer. Every answer the +/// service can give is driven end to end in HouseScanKit's `PhotoProcessingControllerTests`, and +/// rendered here from demo mode. +/// +/// `-processingBackend` keeps the choice in memory for the run, so nothing here changes the stored +/// setting the Legacy tests run with. +final class PhotoProcessingUITests: XCTestCase { + private static let largestText = ["-UIPreferredContentSizeCategoryName", "UICTContentSizeCategoryAccessibilityXXXL"] + + override func setUp() { + continueAfterFailure = false + } + + // MARK: The real engine + + /// Photo processing chosen in Developer options before the scan, a yes, photos sent during the + /// walk, and the scan ends on the processing screen, never on the Legacy upload. A change of + /// choice on that screen leaves this scan on photo processing; after Start over the next scan + /// is Legacy, asks nothing and sends nothing. + @MainActor + func testTheChoiceHoldsForItsScanAndTheNextScanUsesTheNewOne() throws { + let run = try Run.launch(backend: "legacy", fixture: "candidate", holding: ["onboarding"]) + defer { run.finish() } + run.waitFor("onboarding") + run.choose("photoProcessing", footer: "Applies to the next scan you start.", shot: "selection") + try run.open("onboarding") + + run.waitFor("findMeter") + XCTAssertTrue(run.send.waitForExistence(timeout: 20), "no photo question before the meter was marked") + run.app.buttons["photoConsent.send"].tap() + XCTAssertTrue(run.send.waitForNonExistence(timeout: 10)) + run.waitFor("processing", timeout: 300) + XCTAssertFalse(run.app.descendants(matching: .any)["screen.uploading"].exists) + let ended = run.state("notPrepared", timeout: 60) + XCTAssertTrue(ended.exists, "the replay's packet didn't end as not prepared") + run.attach("processing-notPrepared") + let routes = run.routes() + XCTAssertEqual(routes.filter { $0 == "POST captures" }.count, 1, "\(routes)") + XCTAssertTrue(routes.contains("PUT upload"), "no photo went up during the walk: \(routes)") + XCTAssertFalse(routes.contains("POST captures/finalize"), "a packet the phone refused was finalized: \(routes)") + XCTAssertFalse(FileManager.default.fileExists(atPath: run.gate.appending(path: "uploading.held").path), "the Legacy upload ran") + + // Changing the choice mid-scan: this scan stays on photo processing. + run.choose("legacy", footer: "This scan uses Photo processing (beta). A change applies to the next scan.", shot: "selection-midScan") + XCTAssertTrue(run.state("notPrepared", timeout: 5).exists, "the scan left photo processing when the choice changed") + try run.open("processing") + + run.tap("action.startOver") + run.waitFor("onboarding") + XCTAssertEqual(run.app.buttons["action.developerOptions"].label, "Developer options") + if run.app.buttons["action.onboardingSkip"].waitForExistence(timeout: 5) { run.app.buttons["action.onboardingSkip"].tap() } + run.tap("action.finishOnboarding") + run.waitFor("findMeter") + XCTAssertFalse(run.send.waitForExistence(timeout: 5), "a Legacy scan asked to send photos") + Thread.sleep(forTimeInterval: 2) + XCTAssertEqual(run.routes(), routes, "the next scan sent something to photo processing") + } + + /// A no sends nothing for the whole scan, and the scan ends saying so. + @MainActor + func testANoSendsNothingAndEndsSayingSo() throws { + let run = try Run.launch(backend: "photoProcessing", fixture: "candidate") + defer { run.finish() } + run.waitFor("findMeter") + XCTAssertTrue(run.send.waitForExistence(timeout: 20)) + run.app.buttons["photoConsent.decline"].tap() + run.waitFor("processing", timeout: 300) + XCTAssertTrue(run.state("consentNotGiven", timeout: 30).exists) + run.attach("processing-consentNotGiven") + XCTAssertEqual(run.routes(), []) + } + + /// Start over after a sent scan asks the next scan again, shows nothing of the old one and + /// sends nothing more for it. + @MainActor + func testStartingOverNeverRevivesTheOldCapture() throws { + let run = try Run.launch(backend: "photoProcessing", fixture: "hold") + defer { run.finish() } + run.waitFor("findMeter") + XCTAssertTrue(run.send.waitForExistence(timeout: 20)) + run.app.buttons["photoConsent.send"].tap() + run.waitFor("processing", timeout: 300) + // The replay's packet can't be finished, so the upload ends on the phone. + XCTAssertTrue(run.state("notPrepared", timeout: 60).exists) + run.attach("processing-ended") + let sent = run.routes() + Thread.sleep(forTimeInterval: 3) + XCTAssertEqual(run.routes(), sent, "requests went on after the scan ended") + try run.open("processing") + + run.tap("action.startOver") + run.waitFor("onboarding") + if run.app.buttons["action.onboardingSkip"].waitForExistence(timeout: 5) { run.app.buttons["action.onboardingSkip"].tap() } + run.tap("action.finishOnboarding") + run.waitFor("findMeter") + XCTAssertTrue(run.send.waitForExistence(timeout: 20), "the next scan didn't ask again") + run.app.buttons["photoConsent.decline"].tap() + XCTAssertTrue(run.send.waitForNonExistence(timeout: 10)) + Thread.sleep(forTimeInterval: 2) + XCTAssertEqual(run.routes(), sent, "the next scan revived the old capture or sent without a yes") + } + + /// Photo processing chosen in a build without its setup: Developer options say so, and the + /// scan stops before the camera starts, says why, and offers a Legacy scan, which starts as a + /// new scan. No photo question, no capture and nothing sent. + @MainActor + func testWithoutSetupTheScanStopsBeforeCaptureAndOffersLegacy() throws { + let run = try Run.launch(backend: "photoProcessing", fixture: nil, holding: ["onboarding"]) + defer { run.finish() } + run.waitFor("onboarding") + run.openOptions() + let row = run.reveal("developer.processing.photoProcessing") + XCTAssertTrue(row.label.contains("Not set up in this build"), "the option doesn't say it isn't set up: \(row.label)") + run.attach("selection-notSetUp") + run.closeOptions() + try run.open("onboarding") + run.waitFor("processing", timeout: 30) + XCTAssertTrue(run.state("notSetUp", timeout: 5).exists) + let any = run.app.descendants(matching: .any) + XCTAssertFalse(any["screen.findMeter"].exists, "the camera started for a scan that can't be sent") + XCTAssertFalse(run.send.exists, "asked to send photos with nothing to send them to") + XCTAssertEqual(run.routes(), [], "a capture request was made") + run.attach("processing-notSetUp-beforeCapture") + + run.tap("action.scanWithLegacy") + run.waitFor("findMeter", timeout: 20) + XCTAssertFalse(run.send.waitForExistence(timeout: 3), "the Legacy scan asked to send photos") + XCTAssertFalse(FileManager.default.fileExists(atPath: run.gate.appending(path: "uploading.held").path)) + } + + // MARK: The real engine with the synthetic capture + + /// The real coordinator and uploader carry `SyntheticCapture` (labelled synthetic in its + /// packet) to the fixture's answer, which the processing screen shows as the scan's main + /// result: provisional, quoted, and marked as a test answer about a synthetic capture. + @MainActor + func testTheSyntheticCaptureReachesATypedAnswerAsTheMainResult() throws { + let run = try Run.launch(backend: "photoProcessing", fixture: "candidate", extra: ["-photoProcessingSyntheticCapture"]) + defer { run.finish() } + run.waitFor("findMeter") + XCTAssertTrue(run.send.waitForExistence(timeout: 20)) + run.app.buttons["photoConsent.send"].tap() + run.waitFor("processing", timeout: 300) + XCTAssertTrue(run.state("candidate", timeout: 60).exists, "no answer reached the screen") + let any = run.app.descendants(matching: .any) + let message = any["photo.serviceMessage"].firstMatch + XCTAssertTrue(message.label.contains("Fixture answer: a possible spot to the left of the meter."), "the service's words aren't shown as written: \(message.label)") + XCTAssertTrue(any["photo.standIn"].firstMatch.exists, "the fixture's answer isn't marked as a test one") + XCTAssertTrue(run.app.staticTexts.containing(NSPredicate(format: "label CONTAINS 'synthetic capture'")).firstMatch.exists, "no word that the capture was synthetic") + run.attach("processing-candidate-engine") + let routes = run.routes() + XCTAssertTrue(routes.contains("POST captures/finalize"), "\(routes)") + XCTAssertTrue(routes.contains("GET captures/result"), "\(routes)") + XCTAssertFalse(FileManager.default.fileExists(atPath: run.gate.appending(path: "uploading.held").path), "the Legacy upload ran") + } + + /// An answer naming another run than the capture started is refused, and the scan says so + /// instead of showing it. + @MainActor + func testAnAnswerForAnotherRunIsRefused() throws { + let run = try Run.launch(backend: "photoProcessing", fixture: "wrongRun", extra: ["-photoProcessingSyntheticCapture"]) + defer { run.finish() } + run.waitFor("findMeter") + XCTAssertTrue(run.send.waitForExistence(timeout: 20)) + run.app.buttons["photoConsent.send"].tap() + run.waitFor("processing", timeout: 300) + XCTAssertTrue(run.state("answerMismatch", timeout: 60).exists) + XCTAssertFalse(run.app.descendants(matching: .any)["photo.serviceMessage"].firstMatch.exists, "the refused answer's words were shown") + run.attach("processing-answerMismatch-engine") + } + + /// "Stop sending photos" while the service works, with a capture folder the phone can't write + /// (`-photoProcessingFixtureReadOnlyCapture`): sending stops and the screen says the phone + /// couldn't save the choice. + @MainActor + func testStoppingWhenThePhoneCantSaveItSaysSo() throws { + let run = try Run.launch( + backend: "photoProcessing", fixture: "hold", extra: ["-photoProcessingSyntheticCapture", "-photoProcessingFixtureReadOnlyCapture"]) + defer { run.finish() } + run.waitFor("findMeter") + XCTAssertTrue(run.send.waitForExistence(timeout: 20)) + run.app.buttons["photoConsent.send"].tap() + run.waitFor("processing", timeout: 300) + XCTAssertTrue(run.state("processing", timeout: 60).exists, "the capture never reached processing") + run.attach("processing-engine") + run.tap("action.stopSendingPhotos") + let confirm = run.app.buttons["Stop sending"].firstMatch + XCTAssertTrue(confirm.waitForExistence(timeout: 5), "stopping didn't ask first") + confirm.tap() + XCTAssertTrue(run.state("withdrawalNotRecorded", timeout: 10).exists) + run.attach("processing-withdrawalNotRecorded-engine") + let sent = run.routes() + // Longer than the uploader waits between polls of a capture still processing. + Thread.sleep(forTimeInterval: 3) + XCTAssertEqual(run.routes(), sent, "requests went on after the homeowner stopped sending") + } + + // MARK: Every state, from demo mode + + /// The consent question and each processing state, at the default text size and the largest, + /// in demo mode with the capture fixture's words. States a replay can't reach (every service + /// answer, the withdrawal the phone couldn't save) are covered here as they render, and in the + /// package tests as they are reached. + @MainActor + func testEveryStateAtDefaultAndLargestText() throws { + let consent = Self.demo(["-uiDemoPhase", "findMeter", "-uiDemoPhotoConsent"]) + XCTAssertTrue(consent.buttons["photoConsent.send"].waitForExistence(timeout: 15)) + XCTAssertGreaterThanOrEqual(consent.buttons["photoConsent.decline"].frame.height, 44) + attach(consent, "consent") + try audit(consent, "consent") + consent.terminate() + let consentLarge = Self.demo(["-uiDemoPhase", "findMeter", "-uiDemoPhotoConsent"] + Self.largestText) + XCTAssertTrue(consentLarge.buttons["photoConsent.send"].waitForExistence(timeout: 15)) + attach(consentLarge, "consent-AX5") + consentLarge.terminate() + + let large: Set = ["uploading", "processing", "candidate", "needsMorePhotos", "installerReview", "withdrawalNotRecorded", "notSetUp", "setupRefused", "uploadStateUnsaved"] + for state in [ + "queued", "uploading", "processing", "candidate", "needsMorePhotos", "installerReview", "noCandidate", "unrecognized", + "notSetUp", "consentNotGiven", "withdrawn", "withdrawalNotRecorded", "notPrepared", "processingFailed", "expired", "refused", "setupRefused", + "uploadStateUnsaved", "answerUnreadable", "answerNotReady", "answerMismatch", + ] { + for big in large.contains(state) ? [false, true] : [false] { + let app = Self.demo(["-uiDemoPhase", "processing", "-uiDemoPhotoState", state] + (big ? Self.largestText : [])) + XCTAssertTrue( + app.descendants(matching: .any)["photo.state.\(state)"].waitForExistence(timeout: 15), "photo.state.\(state) never appeared") + XCTAssertTrue(app.descendants(matching: .any)["photo.standIn"].exists || state == "notSetUp", "\(state) doesn't say its answer is a test one") + if state == "setupRefused", !big { + // Shown here at the result step, after the service may have processed the scan: + // the words claim neither way, and don't rule out trying again once it's fixed. + let words = app.staticTexts.allElementsBoundByIndex.map(\.label).joined(separator: " ") + XCTAssertTrue(words.contains("This is a setup problem, not a problem with your photos or marks."), words) + XCTAssertTrue(words.contains("The connection needs to be fixed before you try again."), words) + XCTAssertFalse(words.contains("wasn't processed"), words) + XCTAssertFalse(words.contains("won't help"), words) + } + if state == "uploadStateUnsaved", !big { + // The phone's own failure: it says sending stopped here, and promises nothing + // about what the service does with photos it already has. + let words = app.staticTexts.allElementsBoundByIndex.map(\.label).joined(separator: " ") + XCTAssertTrue(words.contains("so it stopped sending from this scan."), words) + XCTAssertTrue(words.contains("Photos already sent may still be processed by the service."), words) + XCTAssertFalse(words.contains("refused"), words) + XCTAssertFalse(words.contains("cancel"), words) + } + attach(app, "processing-\(state)\(big ? "-AX5" : "")") + if !big, ["uploading", "candidate", "needsMorePhotos", "withdrawalNotRecorded", "setupRefused", "uploadStateUnsaved"].contains(state) { try audit(app, "processing-\(state)") } + app.terminate() + } + } + } + + /// "Stop sending photos" asks first, then ends the scan's processing. The demo stands in for the + /// engine, whose replay can't stay in processing long enough to stop; the engine's withdrawal, + /// saved or not, is driven in HouseScanKit's `PhotoProcessingControllerTests`. + @MainActor + func testStopSendingAsksFirstThenEnds() throws { + let app = Self.demo(["-uiDemoPhase", "processing", "-uiDemoPhotoState", "processing"]) + defer { app.terminate() } + let stop = app.buttons["action.stopSendingPhotos"] + XCTAssertTrue(stop.waitForExistence(timeout: 15), "no way to stop sending while processing") + stop.tap() + let confirm = app.buttons["Stop sending"].firstMatch + XCTAssertTrue(confirm.waitForExistence(timeout: 5), "stopping didn't ask first") + // The question says only what House Scan stops doing: photos already sent may still be + // processed, since the app can't cancel the service's job. + let detail = app.staticTexts.matching(NSPredicate(format: "label CONTAINS %@", "Photos already sent may still be processed by the service.")).firstMatch + XCTAssertTrue(detail.waitForExistence(timeout: 5), "the stop question must say photos already sent may still be processed") + XCTAssertTrue(detail.label.contains("House Scan will stop sending from this scan and won't resume it."), detail.label) + XCTAssertFalse(app.staticTexts.matching(NSPredicate(format: "label CONTAINS %@", "won't be processed")).firstMatch.exists, "the stop question must not promise the scan won't be processed") + attach(app, "processing-stopQuestion") + confirm.tap() + XCTAssertTrue(app.descendants(matching: .any)["photo.state.withdrawn"].waitForExistence(timeout: 5)) + let ended = app.staticTexts.matching(NSPredicate(format: "label CONTAINS %@", "House Scan won't send anything more from this scan.")).firstMatch + XCTAssertTrue(ended.waitForExistence(timeout: 5), "the stopped screen must say only what House Scan stops doing") + XCTAssertTrue(ended.label.contains("Photos already sent may still be processed by the service."), ended.label) + XCTAssertFalse(app.staticTexts.matching(NSPredicate(format: "label CONTAINS %@", "won't be processed")).firstMatch.exists, "the stopped screen must not promise the scan won't be processed") + XCTAssertFalse(app.buttons["action.stopSendingPhotos"].exists) + } + + /// The selection sheet at the largest text size. + @MainActor + func testTheSelectionAtLargestText() throws { + let app = Self.demo(Self.largestText) + let run = Run(app: app, gate: URL(fileURLWithPath: NSTemporaryDirectory())) + run.openOptions() + _ = run.reveal("developer.processing.photoProcessing") + attach(app, "selection-AX5") + app.terminate() + } + + // MARK: Helpers + + @MainActor + private static func demo(_ extra: [String]) -> XCUIApplication { + let app = XCUIApplication() + app.launchArguments = ["-uiDemo", "-uiDemoFreeze", "-practiceMeter", "NO", "-processingBackend", "photoProcessing"] + extra + app.launch() + return app + } + + @MainActor + private func attach(_ app: XCUIApplication, _ name: String) { + let shot = XCTAttachment(screenshot: app.screenshot()) + shot.name = name + shot.lifetime = .keepAlways + add(shot) + } + + @MainActor + private func audit(_ app: XCUIApplication, _ screen: String) throws { + let outcome = try AccessibilityAudit.run(app) { _ in Thread.sleep(forTimeInterval: 2.0) } + for (_, finding) in outcome.persistent { + let text = "\(screen): \(finding.message)" + if ProcessInfo.processInfo.environment["HOUSESCAN_AUDIT_REPORT_ONLY"] == "1" { + let note = XCTAttachment(string: text) + note.name = "audit-\(screen)" + note.lifetime = .keepAlways + add(note) + } else { + XCTFail(text) + } + } + } + + /// One app run on the synthetic replay with the autopilot. Every screen's gate is open from + /// the start except those in `holding` and the processing screen, which the test opens. + @MainActor + struct Run { + let app: XCUIApplication + let gate: URL + + @MainActor + static func launch(backend: String, fixture: String?, holding: Set = [], extra: [String] = []) throws -> Run { + let gate = URL(fileURLWithPath: NSTemporaryDirectory()).appending(path: "housescan-gate-\(UUID().uuidString)", directoryHint: .isDirectory) + try FileManager.default.createDirectory(at: gate, withIntermediateDirectories: true) + for phase in ["onboarding", "findMeter", "meterCloseUp", "wallWalk", "markFeatures", "gapRequest"] where !holding.contains(phase) { + try Data().write(to: gate.appending(path: phase)) + } + var arguments = [ + "-replay", FullFlowUITests.fixture, "-autopilot", "-autopilotHold", "1.0", "-autopilotGate", gate.path, "-sampleResult", + "-practiceMeter", "NO", "-processingBackend", backend, + ] + if let fixture { arguments += ["-photoProcessingFixture", fixture] } + arguments += extra + let app = XCUIApplication() + app.launchArguments = arguments + app.launch() + return Run(app: app, gate: gate) + } + + var send: XCUIElement { app.buttons["photoConsent.send"] } + + @MainActor + func waitFor(_ phase: String, timeout: TimeInterval = 60) { + XCTAssertTrue(app.descendants(matching: .any)["screen.\(phase)"].waitForExistence(timeout: timeout), "screen.\(phase) never appeared") + } + + /// Lets the autopilot leave `phase`. + func open(_ phase: String) throws { + try Data().write(to: gate.appending(path: phase)) + } + + @MainActor + func state(_ id: String, timeout: TimeInterval) -> XCUIElement { + let element = app.descendants(matching: .any)["photo.state.\(id)"] + _ = element.waitForExistence(timeout: timeout) + return element + } + + /// Every capture-API route the fixture answered, in order. + func routes() -> [String] { + guard let text = try? String(contentsOf: gate.appending(path: "photo-transport.log"), encoding: .utf8) else { return [] } + return text.split(separator: "\n").map(String.init) + } + + @MainActor + func tap(_ identifier: String) { + let target = app.descendants(matching: .any)[identifier].firstMatch + XCTAssertTrue(target.waitForExistence(timeout: 10), "missing \(identifier)") + for _ in 0..<6 where !target.isHittable { app.swipeUp() } + target.tap() + } + + @MainActor + func openOptions() { + let open = app.buttons["action.developerOptions"] + XCTAssertTrue(open.waitForExistence(timeout: 10), "no developer options here") + open.tap() + XCTAssertTrue(app.buttons["action.closeDeveloperOptions"].waitForExistence(timeout: 10), "the developer options didn't open") + _ = reveal("developer.processing.legacy") + } + + /// Scrolls the options until the row is on screen: at the largest text sizes the sheet + /// opens with the processing choice below its fold. + func reveal(_ identifier: String) -> XCUIElement { + let row = app.buttons[identifier] + let form = app.collectionViews.firstMatch + for _ in 0..<8 where !(row.exists && row.isHittable) { + if form.exists { form.swipeUp() } else { app.swipeUp() } + } + XCTAssertTrue(row.exists && row.isHittable, "\(identifier) never came on screen") + return row + } + + @MainActor + func closeOptions() { + app.buttons["action.closeDeveloperOptions"].tap() + XCTAssertTrue(app.buttons["developer.processing.legacy"].waitForNonExistence(timeout: 10)) + } + + /// Picks `backend` in Developer options and checks the footer, then closes them. + @MainActor + func choose(_ backend: String, footer: String, shot: String) { + openOptions() + let row = reveal("developer.processing.\(backend)") + // The sheet can still be presenting when the row is first on screen (see + // `FullFlowUITests.setPracticeMeter`), so a tap that changed nothing is repeated. + for _ in 0..<3 where !row.isSelected { row.tap() } + XCTAssertTrue(row.isSelected, "\(backend) wasn't selected") + let footnote = app.staticTexts["developer.processing.footer"] + for _ in 0..<4 where !footnote.exists { app.collectionViews.firstMatch.swipeUp() } + XCTAssertEqual(footnote.label, footer) + let attachment = XCTAttachment(screenshot: app.screenshot()) + attachment.name = shot + attachment.lifetime = .keepAlways + XCTContext.runActivity(named: shot) { $0.add(attachment) } + closeOptions() + } + + @MainActor + func attach(_ name: String) { + let shot = XCTAttachment(screenshot: app.screenshot()) + shot.name = name + shot.lifetime = .keepAlways + XCTContext.runActivity(named: name) { $0.add(shot) } + } + + @MainActor + func finish() { + try? open("processing") + app.terminate() + try? FileManager.default.removeItem(at: gate) + } + } +} diff --git a/ios/HouseScanUITests/ProvisionalPlacementUITests.swift b/ios/HouseScanUITests/ProvisionalPlacementUITests.swift new file mode 100644 index 000000000..5e84bef0c --- /dev/null +++ b/ios/HouseScanUITests/ProvisionalPlacementUITests.swift @@ -0,0 +1,327 @@ +import Foundation +import XCTest + +/// A spot whose checks all pass is shown as a possible spot, never as a fit: the server's checks +/// read space beyond what the homeowner confirms in the spot check (B17). Every surface agrees: +/// the card, Details, the AR view and the 3D model. The answer itself is never changed. +final class ProvisionalPlacementUITests: XCTestCase { + private static let largestText = ["-UIPreferredContentSizeCategoryName", "UICTContentSizeCategoryAccessibilityXXXL"] + private static let candidate = "A possible battery spot" + + /// Words that would say the scan established a fit, or celebrate a pass as confirmed space. + private static let fitWords = ["fits here", "fits every", "could go here", "looks good", "your battery spot", "confirmed clear"] + + override func setUp() { + continueAfterFailure = false + } + + // MARK: The real engine + + /// The real engine on the synthetic replay, the test answering as the server through the gate + /// folder (`-answersFromGate`). The first answer names another scene, so "Try again" asks + /// again; the second is the bundled sample turned into a pass naming the scene sent. The + /// autopilot answers the spot check "It's clear". The result is a possible spot, and so is + /// the AR view's title: a retry and a clear answer don't make it a fit. + @MainActor + func testAPassAfterTryAgainAndItsClearIsAPossibleSpot() throws { + let gate = try Self.gate() + defer { try? FileManager.default.removeItem(at: gate) } + let server = GateServer(gate: gate) { body, index in + Self.passingAnswer(binding: index == 0 ? Data("another scene".utf8) : body) + } + defer { server.stop() } + let app = Self.launchEngine(gate: gate) + defer { app.terminate() } + let any = app.descendants(matching: .any) + + let tryAgain = app.buttons["action.retryUpload"] + XCTAssertTrue(tryAgain.waitForExistence(timeout: 300), "the unbound answer never offered Try again") + tryAgain.tap() + XCTAssertTrue(any["screen.result"].firstMatch.waitForExistence(timeout: 120), "the passing answer never reached the result") + XCTAssertEqual(server.requests.count, 2) + XCTAssertEqual(any["result.headline"].firstMatch.label, Self.candidate) + XCTAssertFalse(any["result.spotRefused"].firstMatch.exists) + assertCandidateCard(app) + attach(app, name: "result-candidate-engine") + + // The autopilot opens the AR view once the result's gate opens. + try Data().write(to: gate.appending(path: "result")) + let instruction = any["instruction"].firstMatch + XCTAssertTrue(any["screen.resultAR"].firstMatch.waitForExistence(timeout: 20)) + XCTAssertTrue(instruction.waitForExistence(timeout: 10)) + XCTAssertTrue(instruction.label.contains(Self.candidate), instruction.label) + XCTAssertTrue(instruction.label.contains("An installer needs to check the fit on site."), instruction.label) + assertNoFitWords(instruction.label) + attach(app, name: "resultAR-candidate-engine") + } + + /// "Something's there" withdraws the area and sends the scan again; the passing answer that + /// comes back names the same spot, so the earlier answer settles it and the result shows + /// without asking (`SpotConfirmations.settling`). That shortcut still shows a possible spot, + /// beside the notice that the homeowner said something stands there. At the largest text + /// size, so the AR view's longer instruction for a real candidate (placement, then what the + /// fit still needs) is captured where it is longest; a demo answer is a sample, whose AR + /// instruction differs. + @MainActor + func testASpotSettledByAnEarlierAnswerIsStillAPossibleSpot() throws { + let gate = try Self.gate() + defer { try? FileManager.default.removeItem(at: gate) } + let server = GateServer(gate: gate) { body, _ in Self.passingAnswer(binding: body) } + defer { server.stop() } + let app = Self.launchEngine(gate: gate, extra: ["-autopilotSomethingThere"] + Self.largestText) + defer { app.terminate() } + let any = app.descendants(matching: .any) + + XCTAssertTrue(any["screen.result"].firstMatch.waitForExistence(timeout: 420), "the second passing answer never reached the result") + XCTAssertEqual(server.requests.count, 2, "the spot check's answer should send the scan once more") + XCTAssertEqual(any["result.headline"].firstMatch.label, Self.candidate) + XCTAssertTrue(any["result.spotRefused"].firstMatch.exists, "the settled answer's notice is missing") + assertCandidateCard(app) + attach(app, name: "result-candidate-settled-AX5") + scroll(to: any["result.candidateNote"].firstMatch, in: app) + attach(app, name: "result-candidate-settled-AX5-note") + + // The AR view at AX-XXXL: the card folds to its title and Details, so the camera stays + // open on the spot; Details holds the placement and what the fit still needs. + try Data().write(to: gate.appending(path: "result")) + let instruction = any["instruction"].firstMatch + let details = any["instruction.details"].firstMatch + let detail = any["instruction.detail"].firstMatch + let done = app.buttons["action.closeAR"] + XCTAssertTrue(any["screen.resultAR"].firstMatch.waitForExistence(timeout: 20)) + XCTAssertTrue(instruction.waitForExistence(timeout: 10)) + XCTAssertTrue(details.waitForExistence(timeout: 10), "the AR card didn't fold at the largest text size") + XCTAssertEqual(instruction.label, Self.candidate) + XCTAssertFalse(detail.exists) + assertOpenCamera(below: details, above: done, in: app, "folded") + attach(app, name: "resultAR-candidate-settled-AX5-folded") + + details.tap() + XCTAssertTrue(detail.waitForExistence(timeout: 5), "Details didn't open the placement and caveat") + XCTAssertTrue(detail.label.contains("right of your meter"), detail.label) + XCTAssertTrue(detail.label.contains("An installer needs to check the fit on site."), detail.label) + assertNoFitWords(detail.label) + attach(app, name: "resultAR-candidate-settled-AX5-details") + + details.tap() + XCTAssertTrue(detail.waitForNonExistence(timeout: 5), "Details didn't close") + assertOpenCamera(below: details, above: done, in: app, "folded again") + attach(app, name: "resultAR-candidate-settled-AX5-refolded") + XCTAssertTrue(done.isHittable) + done.tap() + XCTAssertTrue(any["screen.result"].firstMatch.waitForExistence(timeout: 10), "Done didn't return to the result") + } + + /// The open camera between the folded card and Done, measured from the elements' frames and + /// clipped to the window, is at least the chrome's least open camera window + /// (`CameraChromeLayout.minCameraWindow`, 180 pt). The card's bottom is its Details label + /// plus the card's padding under it (8 pt, `InstructionCard.details`), so the area isn't + /// overstated; Done must be on screen and hittable first. + @MainActor + private func assertOpenCamera(below details: XCUIElement, above done: XCUIElement, in app: XCUIApplication, _ state: String, + file: StaticString = #filePath, line: UInt = #line) { + XCTAssertTrue(done.isHittable, "\(state): Done isn't reachable", file: file, line: line) + let window = app.windows.firstMatch.frame + let top = max(details.frame.maxY + 8, window.minY) + let bottom = min(done.frame.minY, window.maxY) + let open = bottom - top + XCTAssertGreaterThanOrEqual(open, 180, "\(state): \(open) pt of camera between the card (bottom \(top)) and Done (top \(bottom))", + file: file, line: line) + let note = XCTAttachment(string: "\(state): open camera \(open) pt, card bottom \(top), Done top \(bottom), window \(window)") + note.name = "resultAR-open-camera-\(state)" + note.lifetime = .keepAlways + add(note) + } + + // MARK: Server answers in Fixtures/results + + /// pass.json, every check passing: the card, Details and the AR overlay at the default size. + @MainActor + func testAPassingAnswerReadsAsAPossibleSpotEverywhere() throws { + try checkPassFixture(textSize: [], name: "result-candidate") + } + + @MainActor + func testAPassingAnswerReadsAsAPossibleSpotAtLargestTextSize() throws { + try checkPassFixture(textSize: Self.largestText, name: "result-candidate-AX5") + } + + /// A result that isn't a possible spot keeps its summary in Details, and the summary never + /// claims a fit: the demo's review answer and its working-space overlap. + @MainActor + func testAReviewSummaryNeverClaimsAFit() throws { + for extra in [[], ["-uiDemoOverlap"]] { + let app = XCUIApplication() + app.launchArguments = ["-practiceMeter", "NO", "-uiDemo", "-uiDemoFreeze", "-uiDemoPhase", "result"] + extra + app.launch() + let any = app.descendants(matching: .any) + XCTAssertTrue(any["screen.result"].firstMatch.waitForExistence(timeout: 15)) + XCTAssertNotEqual(any["result.headline"].firstMatch.label, Self.candidate, "\(extra)") + let details = any["result.details"].firstMatch + scroll(to: details, in: app) + details.tap() + let summary = any["result.summary"].firstMatch + XCTAssertTrue(summary.waitForExistence(timeout: 10), "\(extra): a review answer's summary is missing") + assertNoFitWords(summary.label) + XCTAssertFalse(summary.label.lowercased().contains("fits"), summary.label) + app.terminate() + } + } + + /// A rejected wall keeps its negative answer and its reasons. + @MainActor + func testARejectStaysNegative() throws { + let app = Self.launchFixture("reject-nearest", textSize: []) + defer { app.terminate() } + let any = app.descendants(matching: .any) + XCTAssertEqual(any["result.headline"].firstMatch.label, "Not on this wall") + XCTAssertFalse(any["result.candidateNote"].firstMatch.exists) + XCTAssertTrue(any["result.nearest"].firstMatch.label.hasPrefix("The closest spot")) + attach(app, name: "result-reject") + } + + @MainActor + private func checkPassFixture(textSize: [String], name: String) throws { + let app = Self.launchFixture("pass", textSize: textSize) + defer { app.terminate() } + let any = app.descendants(matching: .any) + XCTAssertEqual(any["result.headline"].firstMatch.label, Self.candidate) + assertCandidateCard(app) + let model = any.matching(NSPredicate(format: "label == '3D view of your wall'")).firstMatch + XCTAssertTrue(model.waitForExistence(timeout: 10)) + let modelValue = model.value as? String ?? "" + XCTAssertTrue(modelValue.hasPrefix("Proposed battery spot"), modelValue) + attach(app, name: name) + + // Details: the checks as calculations on the recorded scan, a pass without the server's + // positive reason, and no server summary that says the spot fits. + let details = any["result.details"].firstMatch + scroll(to: details, in: app) + details.tap() + let note = any["result.calculatedNote"].firstMatch + XCTAssertTrue(note.waitForExistence(timeout: 10)) + XCTAssertFalse(any["result.summary"].firstMatch.exists, "a possible spot's summary is the server's pass") + let row = any["detail.check.gas_clearance"].firstMatch + XCTAssertTrue(row.exists) + XCTAssertEqual(row.label, "Distance from gas equipment: Passes on recorded data", row.label) + let rowValue = row.value as? String ?? "" + XCTAssertFalse(rowValue.contains("well clear"), "a pass kept the server's reason: \(rowValue)") + scroll(to: note, in: app) + attach(app, name: "\(name)-details") + + // The AR view draws the proposed spot. The demo's answer is a sample, so the AR title + // says so (`ResultARScreen`); the drawing names the spot as proposed. + scroll(to: any["action.showAR"].firstMatch, in: app) + app.buttons["action.showAR"].tap() + let overlay = any["ar.overlay"].firstMatch + XCTAssertTrue(overlay.waitForExistence(timeout: 10)) + XCTAssertEqual(overlay.label, "A proposed battery spot, drawn on your wall") + assertNoFitWords(any["instruction"].firstMatch.label) + attach(app, name: "\(name)-AR") + } + + // MARK: Helpers + + /// The card of a possible spot: headline, placement, the note that says what isn't confirmed, + /// no passing check lines, no repeat of the installer footnote, and the AR button's words. + @MainActor + private func assertCandidateCard(_ app: XCUIApplication) { + let any = app.descendants(matching: .any) + XCTAssertTrue(any["result.placement"].firstMatch.exists) + let note = any["result.candidateNote"].firstMatch + XCTAssertTrue(note.exists, "the candidate note is missing") + XCTAssertTrue(note.label.contains("couldn't confirm all the space a battery needs"), note.label) + let lines = any.matching(NSPredicate(format: "identifier BEGINSWITH 'check.'")) + XCTAssertEqual(lines.count, 0, "a passing check reached the card") + XCTAssertFalse(any["result.installerConfirms"].firstMatch.exists) + XCTAssertEqual(app.buttons["action.showAR"].label, "See this spot on your wall") + // Details' scope note says, truthfully, that a pass is "not ... confirmed clear": the sweep + // leaves that one qualified sentence out, whether or not Details is open. + let texts = app.staticTexts.allElementsBoundByIndex + .filter { $0.identifier != "result.calculatedNote" } + .map(\.label).joined(separator: " | ") + assertNoFitWords(texts) + } + + private func assertNoFitWords(_ text: String, file: StaticString = #filePath, line: UInt = #line) { + let lowered = text.lowercased() + for words in Self.fitWords { + XCTAssertFalse(lowered.contains(words), "says \"\(words)\": \(text)", file: file, line: line) + } + } + + /// The bundled sample turned into a pass: every check passing, nothing more needed, naming + /// `binding` as the scene it answers. + private static func passingAnswer(binding: Data) -> Data { + let sample = try! Data(contentsOf: UploadRecoveryUITests.sampleResult) + var object = try! JSONSerialization.jsonObject(with: sample) as! [String: Any] + object["decision"] = "pass" + object["checks"] = (object["checks"] as! [[String: Any]]).map { check in + var check = check + check["outcome"] = "pass" + check.removeValue(forKey: "unsure_cause") + return check + } + object["missing_evidence"] = [Any]() + var stats = object["stats"] as! [String: Any] + stats["input_sha256"] = UploadRecoveryUITests.sha256(binding) + object["stats"] = stats + return try! JSONSerialization.data(withJSONObject: object) + } + + /// A gate folder with every screen up to the spot check let through; the result's stays shut. + private static func gate() throws -> URL { + let gate = URL(fileURLWithPath: NSTemporaryDirectory()).appending(path: "housescan-gate-\(UUID().uuidString)", directoryHint: .isDirectory) + try FileManager.default.createDirectory(at: gate, withIntermediateDirectories: true) + for phase in ["onboarding", "findMeter", "meterCloseUp", "wallWalk", "markFeatures", "gapRequest", "uploading", "spotConfirm"] { + try Data().write(to: gate.appending(path: phase)) + } + return gate + } + + @MainActor + private static func launchEngine(gate: URL, extra: [String] = []) -> XCUIApplication { + let app = XCUIApplication() + app.launchArguments = [ + "-replay", FullFlowUITests.fixture, "-autopilot", "-autopilotHold", "1.5", "-autopilotGate", gate.path, + "-practiceMeter", "NO", "-serverURL", "http://placement.invalid", "-answersFromGate", + ] + extra + app.launch() + return app + } + + @MainActor + private static func launchFixture(_ name: String, textSize: [String]) -> XCUIApplication { + let file = URL(fileURLWithPath: #filePath).deletingLastPathComponent().appending(path: "Fixtures/results/\(name).json").path + let app = XCUIApplication() + app.launchArguments = ["-practiceMeter", "NO", "-uiDemo", "-uiDemoFreeze", "-uiDemoPhase", "result", + "-uiDemoResultFile", file] + textSize + app.launch() + XCTAssertTrue(app.descendants(matching: .any)["screen.result"].firstMatch.waitForExistence(timeout: 15)) + return app + } + + /// Scrolls until the element is inside the window, a third of the window at most per drag. + /// Lookups here take the first match: a DisclosureGroup's identifier ("result.details") + /// matches more than one element, and reading a frame needs exactly one. + @MainActor + private func scroll(to target: XCUIElement, in app: XCUIApplication) { + let window = app.windows.firstMatch.frame + let start = app.scrollViews.firstMatch.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.6)) + for _ in 0..<12 where !window.contains(target.frame) { + let frame = target.frame + let shift: CGFloat = frame.maxY > window.maxY + ? -min(frame.maxY - window.maxY + 24, window.height / 3) + : min(window.minY - frame.minY + 24, window.height / 3) + start.press(forDuration: 0.05, thenDragTo: start.withOffset(CGVector(dx: 0, dy: shift))) + } + } + + @MainActor + private func attach(_ app: XCUIApplication, name: String) { + let shot = XCTAttachment(screenshot: app.screenshot()) + shot.name = name + shot.lifetime = .keepAlways + add(shot) + } +} diff --git a/ios/HouseScanUITests/ScreenStatesUITests.swift b/ios/HouseScanUITests/ScreenStatesUITests.swift index f1cc52f85..0f08148a0 100644 --- a/ios/HouseScanUITests/ScreenStatesUITests.swift +++ b/ios/HouseScanUITests/ScreenStatesUITests.swift @@ -108,6 +108,8 @@ final class ScreenStatesUITests: XCTestCase { // At AX5 the meter's description folds under Details, and the camera stays open. "findMeter-AX5": [("instruction", "Find your electric meter"), ("instruction.details", "Details")], "gapRequest-followUp": [("instruction", "One more view to finish")], + // At AX5 the side to aim at leads, and the follow-up and the stretch fold under Details. + "gapRequest-followUp-AX5": [("instruction", "Show the ground right of your meter"), ("instruction.details", "Details")], // #75: a server request's stretch by its two ends, not its middle. "gapRequest-groundOut": [("instruction", "From 4 ft to 7 ft right of your meter.")], "uploading-followUp": [(nil, "One more view to finish")], diff --git a/ios/HouseScanUITests/WalkRecoveryUITests.swift b/ios/HouseScanUITests/WalkRecoveryUITests.swift index 3f2b45f07..e13626a39 100644 --- a/ios/HouseScanUITests/WalkRecoveryUITests.swift +++ b/ios/HouseScanUITests/WalkRecoveryUITests.swift @@ -16,6 +16,7 @@ final class WalkRecoveryUITests: XCTestCase { func testTheEndCardAnswersThatTheWallKeepsGoing() throws { let app = launch(["-uiDemoPhase", "wallWalk", "-uiDemoMarkEnd"]) XCTAssertTrue(label(app, "instruction").contains("Is this the right end of the wall?")) + assertEndCircleOpen(app, folded: false, reply: "action.cannotAccess", covers: Self.endCovers, "walk mark-end") snap(app, "wallWalk-markEnd-keepsGoing") XCTAssertTrue(element(app, "action.markEnd").exists, "Wall ends here must stay on offer") let keepsGoing = reply(app, "The wall keeps going") @@ -39,6 +40,105 @@ final class WalkRecoveryUITests: XCTestCase { XCTAssertTrue(card.contains("Aim it at the wall where it stops or turns"), "card reads: \(card)") XCTAssertTrue(element(app, "action.markEnd").exists) XCTAssertTrue(reply(app, "The wall keeps going").waitForExistence(timeout: 5)) + assertEndCircleOpen(app, folded: false, reply: "action.cannotAccess", covers: Self.endCovers, "walk mark-end refused") + } + + /// At AX5 the card covered the circle "Wall ends here" marks at (CI run 37165788075, + /// `wallWalk-markEnd-AX5-stacked`). Folded, it leads with where to aim, every word is under + /// Details, and "The wall keeps going" moves to the actions under "Wall ends here", so with + /// "Wall ends here" in reach the circle is in view. + @MainActor + func testTheEndCircleStaysOpenAtLargestTextSize() throws { + let app = launch(["-uiDemoPhase", "wallWalk", "-uiDemoMarkEnd"] + Self.largestText) + let card = label(app, "instruction") + XCTAssertTrue(card.contains("Aim at the right end"), "card reads: \(card)") + XCTAssertFalse(card.contains("Is this the right end of the wall?"), "the question folds under Details: \(card)") + let mark = element(app, "action.markEnd") + scrollIntoView(mark, in: app) + XCTAssertTrue(mark.isHittable, "Wall ends here can't be reached") + assertEndCircleOpen(app, folded: true, reply: "action.cannotAccess", covers: Self.endCovers, "walk mark-end, AX5") + snap(app, "wallWalk-markEnd-AX5") + + scrollToTop(app) + let details = element(app, "instruction.details") + tapWhenReady(details) + let detail = element(app, "instruction.detail") + XCTAssertTrue(detail.waitForExistence(timeout: 5), "Details must open the folded words") + XCTAssertTrue( + detail.label.contains("Is this the right end of the wall? Aim where it stops or turns a corner and tap Wall ends here. If it goes on, tap The wall keeps going."), + "Details reads: \(detail.label)") + snap(app, "wallWalk-markEnd-AX5-details") + tapWhenReady(details) + let gone = XCTNSPredicateExpectation(predicate: NSPredicate(format: "exists == false"), object: detail) + XCTAssertEqual(XCTWaiter().wait(for: [gone], timeout: 5), .completed, "Details must close again") + scrollIntoView(mark, in: app) + assertEndCircleOpen(app, folded: true, reply: "action.cannotAccess", covers: Self.endCovers, "walk mark-end, AX5, Details closed") + + let keepsGoing = reply(app, "The wall keeps going") + XCTAssertTrue(keepsGoing.waitForExistence(timeout: 5), "the wall that goes on still needs its answer") + scrollIntoView(keepsGoing, in: app) + XCTAssertTrue(keepsGoing.isHittable, "The wall keeps going can't be reached") + XCTAssertGreaterThan(keepsGoing.frame.minY, mark.frame.maxY - 0.5, "the answer sits under Wall ends here") + } + + /// Folded, a refusal leads with what to do, and keeps the circle in view. + @MainActor + func testARefusedWallEndKeepsTheCircleOpenAtLargestTextSize() throws { + let app = launch(["-uiDemoPhase", "wallWalk", "-uiDemoMarkEnd", "-uiDemoEndMarkRefusal"] + Self.largestText) + let card = label(app, "instruction") + XCTAssertTrue(card.contains("Aim at the wall"), "card reads: \(card)") + let mark = element(app, "action.markEnd") + scrollIntoView(mark, in: app) + XCTAssertTrue(mark.isHittable) + assertEndCircleOpen(app, folded: true, reply: "action.cannotAccess", covers: Self.endCovers, "walk mark-end refused, AX5") + snap(app, "wallWalk-markEnd-refused-AX5") + } + + /// The walk's end with coaching riding along: the coaching leads in a few words, its note + /// opens Details, and the circle stays open. + @MainActor + func testCoachingOnTheFoldedEndCardKeepsTheCircleOpen() throws { + let app = launch(["-uiDemoPhase", "wallWalk", "-uiDemoMarkEnd", "-uiDemoCoaching", "tooDark"] + Self.largestText) + let card = label(app, "instruction") + XCTAssertTrue(card.contains("Try your flashlight"), "card reads: \(card)") + XCTAssertFalse(card.contains("Try your phone's flashlight"), "the coaching's note must fold under Details: \(card)") + let mark = element(app, "action.markEnd") + scrollIntoView(mark, in: app) + XCTAssertTrue(mark.isHittable) + assertEndCircleOpen(app, folded: true, reply: "action.cannotAccess", covers: Self.endCovers, "coached walk mark-end, AX5") + snap(app, "wallWalk-markEnd-tooDark-AX5") + } + + private static let largestText = ["-UIPreferredContentSizeCategoryName", "UICTContentSizeCategoryAccessibilityXXXL"] + /// What may cover the circle on the walk's end: its actions and the card's reply. + private static let endCovers = ["action.markEnd", "action.markSomething", "action.cannotAccess"] + + /// Scrolls in measured steps until the whole element is in the window: at the largest text + /// sizes the actions sit below the card, reached by scrolling. + @MainActor + private func scrollIntoView(_ target: XCUIElement, in app: XCUIApplication) { + let window = app.windows.firstMatch.frame + let scroll = app.scrollViews.firstMatch + guard scroll.exists else { return } + let start = scroll.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.6)) + for _ in 0..<12 where !window.contains(target.frame) { + let frame = target.frame + let shift: CGFloat = frame.maxY > window.maxY + ? -min(frame.maxY - window.maxY + 24, window.height / 3) + : min(window.minY - frame.minY + 24, window.height / 3) + start.press(forDuration: 0.05, thenDragTo: start.withOffset(CGVector(dx: 0, dy: shift))) + } + } + + /// Drags the screen down until its top shows. + @MainActor + private func scrollToTop(_ app: XCUIApplication) { + let scroll = app.scrollViews.firstMatch + guard scroll.exists else { return } + let start = scroll.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.3)) + for _ in 0..<4 { + start.press(forDuration: 0.05, thenDragTo: start.withOffset(CGVector(dx: 0, dy: app.windows.firstMatch.frame.height * 0.4))) + } } /// B-23: "Point at the meter like this." needs the photo it points to. The follow-up view diff --git a/ios/README.md b/ios/README.md index 601d2da9c..cdd5e143d 100644 --- a/ios/README.md +++ b/ios/README.md @@ -25,10 +25,18 @@ Native iPhone app for the AR capture walk. The homeowner marks the electric mete | `-simulateAppStore` | Runs as an App Store install would: no developer options, and practice meter stays off whatever the stored switch says. `PracticeMeterUITests` uses it. | | `-practiceMeter YES` or `NO` | Overrides the stored practice meter switch for one run (UserDefaults' argument domain). The full-flow UI tests pass `NO`, since the switch outlives a run in the Simulator. | | `-savedScansRoot ` | Debug builds only. Saved scans lists the scan folders in this folder instead of the app's Caches/Scans. It only changes what is listed; the scan store still writes and cleans up Caches/Scans. `SavedScansUITests` passes it with synthetic scans. | +| `-processingBackend ` | Debug builds only. The processing choice in Developer options starts at this value and stays in memory for the run, so a UI test's choice never reaches the stored setting. | +| `-photoProcessingFixture ` | Debug builds only, with `-replay`. Photo processing sends to a capture API answered inside the app (HouseScanKit's `FixtureCaptureHTTP`), which ends each capture with this answer: `candidate`, `needsViews`, `manualReview`, `notEligible`, `failed`, `expired`, `notReady`, `unreadable`, `refuseCreate`, `hold` or `wrongRun`. Nothing leaves the phone. With `-autopilotGate`, the routes it answered are listed in `photo-transport.log` there. | +| `-photoProcessingSyntheticCapture` | Debug builds only, with `-photoProcessingFixture`. Sends HouseScanKit's `SyntheticCapture` (packet `source.kind: synthetic`) in place of the replay's photos, so the fixture's answer comes back through the real upload. | +| `-photoProcessingFixtureReadOnlyCapture` | Debug builds only. Makes the capture's folder read-only just before "Stop sending photos" takes effect, so the phone can't save the withdrawal and says so. | | `-uiDemo` | Runs the screens on a scripted fake engine instead of the capture engine, for design work and for auditing states a replay can't reach. The arguments it takes are listed in `HouseScan/UI/Preview/UIDemo.swift`; for example `-uiDemoGroundQuestion` opens the feature review with the ground question showing, `-uiDemoPhase meterCloseUp -uiDemoMeterChoose` shows the meter number picker, and `-uiDemoPhase result -uiDemoCorner` shows the result model of a wall that turns a corner. | **Spot check.** Before the result, the app shows the kept photo that best sees the answer's spot, with the spot and its clearance area outlined, and asks whether anything stands in front of the wall or on the ground there (`STATE=spotConfirm`). When no kept photo shows the area, it shows no photo and gives the area in words: where it runs beside the meter, how far out from the wall, and how high up the wall. Photos without depth and the walked path claim wall, ground and clear space they never saw; "It's clear" backs those claims for that spot, and "Something's there" takes them back over the area, so scene.json reports it unseen, and sends the scan again. "I can't check this area", for a homeowner who can't see or reach it, does the same without saying anything stands there, and the result says the area wasn't checked and needs a person to check it. When the scan sent again names the same spot, the earlier answer settles it and the homeowner isn't asked again. When it names no spot, or another one, the result still says an area along the wall went unchecked, until Start over or a new wall clears the scan's spot checks. The rules are in `Runtime/ScanEngine+Confirm.swift` and HouseScanKit's `Confirm/SpotConfirmation.swift`. +**Processing (beta).** Developer options offer two backends, both remote: **Legacy (remote checker)** sends scene.json to the placement server, and **Photo processing (beta)** sends the scan's photos and camera path to the capture API. Legacy is the default. A scan reads the choice once, when it starts, and keeps it with its endpoint in its `ScanContext` (HouseScanKit); a change applies to the next scan, and an App Store install always uses Legacy. A photo-processing scan asks before sending (`PhotoConsentCard`), sends photos as they are kept, and after the review goes to `STATE=processing` instead of the upload: its progress, then the service's answer quoted as written (`PhotoProcessingScreen`). It never falls back to the Legacy checker. An answer is shown only when the run it names, and the phone's own record of what it sent (the recording session and world epoch, the coordinator session, packet and capture), match this scan; that is a local association, since the service returns no digest of its input and no attestation. AR is withheld for every answer. Sending off the phone is off in every build for now, so choosing photo processing ends the scan saying it isn't set up. + +Two debug-only switches run the path on a replay against `FixtureCaptureHTTP`, a stand-in for the capture API inside the app (the type is in every build; only its activation is debug-only): `-photoProcessingFixture` alone sends the replay's photos, whose packet always fails the phone's 0.4 checks because a replay has no motion data and too few poses, so the scan ends "couldn't be prepared"; adding `-photoProcessingSyntheticCapture` sends HouseScanKit's `SyntheticCapture` instead, labelled `source.kind: synthetic`, so the real coordinator and uploader carry it to the fixture's answer. Either way the answer is the fixture's, never a service's or a GPU reconstruction's, and the screens say so. Live captures now record each frame's intrinsics with its pose for the 0.4 pose stream, and an accepted meter mark adds the packet's meter tap from that mark's own ARKit frame (`Runtime/LiveCapture+MeterTap.swift`); a mark that can't give one ends the capture as not prepared. Neither is checked on a phone yet. Live use still requires a scoped server-issued credential, the clock and privacy contract, and an input digest or attestation from the service. The code is in `Runtime/ScanEngine+Processing.swift` and HouseScanKit's `Processing/`. + **Practice meter.** For trying the whole scan where there is no electric meter, such as indoors. Debug and TestFlight builds show a wrench button at the top of the first screen. Tap it, turn on **Practice meter**, tap Done, then start the scan. At "find the meter", tap any spot on a wall: the usual raycast and refusals apply, and a drawn sample meter appears there, fixed to the meter's anchor, on every camera screen after that. The close-up keeps the live camera's pose and depth, but stores a photo of the sample (made-up maker SAMPLEWORKS, number 12345678), and the real reader reads it. The walk, fog, marks, requests, spot check, upload, result and AR all run for real. Every screen shows a "Practice meter" badge, and `scan-stamp.json` has `"practice": true`; the packet and scene.json are unchanged. The switch is kept until turned off, and the button reads "Practice meter on" while it is. App Store installs never show the button and ignore a stored switch: the app asks StoreKit's `AppTransaction` which server signed the install (sandbox means TestFlight) and treats no answer as the App Store. It is not in the iOS Settings app, because a Settings.bundle ships in the same binary that TestFlight later promotes to the App Store. The code is in `Runtime/Practice/`, `UI/Practice/` and HouseScanKit's `Practice/PracticeMeter.swift`. **Live fog.** During the walk and a gap request, a soft fog covers what the coverage map has not counted and lifts only where it counts a cell, the same rule the scan's decisions use (`FogValue.target` in `UI/Camera/LiveFog/LiveFogScene.swift`). On a LiDAR phone, dots settle on the surfaces the kept keyframes' depth measured (HouseScanKit's `Overlay/SurfaceDots.swift`, run by `Runtime/LiveDotsFeed.swift`); they are guidance and feed nothing. The look is the prototype's (experiments/live-dots on t3/experience). Phones without LiDAR get the same fog without dots. The shaders compile at launch, and the old frosted strip (`FogOverlay`) draws until they are ready, and for good if Metal fails. @@ -98,7 +106,7 @@ Git ignores `Local.xcconfig`. Leave the team field in Xcode's Signing & Capabili **Logs.** Open Console.app on the Mac, select the iPhone, press Start and search for `subsystem:dev.housescanning.housescan`. Every screen change logs as `STATE=`; the `engine` category records the capture's decisions, the scan bundle and upload failures. The same lines show in Xcode's console while it runs the app. -**Getting a scan off the phone.** The result screen, and the screens for a failed or refused upload, have a Share scan button. It shares the scan bundle (below); AirDrop it to your Mac. It holds photos of a real home, so keep it out of git. After the app is closed, **Saved scans** in the first screen's top bar lists the completed scans the phone still keeps (the two most recent whose `scan.zip` was written to the end, as `ScanFolderCleanup` keeps them) with when each bundle was saved, its size and a Practice tag when its `scan-stamp.json` says so. A bundle cut short by a quit during its write is not listed, holds no kept place and is not deleted. Share there sends the same bundle. It copies the bundle to the app's tmp folder first, so a new scan's cleanup can't delete it while the share sheet reads it, and deletes the copy when the sheet closes; a launch deletes copies earlier runs left. A saved scan can only be shared, never reopened or continued. The code is in `UI/SavedScans/` and HouseScanKit's `SavedScans/`. +**Getting a scan off the phone.** The result screen, and the screens for a failed or refused upload, have a Share scan button. It shares the scan bundle (below); AirDrop it to your Mac. It holds photos of a real home, so keep it out of git. After the app is closed, **Saved scans** in the first screen's top bar lists the completed scans the phone still keeps (the two most recent whose `scan.zip` was written to the end, as `ScanFolderCleanup` keeps them) with when each bundle was saved, its size and a Practice tag when its `scan-stamp.json` says so. A bundle cut short by a quit during its write is not listed, holds no kept place and is not deleted. Start over doesn't wait for a bundle still being written, but the next scan's cleanup does before it deletes anything. If that write finishes the zip, the scan reaches Saved scans; if it fails before zipping, its folder is deleted, and if it leaves a partial zip, the scan isn't listed. Share there sends the same bundle. It copies the bundle to the app's tmp folder first, so a new scan's cleanup can't delete it while the share sheet reads it, and deletes the copy when the sheet closes; a launch deletes copies earlier runs left. A saved scan can only be shared, never reopened or continued. The code is in `UI/SavedScans/` and HouseScanKit's `SavedScans/`. ## Scan bundle diff --git a/ios/Tools/snapshot-xcresult.py b/ios/Tools/snapshot-xcresult.py new file mode 100644 index 000000000..04027cc23 --- /dev/null +++ b/ios/Tools/snapshot-xcresult.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Copy a UI test result bundle to a stable folder before CI uploads it, and say what is missing. + + snapshot-xcresult.py + +Why: when the job's time cap cancels xcodebuild, the bundle is left half-written, and files under +its `Staging/` folder go away while they are being read. Run 37221532164 (#216 at 6fa43b9b) lost +its whole journey artifact that way: upload-artifact stopped at the first vanished file (ENOENT on +`Staging/1_Test/Diagnostics/...`) and kept nothing, screenshots of the failed audit included. + +What it does: copies every file of `` into `/`, then writes +`/SNAPSHOT.txt` saying "complete" or "incomplete" and listing each path that vanished. +Only a file or folder under the bundle's top-level `Staging/` may vanish; that is the transient +state of a cancelled run. Any other error stops with exit 1 and names the path; SNAPSHOT.txt then +says "failed" and that the copy beside it is partial, since the upload after this step keeps +whatever was copied. A broken bundle is never passed off as a complete or merely incomplete one. A missing bundle (the UI step never started) writes nothing +and exits 0, as the upload's `if-no-files-found: ignore` did. The job's own result, a cancellation +included, is not changed by this step. +""" +import os +import pathlib +import shutil +import sys + +EPHEMERAL = "Staging" + + +def is_ephemeral(relative): + parts = pathlib.PurePath(relative).parts + return bool(parts) and parts[0] == EPHEMERAL + + +def snapshot(bundle, out, copy=shutil.copy2): + """Copies `bundle` into `out`. Returns (copied, vanished): a file count and the relative paths + under Staging/ that disappeared mid-copy. Raises on any other error.""" + bundle = pathlib.Path(bundle) + out = pathlib.Path(out) + copied = 0 + vanished = [] + walk_errors = [] + for root, dirs, files in os.walk(bundle, onerror=walk_errors.append): + rel_root = pathlib.Path(root).relative_to(bundle) + (out / rel_root).mkdir(parents=True, exist_ok=True) + for name in sorted(dirs): + if os.path.islink(os.path.join(root, name)): + os.symlink(os.readlink(os.path.join(root, name)), out / rel_root / name) + for name in sorted(files): + relative = rel_root / name + try: + copy(os.path.join(root, name), out / relative, follow_symlinks=False) + copied += 1 + except FileNotFoundError: + if not is_ephemeral(relative): + raise + vanished.append(str(relative)) + for error in walk_errors: + relative = pathlib.Path(error.filename).relative_to(bundle) + if not (isinstance(error, FileNotFoundError) and is_ephemeral(relative)): + raise error + vanished.append(f"{relative}/") + return copied, sorted(vanished) + + +def main(argv): + if len(argv) != 3: + sys.exit(__doc__) + bundle, folder = pathlib.Path(argv[1]), pathlib.Path(argv[2]) + if not bundle.exists(): + print(f"no result bundle at {bundle}; nothing to keep") + return 0 + try: + copied, vanished = snapshot(bundle, folder / bundle.name) + except OSError as error: + print(f"::error::copying {bundle} failed outside its transient {EPHEMERAL}/ folder: {error}") + folder.mkdir(parents=True, exist_ok=True) + (folder / "SNAPSHOT.txt").write_text( + f"failed: copying stopped at an error outside {EPHEMERAL}/; the copy beside this file is partial\n{error}\n", + encoding="utf-8") + return 1 + if vanished: + report = [f"incomplete: copied {copied} files; {len(vanished)} entries under {EPHEMERAL}/ vanished while copying"] + report += vanished + print(f"::warning::The UI test results are incomplete: {len(vanished)} entries under {EPHEMERAL}/ vanished while copying (listed in SNAPSHOT.txt).") + else: + report = [f"complete: copied {copied} files"] + folder.mkdir(parents=True, exist_ok=True) + (folder / "SNAPSHOT.txt").write_text("\n".join(report) + "\n", encoding="utf-8") + print(report[0]) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/ios/Tools/test_ci_tools.py b/ios/Tools/test_ci_tools.py new file mode 100644 index 000000000..749a5dc09 --- /dev/null +++ b/ios/Tools/test_ci_tools.py @@ -0,0 +1,151 @@ +"""Tests for the CI tools in this folder: the UI test groups' check and the result-bundle snapshot. + + python3 -m unittest discover -s ios/Tools -p 'test_*.py' +""" +import contextlib +import importlib.util +import io +import pathlib +import shutil +import tempfile +import unittest + +HERE = pathlib.Path(__file__).resolve().parent + + +def load(filename, name): + spec = importlib.util.spec_from_file_location(name, HERE / filename) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +groups = load("ui-test-groups.py", "ui_test_groups") +snap = load("snapshot-xcresult.py", "snapshot_xcresult") + + +def quiet(function, *args): + with contextlib.redirect_stdout(io.StringIO()) as out: + result = function(*args) + return result, out.getvalue() + + +class UITestGroupsTests(unittest.TestCase): + def test_the_committed_groups_pass(self): + result, out = quiet(groups.check) + self.assertEqual(result, 0, out) + self.assertIn("overlap: 0", out) + self.assertIn("missing: 0", out) + + def test_a_class_in_two_groups_fails(self): + original = groups.selectors + def overlapping(group, full_ui): + args = original(group, full_ui) + return [a for a in args if a != f"-skip-testing:{groups.TARGET}/{groups.SCREEN_CLASS}"] if group == "journey" else args + groups.selectors = overlapping + try: + result, out = quiet(groups.check) + finally: + groups.selectors = original + self.assertEqual(result, 1) + self.assertIn("in more than one group", out) + + def test_a_class_in_no_group_fails(self): + original = groups.selectors + def dropping(group, full_ui): + return [f"-only-testing:{groups.TARGET}/{groups.PHOTO_CLASS}"] if group == "screenStates" else original(group, full_ui) + groups.selectors = dropping + try: + result, out = quiet(groups.check) + finally: + groups.selectors = original + self.assertEqual(result, 1) + self.assertIn("in no group", out) + self.assertIn("screenStates must hold ScreenStatesUITests and nothing else", out) + + +class SnapshotTests(unittest.TestCase): + def setUp(self): + self.root = pathlib.Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.root) + self.bundle = self.root / "HouseScanUITests.xcresult" + for path in ["Info.plist", "Data/data.0", "Staging/1_Test/Attachments/shot.png", "Staging/1_Test/Diagnostics/log.txt", "Staging/note.txt"]: + file = self.bundle / path + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(path) + self.out = self.root / "out" + + def test_a_finished_bundle_is_copied_whole(self): + copied, vanished = snap.snapshot(self.bundle, self.out / self.bundle.name) + self.assertEqual((copied, vanished), (5, [])) + self.assertEqual((self.out / self.bundle.name / "Staging/1_Test/Attachments/shot.png").read_text(), "Staging/1_Test/Attachments/shot.png") + + def test_a_staging_file_that_vanishes_is_listed_and_the_rest_kept(self): + def copy(source, target, follow_symlinks): + if source.endswith("log.txt"): + raise FileNotFoundError(source) + shutil.copy2(source, target, follow_symlinks=follow_symlinks) + copied, vanished = snap.snapshot(self.bundle, self.out, copy=copy) + self.assertEqual(copied, 4) + self.assertEqual(vanished, ["Staging/1_Test/Diagnostics/log.txt"]) + self.assertTrue((self.out / "Staging/1_Test/Attachments/shot.png").exists()) + + def test_a_staging_folder_that_vanishes_is_listed(self): + def copy(source, target, follow_symlinks): + shutil.copy2(source, target, follow_symlinks=follow_symlinks) + if source.endswith("note.txt"): + shutil.rmtree(self.bundle / "Staging/1_Test") + copied, vanished = snap.snapshot(self.bundle, self.out, copy=copy) + self.assertEqual(vanished, ["Staging/1_Test/"]) + self.assertEqual(copied, 3) + + def test_a_file_outside_staging_that_vanishes_stops_the_copy(self): + def copy(source, target, follow_symlinks): + if source.endswith("data.0"): + raise FileNotFoundError(source) + shutil.copy2(source, target, follow_symlinks=follow_symlinks) + with self.assertRaises(FileNotFoundError): + snap.snapshot(self.bundle, self.out, copy=copy) + + def test_main_reports_incomplete_and_exits_zero(self): + original = snap.snapshot + snap.snapshot = lambda bundle, out: (4, ["Staging/1_Test/Diagnostics/log.txt"]) + try: + result, out = quiet(snap.main, ["snapshot", str(self.bundle), str(self.out)]) + finally: + snap.snapshot = original + self.assertEqual(result, 0) + self.assertIn("::warning::", out) + report = (self.out / "SNAPSHOT.txt").read_text() + self.assertTrue(report.startswith("incomplete: copied 4 files; 1 entries under Staging/ vanished")) + self.assertIn("Staging/1_Test/Diagnostics/log.txt", report) + + def test_main_reports_complete(self): + result, out = quiet(snap.main, ["snapshot", str(self.bundle), str(self.out)]) + self.assertEqual(result, 0) + self.assertEqual((self.out / "SNAPSHOT.txt").read_text(), "complete: copied 5 files\n") + + def test_main_fails_loudly_outside_staging(self): + original = snap.snapshot + def broken(bundle, out): + raise PermissionError(13, "denied", str(self.bundle / "Data/data.0")) + snap.snapshot = broken + try: + result, out = quiet(snap.main, ["snapshot", str(self.bundle), str(self.out)]) + finally: + snap.snapshot = original + self.assertEqual(result, 1) + self.assertIn("::error::", out) + # The upload keeps whatever was copied, so the partial copy is marked as failed. + report = (self.out / "SNAPSHOT.txt").read_text() + self.assertTrue(report.startswith("failed: copying stopped at an error outside Staging/; the copy beside this file is partial")) + self.assertIn("data.0", report) + + def test_no_bundle_keeps_nothing(self): + result, out = quiet(snap.main, ["snapshot", str(self.root / "missing.xcresult"), str(self.out)]) + self.assertEqual(result, 0) + self.assertFalse(self.out.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/ios/Tools/ui-test-groups.py b/ios/Tools/ui-test-groups.py new file mode 100755 index 000000000..f190422ae --- /dev/null +++ b/ios/Tools/ui-test-groups.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""The three hosted UI test groups, and a check that together they run every UI test once. + +CI splits HouseScanUITests across three jobs so none passes the 90-minute cap: `screenStates`, +ScreenStatesUITests alone; `photoProcessing`, PhotoProcessingUITests alone; and `journey`, every +other class. With ScreenStatesUITests in `journey`, run 37221532164 (#216 at 6fa43b9b) hit the +cap: 12 min of package tests and build before the UI step, which ran 75.3 min and stopped with +107 of 109 tests run. That run's per-class test times put ScreenStatesUITests at 1892 s (its +every-state audit 905 s) and the remaining journey classes at about 2475 s. The workflow takes +each group's xcodebuild selectors from here, so the check below reads the same selectors CI +passes. + + ui-test-groups.py args [--full-ui true|false] print the group's selectors, one per line + ui-test-groups.py check list the tests from source and check them + +`check` finds every `func test...()` in an XCTestCase class under ios/HouseScanUITests, works out +which tests each group's selectors select as xcodebuild does (an -only-testing prefix includes, a +-skip-testing prefix excludes), and fails unless, with the full audit on, each test is in exactly +one group; with it off, the only test left out must be the every-state audit, the existing +pull-request policy. `screenStates` and `photoProcessing` must each hold their one class and +nothing else. It also fails on a selector that names no class or test. It reads source +only, so a test hidden behind `#if` would be listed although the build leaves it out; there is none +today. +""" +import pathlib +import re +import sys + +TARGET = "HouseScanUITests" +PHOTO_CLASS = "PhotoProcessingUITests" +SCREEN_CLASS = "ScreenStatesUITests" +# Pull requests skip this without the full-ui label (CONTRIBUTING.md, "Checks"). +FULL_UI_ONLY = f"{TARGET}/{SCREEN_CLASS}/testEveryStatePassesTheAccessibilityAudit" +GROUPS = ("screenStates", "journey", "photoProcessing") +# The groups that hold exactly one class, and that class. +SINGLE_CLASS = {"screenStates": SCREEN_CLASS, "photoProcessing": PHOTO_CLASS} + + +def selectors(group, full_ui): + if group == "journey": + return [f"-only-testing:{TARGET}", f"-skip-testing:{TARGET}/{PHOTO_CLASS}", f"-skip-testing:{TARGET}/{SCREEN_CLASS}"] + if group == "screenStates": + args = [f"-only-testing:{TARGET}/{SCREEN_CLASS}"] + if not full_ui: + args.append(f"-skip-testing:{FULL_UI_ONLY}") + return args + if group == "photoProcessing": + # Its default and largest-text audits always run: they are this group's own evidence. + return [f"-only-testing:{TARGET}/{PHOTO_CLASS}"] + sys.exit(f"unknown group {group!r}; the groups are {', '.join(GROUPS)}") + + +def tests_in_source(folder): + """Every `Class/testMethod` XCTest would run, read from the Swift sources.""" + found = [] + for path in sorted(folder.glob("*.swift")): + current = None + for line in path.read_text(encoding="utf-8").splitlines(): + declared = re.match(r"\s*(?:final\s+)?class\s+(\w+)\s*:\s*([\w ,]+)", line) + if declared: + current = declared.group(1) if "XCTestCase" in declared.group(2) else None + continue + if re.match(r"\s*(?:final\s+)?class\s+\w+", line): + current = None + continue + extended = re.match(r"\s*extension\s+(\w+)", line) + if extended: + current = extended.group(1) if extended.group(1).endswith("Tests") else None + continue + method = re.match(r"\s*(?:@MainActor\s+)?func\s+(test\w*)\s*\(\s*\)", line) + if method and current: + found.append(f"{TARGET}/{current}/{method.group(1)}") + return found + + +def selected(tests, args): + only = [a.split(":", 1)[1] for a in args if a.startswith("-only-testing:")] + skip = [a.split(":", 1)[1] for a in args if a.startswith("-skip-testing:")] + covers = lambda prefix, test: test == prefix or test.startswith(prefix + "/") + return {t for t in tests if any(covers(p, t) for p in only) and not any(covers(p, t) for p in skip)} + + +def check(): + folder = pathlib.Path(__file__).resolve().parent.parent / TARGET + tests = tests_in_source(folder) + problems = [] + if len(tests) != len(set(tests)): + problems.append("a test name appears twice in source") + names = set(tests) + for full_ui in (True, False): + for group in GROUPS: + for arg in selectors(group, full_ui): + prefix = arg.split(":", 1)[1] + if prefix != TARGET and not any(t == prefix or t.startswith(prefix + "/") for t in names): + problems.append(f"{group}: {arg} names no test") + full = {g: selected(names, selectors(g, True)) for g in GROUPS} + pr = {g: selected(names, selectors(g, False)) for g in GROUPS} + overlap = {t for t in names if sum(t in full[g] for g in GROUPS) > 1} + missing = names - set().union(*full.values()) + left_out = names - set().union(*pr.values()) + print(f"tests in source: {len(names)}") + for group in GROUPS: + print(f"{group}: {len(full[group])} (pull request without full-ui: {len(pr[group])})") + print(f"overlap: {len(overlap)}") + print(f"missing: {len(missing)}") + print(f"left out without full-ui: {sorted(left_out)}") + problems += [f"in more than one group: {t}" for t in sorted(overlap)] + problems += [f"in no group: {t}" for t in sorted(missing)] + if left_out != {FULL_UI_ONLY}: + problems.append(f"without full-ui exactly {FULL_UI_ONLY} should be left out, not {sorted(left_out)}") + for group, cls in SINGLE_CLASS.items(): + if not full[group] or any(not t.startswith(f"{TARGET}/{cls}/") for t in full[group]): + problems.append(f"{group} must hold {cls} and nothing else") + for group in GROUPS: + if not pr[group]: + problems.append(f"{group} runs no test on a pull request without full-ui") + for problem in problems: + print(f"error: {problem}") + return 1 if problems else 0 + + +def main(argv): + if argv[1:2] == ["check"] and len(argv) == 2: + return check() + if argv[1:2] == ["args"] and len(argv) in (3, 5): + full_ui = True + if len(argv) == 5: + if argv[3] != "--full-ui" or argv[4] not in ("true", "false"): + sys.exit(__doc__) + full_ui = argv[4] == "true" + print("\n".join(selectors(argv[2], full_ui))) + return 0 + sys.exit(__doc__) + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/packet/README.md b/packet/README.md index 98fa39619..541ce1982 100644 --- a/packet/README.md +++ b/packet/README.md @@ -9,7 +9,13 @@ a format of its own. This folder holds the app side's proposals for the next revision (0.5): what the app records that 0.4 has no field for, and three arrival checks for the server's validator ([validator-checks.md](validator-checks.md)). Packet 1.1, which this folder used to define, is -retired; it is still readable at `t3/packet` commit `d5439cf`. +retired as a contract proposal; that version is still readable at `t3/packet` commit `d5439cf`. + +The app still writes a packet 1.1 for its own use: the `scan.zip` that "Share scan" offers, +for looking into a scan by hand. Nothing uploads it, the server's intake does not accept it, +and it is not an approved revision of 0.4. Fields it carries that 0.4 lacks, such as the end +provenance and guidance log below, are additive local diagnostics. Each is also listed here as +a proposal for 0.5, which the contract's owners may adopt, change or decline. ## Proposals for 0.5 @@ -39,18 +45,37 @@ performance. homeowner gave about it: - whether a wall end is a real end (something blocks it) or unexplored (it turns a corner, or no answer), which decides whether the server may reject for want of space; + - whether the homeowner marked a wall end ("Wall ends here") or the app inferred it from the + walk: where the walk reached after "Can't get there" or "The wall keeps going", or past the + ground a `past_end` request showed. An inferred end has the same place and kind as a marked + one, but nobody pointed at it, so it says where the scan stopped, not where the wall does; - whether a window opens, which the rules may use (whether a fixed window is exempt from the opening clearance is the rules' choice, not the app's); - which corner taps belong to the same door or window. - Proposal: `taps[].attrs` with `endKind` (`limit`, `unexplored`) and `operable`, and - `taps[].group` for taps that outline one object. + Proposal: `taps[].attrs` with `endKind` (`limit`, `unexplored`), `operable` and `inferred`, + and `taps[].group` for taps that outline one object. + + The app's local packet 1.1 (above) already writes the provenance: a wall end the homeowner + marked has its mark time `t`; an inferred one has no `t` and `attrs.inferred: true`. A + missing `t` alone never means the homeowner marked it. 4. **The guidance log.** Every request the homeowner was shown (walk, tilt to the ground, mark an end, close-up, a gap the phone or the server asked for), when, and its outcome: `met`, `skipped`, `cannot_reach`, `superseded` or `unresolved`. Today "I can't get there" never reaches the server, so the server can ask for the same unreachable view again. Proposal: `guidance[]` with `id`, `kind`, `origin` (`phone`, `server`), `message`, optional `band` and `span`, `tShown`, `tResolved` and `outcome`. + + For the wall's ends, as the app's local packet 1.1 records them: `mark_end` is `met` only + when the homeowner marked that end. "The wall keeps going" and "Can't get there" close it as + `cannot_reach`, because the end lay beyond where the homeowner got, and the end placed there + is inferred. `gap_past_end` is `met` when views past the end settle it, or when the homeowner + marks the end again and says the wall stops there (something blocks it, or it just ends); the + wall end's provenance tells those apart. A corner the homeowner marks during the request is + not `met`: the wall goes on round it, the request can't follow it there, and the server + settles an unexplored end only with views past it. It closes as `cannot_reach`, and the + corner stays where the homeowner marked it, unexplored. "I can't get there" is `cannot_reach` + too, and the end the request had cleared comes back as it was. 5. **Depth between keyframes** (from S6's live map, draft PR #21). Keyframes are kept about every 0.5 m, but the phone fuses depth at about 10 Hz to decide what was seen. With keyframe depth alone the server cannot rebuild that decision. Proposal: optional `depthFrames[]` with `t`,