From ca98508371943576f43724a638acfe43a06d6b46 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Thu, 1 Oct 2026 14:42:59 -0500 Subject: [PATCH 01/71] chore(review): run Greptile again when PR commits change --- .greptile/config.json | 3 +++ CONTRIBUTING.md | 2 ++ 2 files changed, 5 insertions(+) create mode 100644 .greptile/config.json diff --git a/.greptile/config.json b/.greptile/config.json new file mode 100644 index 000000000..9d6735365 --- /dev/null +++ b/.greptile/config.json @@ -0,0 +1,3 @@ +{ + "autoReview": ["open", "push"] +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index feca4fe13..c0b53fc1a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,6 +22,8 @@ State what the old code did before describing the new behavior. Link the reviewe The `size:*` label describes the effective diff. It is a review signal, not a merge gate. Do not set it by hand. +Greptile reviews new commits through [`.greptile/config.json`](.greptile/config.json). The dashboard's open-only trigger left follow-up commits unreviewed. This repository override adds push reviews without changing other repositories' settings. Greptile reads this setting from the PR's source branch, so older branches need the config too. For a branch without it, request a current review with `@greptileai review` after pushing. Use `@greptileai review this draft` for drafts. + ## Checks | Check | Runs on GitHub when | Local command | From 6ec4016b9201c62f4511ff374ff334354194fd1e Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 18:15:17 -0500 Subject: [PATCH 02/71] ios: Start over keeps a scan whose bundle is still being written Start over makes a new store, whose cleanup lists the previous scan's folder at once. While that scan's packet is still being assembled there is no scan.zip yet, so the folder was listed as never packaged and deleted from under the write, and the scan never reached Saved scans. Start over still doesn't wait. The new store's deferred deletion waits for the bundle write chain as it stood at reset (ScanFolderCleanup.run(after:)), then rechecks as before: a bundle that landed keeps its folder, a write that failed before zipping leaves it to be deleted, and a partial zip stays protected. Launch passes no writer and deletes at once. saveBundle also refuses inputs read from an earlier store before it touches the serial or the offered bundle, so the folder ownership holds locally rather than through its callers. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/Runtime/KeyframeStore.swift | 31 ++-- ios/HouseScan/Runtime/ScanEngine.swift | 17 +- .../Scene/ScanFolderCleanup.swift | 20 ++- .../HouseScanKitTests/KeptScansTests.swift | 167 ++++++++++++++++++ ios/README.md | 2 +- 5 files changed, 222 insertions(+), 15 deletions(-) diff --git a/ios/HouseScan/Runtime/KeyframeStore.swift b/ios/HouseScan/Runtime/KeyframeStore.swift index c75157660..957bbc289 100644 --- a/ios/HouseScan/Runtime/KeyframeStore.swift +++ b/ios/HouseScan/Runtime/KeyframeStore.swift @@ -51,16 +51,21 @@ final class KeyframeStore { /// Bumped by `discardKeyframes`, so a write that started before it doesn't land in the list. private var epoch = 0 - /// Makes a new, empty scan folder and deletes every other one: only the current scan is kept - /// on the phone. Covers both a start over (the previous scan's folder) and launch (folders a - /// quit or crashed run left behind), since both make a new store. The folders to delete are - /// listed here, before a newer store can exist, and deleted later off the main actor - /// (`ScanFolderCleanup`), so a deletion that runs late can't take a newer scan's folder. - init() { + /// Makes a new, empty scan folder and deletes the old ones `ScanFolderCleanup` lists: it keeps + /// the two most recent completed scans (Saved scans offers them) and any folder whose bundle + /// is there but not whole. Covers both a start over (the previous scan's folder) and launch + /// (folders a quit or crashed run left behind), since both make a new store. + /// + /// The folders are listed here, before a newer store can exist, so a deletion that runs late + /// can't take a newer scan's folder. They are deleted later off the main actor, once + /// `bundleWrites` ends: the scan bundle writes started before this store, of which the + /// previous scan's may still be assembling its packet in a folder listed as never packaged. + /// Start over doesn't wait for it; only the deletion does. Nil, as at launch, waits for none. + init(deletingAfter bundleWrites: Task? = nil) { let scans = Self.scansRoot directory = scans.appending(path: UUID().uuidString, directoryHint: .isDirectory) try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - Self.delete(ScanFolderCleanup(root: scans, keeping: directory.lastPathComponent)) + Self.delete(ScanFolderCleanup(root: scans, keeping: directory.lastPathComponent), after: bundleWrites) } /// Caches/Scans: one folder per scan. Saved scans lists the completed ones here. @@ -246,7 +251,8 @@ final class KeyframeStore { /// Zips the packet folder into `scan.zip`, manifest.json at the zip's root, and removes the /// folder. The zip is what "Share scan" offers; the upload sends scene.json alone. It stays on - /// the phone until the next scan unless the homeowner shares it. + /// the phone, listed in Saved scans, until two newer completed scans push it out + /// (`ScanFolderCleanup`). /// /// Streamed to disk one file at a time: a long walk's JPEGs held in memory twice (the entries /// and the archive) is what an in-memory build cost. @@ -267,11 +273,12 @@ final class KeyframeStore { try? files.removeItem(at: folder) } - /// Off the main actor. A write still in flight for a deleted folder fails, because its - /// directory is gone, and is dropped like any failed write. - nonisolated private static func delete(_ cleanup: ScanFolderCleanup) { + /// Off the main actor, after `bundleWrites`. A keyframe write still in flight for a deleted + /// folder fails, because its directory is gone, and is dropped like any failed write; the + /// bundle lists only keyframes already written. + nonisolated private static func delete(_ cleanup: ScanFolderCleanup, after bundleWrites: Task?) { Task.detached(priority: .utility) { - for (url, error) in cleanup.run() { + for (url, error) in await cleanup.run(after: { await bundleWrites?.value }) { RuntimeLog.engine.error("could not delete old scan \(url.lastPathComponent, privacy: .public): \(String(describing: error), privacy: .public)") } } diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index d43244723..6405337fb 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -2220,7 +2220,18 @@ final class ScanEngine { /// place, so it is not offered while a write is under way, and writes run one after another: /// a retry's write waits for the last one, and a write already superseded is skipped. /// `inputs` were captured with the upload's scene (`captureUpload`); this rereads nothing. + /// + /// Inputs read from an earlier store write into that store's folder, which the current + /// store's cleanup may delete; they are refused before they change what this scan offers. + /// The callers already drop a capture from before Start over; this keeps that true here. + /// + /// Start over doesn't wait for a write in flight: the next store's cleanup waits for + /// `bundleTask` before deleting the folder, so the write's bundle lands and is kept. func saveBundle(_ inputs: PacketInputs?) { + if let inputs, inputs.storeDirectory != store.directory { + RuntimeLog.engine.error("scan bundle not written: its capture belongs to a scan already started over") + return + } state.shareableScan = nil bundleSerial += 1 let serial = bundleSerial @@ -2271,7 +2282,11 @@ final class ScanEngine { meterTracking = nil meterAnchorPresence = MeterAnchorPresence() meterPlaneSource = .detectedPlane - store = KeyframeStore() + // The old scan's bundle may still be packing in the folder the new store's cleanup lists + // as never packaged. Start over doesn't wait for it; the cleanup's deletion does, for this + // chain head (every write waits for the one before it). `generation` was raised above, + // so a queued write for the old scan still skips, and a finished one isn't offered here. + store = KeyframeStore(deletingAfter: bundleTask) recorder = Self.makeRecorder(store) live?.setRecorder(recorder) motion.stop() diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Scene/ScanFolderCleanup.swift b/ios/HouseScanKit/Sources/HouseScanKit/Scene/ScanFolderCleanup.swift index 3335f8361..927e58c00 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Scene/ScanFolderCleanup.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Scene/ScanFolderCleanup.swift @@ -26,6 +26,13 @@ import Foundation /// can exist, and only those are deleted, whenever the deletion gets to run. Listing at deletion /// time instead let a cleanup that ran late delete a scan started after it: store A is made, /// Start over makes B, and A's cleanup then found B in the listing and deleted the scan in use. +/// +/// A folder whose bundle is still being built has no `scan.zip` yet, so it is listed as never +/// packaged. The packet is assembled in the folder first and zipped last, and Start over doesn't +/// wait for that write, so deleting on the listing alone took the folder from under it and the +/// scan never reached Saved scans. The app therefore deletes with `run(after:)`, waiting for the +/// bundle writes that were started before the listing; the recheck in `run` then keeps a folder +/// whose bundle landed meanwhile. public struct ScanFolderCleanup: Sendable { /// The bundle Share scan offers. A scan is completed when this file is a whole capture packet /// (`PacketArchiveCheck`); the file being there is not enough. @@ -132,7 +139,8 @@ public struct ScanFolderCleanup: Sendable { /// The deletion runs later than the listing, off the main actor, and a folder can change in /// between: a bundle write can start in a folder listed as never packaged, or finish in one. /// Each entry is read again and deleted only if it is still what it was when listed; one that - /// changed is left for the next cleanup. A change after this second read is not caught. + /// changed is left for the next cleanup. A change after this second read is not caught, so a + /// write still going when this runs can lose its folder: `run(after:)` waits for known writers. public func run() -> [(url: URL, error: any Error)] { let files = FileManager.default return obsolete.compactMap { url in @@ -146,4 +154,14 @@ public struct ScanFolderCleanup: Sendable { } } } + + /// `run()`, once `wait` returns. The app's `wait` is the scan bundle writes started before + /// this cleanup was listed (one chain: each write waits for the one before it), so a folder + /// listed while its packet was still being assembled is judged again only after that write + /// ends: a bundle that landed keeps the folder, and a write that failed before zipping leaves + /// it to be deleted. A `wait` that returns at once, as at launch with no write, is `run()`. + public func run(after wait: () async -> Void) async -> [(url: URL, error: any Error)] { + await wait() + return run() + } } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/KeptScansTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/KeptScansTests.swift index 521549436..579cb2fe7 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/KeptScansTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/KeptScansTests.swift @@ -219,6 +219,173 @@ import Testing #expect(try Set(files.contentsOfDirectory(atPath: root.path)) == ["intact1", "intact2", "rewritten", "startsWriting", "finishesWriting"]) } + // MARK: A bundle still being written when Start over lists its folder (B44) + + /// A scan folder whose bundle is being written: the photos, stamp and raw streams the packet + /// is built from and the half-assembled `packet/`, but no `scan.zip` yet. + static func packing(_ name: String, in root: URL) throws -> URL { + try scan(name, in: root) + let folder = root.appending(path: name) + let files = FileManager.default + try Data("{}".utf8).write(to: folder.appending(path: "scan-stamp.json")) + try files.createDirectory(at: folder.appending(path: "streams-raw"), withIntermediateDirectories: true) + try Data([1, 2, 3]).write(to: folder.appending(path: "streams-raw/trajectory.bin")) + try files.createDirectory(at: folder.appending(path: "packet/photos"), withIntermediateDirectories: true) + try Data("photo".utf8).write(to: folder.appending(path: "packet/photos/p00001.jpg")) + return folder + } + + /// What a bundle write leaves when it ends. + enum WriteEnd: Sendable, CaseIterable { + /// The zip written to the end and `packet/` removed, as `zipPacket` does. + case bundled + /// It failed before zipping: `packet/` is left and there is no zip. + case failedBeforeZipping + /// A zip cut short is left. + case partialZip + + func apply(to folder: URL) throws { + let bundle = folder.appending(path: ScanFolderCleanup.bundleName) + switch self { + case .bundled: + try KeptScansTests.packet.write(to: bundle) + try FileManager.default.removeItem(at: folder.appending(path: "packet")) + case .failedBeforeZipping: + break + case .partialZip: + try KeptScansTests.packet.prefix(KeptScansTests.packet.count / 2).write(to: bundle) + } + } + } + + /// A bundle write the test holds open, then ends as `end` says; like the app's, it starts + /// only after the write before it (`after`). + struct HeldWrite { + let task: Task + private let release: AsyncStream.Continuation + + init(_ folder: URL, ending end: WriteEnd, after earlier: Task? = nil) { + let (gate, release) = AsyncStream.makeStream() + self.release = release + task = Task { + await earlier?.value + for await _ in gate { break } + try? end.apply(to: folder) + } + } + + func finish() { release.finish() } + } + + /// Deletes with `run(after:)`, waiting for `writes` as the app does, and says through + /// `arrived` when it has reached that wait. Returns the folders it failed to delete. + static func deleting(_ cleanup: ScanFolderCleanup, after writes: Task?, arrived: AsyncStream.Continuation) -> Task<[URL], Never> { + Task { + await cleanup.run(after: { + arrived.yield() + await writes?.value + }).map(\.url) + } + } + + /// Start over lists the previous scan's folder while its bundle is still being assembled: + /// with no zip yet it is listed as never packaged. The deletion waits for the write; a zip + /// written to the end keeps the folder, one cut short keeps it without a place (as any + /// partial bundle), and a write that failed before zipping leaves it to be deleted. + @Test(arguments: WriteEnd.allCases) + func aBundleStillBeingWrittenIsJudgedOnlyAfterItsWrite(end: WriteEnd) async throws { + let root = try Self.makeRoot() + defer { try? FileManager.default.removeItem(at: root) } + let folder = try Self.packing("previous", in: root) + try Self.scan("new", in: root) + let write = HeldWrite(folder, ending: end) + + let cleanup = ScanFolderCleanup(root: root, keeping: "new") + // What B44 deleted: the folder being written is listed with the never-packaged ones. + #expect(Self.names(cleanup.obsolete) == ["previous"]) + let (arrived, arrive) = AsyncStream.makeStream() + let deleting = Self.deleting(cleanup, after: write.task, arrived: arrive) + var arrivals = arrived.makeAsyncIterator() + _ = await arrivals.next() + // The cleanup is waiting for the write, which is still assembling the packet. + #expect(FileManager.default.fileExists(atPath: folder.appending(path: "packet/photos/p00001.jpg").path)) + + write.finish() + #expect(await deleting.value.isEmpty) + let next = ScanFolderCleanup(root: root, keeping: "next") + switch end { + case .bundled: + #expect(FileManager.default.fileExists(atPath: folder.appending(path: ScanFolderCleanup.bundleName).path)) + #expect(Self.names(next.keptCompleted) == ["previous"]) + case .failedBeforeZipping: + #expect(!FileManager.default.fileExists(atPath: folder.path)) + case .partialZip: + #expect(FileManager.default.fileExists(atPath: folder.path)) + #expect(Self.names(next.incompleteBundles) == ["previous"]) + #expect(!Self.names(next.obsolete).contains("previous")) + } + } + + /// At launch there is no write to wait for: the deletion is `run()`'s, at once. + @Test func withNoWriteTheDeletionRunsAtOnce() async throws { + let root = try Self.makeRoot() + defer { try? FileManager.default.removeItem(at: root) } + try Self.scan("abandoned", in: root) + try Self.scan("run1", in: root, bundledMinutesAgo: 90) + try Self.scan("run2", in: root, bundledMinutesAgo: 30) + try Self.scan("run3", in: root, bundledMinutesAgo: 5) + try Self.scan("now", in: root) + + let none: Task? = nil + let failed = await ScanFolderCleanup(root: root, keeping: "now").run(after: { await none?.value }) + #expect(failed.isEmpty) + #expect(try Set(FileManager.default.contentsOfDirectory(atPath: root.path)) == ["now", "run2", "run3"]) + } + + /// Start over twice while the first scan's bundle is written. The second scan's write, if it + /// runs, waits for the first; each cleanup waits for the chain as it stood when it listed. + /// The first scan's bundle is kept, the second scan's is kept only if it was written (a write + /// queued for a scan already started over is skipped), and the newest folders are never + /// listed. + @Test(arguments: [true, false]) + func rapidStartOversWaitForTheWholeChain(secondScanWrites: Bool) async throws { + let root = try Self.makeRoot() + defer { try? FileManager.default.removeItem(at: root) } + let first = try Self.packing("first", in: root) + let firstWrite = HeldWrite(first, ending: .bundled) + + try Self.scan("second", in: root) + let firstCleanup = ScanFolderCleanup(root: root, keeping: "second") + #expect(Self.names(firstCleanup.obsolete) == ["first"]) + let second = root.appending(path: "second") + let secondWrite = Task { + await firstWrite.task.value + if secondScanWrites { try? WriteEnd.bundled.apply(to: try Self.packing("second", in: root)) } + } + + try Self.scan("third", in: root) + let secondCleanup = ScanFolderCleanup(root: root, keeping: "third") + #expect(Self.names(secondCleanup.obsolete) == ["first", "second"]) + try Self.scan("fourth", in: root) + + let (arrived, arrive) = AsyncStream.makeStream() + let deletingFirst = Self.deleting(firstCleanup, after: firstWrite.task, arrived: arrive) + let deletingSecond = Self.deleting(secondCleanup, after: secondWrite, arrived: arrive) + var arrivals = arrived.makeAsyncIterator() + _ = await arrivals.next() + _ = await arrivals.next() + #expect(FileManager.default.fileExists(atPath: first.appending(path: "packet").path)) + + firstWrite.finish() + #expect(await deletingFirst.value.isEmpty) + #expect(await deletingSecond.value.isEmpty) + var expected: Set = ["first", "third", "fourth"] + if secondScanWrites { expected.insert("second") } + #expect(try Set(FileManager.default.contentsOfDirectory(atPath: root.path)) == expected) + #expect(FileManager.default.fileExists(atPath: first.appending(path: ScanFolderCleanup.bundleName).path)) + #expect(FileManager.default.fileExists(atPath: second.path) == secondScanWrites) + } + /// Bundles saved at the same moment keep the order by name, newest name first, as before. @Test func equalSaveTimesFallBackToNameOrder() throws { let root = try Self.makeRoot() diff --git a/ios/README.md b/ios/README.md index 601d2da9c..e69553734 100644 --- a/ios/README.md +++ b/ios/README.md @@ -98,7 +98,7 @@ Git ignores `Local.xcconfig`. Leave the team field in Xcode's Signing & Capabili **Logs.** Open Console.app on the Mac, select the iPhone, press Start and search for `subsystem:dev.housescanning.housescan`. Every screen change logs as `STATE=`; the `engine` category records the capture's decisions, the scan bundle and upload failures. The same lines show in Xcode's console while it runs the app. -**Getting a scan off the phone.** The result screen, and the screens for a failed or refused upload, have a Share scan button. It shares the scan bundle (below); AirDrop it to your Mac. It holds photos of a real home, so keep it out of git. After the app is closed, **Saved scans** in the first screen's top bar lists the completed scans the phone still keeps (the two most recent whose `scan.zip` was written to the end, as `ScanFolderCleanup` keeps them) with when each bundle was saved, its size and a Practice tag when its `scan-stamp.json` says so. A bundle cut short by a quit during its write is not listed, holds no kept place and is not deleted. Share there sends the same bundle. It copies the bundle to the app's tmp folder first, so a new scan's cleanup can't delete it while the share sheet reads it, and deletes the copy when the sheet closes; a launch deletes copies earlier runs left. A saved scan can only be shared, never reopened or continued. The code is in `UI/SavedScans/` and HouseScanKit's `SavedScans/`. +**Getting a scan off the phone.** The result screen, and the screens for a failed or refused upload, have a Share scan button. It shares the scan bundle (below); AirDrop it to your Mac. It holds photos of a real home, so keep it out of git. After the app is closed, **Saved scans** in the first screen's top bar lists the completed scans the phone still keeps (the two most recent whose `scan.zip` was written to the end, as `ScanFolderCleanup` keeps them) with when each bundle was saved, its size and a Practice tag when its `scan-stamp.json` says so. A bundle cut short by a quit during its write is not listed, holds no kept place and is not deleted. Start over doesn't wait for a bundle still being written, but the next scan's cleanup does before it deletes anything, so that scan still reaches Saved scans. Share there sends the same bundle. It copies the bundle to the app's tmp folder first, so a new scan's cleanup can't delete it while the share sheet reads it, and deletes the copy when the sheet closes; a launch deletes copies earlier runs left. A saved scan can only be shared, never reopened or continued. The code is in `UI/SavedScans/` and HouseScanKit's `SavedScans/`. ## Scan bundle From 3e75514a625a204e2dfb5a648cd431f0946bcc5d Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 18:16:30 -0500 Subject: [PATCH 03/71] Show a passing spot as a possible spot, not a fit The server's checks pass on observed space beyond the area the homeowner confirms in the spot check, some of it inferred rather than seen (a walked path, a photo's view), and the answer doesn't say which. The app can't tell that all the space a battery needs was confirmed (B17), so no result says a battery fits. ResultReading's affirmative answer becomes `candidate`: "A possible battery spot", with a note that the scan couldn't confirm all the space and an installer needs to check the fit on site. Every path that read as a fit goes through it, policy-only manual review included, so a retry, a clear answer or a settled earlier answer can't restore one. The other answers are unchanged, and so are the server's answer, its binding, the scene and the packet. Every result surface agrees. The card shows only failing and unsure checks. Details calls its list the server's calculations on what the scan recorded, drops a pass's positive server reason (keeping its measurement), and hides a candidate's pass summary. The AR title, overlay and off-screen direction name the proposed spot, and passing clearance zones draw in the proposal's blue rather than clear-ground green. The result arrives with a light tap instead of the success haptic (B26). The card's pass ranking was the only use of ResultReading.margin, so it is removed. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Contract/ResultPresentation+Reading.swift | 14 +- ios/HouseScan/UI/Copy/ScanCopy.swift | 29 ++- ios/HouseScan/UI/Result/ResultScene3D.swift | 16 +- ios/HouseScan/UI/ScanRootView.swift | 19 +- ios/HouseScan/UI/Screens/ResultARScreen.swift | 20 +- ios/HouseScan/UI/Screens/ResultScreen.swift | 59 ++++- ios/HouseScan/UI/Theme/Theme.swift | 7 + .../Scene/ResultCardActions.swift | 2 +- .../HouseScanKit/Scene/ResultReading.swift | 56 ++-- .../ResultCardActionsTests.swift | 6 +- .../ResultReadingTests.swift | 105 ++++++-- .../ProvisionalPlacementUITests.swift | 241 ++++++++++++++++++ 12 files changed, 458 insertions(+), 116 deletions(-) create mode 100644 ios/HouseScanUITests/ProvisionalPlacementUITests.swift diff --git a/ios/HouseScan/Contract/ResultPresentation+Reading.swift b/ios/HouseScan/Contract/ResultPresentation+Reading.swift index 19750bb65..7689343b4 100644 --- a/ios/HouseScan/Contract/ResultPresentation+Reading.swift +++ b/ios/HouseScan/Contract/ResultPresentation+Reading.swift @@ -76,10 +76,7 @@ extension ResultPresentation { checks.map { row in ResultReading.Check( id: row.id, outcome: row.outcome.placementOutcome, needsPerson: row.needsPerson, - viewCapturable: viewToTake(for: row) != nil, - margin: ResultReading.margin( - measured: row.measured.map(Double.init), threshold: row.threshold.map(Double.init), - plusMinus: row.plusMinus.map(Double.init), comparison: row.comparison?.placementComparison) + viewCapturable: viewToTake(for: row) != nil ) } } @@ -102,12 +99,3 @@ private extension CheckOutcome { } } } - -private extension RuleComparison { - var placementComparison: PlacementComparison { - switch self { - case .atLeast: .atLeast - case .atMost: .atMost - } - } -} diff --git a/ios/HouseScan/UI/Copy/ScanCopy.swift b/ios/HouseScan/UI/Copy/ScanCopy.swift index 24f23609e..a8829d756 100644 --- a/ios/HouseScan/UI/Copy/ScanCopy.swift +++ b/ios/HouseScan/UI/Copy/ScanCopy.swift @@ -627,10 +627,11 @@ enum ScanCopy { // MARK: Result - /// The answer in the homeowner's words (`ResultPresentation.answer`). + /// The answer in the homeowner's words (`ResultPresentation.answer`). A passing spot is a + /// possible one, never "fits": see `candidateNote`. static func headline(_ answer: ResultReading.Answer) -> String { switch answer { - case .fits: "A battery fits here" + case .candidate: "A possible battery spot" case .oneMoreLook: "One more look" case .installer: "Needs an installer's review" case .notHere: "Not on this wall" @@ -690,7 +691,18 @@ enum ScanCopy { return "Settles: \(titles.joined(separator: ", "))" } - static let seeOnWall = "See it on your wall" + /// Under a possible spot's headline and placement. The server's checks pass on space the scan + /// recorded, some of it beyond what the homeowner confirmed in the spot check, so the app + /// can't say all the space a battery needs is clear (`ResultReading`, B17). It says what the + /// fit still needs, never that a review was sent: the app contacts nobody. + static let candidateNote = "The scan suggests this spot, but it couldn't confirm all the space a battery needs. An installer needs to check the fit on site." + + /// The AR title over a possible spot, and the words for what the AR view draws there. + static let proposedSpotOverlay = "A proposed battery spot, drawn on your wall" + static let proposedSpotThisWay = "The proposed spot is this way" + static let proposedSpotOffScreen = "The proposed spot is off screen. Turn the phone toward the arrow." + + static let seeOnWall = "See this spot on your wall" /// For a spot an installer still has to confirm against the meter's working space. static let seeClosest = "See the closest spot" static let showMe = "Show me" @@ -703,8 +715,8 @@ enum ScanCopy { static let wallNotMeasuredDetail = "The scan stopped before you walked along the wall on either side of your meter, so we can't tell where a battery would fit. Scan again and walk a few steps each way." static let scanAgain = "Scan again" - /// Shown on every result, with or without a spot ("Not on this wall" has none), so it names - /// no spot. + /// Shown on every result but a possible spot, whose `candidateNote` says it already, with or + /// without a spot ("Not on this wall" has none), so it names no spot. static let installerConfirms = "Before any battery goes in, an installer has to confirm where it goes on site." static let rulesNotFinal = "The placement rules aren't final yet, so every result needs an installer's review for now." // The server's result covers where the battery goes, not the panel itself. @@ -807,9 +819,14 @@ enum ScanCopy { } } + /// The heading over every check in Details, and what a passing one means there: the server's + /// calculation on the scan's measurements, not space anyone confirmed clear. + static let calculatedTitle = "What the server calculated" + static let calculatedNote = "From what your scan recorded. A passing check means those measurements meet the rule, not that the space is confirmed clear." + static func outcomeWord(_ outcome: CheckOutcome) -> String { switch outcome { - case .pass: "Looks good" + case .pass: "Meets the rule on what the scan recorded" case .unsure: "Not sure yet" case .fail: "Doesn't work" } diff --git a/ios/HouseScan/UI/Result/ResultScene3D.swift b/ios/HouseScan/UI/Result/ResultScene3D.swift index a95a45813..678583e7f 100644 --- a/ios/HouseScan/UI/Result/ResultScene3D.swift +++ b/ios/HouseScan/UI/Result/ResultScene3D.swift @@ -220,7 +220,7 @@ struct ResultScene3D: View { } for (index, zone) in result.clearances.enumerated() { - var material = UnlitMaterial(color: SceneColor.outcome(zone.outcome)) + var material = UnlitMaterial(color: SceneColor.zone(zone.outcome)) material.blending = .transparent(opacity: .init(floatLiteral: 0.35)) // Stacked zones sit a few millimeters apart so overlapping ones don't flicker. let lift = ResultMarkLayout.zoneLift(index: index, base: Self.zoneBase) @@ -467,11 +467,11 @@ struct ResultScene3D: View { } var parts: [String] if spot.span.lowerBound > 0 { - parts = ["Battery \(Distance.spoken(spot.span.lowerBound)) right of your meter"] + parts = ["Proposed battery spot \(Distance.spoken(spot.span.lowerBound)) right of your meter"] } else if spot.span.upperBound < 0 { - parts = ["Battery \(Distance.spoken(-spot.span.upperBound)) left of your meter"] + parts = ["Proposed battery spot \(Distance.spoken(-spot.span.upperBound)) left of your meter"] } else { - parts = ["Battery below your meter"] + parts = ["Proposed battery spot below your meter"] } if let cable = result.cableLength { parts.append("cable \(Distance.spoken(cable))") @@ -549,7 +549,7 @@ enum ResultARModel { wall.world(s: s, height: height, out: out) - wall.meter } for (index, zone) in result.clearances.enumerated() { - var material = UnlitMaterial(color: SceneColor.outcome(zone.outcome)) + var material = UnlitMaterial(color: SceneColor.zone(zone.outcome)) material.blending = .transparent(opacity: .init(floatLiteral: 0.35)) let width = zone.span.upperBound - zone.span.lowerBound let middle = zone.span.lowerBound + width / 2 @@ -659,9 +659,11 @@ private enum SceneColor { } } - static func outcome(_ outcome: CheckOutcome) -> UIColor { + /// A clearance zone's tint, as `Palette.zone`: where the server's sweep passes, the + /// proposal's blue, not a green that reads as ground confirmed clear. + static func zone(_ outcome: CheckOutcome) -> UIColor { switch outcome { - case .pass: rgb(0x2FC273) + case .pass: signal case .unsure: rgb(0xF5B53D) case .fail: rgb(0xFF5A4E) } diff --git a/ios/HouseScan/UI/ScanRootView.swift b/ios/HouseScan/UI/ScanRootView.swift index 9cc143054..54c545573 100644 --- a/ios/HouseScan/UI/ScanRootView.swift +++ b/ios/HouseScan/UI/ScanRootView.swift @@ -93,15 +93,20 @@ struct ScanRootView: View { /// Haptics for the moments that matter, fired on the same state change the screen animates: /// a deliberate photo, the meter pinned, a mark placed or refused, a requested view done, the /// result. +/// +/// The result arrives with a light tap, whatever it says. `.success` there celebrated every +/// answer alike, a rejected wall included, and on a possible spot it said "done, it fits" when +/// the scan couldn't confirm the space (B17, B26). Coming back from the AR view taps nothing. private struct ScanHaptics: ViewModifier { let state: ScanViewState - // Two steps: the five-modifier chain took 215 ms to type-check on Swift 6.4, and CI's - // Swift 6.2 has failed on slower expressions before. + // Steps of at most three: the five-modifier chain took 215 ms to type-check on Swift 6.4, and + // CI's Swift 6.2 has failed on slower expressions before. func body(content: Content) -> some View { let captures = content .sensoryFeedback(.impact(weight: .light, intensity: 0.7), trigger: deliberateCaptureID) { _, new in new != nil } - .sensoryFeedback(.success, trigger: state.phase, condition: Self.isMilestone) + .sensoryFeedback(.success, trigger: state.phase, condition: Self.meterPinned) + .sensoryFeedback(.impact(weight: .light), trigger: state.phase, condition: Self.resultArrived) return captures .sensoryFeedback(.success, trigger: state.gap?.isSatisfied ?? false) { _, new in new } .sensoryFeedback(.impact(weight: .medium), trigger: state.features.count) { old, new in new > old } @@ -116,8 +121,12 @@ private struct ScanHaptics: ViewModifier { return capture.id } - private static func isMilestone(_ old: ScanPhase, _ new: ScanPhase) -> Bool { - (old == .findMeter && new == .meterCloseUp) || (old != .resultAR && new == .result) + private static func meterPinned(_ old: ScanPhase, _ new: ScanPhase) -> Bool { + old == .findMeter && new == .meterCloseUp + } + + private static func resultArrived(_ old: ScanPhase, _ new: ScanPhase) -> Bool { + old != .resultAR && new == .result } } diff --git a/ios/HouseScan/UI/Screens/ResultARScreen.swift b/ios/HouseScan/UI/Screens/ResultARScreen.swift index ef697d885..14bb00bbc 100644 --- a/ios/HouseScan/UI/Screens/ResultARScreen.swift +++ b/ios/HouseScan/UI/Screens/ResultARScreen.swift @@ -56,7 +56,7 @@ struct ResultARScreen: View { private static func overlayLabel(_ result: ResultPresentation) -> String { guard result.spot != nil else { return "The cable run and clearances, drawn on your wall" } return result.spotIsClean - ? "The battery, drawn on your wall at its spot" + ? ScanCopy.proposedSpotOverlay : "An outline of the spot an installer needs to check, drawn on your wall" } @@ -67,17 +67,17 @@ struct ResultARScreen: View { guard let result = state.result, result.spot != nil else { return Instruction(title: "Point at your meter", detail: nil) } - // Only a pass under approved rules may sound settled; a spot an installer still has to - // check says so, as the result screen does. - let title = result.decision == .pass && result.policyApproved && result.spotIsClean - ? "Your battery could go here" - : "The spot an installer needs to check" + // The same answer as the result screen: a possible spot says it's possible and what it + // still needs; any other spot an installer still has to check says so. + let candidate = result.answer == .candidate + let title = candidate ? ScanCopy.headline(.candidate) : "The spot an installer needs to check" let placement = ScanCopy.placement(result) guard !result.isSample else { // A sample spot drawn on the homeowner's real wall must not pass for their result. return Instruction(title: "Example spot, not your result", detail: ["No server checked this scan.", placement].compactMap { $0 }.joined(separator: " ")) } - return Instruction(title: title, detail: placement) + let onSite = candidate ? "An installer needs to check the fit on site." : nil + return Instruction(title: title, detail: [placement.map { "\($0)." }, onSite].compactMap { $0 }.joined(separator: " ")) } } @@ -107,7 +107,7 @@ private struct SpotDirection: View { if let chevron = chevronPlacement(in: size) { ZStack { TargetMarker(placement: .offScreen(chevron.point, angle: chevron.angle)) - Text("Your battery spot is this way") + Text(ScanCopy.proposedSpotThisWay) .font(.subheadline.weight(.semibold)) .foregroundStyle(Palette.chalk) .multilineTextAlignment(.center) @@ -122,7 +122,7 @@ private struct SpotDirection: View { } .frame(width: size.width, height: size.height) .accessibilityElement(children: .ignore) - .accessibilityLabel("Your battery spot is off screen. Turn the phone toward the arrow.") + .accessibilityLabel(ScanCopy.proposedSpotOffScreen) .accessibilityIdentifier("ar.spotDirection") } } @@ -187,7 +187,7 @@ struct BatteryOverlay: View, Animatable { private func drawClearances(in context: inout GraphicsContext, _ geometry: WallProjection) { for zone in result.clearances { guard let quad = geometry.groundQuad(s: zone.span, out: 0...zone.depth, height: 0.01) else { continue } - let color = Palette.outcome(zone.outcome) + let color = Palette.zone(zone.outcome) context.fill(quad, with: .color(color.opacity(0.28 * rise))) context.stroke(quad, with: .color(color.opacity(0.9 * rise)), style: StrokeStyle(lineWidth: 2, dash: [8, 6])) } diff --git a/ios/HouseScan/UI/Screens/ResultScreen.swift b/ios/HouseScan/UI/Screens/ResultScreen.swift index 9b8c88ae3..e4234a39b 100644 --- a/ios/HouseScan/UI/Screens/ResultScreen.swift +++ b/ios/HouseScan/UI/Screens/ResultScreen.swift @@ -127,7 +127,9 @@ struct ResultScreen: View { /// Plain small print, never boxes: none of it changes what the homeowner does next. private func footnotes(_ result: ResultPresentation) -> some View { VStack(alignment: .leading, spacing: 6) { - footnote(ScanCopy.installerConfirms, id: "result.installerConfirms") + if result.answer != .candidate { + footnote(ScanCopy.installerConfirms, id: "result.installerConfirms") + } if let notice = result.rulesNotice { footnote(notice, id: "result.rulesNotice") } @@ -160,7 +162,10 @@ struct ResultScreen: View { private func details(_ result: ResultPresentation) -> some View { DisclosureGroup(isExpanded: $detailsExpanded) { VStack(alignment: .leading, spacing: 22) { - if !result.summary.isEmpty { + // A possible spot's summary is the server's own pass ("fits every check"), which + // the card's candidate note contradicts on purpose; the summary of any other answer + // explains what failed or is unsure, so it stays. + if !result.summary.isEmpty, result.answer != .candidate { Text(result.summary) .font(Typeface.hint) .foregroundStyle(Palette.muted) @@ -300,6 +305,12 @@ private struct AnswerCard: View { .foregroundStyle(Palette.signalText) .fixedSize(horizontal: false, vertical: true) .accessibilityIdentifier("result.placement") + if answer == .candidate { + Text(ScanCopy.candidateNote) + .font(Typeface.hint) + .fixedSize(horizontal: false, vertical: true) + .accessibilityIdentifier("result.candidateNote") + } } else if let nearest = ScanCopy.nearest(result) { Text(nearest) .font(Typeface.hint) @@ -420,7 +431,7 @@ private struct CardCheckLine: View { HStack(alignment: .firstTextBaseline, spacing: 12) { Image(systemName: CheckSymbol.name(row.outcome)) .font(.body.weight(.semibold)) - .foregroundStyle(Palette.outcomeInk(row.outcome)) + .foregroundStyle(CheckSymbol.ink(row.outcome)) .frame(width: 22) VStack(alignment: .leading, spacing: 3) { Text(row.title) @@ -469,13 +480,19 @@ private struct Reveal: ViewModifier { } private enum CheckSymbol { + /// A pass is a muted outline: it met the rule on what the scan recorded, and a filled green + /// badge read as space confirmed clear. static func name(_ outcome: CheckOutcome) -> String { switch outcome { - case .pass: "checkmark.circle.fill" + case .pass: "checkmark.circle" case .unsure: "questionmark.circle.fill" case .fail: "xmark.circle.fill" } } + + static func ink(_ outcome: CheckOutcome) -> Color { + outcome == .pass ? Palette.muted : Palette.outcomeInk(outcome) + } } // MARK: - Details @@ -487,9 +504,16 @@ private struct ChecksList: View { var body: some View { VStack(alignment: .leading, spacing: 12) { - Text("What we checked") - .font(Typeface.sectionTitle) - .accessibilityAddTraits(.isHeader) + VStack(alignment: .leading, spacing: 4) { + Text(ScanCopy.calculatedTitle) + .font(Typeface.sectionTitle) + .accessibilityAddTraits(.isHeader) + Text(ScanCopy.calculatedNote) + .font(.subheadline) + .foregroundStyle(Palette.muted) + .fixedSize(horizontal: false, vertical: true) + .accessibilityIdentifier("result.calculatedNote") + } VStack(spacing: 0) { ForEach(Array(checks.enumerated()), id: \.element.id) { index, row in CheckRowView(row: row, photoOffered: photoOffered.contains(row.id)) @@ -507,20 +531,29 @@ private struct CheckRowView: View { var row: CheckRow var photoOffered: Bool + /// The server's reason, except for a pass: its reasons ("Nothing limits the clear spaceā€¦") + /// state as fact what is only a calculation on recorded space. A pass keeps its measurement + /// against the rule. + private var reason: String? { + row.outcome == .pass ? nil : row.reason + } + var body: some View { HStack(alignment: .top, spacing: 12) { Image(systemName: CheckSymbol.name(row.outcome)) .font(.title3.weight(.semibold)) - .foregroundStyle(Palette.outcomeInk(row.outcome)) + .foregroundStyle(CheckSymbol.ink(row.outcome)) .frame(width: 28) .accessibilityHidden(true) VStack(alignment: .leading, spacing: 4) { Text(row.title) .font(Typeface.hint.weight(.semibold)) - Text(row.reason) - .font(.subheadline) - .foregroundStyle(Palette.muted) - .fixedSize(horizontal: false, vertical: true) + if let reason { + Text(reason) + .font(.subheadline) + .foregroundStyle(Palette.muted) + .fixedSize(horizontal: false, vertical: true) + } if let measurement = ScanCopy.measurement(row) { Text(measurement) .font(.subheadline.weight(.medium)) @@ -537,7 +570,7 @@ private struct CheckRowView: View { .padding(14) .accessibilityElement(children: .combine) .accessibilityLabel("\(row.title): \(ScanCopy.outcomeWord(row.outcome))") - .accessibilityValue([row.reason, ScanCopy.measurement(row, spoken: true), row.outcome == .unsure ? ScanCopy.unsureNote(photoOffered: photoOffered) : nil] + .accessibilityValue([reason, ScanCopy.measurement(row, spoken: true), row.outcome == .unsure ? ScanCopy.unsureNote(photoOffered: photoOffered) : nil] .compactMap(\.self).joined(separator: ". ")) // The card shows the deciding checks as `check.`; this is the full list. .accessibilityIdentifier("detail.check.\(row.id)") diff --git a/ios/HouseScan/UI/Theme/Theme.swift b/ios/HouseScan/UI/Theme/Theme.swift index 2313e425f..e11f9ba2d 100644 --- a/ios/HouseScan/UI/Theme/Theme.swift +++ b/ios/HouseScan/UI/Theme/Theme.swift @@ -68,6 +68,13 @@ enum Palette { } } + /// A result's clearance zone in AR. Where the server's sweep passes, the proposal's blue + /// rather than `covered` green, which read as ground confirmed clear: the checks pass there + /// on recorded space, not on space anyone confirmed (B17). + static func zone(_ outcome: CheckOutcome) -> Color { + outcome == .pass ? signal : self.outcome(outcome) + } + static func outcomeInk(_ outcome: CheckOutcome) -> Color { switch outcome { case .pass: passInk diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultCardActions.swift b/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultCardActions.swift index 1a77f1aad..cfb7d8f88 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultCardActions.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultCardActions.swift @@ -23,7 +23,7 @@ public enum ResultCardActions { public static func primary(answer: ResultReading.Answer, hasSpot: Bool, spotIsClean: Bool, hasViewToTake: Bool, sourceAvailable: Bool) -> Primary? { switch answer { - case .fits: + case .candidate: return hasSpot && sourceAvailable ? .showAR(clean: true) : nil case .oneMoreLook: return offersView(capturable: hasViewToTake, sourceAvailable: sourceAvailable) ? .takeView : nil diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultReading.swift b/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultReading.swift index 8330d35cd..008d614d5 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultReading.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultReading.swift @@ -3,8 +3,14 @@ import Foundation // What the result screen leads with, read from the server's answer. The app's screen turns these // into words; the rules that pick them live here so each one has a test against a decoded answer // (ResultReadingTests). They read the checks as well as `decision`: a manual_review held back -// only by rules that aren't approved yet still fits, and a spot that might stand in the meter's -// working space goes to an installer whatever the decision says. +// only by rules that aren't approved yet is still a candidate, and a spot that might stand in the +// meter's working space goes to an installer whatever the decision says. +// +// No answer says a battery fits. A spot whose checks all pass is a candidate: the server's checks +// read observed space beyond the area the homeowner confirms in the spot check, some of it +// inferred rather than seen (a walked path, a photo's view; `CoverageMap`, "Bounded +// exceptions"), and the answer doesn't say which. Until it does, the app can't tell that all the +// space a battery needs was confirmed (B17). The answer itself is unchanged: this reads it. public enum ResultReading { /// The server's check for the NEC 110.26 working space in front of the meter @@ -14,8 +20,9 @@ public enum ResultReading { /// What the screen says first. public enum Answer: String, Equatable, Sendable { - /// The spot passes every check; at most the rules' approval holds it back. - case fits + /// A proposed spot: every check passes on what the scan recorded, and at most the rules' + /// approval holds it back. Not a confirmed fit: an installer checks the space on site. + case candidate /// An unsure check that a view the camera can take now would settle. case oneMoreLook /// A person has to decide: a borderline measurement, an unknown attribute, a spot that @@ -33,16 +40,12 @@ public enum ResultReading { public var needsPerson: Bool /// True when a view the camera can take now would settle it. public var viewCapturable: Bool - /// How far the measurement clears its limit (`margin(measured:threshold:plusMinus:comparison:)`); - /// nil without both a measurement and a limit. - public var margin: Double? - public init(id: String, outcome: PlacementOutcome, needsPerson: Bool = false, viewCapturable: Bool = false, margin: Double? = nil) { + public init(id: String, outcome: PlacementOutcome, needsPerson: Bool = false, viewCapturable: Bool = false) { self.id = id self.outcome = outcome self.needsPerson = needsPerson self.viewCapturable = viewCapturable - self.margin = margin } } @@ -58,29 +61,26 @@ public enum ResultReading { if hasSpot, !spotIsClean(hasSpot: hasSpot, checks: checks) { return .installer } switch decision { case .pass: - return .fits + return .candidate case .reject: return .notHere case .manualReview: // Needs at least one check: over none, "every check passes" is vacuously true. - if hasSpot, !policyApproved, !checks.isEmpty, checks.allSatisfy({ $0.outcome == .pass }) { return .fits } + if hasSpot, !policyApproved, !checks.isEmpty, checks.allSatisfy({ $0.outcome == .pass }) { return .candidate } if checks.contains(where: { $0.outcome == .unsure && !$0.needsPerson && $0.viewCapturable }) { return .oneMoreLook } return .installer } } - /// Indices of the checks the result card shows, in order: every FAIL, then every UNSURE, then - /// the two PASSes closest to their limits (smallest `margin`; a pass without one sorts last). - /// At most `limit`. + /// Indices of the checks the result card shows, in order: every FAIL, then every UNSURE, at + /// most `limit`. No PASS: on the card a passing line read as space confirmed clear, which the + /// scan can't show (see the top of this file). Every check, passing ones included, stays in + /// the result's Details. public static func cardLines(_ checks: [Check], limit: Int = 3) -> [Int] { let indices = checks.indices let fails = indices.filter { checks[$0].outcome == .fail } let unsure = indices.filter { checks[$0].outcome == .unsure } - // Ties keep the server's order. - let passes = indices.filter { checks[$0].outcome == .pass } - .sorted { (checks[$0].margin ?? .infinity, $0) < (checks[$1].margin ?? .infinity, $1) } - .prefix(2) - return Array((fails + unsure + passes).prefix(limit)) + return Array((fails + unsure).prefix(limit)) } /// Whether a check's review line (`PlacementCheck.reviewThresholdFt`) explains its outcome, so @@ -104,20 +104,6 @@ public enum ResultReading { case .atLeast: return reviewThreshold > threshold && !(measured - error > reviewThreshold) } } - - /// How far `measured` clears `threshold` on the passing side, in units of `plusMinus` when it - /// is positive, else in the measurement's own units. Negative when it misses. Without a - /// comparison the side is unknown, so the distance to the limit either way. - public static func margin(measured: Double?, threshold: Double?, plusMinus: Double?, comparison: PlacementComparison?) -> Double? { - guard let measured, let threshold else { return nil } - let clearance = switch comparison { - case .atLeast: measured - threshold - case .atMost: threshold - measured - case nil: abs(measured - threshold) - } - guard let plusMinus, plusMinus > 0 else { return clearance } - return clearance / plusMinus - } } extension PlacementPolicy { @@ -175,9 +161,7 @@ extension PlacementResult { checks.map { check in ResultReading.Check( id: check.id, outcome: check.outcome, needsPerson: check.needsPerson, - viewCapturable: evidenceIndex(settling: check.id).map(capturable) ?? false, - margin: ResultReading.margin(measured: check.measuredFt, threshold: check.thresholdFt, - plusMinus: check.plusMinusFt, comparison: check.comparison) + viewCapturable: evidenceIndex(settling: check.id).map(capturable) ?? false ) } } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/ResultCardActionsTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/ResultCardActionsTests.swift index 9a1cd9a53..36f547ecd 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/ResultCardActionsTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/ResultCardActionsTests.swift @@ -12,7 +12,7 @@ import Testing } @Test func withTheCameraUpEachAnswerKeepsItsButton() { - #expect(Self.primary(.fits) == .showAR(clean: true)) + #expect(Self.primary(.candidate) == .showAR(clean: true)) #expect(Self.primary(.oneMoreLook, hasViewToTake: true) == .takeView) #expect(Self.primary(.installer, spotIsClean: false) == .showAR(clean: false)) #expect(Self.primary(.installer) == .showAR(clean: true)) @@ -21,7 +21,7 @@ import Testing /// No spot, nothing to show in AR; no view to take, no "Show me". @Test func nothingToShowMeansNoButton() { - #expect(Self.primary(.fits, hasSpot: false) == nil) + #expect(Self.primary(.candidate, hasSpot: false) == nil) #expect(Self.primary(.installer, hasSpot: false) == nil) #expect(Self.primary(.oneMoreLook, hasViewToTake: false) == nil) } @@ -31,7 +31,7 @@ import Testing /// button already was. Starting over starts a new camera, so it stays. @Test func aFailedCameraOffersNothingThatNeedsIt() { #expect(Self.primary(.oneMoreLook, hasViewToTake: true, sourceAvailable: false) == nil) - #expect(Self.primary(.fits, sourceAvailable: false) == nil) + #expect(Self.primary(.candidate, sourceAvailable: false) == nil) #expect(Self.primary(.installer, spotIsClean: false, sourceAvailable: false) == nil) #expect(Self.primary(.notHere, hasSpot: false, sourceAvailable: false) == .startOver) } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/ResultReadingTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/ResultReadingTests.swift index 3f525a0ef..e2d07e128 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/ResultReadingTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/ResultReadingTests.swift @@ -112,8 +112,9 @@ import simd #expect(result.answer { _ in true } == .oneMoreLook) } - /// A manual_review held back only by unapproved rules, with every check passing, fits. - @Test func allPassUnderUnapprovedRulesFits() throws { + /// A manual_review held back only by unapproved rules, with every check passing, is a + /// candidate like a pass: the rules' approval was never what made a spot a confirmed fit. + @Test func allPassUnderUnapprovedRulesIsACandidate() throws { var object = try Self.sampleObject() object["checks"] = [ Self.check("wall_backing", "pass"), @@ -123,7 +124,7 @@ import simd let result = try Self.decode(object) #expect(result.decision == .manualReview && !result.policy.autoApprove && result.spot != nil) #expect(result.spotIsClean) - #expect(result.answer { _ in true } == .fits) + #expect(result.answer { _ in true } == .candidate) } /// The same answer with `checks: []`, which the schema allows: no evidence is not a fit. @@ -208,7 +209,7 @@ import simd /// ios/HouseScanUITests/Fixtures/results, the answers the screenshots and UI tests show /// through `-uiDemoResultFile`. Each must stay in the schema and read as its name says. @Test(arguments: [ - ("pass", ResultReading.Answer.fits), ("reject-nearest", .notHere), + ("pass", ResultReading.Answer.candidate), ("reject-nearest", .notHere), ("unsure-view", .oneMoreLook), ("no-clean-spot", .installer), ("no-spot-no-nearest", .notHere), ("review-band", .installer), ("view-not-offered", .oneMoreLook), ]) @@ -236,9 +237,9 @@ import simd ResultReading.answer(decision: decision, policyApproved: approved, hasSpot: spot, checks: checks) } - #expect(answer(.pass, true, true, [pass]) == .fits) + #expect(answer(.pass, true, true, [pass]) == .candidate) // Only the rules' approval holds it back. - #expect(answer(.manualReview, false, true, [pass]) == .fits) + #expect(answer(.manualReview, false, true, [pass]) == .candidate) #expect(answer(.manualReview, false, false, [pass]) == .installer) #expect(answer(.manualReview, true, true, [pass, byView, byPerson]) == .oneMoreLook) #expect(answer(.manualReview, true, true, [pass, byPerson]) == .installer) @@ -255,28 +256,88 @@ import simd #expect(unexplained.needsPerson) } + // MARK: No confirmed fit (B17) + + /// The reading has no answer that says a battery fits: a passing spot rests on space the scan + /// can't show was confirmed, so the most it can be is a candidate. + @Test func noAnswerSaysABatteryFits() { + #expect(ResultReading.Answer(rawValue: "fits") == nil) + typealias Check = ResultReading.Check + let outcomes: [PlacementOutcome] = [.pass, .unsure, .fail] + // Each check set once with a check this app doesn't know, so a new server check can't + // open a shortcut to a stronger answer. + var sets: [[Check]] = [[]] + for id in ["wall_backing", "a_check_this_app_does_not_know", ResultReading.meterWorkingSpaceCheckID] { + sets = sets.flatMap { set in [set] + outcomes.map { set + [Check(id: id, outcome: $0, viewCapturable: true)] } } + } + for decision in [PlacementDecision.pass, .manualReview, .reject] { + for approved in [true, false] { + for hasSpot in [true, false] { + for checks in sets { + let answer = ResultReading.answer(decision: decision, policyApproved: approved, hasSpot: hasSpot, checks: checks) + guard answer == .candidate else { continue } + #expect(hasSpot && ResultReading.spotIsClean(hasSpot: hasSpot, checks: checks) && decision != .reject, + "\(decision) approved=\(approved) checks=\(checks.map { "\($0.id)=\($0.outcome)" })") + } + } + } + } + } + + /// An installer's review can still carry a spot, and the reading keeps it: only the words + /// over it change. + @Test func anInstallerReviewKeepsItsSpot() throws { + let result = try Self.sample(adding: Self.check(ResultReading.meterWorkingSpaceCheckID, "unsure", measured: -0.5, threshold: 0)) + #expect(result.answer { _ in true } == .installer) + #expect(result.spot != nil) + } + + /// Reading an answer changes nothing in it. A passing answer bound to the scene sent stays + /// bound, decodes to the same result before and after it is read, and keeps the server's + /// decision and every check's outcome: the candidate is how the app words it, not a new + /// answer. + @Test func readingAPassLeavesTheAnswerAsTheServerSentIt() throws { + let scene = Data(#"{"note":"the bytes this phone sent"}"#.utf8) + var object = try #require(try JSONSerialization.jsonObject(with: ResultBindingTests.answer(declaring: PacketFiles.sha256(scene))) as? [String: Any]) + object["decision"] = "pass" + object["checks"] = try #require(object["checks"] as? [[String: Any]]).map { check in + var check = check + check["outcome"] = "pass" + check["unsure_cause"] = NSNull() + return check + } + object["missing_evidence"] = [Any]() + let data = try JSONSerialization.data(withJSONObject: object) + #expect(try SceneSchemas.result().validate(data) == []) + try ResultBinding.check(answer: data, submittedScene: scene) + + let result = try PlacementResult.decode(data) + let sent = data + #expect(result.answer { _ in true } == .candidate) + #expect(ResultReading.cardLines(result.readingChecks { _ in true }).isEmpty) + #expect(data == sent) + #expect(try PlacementResult.decode(data) == result) + #expect(result.decision == .pass) + #expect(result.checks.allSatisfy { $0.outcome == .pass }) + try ResultBinding.check(answer: data, submittedScene: scene) + } + // MARK: The card's check lines - @Test func cardLinesPutFailuresFirstThenUnsureThenTheClosestPasses() { + /// The card leads with what failed and what is unsure. A passing check is never on it: the + /// card read a pass as space confirmed clear, which a candidate can't claim (B17). Every check + /// stays in Details. + @Test func cardLinesPutFailuresFirstThenUnsureAndNoPasses() { typealias Check = ResultReading.Check let checks = [ - Check(id: "wide", outcome: .pass, margin: 9), + Check(id: "pass", outcome: .pass), Check(id: "unsure", outcome: .unsure), - Check(id: "close", outcome: .pass, margin: 0.5), Check(id: "fail", outcome: .fail), - Check(id: "unmeasured", outcome: .pass), - Check(id: "closer", outcome: .pass, margin: 0.2), + Check(id: "unsure2", outcome: .unsure), + Check(id: "fail2", outcome: .fail), ] - #expect(ResultReading.cardLines(checks) == [3, 1, 5]) - let passes = checks.filter { $0.outcome == .pass } - #expect(ResultReading.cardLines(passes) == [3, 1]) // closer, close: two passes at most - } - - @Test func marginIsInUnitsOfTheErrorWhenThereIsOne() throws { - #expect(abs(try #require(ResultReading.margin(measured: 3.6, threshold: 3, plusMinus: 0.3, comparison: .atLeast)) - 2) < 1e-9) - #expect(ResultReading.margin(measured: 18, threshold: 20, plusMinus: nil, comparison: .atMost) == 2) - #expect(ResultReading.margin(measured: 2.5, threshold: 3, plusMinus: 0, comparison: .atLeast) == -0.5) - #expect(ResultReading.margin(measured: 2.5, threshold: 3, plusMinus: nil, comparison: nil) == 0.5) - #expect(ResultReading.margin(measured: nil, threshold: 3, plusMinus: 0.3, comparison: .atLeast) == nil) + #expect(ResultReading.cardLines(checks) == [2, 4, 1]) + #expect(ResultReading.cardLines(checks, limit: 5) == [2, 4, 1, 3]) + #expect(ResultReading.cardLines(checks.filter { $0.outcome == .pass }).isEmpty) } } diff --git a/ios/HouseScanUITests/ProvisionalPlacementUITests.swift b/ios/HouseScanUITests/ProvisionalPlacementUITests.swift new file mode 100644 index 000000000..783438126 --- /dev/null +++ b/ios/HouseScanUITests/ProvisionalPlacementUITests.swift @@ -0,0 +1,241 @@ +import Foundation +import XCTest + +/// A spot whose checks all pass is shown as a possible spot, never as a fit: the server's checks +/// read space beyond what the homeowner confirms in the spot check (B17). Every surface agrees: +/// the card, Details, the AR view and the 3D model. The answer itself is never changed. +final class ProvisionalPlacementUITests: XCTestCase { + private static let largestText = ["-UIPreferredContentSizeCategoryName", "UICTContentSizeCategoryAccessibilityXXXL"] + private static let candidate = "A possible battery spot" + + /// Words that would say the scan established a fit, or celebrate a pass as confirmed space. + private static let fitWords = ["fits here", "fits every", "could go here", "looks good", "your battery spot", "confirmed clear"] + + override func setUp() { + continueAfterFailure = false + } + + // MARK: The real engine + + /// The real engine on the synthetic replay, the test answering as the server through the gate + /// folder (`-answersFromGate`). The first answer names another scene, so "Try again" asks + /// again; the second is the bundled sample turned into a pass naming the scene sent. The + /// autopilot answers the spot check "It's clear". The result is a possible spot, and so is + /// the AR view's title: a retry and a clear answer don't make it a fit. + @MainActor + func testAPassAfterTryAgainAndItsClearIsAPossibleSpot() throws { + let gate = try Self.gate() + defer { try? FileManager.default.removeItem(at: gate) } + let server = GateServer(gate: gate) { body, index in + Self.passingAnswer(binding: index == 0 ? Data("another scene".utf8) : body) + } + defer { server.stop() } + let app = Self.launchEngine(gate: gate) + let any = app.descendants(matching: .any) + + let tryAgain = app.buttons["action.retryUpload"] + XCTAssertTrue(tryAgain.waitForExistence(timeout: 300), "the unbound answer never offered Try again") + tryAgain.tap() + XCTAssertTrue(any["screen.result"].waitForExistence(timeout: 120), "the passing answer never reached the result") + XCTAssertEqual(server.requests.count, 2) + XCTAssertEqual(any["result.headline"].label, Self.candidate) + XCTAssertFalse(any["result.spotRefused"].exists) + assertCandidateCard(app) + attach(app, name: "result-candidate-engine") + + // The autopilot opens the AR view once the result's gate opens. + try Data().write(to: gate.appending(path: "result")) + let instruction = any["instruction"] + XCTAssertTrue(any["screen.resultAR"].waitForExistence(timeout: 20)) + XCTAssertTrue(instruction.waitForExistence(timeout: 10)) + XCTAssertTrue(instruction.label.contains(Self.candidate), instruction.label) + XCTAssertTrue(instruction.label.contains("An installer needs to check the fit on site."), instruction.label) + assertNoFitWords(instruction.label) + attach(app, name: "resultAR-candidate-engine") + } + + /// "Something's there" withdraws the area and sends the scan again; the passing answer that + /// comes back names the same spot, so the earlier answer settles it and the result shows + /// without asking (`SpotConfirmations.settling`). That shortcut still shows a possible spot, + /// beside the notice that the homeowner said something stands there. + @MainActor + func testASpotSettledByAnEarlierAnswerIsStillAPossibleSpot() throws { + let gate = try Self.gate() + defer { try? FileManager.default.removeItem(at: gate) } + let server = GateServer(gate: gate) { body, _ in Self.passingAnswer(binding: body) } + defer { server.stop() } + let app = Self.launchEngine(gate: gate, extra: ["-autopilotSomethingThere"]) + let any = app.descendants(matching: .any) + + XCTAssertTrue(any["screen.result"].waitForExistence(timeout: 420), "the second passing answer never reached the result") + XCTAssertEqual(server.requests.count, 2, "the spot check's answer should send the scan once more") + XCTAssertEqual(any["result.headline"].label, Self.candidate) + XCTAssertTrue(any["result.spotRefused"].exists, "the settled answer's notice is missing") + assertCandidateCard(app) + attach(app, name: "result-candidate-settled") + } + + // MARK: Server answers in Fixtures/results + + /// pass.json, every check passing: the card, Details and the AR overlay at the default size. + @MainActor + func testAPassingAnswerReadsAsAPossibleSpotEverywhere() throws { + try checkPassFixture(textSize: [], name: "result-candidate") + } + + @MainActor + func testAPassingAnswerReadsAsAPossibleSpotAtLargestTextSize() throws { + try checkPassFixture(textSize: Self.largestText, name: "result-candidate-AX5") + } + + /// A rejected wall keeps its negative answer and its reasons. + @MainActor + func testARejectStaysNegative() throws { + let app = Self.launchFixture("reject-nearest", textSize: []) + defer { app.terminate() } + let any = app.descendants(matching: .any) + XCTAssertEqual(any["result.headline"].label, "Not on this wall") + XCTAssertFalse(any["result.candidateNote"].exists) + XCTAssertTrue(any["result.nearest"].label.hasPrefix("The closest spot")) + attach(app, name: "result-reject") + } + + @MainActor + private func checkPassFixture(textSize: [String], name: String) throws { + let app = Self.launchFixture("pass", textSize: textSize) + defer { app.terminate() } + let any = app.descendants(matching: .any) + XCTAssertEqual(any["result.headline"].label, Self.candidate) + assertCandidateCard(app) + let model = any.matching(NSPredicate(format: "label == '3D view of your wall'")).firstMatch + XCTAssertTrue(model.waitForExistence(timeout: 10)) + let modelValue = model.value as? String ?? "" + XCTAssertTrue(modelValue.hasPrefix("Proposed battery spot"), modelValue) + attach(app, name: name) + + // Details: the checks as calculations on the recorded scan, a pass without the server's + // positive reason, and no server summary that says the spot fits. + let details = any["result.details"] + scroll(to: details, in: app) + details.tap() + let note = any["result.calculatedNote"] + XCTAssertTrue(note.waitForExistence(timeout: 10)) + XCTAssertFalse(any["result.summary"].exists, "a possible spot's summary is the server's pass") + let row = any["detail.check.gas_clearance"] + XCTAssertTrue(row.exists) + XCTAssertEqual(row.label, "Distance from gas equipment: Meets the rule on what the scan recorded", row.label) + let rowValue = row.value as? String ?? "" + XCTAssertFalse(rowValue.contains("well clear"), "a pass kept the server's reason: \(rowValue)") + scroll(to: note, in: app) + attach(app, name: "\(name)-details") + + // The AR view draws the proposed spot. The demo's answer is a sample, so the AR title + // says so (`ResultARScreen`); the drawing names the spot as proposed. + scroll(to: any["action.showAR"], in: app) + app.buttons["action.showAR"].tap() + let overlay = any["ar.overlay"] + XCTAssertTrue(overlay.waitForExistence(timeout: 10)) + XCTAssertEqual(overlay.label, "A proposed battery spot, drawn on your wall") + assertNoFitWords(any["instruction"].label) + attach(app, name: "\(name)-AR") + } + + // MARK: Helpers + + /// The card of a possible spot: headline, placement, the note that says what isn't confirmed, + /// no passing check lines, no repeat of the installer footnote, and the AR button's words. + @MainActor + private func assertCandidateCard(_ app: XCUIApplication) { + let any = app.descendants(matching: .any) + XCTAssertTrue(any["result.placement"].exists) + let note = any["result.candidateNote"] + XCTAssertTrue(note.exists, "the candidate note is missing") + XCTAssertTrue(note.label.contains("couldn't confirm all the space a battery needs"), note.label) + let lines = any.matching(NSPredicate(format: "identifier BEGINSWITH 'check.'")) + XCTAssertEqual(lines.count, 0, "a passing check reached the card") + XCTAssertFalse(any["result.installerConfirms"].exists) + XCTAssertEqual(app.buttons["action.showAR"].label, "See this spot on your wall") + let texts = app.staticTexts.allElementsBoundByIndex.map(\.label).joined(separator: " | ") + assertNoFitWords(texts) + } + + private func assertNoFitWords(_ text: String, file: StaticString = #filePath, line: UInt = #line) { + let lowered = text.lowercased() + for words in Self.fitWords { + XCTAssertFalse(lowered.contains(words), "says \"\(words)\": \(text)", file: file, line: line) + } + } + + /// The bundled sample turned into a pass: every check passing, nothing more needed, naming + /// `binding` as the scene it answers. + private static func passingAnswer(binding: Data) -> Data { + let sample = try! Data(contentsOf: UploadRecoveryUITests.sampleResult) + var object = try! JSONSerialization.jsonObject(with: sample) as! [String: Any] + object["decision"] = "pass" + object["checks"] = (object["checks"] as! [[String: Any]]).map { check in + var check = check + check["outcome"] = "pass" + check["unsure_cause"] = NSNull() + return check + } + object["missing_evidence"] = [Any]() + var stats = object["stats"] as! [String: Any] + stats["input_sha256"] = UploadRecoveryUITests.sha256(binding) + object["stats"] = stats + return try! JSONSerialization.data(withJSONObject: object) + } + + /// A gate folder with every screen up to the spot check let through; the result's stays shut. + private static func gate() throws -> URL { + let gate = URL(fileURLWithPath: NSTemporaryDirectory()).appending(path: "housescan-gate-\(UUID().uuidString)", directoryHint: .isDirectory) + try FileManager.default.createDirectory(at: gate, withIntermediateDirectories: true) + for phase in ["onboarding", "findMeter", "meterCloseUp", "wallWalk", "markFeatures", "gapRequest", "uploading", "spotConfirm"] { + try Data().write(to: gate.appending(path: phase)) + } + return gate + } + + @MainActor + private static func launchEngine(gate: URL, extra: [String] = []) -> XCUIApplication { + let app = XCUIApplication() + app.launchArguments = [ + "-replay", FullFlowUITests.fixture, "-autopilot", "-autopilotHold", "1.5", "-autopilotGate", gate.path, + "-practiceMeter", "NO", "-serverURL", "http://placement.invalid", "-answersFromGate", + ] + extra + app.launch() + return app + } + + @MainActor + private static func launchFixture(_ name: String, textSize: [String]) -> XCUIApplication { + let file = URL(fileURLWithPath: #filePath).deletingLastPathComponent().appending(path: "Fixtures/results/\(name).json").path + let app = XCUIApplication() + app.launchArguments = ["-practiceMeter", "NO", "-uiDemo", "-uiDemoFreeze", "-uiDemoPhase", "result", + "-uiDemoResultFile", file] + textSize + app.launch() + XCTAssertTrue(app.descendants(matching: .any)["screen.result"].waitForExistence(timeout: 15)) + return app + } + + /// Scrolls until the element is inside the window, a third of the window at most per drag. + @MainActor + private func scroll(to target: XCUIElement, in app: XCUIApplication) { + let window = app.windows.firstMatch.frame + let start = app.scrollViews.firstMatch.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.6)) + for _ in 0..<12 where !window.contains(target.frame) { + let frame = target.frame + let shift: CGFloat = frame.maxY > window.maxY + ? -min(frame.maxY - window.maxY + 24, window.height / 3) + : min(window.minY - frame.minY + 24, window.height / 3) + start.press(forDuration: 0.05, thenDragTo: start.withOffset(CGVector(dx: 0, dy: shift))) + } + } + + @MainActor + private func attach(_ app: XCUIApplication, name: String) { + let shot = XCTAttachment(screenshot: app.screenshot()) + shot.name = name + shot.lifetime = .keepAlways + add(shot) + } +} From 1b7209fc73fcaf3db0e7eedf8849fe5ec8469965 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 18:20:43 -0500 Subject: [PATCH 04/71] Read a pass without a spot or passing checks as installer review ResultReading.answer read any pass decision as a candidate, so a pass with no spot or no checks still said "A possible battery spot", and the new exhaustive test caught it. A candidate now needs a spot and at least one check, every one passing; a pass that contradicts itself goes to an installer, with its checks shown as sent. A reject without a spot stays "Not on this wall", and the exhaustive test now checks that too. Details' pass word is now "Passes on recorded data", under the existing note on what the list is. The real-engine UI tests terminate their app, and the render sweep leaves out the scope note's own "not ... confirmed clear". Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/UI/Copy/ScanCopy.swift | 2 +- .../HouseScanKit/Scene/ResultReading.swift | 14 ++++++++--- .../ResultReadingTests.swift | 23 +++++++++++++++++-- .../ProvisionalPlacementUITests.swift | 10 ++++++-- 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/ios/HouseScan/UI/Copy/ScanCopy.swift b/ios/HouseScan/UI/Copy/ScanCopy.swift index a8829d756..82e9d22f5 100644 --- a/ios/HouseScan/UI/Copy/ScanCopy.swift +++ b/ios/HouseScan/UI/Copy/ScanCopy.swift @@ -826,7 +826,7 @@ enum ScanCopy { static func outcomeWord(_ outcome: CheckOutcome) -> String { switch outcome { - case .pass: "Meets the rule on what the scan recorded" + case .pass: "Passes on recorded data" case .unsure: "Not sure yet" case .fail: "Doesn't work" } diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultReading.swift b/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultReading.swift index 008d614d5..563d9aa32 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultReading.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultReading.swift @@ -61,17 +61,25 @@ public enum ResultReading { if hasSpot, !spotIsClean(hasSpot: hasSpot, checks: checks) { return .installer } switch decision { case .pass: - return .candidate + // A candidate needs what makes one: a spot, and checks that all pass. A pass without a + // spot, with no checks, or with a check that didn't pass contradicts itself, and a + // person reads it; its checks stay as the server sent them. + return hasSpot && allPass(checks) ? .candidate : .installer case .reject: return .notHere case .manualReview: - // Needs at least one check: over none, "every check passes" is vacuously true. - if hasSpot, !policyApproved, !checks.isEmpty, checks.allSatisfy({ $0.outcome == .pass }) { return .candidate } + if hasSpot, !policyApproved, allPass(checks) { return .candidate } if checks.contains(where: { $0.outcome == .unsure && !$0.needsPerson && $0.viewCapturable }) { return .oneMoreLook } return .installer } } + /// At least one check, and every one passed. Over none, "every check passes" is vacuously + /// true and says nothing. + private static func allPass(_ checks: [Check]) -> Bool { + !checks.isEmpty && checks.allSatisfy { $0.outcome == .pass } + } + /// Indices of the checks the result card shows, in order: every FAIL, then every UNSURE, at /// most `limit`. No PASS: on the card a passing line read as space confirmed clear, which the /// scan can't show (see the top of this file). Every check, passing ones included, stays in diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/ResultReadingTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/ResultReadingTests.swift index e2d07e128..47cbc7729 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/ResultReadingTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/ResultReadingTests.swift @@ -275,15 +275,34 @@ import simd for hasSpot in [true, false] { for checks in sets { let answer = ResultReading.answer(decision: decision, policyApproved: approved, hasSpot: hasSpot, checks: checks) + let described = "\(decision) approved=\(approved) spot=\(hasSpot) checks=\(checks.map { "\($0.id)=\($0.outcome)" })" + // A reject without a spot stays negative whatever its checks say. + if decision == .reject, !hasSpot { #expect(answer == .notHere, "\(described)") } guard answer == .candidate else { continue } - #expect(hasSpot && ResultReading.spotIsClean(hasSpot: hasSpot, checks: checks) && decision != .reject, - "\(decision) approved=\(approved) checks=\(checks.map { "\($0.id)=\($0.outcome)" })") + // A candidate has a spot and at least one check, every one passing. + #expect(hasSpot && !checks.isEmpty && checks.allSatisfy { $0.outcome == .pass } && decision != .reject, + "\(described)") } } } } } + /// A pass that contradicts itself (no spot, no checks, or a check that didn't pass) goes to a + /// person rather than reading as a candidate, and its checks stay as sent. + @Test func anInconsistentPassGoesToAnInstaller() { + typealias Check = ResultReading.Check + let answer = { (spot: Bool, checks: [Check]) in + ResultReading.answer(decision: .pass, policyApproved: true, hasSpot: spot, checks: checks) + } + #expect(answer(false, []) == .installer) + #expect(answer(true, []) == .installer) + #expect(answer(false, [Check(id: "a", outcome: .pass)]) == .installer) + #expect(answer(true, [Check(id: "a", outcome: .pass), Check(id: "b", outcome: .unsure, viewCapturable: true)]) == .installer) + #expect(answer(true, [Check(id: "a", outcome: .pass), Check(id: "b", outcome: .fail)]) == .installer) + #expect(answer(true, [Check(id: "a", outcome: .pass)]) == .candidate) + } + /// An installer's review can still carry a spot, and the reading keeps it: only the words /// over it change. @Test func anInstallerReviewKeepsItsSpot() throws { diff --git a/ios/HouseScanUITests/ProvisionalPlacementUITests.swift b/ios/HouseScanUITests/ProvisionalPlacementUITests.swift index 783438126..757c61bf1 100644 --- a/ios/HouseScanUITests/ProvisionalPlacementUITests.swift +++ b/ios/HouseScanUITests/ProvisionalPlacementUITests.swift @@ -31,6 +31,7 @@ final class ProvisionalPlacementUITests: XCTestCase { } defer { server.stop() } let app = Self.launchEngine(gate: gate) + defer { app.terminate() } let any = app.descendants(matching: .any) let tryAgain = app.buttons["action.retryUpload"] @@ -65,6 +66,7 @@ final class ProvisionalPlacementUITests: XCTestCase { let server = GateServer(gate: gate) { body, _ in Self.passingAnswer(binding: body) } defer { server.stop() } let app = Self.launchEngine(gate: gate, extra: ["-autopilotSomethingThere"]) + defer { app.terminate() } let any = app.descendants(matching: .any) XCTAssertTrue(any["screen.result"].waitForExistence(timeout: 420), "the second passing answer never reached the result") @@ -123,7 +125,7 @@ final class ProvisionalPlacementUITests: XCTestCase { XCTAssertFalse(any["result.summary"].exists, "a possible spot's summary is the server's pass") let row = any["detail.check.gas_clearance"] XCTAssertTrue(row.exists) - XCTAssertEqual(row.label, "Distance from gas equipment: Meets the rule on what the scan recorded", row.label) + XCTAssertEqual(row.label, "Distance from gas equipment: Passes on recorded data", row.label) let rowValue = row.value as? String ?? "" XCTAssertFalse(rowValue.contains("well clear"), "a pass kept the server's reason: \(rowValue)") scroll(to: note, in: app) @@ -155,7 +157,11 @@ final class ProvisionalPlacementUITests: XCTestCase { XCTAssertEqual(lines.count, 0, "a passing check reached the card") XCTAssertFalse(any["result.installerConfirms"].exists) XCTAssertEqual(app.buttons["action.showAR"].label, "See this spot on your wall") - let texts = app.staticTexts.allElementsBoundByIndex.map(\.label).joined(separator: " | ") + // Details' scope note says, truthfully, that a pass is "not ... confirmed clear": the sweep + // leaves that one qualified sentence out, whether or not Details is open. + let texts = app.staticTexts.allElementsBoundByIndex + .filter { $0.identifier != "result.calculatedNote" } + .map(\.label).joined(separator: " | ") assertNoFitWords(texts) } From ab7b5829ea811388f6474f605b7ccfb6efada61f Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 18:28:59 -0500 Subject: [PATCH 05/71] Keep fit claims out of every result's summary and footnotes Details hid only a candidate's summary, assuming any other summary explains what failed. A pass the reading sends to an installer still carries the server's pass summary, and the demo's review answer said "A spot 3 ft right of your meter fits" under One more look (Sol review). Details now hides the summary of any pass decision as well as any candidate; manual review and reject summaries stay. The demo's review summary names the best spot without calling it a fit, and the panel footnote says the scan suggests where the battery could go rather than covering where it can. A UI test opens Details on the demo's review and overlap answers and checks the summary is there and claims no fit. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/UI/Copy/ScanCopy.swift | 2 +- ios/HouseScan/UI/Preview/DemoEngine.swift | 2 +- ios/HouseScan/UI/Screens/ResultScreen.swift | 10 +++++---- .../ProvisionalPlacementUITests.swift | 22 +++++++++++++++++++ 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/ios/HouseScan/UI/Copy/ScanCopy.swift b/ios/HouseScan/UI/Copy/ScanCopy.swift index 82e9d22f5..16d7b130f 100644 --- a/ios/HouseScan/UI/Copy/ScanCopy.swift +++ b/ios/HouseScan/UI/Copy/ScanCopy.swift @@ -720,7 +720,7 @@ enum ScanCopy { static let installerConfirms = "Before any battery goes in, an installer has to confirm where it goes on site." static let rulesNotFinal = "The placement rules aren't final yet, so every result needs an installer's review for now." // The server's result covers where the battery goes, not the panel itself. - static let panelReview = "Your electrical panel still needs an electrician's review. This scan only covers where the battery can go." + static let panelReview = "Your electrical panel still needs an electrician's review. This scan only suggests where the battery could go." /// Which rules answered, for a reviewer: "Rules 2f52ec35". static func rulesHash(_ hash: String) -> String { diff --git a/ios/HouseScan/UI/Preview/DemoEngine.swift b/ios/HouseScan/UI/Preview/DemoEngine.swift index 506d3dae4..268defc29 100644 --- a/ios/HouseScan/UI/Preview/DemoEngine.swift +++ b/ios/HouseScan/UI/Preview/DemoEngine.swift @@ -1212,7 +1212,7 @@ final class DemoEngine: ScanActions { /// The offline sample result. Placeholder rules, so the decision is manual review. static let reviewSample = ResultPresentation( decision: .manualReview, - summary: "A spot 3 ft right of your meter fits, but it's close to a window, so it needs an installer's review.", + summary: "The best spot, 3 ft right of your meter, is close to a window, so it needs an installer's review.", policyApproved: false, spot: BatterySpot(span: 0.55...1.34, depth: 0.56, height: 1.1, offsetFromWall: 0.03), cableRoute: [SIMD2(0.16, 1.45), SIMD2(0.95, 1.45), SIMD2(0.95, 1.1)], diff --git a/ios/HouseScan/UI/Screens/ResultScreen.swift b/ios/HouseScan/UI/Screens/ResultScreen.swift index e4234a39b..e5b3e7ad1 100644 --- a/ios/HouseScan/UI/Screens/ResultScreen.swift +++ b/ios/HouseScan/UI/Screens/ResultScreen.swift @@ -162,10 +162,12 @@ struct ResultScreen: View { private func details(_ result: ResultPresentation) -> some View { DisclosureGroup(isExpanded: $detailsExpanded) { VStack(alignment: .leading, spacing: 22) { - // A possible spot's summary is the server's own pass ("fits every check"), which - // the card's candidate note contradicts on purpose; the summary of any other answer - // explains what failed or is unsure, so it stays. - if !result.summary.isEmpty, result.answer != .candidate { + // A pass's summary, and a possible spot's under manual review, is the server's own + // pass ("fits every check"), which the card's candidate note contradicts on + // purpose; a pass the reading sends to an installer contradicts itself and says the + // same. Manual review and reject summaries explain what failed or is unsure, so + // they stay. + if !result.summary.isEmpty, result.decision != .pass, result.answer != .candidate { Text(result.summary) .font(Typeface.hint) .foregroundStyle(Palette.muted) diff --git a/ios/HouseScanUITests/ProvisionalPlacementUITests.swift b/ios/HouseScanUITests/ProvisionalPlacementUITests.swift index 757c61bf1..3f6877753 100644 --- a/ios/HouseScanUITests/ProvisionalPlacementUITests.swift +++ b/ios/HouseScanUITests/ProvisionalPlacementUITests.swift @@ -90,6 +90,28 @@ final class ProvisionalPlacementUITests: XCTestCase { try checkPassFixture(textSize: Self.largestText, name: "result-candidate-AX5") } + /// A result that isn't a possible spot keeps its summary in Details, and the summary never + /// claims a fit: the demo's review answer and its working-space overlap. + @MainActor + func testAReviewSummaryNeverClaimsAFit() throws { + for extra in [[], ["-uiDemoOverlap"]] { + let app = XCUIApplication() + app.launchArguments = ["-practiceMeter", "NO", "-uiDemo", "-uiDemoFreeze", "-uiDemoPhase", "result"] + extra + app.launch() + let any = app.descendants(matching: .any) + XCTAssertTrue(any["screen.result"].waitForExistence(timeout: 15)) + XCTAssertNotEqual(any["result.headline"].label, Self.candidate, "\(extra)") + let details = any["result.details"] + scroll(to: details, in: app) + details.tap() + let summary = any["result.summary"] + XCTAssertTrue(summary.waitForExistence(timeout: 10), "\(extra): a review answer's summary is missing") + assertNoFitWords(summary.label) + XCTAssertFalse(summary.label.lowercased().contains("fits"), summary.label) + app.terminate() + } + } + /// A rejected wall keeps its negative answer and its reasons. @MainActor func testARejectStaysNegative() throws { From 2d39637329c7af4a3504c6a7855eea28ad15ebdd Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 18:34:55 -0500 Subject: [PATCH 06/71] ios: the rapid Start over test reads which cleanup each write released Review found the test held only the first write. The second write ran unheld and rebuilt its folder after waiting, so a cleanup waiting for the wrong write could still pass, by scheduling or by that rebuild. Each write's end now releases the cleanups waiting for it and records which, under a lock, so the test checks that the first write released only the first cleanup. The second scan's inputs are made before the second cleanup lists, its write is held before zipping and checks that its inputs are still there instead of recreating them, and the two async tests have a one-minute time limit. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../HouseScanKitTests/KeptScansTests.swift | 125 ++++++++++++++---- 1 file changed, 101 insertions(+), 24 deletions(-) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/KeptScansTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/KeptScansTests.swift index 579cb2fe7..5e6ca08d1 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/KeptScansTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/KeptScansTests.swift @@ -292,7 +292,7 @@ import Testing /// with no zip yet it is listed as never packaged. The deletion waits for the write; a zip /// written to the end keeps the folder, one cut short keeps it without a place (as any /// partial bundle), and a write that failed before zipping leaves it to be deleted. - @Test(arguments: WriteEnd.allCases) + @Test(.timeLimit(.minutes(1)), arguments: WriteEnd.allCases) func aBundleStillBeingWrittenIsJudgedOnlyAfterItsWrite(end: WriteEnd) async throws { let root = try Self.makeRoot() defer { try? FileManager.default.removeItem(at: root) } @@ -342,48 +342,125 @@ import Testing #expect(try Set(FileManager.default.contentsOfDirectory(atPath: root.path)) == ["now", "run2", "run3"]) } - /// Start over twice while the first scan's bundle is written. The second scan's write, if it - /// runs, waits for the first; each cleanup waits for the chain as it stood when it listed. - /// The first scan's bundle is kept, the second scan's is kept only if it was written (a write - /// queued for a scan already started over is skipped), and the newest folders are never - /// listed. - @Test(arguments: [true, false]) + /// The ends of named bundle writes, and the cleanups waiting for them. Ending a write + /// releases the cleanups waiting for it and records which it released, under the lock, so a + /// test reads which cleanup a write let go instead of inferring it from scheduling. + final class WriteEnds: @unchecked Sendable { + private let lock = NSLock() + private var ended: Set = [] + private var waiting: [(write: String, cleanup: String, resume: CheckedContinuation)] = [] + private var released: [String: [String]] = [:] + + /// Waits until `write` ends. `registered` is told once `cleanup` is on the list, so a + /// write ended after that releases it. + func wait(for write: String, as cleanup: String, registered: AsyncStream.Continuation) async { + await withCheckedContinuation { (resume: CheckedContinuation) in + lock.lock() + let already = ended.contains(write) + if already { released[write, default: []].append(cleanup) } else { waiting.append((write, cleanup, resume)) } + lock.unlock() + registered.yield() + if already { resume.resume() } + } + } + + func end(_ write: String) { + lock.lock() + ended.insert(write) + let ready = waiting.filter { $0.write == write } + waiting.removeAll { $0.write == write } + released[write, default: []] += ready.map(\.cleanup) + lock.unlock() + for waiter in ready { waiter.resume.resume() } + } + + func released(by write: String) -> [String] { + lock.lock() + defer { lock.unlock() } + return released[write] ?? [] + } + } + + /// Start over twice while the first scan's bundle is written, as the app chains them: the + /// second scan's write starts only after the first, so each cleanup waits for the newest + /// write when it listed (the chain head). The first write's end releases only the first + /// cleanup; the second cleanup waits for the second write, which is held before zipping with + /// its inputs in place, and finds them intact. The first scan's bundle is kept, the second's + /// only if its write ran (one queued for a scan already started over is skipped), and the + /// newest folders are never listed. + @Test(.timeLimit(.minutes(1)), arguments: [true, false]) func rapidStartOversWaitForTheWholeChain(secondScanWrites: Bool) async throws { let root = try Self.makeRoot() defer { try? FileManager.default.removeItem(at: root) } + let files = FileManager.default + let ends = WriteEnds() let first = try Self.packing("first", in: root) - let firstWrite = HeldWrite(first, ending: .bundled) + let (firstGate, releaseFirst) = AsyncStream.makeStream() + let firstWrite = Task { + for await _ in firstGate { break } + try? WriteEnd.bundled.apply(to: first) + ends.end("first") + } + // Start over: the second scan's folder is made and the first cleanup lists. try Self.scan("second", in: root) let firstCleanup = ScanFolderCleanup(root: root, keeping: "second") #expect(Self.names(firstCleanup.obsolete) == ["first"]) - let second = root.appending(path: "second") - let secondWrite = Task { - await firstWrite.task.value - if secondScanWrites { try? WriteEnd.bundled.apply(to: try Self.packing("second", in: root)) } + // The second scan captures its packet inputs, and its write queues behind the first. + let second = try Self.packing("second", in: root) + let secondInputs = second.appending(path: "packet/photos/p00001.jpg") + let (secondGate, releaseSecond) = AsyncStream.makeStream() + let secondWrite = Task { () -> Bool? in + await firstWrite.value + guard secondScanWrites else { + ends.end("second") + return nil + } + for await _ in secondGate { break } + // Its inputs must still be there; this write never recreates them. + let intact = FileManager.default.fileExists(atPath: secondInputs.path) + if intact { try? WriteEnd.bundled.apply(to: second) } + ends.end("second") + return intact } + // Start over again: the second cleanup lists both earlier folders. try Self.scan("third", in: root) let secondCleanup = ScanFolderCleanup(root: root, keeping: "third") #expect(Self.names(secondCleanup.obsolete) == ["first", "second"]) try Self.scan("fourth", in: root) - let (arrived, arrive) = AsyncStream.makeStream() - let deletingFirst = Self.deleting(firstCleanup, after: firstWrite.task, arrived: arrive) - let deletingSecond = Self.deleting(secondCleanup, after: secondWrite, arrived: arrive) - var arrivals = arrived.makeAsyncIterator() - _ = await arrivals.next() - _ = await arrivals.next() - #expect(FileManager.default.fileExists(atPath: first.appending(path: "packet").path)) - - firstWrite.finish() + let (registered, register) = AsyncStream.makeStream() + let deletingFirst = Task { + await firstCleanup.run(after: { await ends.wait(for: "first", as: "firstCleanup", registered: register) }).map(\.url) + } + let deletingSecond = Task { + await secondCleanup.run(after: { await ends.wait(for: "second", as: "secondCleanup", registered: register) }).map(\.url) + } + var registrations = registered.makeAsyncIterator() + _ = await registrations.next() + _ = await registrations.next() + #expect(files.fileExists(atPath: first.appending(path: "packet").path)) + + releaseFirst.finish() + await firstWrite.value + #expect(ends.released(by: "first") == ["firstCleanup"]) #expect(await deletingFirst.value.isEmpty) + #expect(files.fileExists(atPath: first.appending(path: ScanFolderCleanup.bundleName).path)) + if secondScanWrites { + // The first write has ended and its cleanup has run; the second cleanup still waits. + #expect(files.fileExists(atPath: secondInputs.path)) + } + + releaseSecond.finish() + let wrote = await secondWrite.value + #expect(wrote == (secondScanWrites ? true : nil)) + #expect(ends.released(by: "second") == ["secondCleanup"]) #expect(await deletingSecond.value.isEmpty) var expected: Set = ["first", "third", "fourth"] if secondScanWrites { expected.insert("second") } - #expect(try Set(FileManager.default.contentsOfDirectory(atPath: root.path)) == expected) - #expect(FileManager.default.fileExists(atPath: first.appending(path: ScanFolderCleanup.bundleName).path)) - #expect(FileManager.default.fileExists(atPath: second.path) == secondScanWrites) + #expect(try Set(files.contentsOfDirectory(atPath: root.path)) == expected) + #expect(files.fileExists(atPath: second.appending(path: ScanFolderCleanup.bundleName).path) == secondScanWrites) } /// Bundles saved at the same moment keep the order by name, newest name first, as before. From a7c2a64cfd73447a1d90922c8ab04ef258b5a946 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 18:35:18 -0500 Subject: [PATCH 07/71] ios: the README says a scan started over reaches Saved scans only if its zip is finished Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ios/README.md b/ios/README.md index e69553734..788019214 100644 --- a/ios/README.md +++ b/ios/README.md @@ -98,7 +98,7 @@ Git ignores `Local.xcconfig`. Leave the team field in Xcode's Signing & Capabili **Logs.** Open Console.app on the Mac, select the iPhone, press Start and search for `subsystem:dev.housescanning.housescan`. Every screen change logs as `STATE=`; the `engine` category records the capture's decisions, the scan bundle and upload failures. The same lines show in Xcode's console while it runs the app. -**Getting a scan off the phone.** The result screen, and the screens for a failed or refused upload, have a Share scan button. It shares the scan bundle (below); AirDrop it to your Mac. It holds photos of a real home, so keep it out of git. After the app is closed, **Saved scans** in the first screen's top bar lists the completed scans the phone still keeps (the two most recent whose `scan.zip` was written to the end, as `ScanFolderCleanup` keeps them) with when each bundle was saved, its size and a Practice tag when its `scan-stamp.json` says so. A bundle cut short by a quit during its write is not listed, holds no kept place and is not deleted. Start over doesn't wait for a bundle still being written, but the next scan's cleanup does before it deletes anything, so that scan still reaches Saved scans. Share there sends the same bundle. It copies the bundle to the app's tmp folder first, so a new scan's cleanup can't delete it while the share sheet reads it, and deletes the copy when the sheet closes; a launch deletes copies earlier runs left. A saved scan can only be shared, never reopened or continued. The code is in `UI/SavedScans/` and HouseScanKit's `SavedScans/`. +**Getting a scan off the phone.** The result screen, and the screens for a failed or refused upload, have a Share scan button. It shares the scan bundle (below); AirDrop it to your Mac. It holds photos of a real home, so keep it out of git. After the app is closed, **Saved scans** in the first screen's top bar lists the completed scans the phone still keeps (the two most recent whose `scan.zip` was written to the end, as `ScanFolderCleanup` keeps them) with when each bundle was saved, its size and a Practice tag when its `scan-stamp.json` says so. A bundle cut short by a quit during its write is not listed, holds no kept place and is not deleted. Start over doesn't wait for a bundle still being written, but the next scan's cleanup does before it deletes anything. If that write finishes the zip, the scan reaches Saved scans; if it fails before zipping, its folder is deleted, and if it leaves a partial zip, the scan isn't listed. Share there sends the same bundle. It copies the bundle to the app's tmp folder first, so a new scan's cleanup can't delete it while the share sheet reads it, and deletes the copy when the sheet closes; a launch deletes copies earlier runs left. A saved scan can only be shared, never reopened or continued. The code is in `UI/SavedScans/` and HouseScanKit's `SavedScans/`. ## Scan bundle From 12cf77fc6b6aa117cc3fbed99cea9fcde30a844f Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 18:35:43 -0500 Subject: [PATCH 08/71] Capture the real candidate's AR instruction at the largest text size The prep-for-prod review found the AX5 check ran only on a demo answer, which is a sample with a different AR instruction, while the real candidate's longer instruction ran only at the default size. The settled engine test now runs at the largest text size, opens the AR view, checks its instruction names a possible spot and what the fit still needs, and that Done stays reachable, with screenshots of the card, its note and the AR view. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../ProvisionalPlacementUITests.swift | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/ios/HouseScanUITests/ProvisionalPlacementUITests.swift b/ios/HouseScanUITests/ProvisionalPlacementUITests.swift index 3f6877753..1e9403581 100644 --- a/ios/HouseScanUITests/ProvisionalPlacementUITests.swift +++ b/ios/HouseScanUITests/ProvisionalPlacementUITests.swift @@ -58,14 +58,17 @@ final class ProvisionalPlacementUITests: XCTestCase { /// "Something's there" withdraws the area and sends the scan again; the passing answer that /// comes back names the same spot, so the earlier answer settles it and the result shows /// without asking (`SpotConfirmations.settling`). That shortcut still shows a possible spot, - /// beside the notice that the homeowner said something stands there. + /// beside the notice that the homeowner said something stands there. At the largest text + /// size, so the AR view's longer instruction for a real candidate (placement, then what the + /// fit still needs) is captured where it is longest; a demo answer is a sample, whose AR + /// instruction differs. @MainActor func testASpotSettledByAnEarlierAnswerIsStillAPossibleSpot() throws { let gate = try Self.gate() defer { try? FileManager.default.removeItem(at: gate) } let server = GateServer(gate: gate) { body, _ in Self.passingAnswer(binding: body) } defer { server.stop() } - let app = Self.launchEngine(gate: gate, extra: ["-autopilotSomethingThere"]) + let app = Self.launchEngine(gate: gate, extra: ["-autopilotSomethingThere"] + Self.largestText) defer { app.terminate() } let any = app.descendants(matching: .any) @@ -74,7 +77,18 @@ final class ProvisionalPlacementUITests: XCTestCase { XCTAssertEqual(any["result.headline"].label, Self.candidate) XCTAssertTrue(any["result.spotRefused"].exists, "the settled answer's notice is missing") assertCandidateCard(app) - attach(app, name: "result-candidate-settled") + attach(app, name: "result-candidate-settled-AX5") + scroll(to: any["result.candidateNote"], in: app) + attach(app, name: "result-candidate-settled-AX5-note") + + try Data().write(to: gate.appending(path: "result")) + let instruction = any["instruction"] + XCTAssertTrue(any["screen.resultAR"].waitForExistence(timeout: 20)) + XCTAssertTrue(instruction.waitForExistence(timeout: 10)) + XCTAssertTrue(instruction.label.contains(Self.candidate), instruction.label) + XCTAssertTrue(instruction.label.contains("An installer needs to check the fit on site."), instruction.label) + XCTAssertTrue(app.buttons["action.closeAR"].isHittable, "Done is out of reach under the AR instruction") + attach(app, name: "resultAR-candidate-settled-AX5") } // MARK: Server answers in Fixtures/results From 8e9b07c9315c71dac90f15fe5817e7296bcebb9d Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 18:43:16 -0500 Subject: [PATCH 09/71] Leave unsure_cause out of the tests' passing answers The result schema's unsure_cause is an enum without null, so the bound all-pass answer in readingAPassLeavesTheAnswerAsTheServerSentIt failed validation (Actions run 37162046960, the run's only failure out of 714 package tests). Both that test and the UI test's passing answer now remove the key instead of setting it to null. Both answers validate against the schema with Python's jsonschema. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Tests/HouseScanKitTests/ResultReadingTests.swift | 2 +- ios/HouseScanUITests/ProvisionalPlacementUITests.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/ResultReadingTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/ResultReadingTests.swift index 47cbc7729..d6d6b2496 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/ResultReadingTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/ResultReadingTests.swift @@ -322,7 +322,7 @@ import simd object["checks"] = try #require(object["checks"] as? [[String: Any]]).map { check in var check = check check["outcome"] = "pass" - check["unsure_cause"] = NSNull() + check.removeValue(forKey: "unsure_cause") return check } object["missing_evidence"] = [Any]() diff --git a/ios/HouseScanUITests/ProvisionalPlacementUITests.swift b/ios/HouseScanUITests/ProvisionalPlacementUITests.swift index 1e9403581..5ed0e8fab 100644 --- a/ios/HouseScanUITests/ProvisionalPlacementUITests.swift +++ b/ios/HouseScanUITests/ProvisionalPlacementUITests.swift @@ -217,7 +217,7 @@ final class ProvisionalPlacementUITests: XCTestCase { object["checks"] = (object["checks"] as! [[String: Any]]).map { check in var check = check check["outcome"] = "pass" - check["unsure_cause"] = NSNull() + check.removeValue(forKey: "unsure_cause") return check } object["missing_evidence"] = [Any]() From a7c65bd22f89a6b261b9928a9e1d1f4bf2411899 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 19:09:20 -0500 Subject: [PATCH 10/71] ios: a past-end request can mark the real wall end; inferred ends say so in the packet (B-12) UNVALIDATED SOURCE CHECKPOINT: not compiled, and no package, app or UI tests have run. Local admission was refused (memory pressure, then a heavy process); validation is to run on hosted CI after publication. A past_end request asked the homeowner to walk on past an end, and its screen offered only "I can't get there": a wall that really stops sooner had no true answer. The engine already accepted an end marked during the request, but no control reached it, and that path skipped the walk's aim check. Now the follow-up screen offers the walk's own "Wall ends here" for that end, with the circle, the tape's preview, the refusal card and the end question (shared `EndQuestionAnswers`). The mark goes through `EndAim` (tracking, side, below ground, reach), from the circle or a point. A nearer end is allowed and bounds the scene; it keeps the walk's minimum wall length (`CoverageMap.endWouldLeaveTooLittle`, refused as `tooLittleWall`). "I can't get there" stays the decline and restores the cleared end. The question holds the decline and "Show my result" until answered, and is cleared if the request is left otherwise. Every wall end was stamped with a mark time, so ends the app placed from the walk ("Can't get there", "The wall keeps going", a past-end request met by views) read as homeowner marks. `WallEndStamp` records the source: a homeowner's mark keeps its time; an inferred end has no time and `attrs.inferred: true` in the packet (declared in the vendored manifest schema, pin updated; packet/README explains it and the end outcomes). A restored end keeps its stamp. `mark_end` is met only for a homeowner mark; "The wall keeps going" stays `cannot_reach`. Scene, server and recon unchanged. "Wall ends here" also requires the phone's current tracking to be normal, not only the last sampled frame's. Tests (not yet run): WallEndProvenanceTests (stamps, settlement, encoding, schema, writer refusal, nearer-end clipping, minimum length) and PastEndRecoveryUITests (demo at default and largest text: offered, refused, question; other requests offer nothing; two replays on the real engine through the answer gate, marking again nearer and declining, checking scene bounds and kinds and the packet marks and guidance via a debug-only gate export of the last upload's packet inputs). Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/Contract/ScanContract.swift | 7 + ios/HouseScan/Runtime/Autopilot.swift | 61 ++- ios/HouseScan/Runtime/LaunchOptions.swift | 6 + .../Runtime/ScanEngine+Actions.swift | 43 ++- ios/HouseScan/Runtime/ScanEngine+Ends.swift | 46 ++- .../Runtime/ScanEngine+GuidanceLog.swift | 5 +- ios/HouseScan/Runtime/ScanEngine+Packet.swift | 22 +- ios/HouseScan/Runtime/ScanEngine.swift | 77 ++-- .../UI/Components/EndQuestionAnswers.swift | 42 +++ ios/HouseScan/UI/Copy/ScanCopy.swift | 11 +- ios/HouseScan/UI/Preview/DemoEngine.swift | 54 +++ .../UI/Screens/GapRequestScreen.swift | 77 +++- ios/HouseScan/UI/Screens/WallWalkScreen.swift | 35 +- .../HouseScanKit/Guidance/WalkedEnd.swift | 10 + .../HouseScanKit/Packet/PacketMarks.swift | 30 +- .../Packet/WallEndProvenance.swift | 64 ++++ .../HouseScanKitTests/PacketWriterTests.swift | 23 +- .../Schemas/manifest.schema.json | 5 + .../WallEndProvenanceTests.swift | 198 ++++++++++ .../PastEndRecoveryUITests.swift | 351 ++++++++++++++++++ packet/README.md | 19 +- 21 files changed, 1078 insertions(+), 108 deletions(-) create mode 100644 ios/HouseScan/UI/Components/EndQuestionAnswers.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Packet/WallEndProvenance.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/WallEndProvenanceTests.swift create mode 100644 ios/HouseScanUITests/PastEndRecoveryUITests.swift diff --git a/ios/HouseScan/Contract/ScanContract.swift b/ios/HouseScan/Contract/ScanContract.swift index 141a07113..e70e8ec97 100644 --- a/ios/HouseScan/Contract/ScanContract.swift +++ b/ios/HouseScan/Contract/ScanContract.swift @@ -567,6 +567,9 @@ enum EndMarkRefusal: Error, Equatable, Sendable { case otherSide(WallSide) /// Tracking isn't normal, as for a feature mark (`MarkRefusal.trackingNotReady`). case trackingNotReady + /// During a past_end request: an end there would leave less wall than a battery needs + /// (`CoverageMap.endWouldLeaveTooLittle`), the walk's own minimum (`WallFrame.minWallLength`). + case tooLittleWall } enum MarkRefusal: Equatable, Sendable { @@ -631,6 +634,10 @@ struct GapRequest: Identifiable, Equatable, Sendable { /// Set while a walk-out request's line lies past where the space ends /// (`GapPlanner.walkOutBlock`): no walk can meet it, and the card says so. var spaceEnds: SpaceEnds? = nil + /// A server past_end request's side: the walk stopped there, and the request asks to walk on + /// past it. The screen also offers "Wall ends here" for that end, with its question, as the + /// walk does (B-12). Nil for every other request. + var pastEndSide: WallSide? = nil } // MARK: - Upload and result diff --git a/ios/HouseScan/Runtime/Autopilot.swift b/ios/HouseScan/Runtime/Autopilot.swift index cfad487a3..e92ced56b 100644 --- a/ios/HouseScan/Runtime/Autopilot.swift +++ b/ios/HouseScan/Runtime/Autopilot.swift @@ -192,8 +192,9 @@ final class Autopilot { let intervals = map.wallSeenSpans().map(\.span) + map.coveredIntervals(.ground) guard let low = intervals.map(\.lowerBound).min(), let high = intervals.map(\.upperBound).max() else { log("nothing covered; marking ends 1 m either side of the meter") - engine.setEnd(.left, at: -1, kind: .limit) - engine.setEnd(.right, at: 1, kind: .limit) + // These stand in for the homeowner's marks, as the taps below do. + engine.setEnd(.left, at: -1, kind: .limit, source: .homeowner) + engine.setEnd(.right, at: 1, kind: .limit, source: .homeowner) return } for (side, s) in [(WallSide.left, min(low, -0.2)), (.right, max(high, 0.2))] { @@ -204,7 +205,7 @@ final class Autopilot { let marked = side == .left ? engine.coverage?.leftEnd : engine.coverage?.rightEnd if !tapped || marked == nil { log("no replay frame shows the \(side.rawValue) end; setting it directly at s=\(s)") - engine.setEnd(side, at: s, kind: .limit) + engine.setEnd(side, at: s, kind: .limit, source: .homeowner) } if let asked = engine.state.endQuestion { // Every end counts as a corner here: a replay has no fence or gate to report. @@ -305,6 +306,14 @@ final class Autopilot { do { try engine.sceneJSON().write(to: gate.appending(path: "scene.json")) try engine.spotConfirm.lastScene?.write(to: gate.appending(path: "uploaded-scene.json")) + // The last upload's packet marks and guidance log, from the inputs its packet is + // written from (`ScanEngine.packetMarksAndGuidance`), for the UI test to check ends' + // provenance on the real path. + #if DEBUG + if let packet = engine.packetMarksAndGuidance() { + try packet.write(to: gate.appending(path: "packet-marks-guidance.json")) + } + #endif log("wrote scene.json to the gate folder") } catch { log("could not write scene.json to the gate folder: \(error)") @@ -449,8 +458,17 @@ final class Autopilot { private func driveServerRequest(_ replay: ReplayPlayer) async { guard let request = engine.state.gap else { return } log("server request \(request.id) over \(format(request.span))") + #if DEBUG + if request.pastEndSide != nil, let gate = engine.options.autopilotGate, let packet = engine.packetMarksAndGuidance() { + // The packet of the upload that raised the request, with the end it then cleared, for + // the UI test to compare the end that comes back with. + try? packet.write(to: gate.appending(path: "packet-first.json")) + } + #endif if request.reason == .overhead { await answerOverheadGap() + } else if request.pastEndSide != nil, engine.options.autopilotMarkPastEnd { + await markPastEnd(request, replay: replay) } else { await playGapFrames(replay) } @@ -458,6 +476,43 @@ final class Autopilot { _ = await waitUntil(timeout: 10) { self.engine.state.phase != .gapRequest || self.engine.state.gap?.id != request.id } } + /// `-autopilotMarkPastEnd`: marks a past_end request's end again, 60% of the way from the meter + /// to the end the request cleared, and answers "Something blocks it", so the wall comes out + /// shorter and its end a limit. The mark is a tap where the replay shows that place on the + /// wall, not the circle in the middle of the view: it goes through the same `EndAim` check + /// from that point (`ScanEngine.markWallEnd`), but no camera was aimed. Where it marked goes + /// to the gate folder as `past-end-mark.json` (side, s in meters), for the UI test. + private func markPastEnd(_ request: GapRequest, replay: ReplayPlayer) async { + guard let side = request.pastEndSide, let wall = engine.coverage?.wall else { return } + let cleared = side == .left ? request.span.upperBound : request.span.lowerBound + let s = side == .left ? min(cleared * 0.6, -0.5) : max(cleared * 0.6, 0.5) + // The engine plays the request's frames first; showing another frame meanwhile would + // race it. A replay that settled the request by itself leaves nothing to mark. + _ = await waitUntil(timeout: 60) { !replay.isPlaying || self.engine.state.gap?.id != request.id } + await pause(hold) + guard engine.state.phase == .gapRequest, let gap = engine.state.gap, gap.id == request.id, !gap.isSatisfied else { + fail("past_end request \(request.id) settled before its end could be marked again") + return + } + let tapped = await tap(wall.world(s: s, height: 0.5), replay: replay) { viewPoint in + self.engine.markWallEnd(at: viewPoint, viewSize: self.viewSize) + } + guard tapped, engine.state.endQuestion == side else { + let why = engine.state.endMarkRefusal.map { String(describing: $0) } ?? (tapped ? "no question" : "no frame shows it") + fail("could not mark the \(side.rawValue) end again at s=\(s): \(why)") + return + } + let marked = side == .left ? engine.coverage?.leftEnd : engine.coverage?.rightEnd + log("marked the \(side.rawValue) end again at s=\(marked ?? .nan) (cleared at s=\(cleared))") + if let gate = engine.options.autopilotGate, let marked { + let record: [String: Any] = ["side": side.rawValue, "s": Double(marked), "cleared_s": Double(cleared)] + try? JSONSerialization.data(withJSONObject: record).write(to: gate.appending(path: "past-end-mark.json")) + } + await pause(hold) + engine.answerWallEnd(turnsCorner: false) + log("answered the \(side.rawValue) end: something blocks it") + } + /// Waits while the engine plays the replay's frames for the current request. A replay shows /// only what it recorded, so when they don't settle the request the autopilot says "I can't /// get there", as a homeowner would. diff --git a/ios/HouseScan/Runtime/LaunchOptions.swift b/ios/HouseScan/Runtime/LaunchOptions.swift index d0e023d60..665b1b400 100644 --- a/ios/HouseScan/Runtime/LaunchOptions.swift +++ b/ios/HouseScan/Runtime/LaunchOptions.swift @@ -22,6 +22,10 @@ import OSLog /// marking the ends (`Autopilot.endWalkByCantGetThere`). /// - `-autopilotSomethingThere`: the autopilot answers the first spot check "Something's there" /// instead of "It's clear", so the scan is checked again without that area. +/// - `-autopilotMarkPastEnd`: on a server past_end request the autopilot marks that end again, +/// nearer than the end the request cleared, with a tap where the replay shows that place, and +/// answers "Something blocks it" (`Autopilot.markPastEnd`). Without it the autopilot plays the +/// request's frames and says "I can't get there" when they don't settle it. /// - `-autopilotCannotCheck`: the autopilot answers the first spot check "I can't check this /// area" instead of "It's clear". It can't be combined with `-autopilotSomethingThere`. /// - `-sampleResultAfterSpotAnswer ` (debug builds only): with the bundled sample, the @@ -54,6 +58,7 @@ struct LaunchOptions: Equatable { var autopilotCantGetThere = false var autopilotSomethingThere = false var autopilotCannotCheck = false + var autopilotMarkPastEnd = false var sampleResultAfterSpotAnswer: URL? var simulateAppStore = false var injectGroundRise: Float? @@ -75,6 +80,7 @@ struct LaunchOptions: Equatable { autopilotCantGetThere = arguments.contains("-autopilotCantGetThere") autopilotSomethingThere = arguments.contains("-autopilotSomethingThere") autopilotCannotCheck = arguments.contains("-autopilotCannotCheck") + autopilotMarkPastEnd = arguments.contains("-autopilotMarkPastEnd") precondition(!(autopilotSomethingThere && autopilotCannotCheck), "-autopilotSomethingThere and -autopilotCannotCheck each choose the first spot answer; pass one") #if DEBUG sampleResultAfterSpotAnswer = value(after: "-sampleResultAfterSpotAnswer").map { URL(fileURLWithPath: $0) } diff --git a/ios/HouseScan/Runtime/ScanEngine+Actions.swift b/ios/HouseScan/Runtime/ScanEngine+Actions.swift index 100181f83..1f970742d 100644 --- a/ios/HouseScan/Runtime/ScanEngine+Actions.swift +++ b/ios/HouseScan/Runtime/ScanEngine+Actions.swift @@ -138,19 +138,24 @@ extension ScanEngine: ScanActions { } /// Marks a wall end during the walk, or, during a server past_end request, marks that - /// request's end again at wherever the wall is now seen to stop. - /// The end at `point`, or with no point at the circle in the middle of the view: the walk's - /// "Wall ends here" button. The button answers the card, which names a side, so on the walk - /// it marks only that side, only with the phone's place known, and only where the circle is - /// on the wall; otherwise the card says why (`EndMarkRefusal`, B-06). Before, each of those - /// did nothing, and aiming across the meter ended the other side. A tap at a point (the - /// autopilot's) marks the side it lands on, as before. + /// request's end again at wherever the wall really stops (B-12), nearer or farther than the + /// end the request cleared: that end only said how far the walk had seen. + /// The end at `point`, or with no point at the circle in the middle of the view: the + /// "Wall ends here" button. The button answers the card, which names a side, so it marks only + /// that side, only with the phone's place known, and only where the circle is on the wall; + /// otherwise the card says why (`EndMarkRefusal`, B-06). Before, each of those did nothing, + /// and aiming across the meter ended the other side. During a past_end request a tap at a + /// point (the autopilot's) goes through the same check from that point; on the walk it marks + /// the side it lands on, as before. func markWallEnd(at point: CGPoint?, viewSize: CGSize) { - guard state.phase == .wallWalk || (state.phase == .gapRequest && pastEndSide != nil), + guard state.phase == .wallWalk || askedEndSide != nil, + // During a request, once the end is marked its question is what is left to answer. + state.phase != .gapRequest || state.endQuestion == nil, let map = coverage, let frame = currentFrame else { return } let hit: WallPoint - if point == nil, state.phase == .wallWalk, case .markEnd(let asked) = state.guidance { - switch circleEnd(asked: asked, frame: frame, map: map) { + if let asked = askedEndSide, point == nil || state.phase == .gapRequest { + let pixel = point.map { frame.projection.imagePixel(forViewPoint: $0, in: viewSize) } ?? frame.camera.imageSize / 2 + switch aimedEnd(asked: asked, pixel: pixel, frame: frame, map: map) { case .success(let found): hit = found case .failure(let refusal): return refuseEndMark(refusal) } @@ -159,14 +164,22 @@ extension ScanEngine: ScanActions { hit = found } let side: WallSide = hit.s < 0 ? .left : .right - if state.phase == .gapRequest, side != pastEndSide { return } + // The walk checks the wall's length when it finishes ("Done with this wall"); a request + // settles without that step, so its end is checked here against the other end, which + // stays where it is. + if state.phase == .gapRequest, map.endWouldLeaveTooLittle(side.walk, at: hit.s) { + return refuseEndMark(.tooLittleWall) + } state.endMarkRefusal = nil // Unexplored until the homeowner says something blocks the wall there: an unanswered // question must not tell the server the usable wall stops at this point. state.endQuestion = side state.endQuestionLeavesOut = nil state.endQuestionLeavesOutSeen = false - setEnd(side, at: hit.s, kind: .unexplored) + setEnd(side, at: hit.s, kind: .unexplored, source: .homeowner) + if state.phase == .gapRequest { + RuntimeLog.engine.info("gap \(self.state.gap?.id ?? 0): the \(side.rawValue, privacy: .public) end marked again at s=\(hit.s)") + } } private func refuseEndMark(_ refusal: EndMarkRefusal) { @@ -474,7 +487,9 @@ extension ScanEngine: ScanActions { } func skipGap() { - guard state.phase == .gapRequest else { return } + // With the end just marked, the end question answers the request; the screen hides this + // reply until it is answered. + guard state.phase == .gapRequest, state.endQuestion == nil else { return } // The card said the space ends short of the walk-out line (#164): the same answer, with // why in the log. if let ends = state.gap?.spaceEnds { @@ -485,7 +500,7 @@ extension ScanEngine: ScanActions { } func showResultNow() { - guard state.phase == .gapRequest else { return } + guard state.phase == .gapRequest, state.endQuestion == nil else { return } stopGapRequests() } diff --git a/ios/HouseScan/Runtime/ScanEngine+Ends.swift b/ios/HouseScan/Runtime/ScanEngine+Ends.swift index b61cf80cd..c3f749f85 100644 --- a/ios/HouseScan/Runtime/ScanEngine+Ends.swift +++ b/ios/HouseScan/Runtime/ScanEngine+Ends.swift @@ -4,13 +4,35 @@ import OSLog import simd /// Ending the wall during the walk without pointing at the end ("Can't get there", "Wall ends -/// here"), and the preview of where an end would land. +/// here"), marking it at the circle, during the walk or a past-end request, and the preview of +/// where an end would land. extension ScanEngine { + /// The end a card asks the homeowner to mark at the circle: the walk's "Is this the right end + /// of the wall?", or a past_end request's side while it is unsettled (B-12). "Wall ends here" + /// marks only this side, through the same aim check either way (`circleEnd`). + var askedEndSide: WallSide? { + switch state.phase { + case .wallWalk: + if case .markEnd(let side) = state.guidance { return side } + return nil + case .gapRequest: + guard let side = pastEndSide, state.gap?.isSatisfied == false else { return nil } + return side + default: + return nil + } + } + /// The side ending the wall applies to now, and whether the end lands at the reticle. The /// walk's own side while it asks to walk or to mark the end; during a request about the wall /// in front of the homeowner (tilt down, tilt up, step back), the side of the meter the phone - /// is on (`WalkedEnd.side`). Nil when both ends are marked or the walk is doing something else. + /// is on (`WalkedEnd.side`). During a past_end request its side, at the reticle. Nil when both + /// ends are marked or the walk is doing something else. private var endOnOffer: (side: WallSide, atReticle: Bool)? { + if state.phase == .gapRequest { + guard let side = askedEndSide, state.endQuestion == nil, !state.overheadQuestion, coverage != nil else { return nil } + return (side, true) + } guard state.phase == .wallWalk, state.marking == nil, state.endQuestion == nil, !state.overheadQuestion, nextWallSide == nil, let map = coverage else { return nil } switch state.guidance { @@ -103,10 +125,18 @@ extension ScanEngine { /// middle whatever the view's size. Shared by the button and its preview, so the tape never /// shows an end the button would refuse. Only normal tracking counts, as for feature marks. func circleEnd(asked: WallSide, frame: SourceFrame, map: CoverageMap) -> Result { - let hit = map.wall.intersectWall(frame.camera.ray(throughPixel: frame.camera.imageSize / 2)) + aimedEnd(asked: asked, pixel: frame.camera.imageSize / 2, frame: frame, map: map) + } + + /// `circleEnd` for the ray through `pixel` of the sensor image. A tap at a point (the + /// autopilot's) during a past_end request goes through the same check as the circle. + func aimedEnd(asked: WallSide, pixel: SIMD2, frame: SourceFrame, map: CoverageMap) -> Result { + let hit = map.wall.intersectWall(frame.camera.ray(throughPixel: pixel)) switch EndAim.verdict( hit: hit, camera: frame.camera.position, wall: map.wall, reach: map.config.maxDistance, - groundError: map.heightError, askedLeft: asked == .left, trackingNormal: frame.tracking == .normal + // The frame and the phone now: a pose-only frame after it can report tracking lost + // without replacing the sampled frame whose ray this is. + groundError: map.heightError, askedLeft: asked == .left, trackingNormal: frame.tracking == .normal && state.tracking == .normal ) { case .end(let point): return .success(point) case .trackingLimited: return .failure(.trackingNotReady) @@ -122,7 +152,7 @@ extension ScanEngine { // A refusal stands until the circle is on the end the card asks for, or the card moves on. if state.endMarkRefusal != nil { var fixed = true - if case .markEnd(let side) = state.guidance { fixed = preview?.atReticle == true && preview?.side == side } + if let side = askedEndSide { fixed = preview?.atReticle == true && preview?.side == side } if fixed { state.endMarkRefusal = nil } } } @@ -141,7 +171,8 @@ extension ScanEngine { state.endQuestionLeavesOut = leavesOut state.endQuestionLeavesOutSeen = leavesOut != nil && WalkedEnd.leftOutIsSeen(preview.side.walk, s: preview.s, walked: map.walkedPositions, wall: map.wall, seen: seen) - setEnd(preview.side, at: preview.s, kind: .unexplored) + // The homeowner said the wall ends where they stand: their mark. + setEnd(preview.side, at: preview.s, kind: .unexplored, source: .homeowner) } /// "Can't get there" while the walk asks to walk `side` or to mark its end: the end goes where @@ -151,7 +182,8 @@ extension ScanEngine { guard let s = walkedEnd(side), let map = coverage else { return } logEnd(refused ? "can't get there" : "the wall keeps going", side: side, at: s) let walked = map.walkedFarthest(side.walk) - setEnd(side, at: s, kind: .unexplored) + // Where the walk reached, not a place the homeowner marked: inferred (B-12). + setEnd(side, at: s, kind: .unexplored, source: .inferred) guard refused else { return } walkRefusals.ended(side.walk, at: s, walked: walked, time: ScanEngine.refusalClock) if walkRefusals.wasRefused(side.walk, end: s) { diff --git a/ios/HouseScan/Runtime/ScanEngine+GuidanceLog.swift b/ios/HouseScan/Runtime/ScanEngine+GuidanceLog.swift index fb6bbdb59..b294b4f63 100644 --- a/ios/HouseScan/Runtime/ScanEngine+GuidanceLog.swift +++ b/ios/HouseScan/Runtime/ScanEngine+GuidanceLog.swift @@ -112,7 +112,10 @@ extension ScanEngine { // before it leaves. return nextWallSide == nil ? .met : .superseded case .markEnd(let side): - return wallEndKinds[side] != nil ? .met : .superseded + // Met only by the homeowner's mark: an end the walk inferred answered another request + // ("Can't get there", or "The wall keeps going", closed as cannot_reach) (B-12). + guard wallEndKinds[side] != nil, let stamp = endStamps[side] else { return .superseded } + return stamp.markEndOutcome case .stepBack: guard let camera = currentFrame?.camera, let wall = coverage?.wall else { return .superseded } return wall.wallPoint(camera.position).out > GuidancePlanner().config.tooClose ? .met : .superseded diff --git a/ios/HouseScan/Runtime/ScanEngine+Packet.swift b/ios/HouseScan/Runtime/ScanEngine+Packet.swift index 1e515631b..253daedb8 100644 --- a/ios/HouseScan/Runtime/ScanEngine+Packet.swift +++ b/ios/HouseScan/Runtime/ScanEngine+Packet.swift @@ -114,6 +114,23 @@ extension ScanEngine { return inputs } + /// The marks and guidance log of the last upload's packet, as `writePacket` writes them into + /// manifest.json (`{"marks": [...], "guidance": [...]}`): `withCurrentGuidance` of the inputs + /// the upload captured, the same projection `saveBundle` is given. Only the autopilot's UI-test + /// gate reads it (`Autopilot.writeSceneForTest`); nil before an upload. Debug builds only. + #if DEBUG + func packetMarksAndGuidance() -> Data? { + guard let inputs = spotConfirm.lastPacket.map(withCurrentGuidance) else { return nil } + struct Projection: Encodable { + var marks: [PacketMark] + var guidance: [PacketGuidanceEntry] + } + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] + return try? encoder.encode(Projection(marks: inputs.marks, guidance: inputs.guidance)) + } + #endif + /// `utsname.machine`, such as "iPhone16,1": the hardware, never the phone's name. "arm64" in /// the Simulator. nonisolated static func hardwareModel() -> String { @@ -137,7 +154,9 @@ extension ScanEngine { guard let s else { continue } marks.append(.wallEnd( id: "wall_end_\(side.rawValue)", side: side == .left ? .left : .right, - endKind: wallEndKinds[side] == .limit ? .limit : .unexplored, s: s, wall: wall, frame: frame, t: markTimes[MarkKey.end(side)] + endKind: wallEndKinds[side] == .limit ? .limit : .unexplored, s: s, wall: wall, frame: frame, + // No stamp is never read as the homeowner's mark (B-12). + stamp: endStamps[side] ?? .inferred )) } for feature in state.features { @@ -170,7 +189,6 @@ extension ScanEngine { /// Keys of `markTimes`. enum MarkKey { static let meter = "meter" - static func end(_ side: WallSide) -> String { "end.\(side.rawValue)" } static func feature(_ id: UUID) -> String { id.uuidString } } diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index d43244723..6901aaaf3 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -133,9 +133,10 @@ final class ScanEngine { /// The side of a server past_end request being captured: that end was cleared, and marking /// it again settles the request (see `markWallEnd`). var pastEndSide: WallSide? - /// The end the past_end request cleared: where it was, its kind and when it was marked, put - /// back or moved on when the request ends without it marked again (`settleClearedEnd`). - private var clearedEnd: (s: Float, kind: EndKind, t: Double?)? + /// The end the past_end request cleared: where it was, its kind and its stamp (marked, with + /// when, or inferred), put back or moved on when the request ends without it marked again + /// (`settleClearedEnd`). + private var clearedEnd: (s: Float, kind: EndKind, stamp: WallEndStamp)? /// Server requests raised without a tap since the review was confirmed (`automaticGapQueue`), /// oldest first. Each is raised once, whether its view was taken or the homeowner couldn't /// get there; the result still offers it as a capture. @@ -232,8 +233,12 @@ final class ScanEngine { } } } - /// When each mark was made, on the capture clock (`MarkKey`). + /// When each mark was made, on the capture clock (`MarkKey`). Wall ends keep theirs in + /// `endStamps`, with whether the homeowner marked them at all. var markTimes: [String: Double] = [:] + /// Whether each wall end was marked by the homeowner, with when, or inferred from the walk + /// (`WallEndStamp`, B-12). Written by every `setEnd`, so it never outlives its end's source. + var endStamps: [WallSide: WallEndStamp] = [:] /// The packet's clock for guidance and marks: the latest frame's time, ARFrame.timestamp /// live. A replay plays parts of its recording more than once, so its clock is the latest /// frame time seen and never runs back. Nil until the first frame. @@ -1383,7 +1388,9 @@ final class ScanEngine { } else { store.keyframes.count > keyframesAtGapStart } - let satisfied = gapPlanner.isSatisfied(plan, map) && fresh + // Not while the end question is up: the homeowner marked the end and is saying what is + // there, and that answer settles the request (`settlePastEnd`). + let satisfied = gapPlanner.isSatisfied(plan, map) && fresh && state.endQuestion == nil state.guidance = .gap let center = (plan.span.lowerBound + plan.span.upperBound) / 2 let cue = gapCue(plan, map, center: center) @@ -1675,6 +1682,7 @@ final class ScanEngine { coverage?.heightError = groundMeasured ? 0 : Self.estimatedGroundError self.groundMeasured = groundMeasured endKinds = [:] + endStamps = [:] state.endQuestion = nil state.wallTooShort = false nextWallSide = nil @@ -1867,9 +1875,16 @@ final class ScanEngine { ) } - func setEnd(_ side: WallSide, at s: Float, kind: EndKind) { + /// Sets the end on `side`. `source` says who put it there: the homeowner's mark is stamped + /// with the capture clock; an end inferred from the walk has no mark time (B-12). + func setEnd(_ side: WallSide, at s: Float, kind: EndKind, source: WallEndSource) { + setEnd(side, at: s, kind: kind, stamp: source == .homeowner ? .marked(at: captureClock) : .inferred) + } + + /// Sets the end on `side` with a given stamp: a restored end keeps the one it had. + func setEnd(_ side: WallSide, at s: Float, kind: EndKind, stamp: WallEndStamp) { guard var map = coverage else { return } - markTimes[MarkKey.end(side)] = captureClock + endStamps[side] = stamp map.setEnd(side == .left ? .left : .right, at: s) // Ground past a limit end still counts toward clearances (server contract, "Ends and // corners"); past an unexplored end it doesn't. @@ -1880,7 +1895,7 @@ final class ScanEngine { state.endMarkRefusal = nil publishWall() publishCoverage() - RuntimeLog.engine.info("end \(side.rawValue, privacy: .public) at s=\(s) (\(kind == .limit ? "limit" : "unexplored", privacy: .public))") + RuntimeLog.engine.info("end \(side.rawValue, privacy: .public) at s=\(s) (\(kind == .limit ? "limit" : "unexplored", privacy: .public), \(stamp.isInferred ? "inferred" : "marked", privacy: .public))") if let camera = lastFrame?.camera, state.phase == .wallWalk { updateGuidance(camera: camera, time: lastFrame?.timestamp ?? 0) } @@ -1894,7 +1909,7 @@ final class ScanEngine { } func clearEnd(_ side: WallSide) { - markTimes[MarkKey.end(side)] = nil + endStamps[side] = nil updateCoverage { $0.clearEnd(side == .left ? .left : .right) } endKinds[side] = nil if state.endQuestion == side { state.endQuestion = nil } @@ -1939,7 +1954,9 @@ final class ScanEngine { keyframesAtGapStart = store.keyframes.count overheadViewsAtGapStart = coverage?.overheadCameras.count ?? 0 let progress = coverage.map { gapPlanner.progress(of: plan, $0) } ?? 0 - state.gap = GapRequest(id: gapCounter, origin: origin, reason: reason, band: plan.band == .ground ? .ground : .wall, span: plan.span, progress: progress, isSatisfied: false) + state.gap = GapRequest( + id: gapCounter, origin: origin, reason: reason, band: plan.band == .ground ? .ground : .wall, span: plan.span, + progress: progress, isSatisfied: false, pastEndSide: pastEndSide) state.guidance = .gap go(.gapRequest) updateGap(camera: lastFrame?.camera) @@ -1949,6 +1966,15 @@ final class ScanEngine { /// with the new evidence (the closed loop: gap, instruction, capture, updated result). The /// answer then leads to the next capturable request or to the result (`upload`). private func afterGapResolved() { + // A request left with its end question unanswered (the phone lost its place, say): the + // end stays the homeowner's mark, unexplored as for any unanswered end, and the question + // must not outlive the request and hold the next one. + if let side = state.endQuestion, side == pastEndSide { + state.endQuestion = nil + state.endQuestionLeavesOut = nil + state.endQuestionLeavesOutSeen = false + } + state.endMarkRefusal = nil settleClearedEnd() gapPlan = nil pastEndSide = nil @@ -2187,7 +2213,8 @@ final class ScanEngine { let wallSide: WallSide = side == .left ? .left : .right pastEnd = wallSide let old = wallSide == .left ? coverage?.leftEnd : coverage?.rightEnd - clearedEnd = old.map { (s: $0, kind: endKinds[wallSide] ?? EndKind.unexplored, t: markTimes[MarkKey.end(wallSide)]) } + // An end without a stamp comes back as inferred: no source is ever made up for it. + clearedEnd = old.map { (s: $0, kind: endKinds[wallSide] ?? EndKind.unexplored, stamp: endStamps[wallSide] ?? .inferred) } clearEnd(wallSide) } // Set first, so the guidance log records the request as a past-end one. @@ -2195,21 +2222,24 @@ final class ScanEngine { beginGap(plan, origin: .server, reason: .server(detail: item.message)) } - /// A past_end request ending without its end marked again (the gap screen offers only "I - /// can't get there") must not leave that side without an end: the export would run the wall - /// out to whatever the fog saw, and the next past_end request would be planned from the - /// meter (issue #35). Met, the end moves on past the ground the request showed, still - /// unexplored (`GapPlanner.endAfterPastEnd`); skipped, the end it cleared comes back with its - /// kind and mark time. + /// A past_end request leaving the screen settles the end it cleared (`PastEndSettlement`). + /// Marked again during the request ("Wall ends here"), the homeowner's end stands, nearer or + /// farther than the cleared one, which only said how far the walk had seen. Met by views, + /// the end moves on past the ground the request showed, unexplored and inferred + /// (`GapPlanner.endAfterPastEnd`). Skipped, the cleared end comes back with its kind, source + /// and mark time. Left without an end, the export would run the wall out to whatever the fog + /// saw, and the next past_end request would be planned from the meter (issue #35). private func settleClearedEnd() { guard let side = pastEndSide, let old = clearedEnd, let plan = gapPlan else { return } clearedEnd = nil - guard (side == .left ? coverage?.leftEnd : coverage?.rightEnd) == nil else { return } - if state.gap?.isSatisfied == true { - setEnd(side, at: gapPlanner.endAfterPastEnd(plan, side: side == .left ? .left : .right, clearedAt: old.s), kind: .unexplored) - } else { - setEnd(side, at: old.s, kind: old.kind) - markTimes[MarkKey.end(side)] = old.t + let marked = (side == .left ? coverage?.leftEnd : coverage?.rightEnd) != nil + switch PastEndSettlement.when(endMarkedDuringRequest: marked, met: state.gap?.isSatisfied == true) { + case .keepMarked: + return + case .moveOn: + setEnd(side, at: gapPlanner.endAfterPastEnd(plan, side: side == .left ? .left : .right, clearedAt: old.s), kind: .unexplored, source: .inferred) + case .restore: + setEnd(side, at: old.s, kind: old.kind, stamp: old.stamp) } } @@ -2388,6 +2418,7 @@ final class ScanEngine { private func resetPacketLog() { guidanceLog = GuidanceLog() markTimes = [:] + endStamps = [:] captureClock = nil } diff --git a/ios/HouseScan/UI/Components/EndQuestionAnswers.swift b/ios/HouseScan/UI/Components/EndQuestionAnswers.swift new file mode 100644 index 000000000..068b175d2 --- /dev/null +++ b/ios/HouseScan/UI/Components/EndQuestionAnswers.swift @@ -0,0 +1,42 @@ +import SwiftUI + +/// The three answers to "What's at the right end?" (`ScanViewState.endQuestion`), shared by the +/// walk and a past-end request, where marking the end again asks the same question (B-12). One +/// question, three equal full-width answers that say what they mean (checklist I4). +struct EndQuestionAnswers: View { + let actions: any ScanActions + + var body: some View { + VStack(spacing: 8) { + Button { + actions.answerWallEnd(turnsCorner: true) + } label: { + Label("It turns a corner", systemImage: "arrow.turn.up.right") + .frame(maxWidth: .infinity) + } + .buttonStyle(.secondaryProminent) + .accessibilityIdentifier("action.endCorner") + Button { + actions.answerWallEnd(turnsCorner: false) + } label: { + Label("Something blocks it", systemImage: "xmark.octagon") + .frame(maxWidth: .infinity) + } + .buttonStyle(.secondaryProminent) + .accessibilityHint("A fence, gate, or your neighbor's yard") + .accessibilityIdentifier("action.endBlocked") + // A garden wall or a fence can just stop, with no corner and nothing in the way, + // and neither answer above fits it (#70). Ends the wall as "Something blocks it" + // does: the usable wall stops here. + Button { + actions.answerWallEnd(turnsCorner: false) + } label: { + Label("The wall just ends", systemImage: "arrow.right.to.line") + .frame(maxWidth: .infinity) + } + .buttonStyle(.secondaryProminent) + .accessibilityHint("No corner and nothing in the way. The wall stops here.") + .accessibilityIdentifier("action.endEnds") + } + } +} diff --git a/ios/HouseScan/UI/Copy/ScanCopy.swift b/ios/HouseScan/UI/Copy/ScanCopy.swift index 24f23609e..fcfcf3995 100644 --- a/ios/HouseScan/UI/Copy/ScanCopy.swift +++ b/ios/HouseScan/UI/Copy/ScanCopy.swift @@ -397,6 +397,8 @@ enum ScanCopy { Instruction(title: "That's the \(landed.rawValue) side of your meter", detail: "Turn to the \(side.rawValue) end, then tap Wall ends here.") case .trackingNotReady: Instruction(title: "One moment, your phone is still finding its place", detail: "Then aim at the \(side.rawValue) end.") + case .tooLittleWall: + Instruction(title: "That leaves too little wall for a battery", detail: "Aim where the wall really stops, or tap I can't get there.") } } @@ -453,7 +455,11 @@ enum ScanCopy { case .wallAboveCandidate: return Instruction(title: "Show the wall \(place)", detail: "Tilt up so the wall above this spot is in view.") case .server(let detail): - return Instruction(title: gap.band == .ground ? "Show the ground \(stretch)" : "Show the wall \(stretch)", detail: detail) + let title = gap.band == .ground ? "Show the ground \(stretch)" : "Show the wall \(stretch)" + // A past_end request asks to walk on past an end; the wall may really stop before + // that, and the screen offers the walk's own "Wall ends here" for it (B-12). + guard gap.pastEndSide != nil else { return Instruction(title: title, detail: detail) } + return Instruction(title: title, detail: "\(detail) \(pastEndAlternative)") case .groundOut(let out): return Instruction( title: "Show the ground out to about \(Distance.feetAtLeast(out)) from the wall", @@ -489,6 +495,9 @@ enum ScanCopy { /// The request a gap card asks for, without the walk-out's reading from where the phone is /// now: what the card's reply answers (`InstructionCard.Reply.task`) and what the guidance log /// keeps. The reading changes as the phone moves, and each change would lock the reply again. + /// After a past_end request's own words: the other true answer, in the walk's vocabulary. + static let pastEndAlternative = "If the wall stops sooner, aim where it stops and tap Wall ends here." + static func gapTask(_ gap: GapRequest) -> Instruction { var steady = gap steady.walkOut = nil diff --git a/ios/HouseScan/UI/Preview/DemoEngine.swift b/ios/HouseScan/UI/Preview/DemoEngine.swift index 506d3dae4..ab27474d3 100644 --- a/ios/HouseScan/UI/Preview/DemoEngine.swift +++ b/ios/HouseScan/UI/Preview/DemoEngine.swift @@ -26,7 +26,11 @@ final class DemoEngine: ScanActions { /// `-uiDemoUnusableAnswer `: how many answers in a row House Scan can't use; 0 when off. private let unusableAnswers: Int /// Which request the gap screen shows (`-uiDemoGap`); the phone's ground request by default. + /// `pastEnd` is a server past_end request on the right, with "Wall ends here" (B-12). private let gapKind: String? + /// `-uiDemoEndMarkRefusal`: "Wall ends here" was pressed with the circle off the wall, and the + /// demo's circle stays off it, so pressing again refuses again. + private let endMarkRefused: Bool /// The tilt-up step was answered or skipped. private var tiltUpSettled = false /// Walk-script ticks spent on the tilt-up step, standing in for the phone being tilted up. @@ -84,6 +88,7 @@ final class DemoEngine: ScanActions { rejectUpload = arguments.contains("-uiDemoRejected") unusableAnswers = arguments.contains("-uiDemoUnusableAnswer") ? max(value("-uiDemoUnusableAnswer").flatMap(Int.init) ?? 1, 1) : 0 gapKind = value("-uiDemoGap") + endMarkRefused = arguments.contains("-uiDemoEndMarkRefusal") state.feed = DemoScene.image.map(CameraFeed.still) ?? .none state.isReplay = true state.tracking = .normal @@ -417,6 +422,27 @@ final class DemoEngine: ScanActions { state.gap?.band = .wall state.target = DemoScene.wall.world(s: 1.7, height: 3.0) state.path = [] + case "pastEnd": + // The check asks to walk on past the right end, which it clears, as the real engine's + // `beginServerGap` does; the wall may stop sooner, so "Wall ends here" is offered. + let end = demoRightEnd + let past: ClosedRange = end...(end + 0.6) + state.wall?.rightEnd = nil + state.gap?.origin = .server + state.gap?.reason = .server(detail: "Keep walking past the right end of the scan (14 ft 1 in right of the meter): a spot within reach may be there.") + state.gap?.span = past + state.gap?.pastEndSide = .right + state.target = DemoScene.wall.world(s: end + 0.3, height: 0, out: 0.5) + state.path = DemoScene.path(toward: end + 0.3) + if endMarkRefused { + // As in the real engine, a circle off the wall has no end preview (`circleEnd`). + state.endMarkRefusal = .noWall + state.endPreview = nil + } else { + // The circle on the wall a little before the old end: where the wall really stops. + state.endPreview = EndPreview(side: .right, s: end - 0.4, atReticle: true) + } + return default: break } @@ -855,6 +881,20 @@ final class DemoEngine: ScanActions { } func markWallEnd(at point: CGPoint?, viewSize: CGSize) { + if state.phase == .gapRequest, let side = state.gap?.pastEndSide, state.gap?.isSatisfied == false, state.endQuestion == nil { + if endMarkRefused { + state.endMarkRefusal = .noWall + state.endPreview = nil + return + } + let s = state.endPreview?.s ?? (side == .right ? demoRightEnd : demoLeftEnd) + if side == .right { state.wall?.rightEnd = s } else { state.wall?.leftEnd = s } + state.endMarkRefusal = nil + state.endQuestion = side + state.endQuestionLeavesOut = nil + state.endQuestionLeavesOutSeen = false + return + } guard case .markEnd(let side) = state.guidance else { return } state.endMarkRefusal = nil if side == .right { @@ -891,6 +931,20 @@ final class DemoEngine: ScanActions { state.endQuestion = nil state.endQuestionLeavesOut = nil state.endQuestionLeavesOutSeen = false + if state.phase == .gapRequest, var gap = state.gap, gap.pastEndSide == side { + // The end marked again settles the request, as `ScanEngine.settlePastEnd` does; the + // check runs again after the request's check mark has been seen. + gap.isSatisfied = true + gap.progress = 1 + state.gap = gap + let id = gap.id + Task { [weak self] in + try? await Task.sleep(for: .seconds(1)) + guard let self, self.state.phase == .gapRequest, self.state.gap?.id == id else { return } + self.enterUpload() + } + return + } let followed = state.wall?.cornerSegments.contains { side == .right ? $0.span.lowerBound > 0 : $0.span.upperBound < 0 } ?? true if turnsCorner, !followed { state.guidance = .markNextWall(side: side, refusal: nil) diff --git a/ios/HouseScan/UI/Screens/GapRequestScreen.swift b/ios/HouseScan/UI/Screens/GapRequestScreen.swift index 41417cb76..bc94f342b 100644 --- a/ios/HouseScan/UI/Screens/GapRequestScreen.swift +++ b/ios/HouseScan/UI/Screens/GapRequestScreen.swift @@ -8,6 +8,11 @@ import SwiftUI /// review) carries "One more view to finish" above the instruction, the words the upload screen /// just said. "I can't get there" skips only this view and goes on to the check's next one; /// "Show my result", on every such request, stops asking and leads to the result. +/// +/// A past_end request (the walk stopped at an end, and the check asks to walk on past it) also +/// offers the walk's own "Wall ends here" for that end, with the circle, the end's preview on the +/// tape, the refusal when the circle isn't on that end, and the end question (B-12). Walking on +/// stays the request; marking the end is the other true answer. struct GapRequestScreen: View { let state: ScanViewState let actions: any ScanActions @@ -27,6 +32,13 @@ struct GapRequestScreen: View { SavedMeterPhoto(image: meterPhoto) .transition(.opacity) } + // The end lands under the circle, as on the walk's "Is this the right end?". + if offersEndMark { + Reticle(diameter: 56) + .frame(maxWidth: .infinity, maxHeight: .infinity) + .ignoresSafeArea() + .transition(.opacity) + } if state.gap?.isSatisfied == true { SuccessBadge() .frame(maxWidth: .infinity, maxHeight: .infinity) @@ -35,8 +47,8 @@ struct GapRequestScreen: View { } CameraChrome( instruction: instruction, - tone: asking ? .normal : coaching.map { .coaching(symbol: ScanCopy.coachingSymbol($0)) } ?? .normal, - reply: state.gap?.isSatisfied == true || asking ? nil : InstructionCard.Reply( + tone: tone, + reply: state.gap?.isSatisfied == true || asking || askingEnd ? nil : InstructionCard.Reply( title: "I can't get there", identifier: "action.skipGap", hint: skipHint, @@ -58,6 +70,13 @@ struct GapRequestScreen: View { OverheadAnswers(actions: actions) .transition(.opacity) } + if askingEnd { + EndQuestionAnswers(actions: actions) + .transition(.opacity) + } + if offersEndMark { + markEndButton + } if !typeSize.isAccessibilitySize { showResult } @@ -71,7 +90,8 @@ struct GapRequestScreen: View { features: state.features, cameraS: state.projection.map { WallProjection(projection: $0, wall: wall, size: cameraSize).cameraS }, highlight: state.gap, - depthChecked: state.depthAvailable + depthChecked: state.depthAvailable, + endPreview: offersEndMark ? state.endPreview : nil ) } if typeSize.isAccessibilitySize { @@ -79,6 +99,7 @@ struct GapRequestScreen: View { } } .animation(reduceMotion ? .easeOut(duration: 0.15) : Motion.settle, value: asking) + .animation(reduceMotion ? .easeOut(duration: 0.15) : Motion.settle, value: askingEnd) } } .animation(reduceMotion ? .easeOut(duration: 0.15) : Motion.pin, value: state.gap?.isSatisfied) @@ -96,7 +117,7 @@ struct GapRequestScreen: View { /// sizes it goes below the tape, the last thing on the screen, reached by scrolling like /// everything else below the card at that size. @ViewBuilder private var showResult: some View { - if followUps > 0, state.gap?.isSatisfied != true, !asking { + if followUps > 0, state.gap?.isSatisfied != true, !asking, !askingEnd { Button { actions.showResultNow() } label: { @@ -113,6 +134,47 @@ struct GapRequestScreen: View { /// until answered, as on the walk. private var asking: Bool { state.overheadQuestion && state.gap?.isSatisfied != true } + /// The end question after "Wall ends here" on a past_end request: it replaces the request, + /// "I can't get there" and "Show my result" until answered, and the answer settles the request. + private var askingEnd: Bool { + guard let side = state.endQuestion, let gap = state.gap else { return false } + return gap.pastEndSide == side && !gap.isSatisfied + } + + /// "Wall ends here" with the circle, on a past_end request not yet settled or answered. + private var offersEndMark: Bool { + guard let gap = state.gap, gap.pastEndSide != nil, !gap.isSatisfied else { return false } + return !asking && !askingEnd + } + + /// The walk's "Wall ends here" at the circle, the same words and flag. Beside walking on, which + /// stays the request, it is the other answer, so it is not the screen's primary button. + private var markEndButton: some View { + Button { + actions.markWallEnd(at: nil, viewSize: cameraSize) + } label: { + Label("Wall ends here", systemImage: "flag.fill") + .frame(maxWidth: .infinity) + } + .buttonStyle(.secondaryProminent) + .accessibilityHint("Marks the end of the wall at the circle in the middle of the screen") + .accessibilityIdentifier("action.markEnd") + .transition(.opacity) + } + + /// A refusal on "Wall ends here" keeps its red triangle, as on the walk; otherwise coaching + /// marks the card. + private var tone: InstructionCard.Tone { + if asking || askingEnd { return .normal } + if refusal != nil, !(coaching.map(ScanCopy.coachingReplacesTask) ?? false) { return .refusal } + return coaching.map { .coaching(symbol: ScanCopy.coachingSymbol($0)) } ?? .normal + } + + /// Why the last "Wall ends here" marked nothing, while the request still asks for that end. + private var refusal: EndMarkRefusal? { + offersEndMark ? state.endMarkRefusal : nil + } + /// Views the finished check still wants, counting this one, when this request is one of /// them: a server request while the check's answer is in. Zero otherwise. private var followUps: Int { @@ -139,7 +201,7 @@ struct GapRequestScreen: View { /// own words, which this screen can't judge. Nor the overhead question, a request already /// seen, or coaching that replaces the request. private var aims: Bool { - guard let gap = state.gap, !asking, !gap.isSatisfied else { return false } + guard let gap = state.gap, !asking, !askingEnd, refusal == nil, !gap.isSatisfied else { return false } if let coaching, ScanCopy.coachingReplacesTask(coaching) { return false } if case .groundNearCandidate = gap.reason { return true } return false @@ -151,11 +213,16 @@ struct GapRequestScreen: View { /// thanks" for as long as it did (field test 4.1, run 3). private var instruction: Instruction { if asking { return ScanCopy.overheadQuestion } + if askingEnd, let side = state.endQuestion { return ScanCopy.endQuestion(side) } if let coaching, ScanCopy.coachingReplacesTask(coaching) { return ScanCopy.coaching(coaching, meterPhoto: meterPhoto != nil) } guard let gap = state.gap else { return ScanCopy.withCoaching(ScanCopy.guidance(.gap), coaching) } if gap.isSatisfied { return Instruction(title: "Got it, thanks", detail: followUps > 0 ? "Updating your result." : "That's the view we needed.") } + // "Wall ends here" marked nothing: why, and what to do (B-06, B-12). + if let refusal, let side = gap.pastEndSide { + return ScanCopy.withCoaching(ScanCopy.endMarkRefusal(refusal, asked: side), coaching) + } return ScanCopy.withCoaching(ScanCopy.gap(gap), coaching) } diff --git a/ios/HouseScan/UI/Screens/WallWalkScreen.swift b/ios/HouseScan/UI/Screens/WallWalkScreen.swift index c030f71b9..ae42499a6 100644 --- a/ios/HouseScan/UI/Screens/WallWalkScreen.swift +++ b/ios/HouseScan/UI/Screens/WallWalkScreen.swift @@ -254,39 +254,8 @@ struct WallWalkScreen: View { } .transition(.opacity) case .endQuestion: - // One question, three equal full-width answers that say what they mean (checklist I4). - VStack(spacing: 8) { - Button { - actions.answerWallEnd(turnsCorner: true) - } label: { - Label("It turns a corner", systemImage: "arrow.turn.up.right") - .frame(maxWidth: .infinity) - } - .buttonStyle(.secondaryProminent) - .accessibilityIdentifier("action.endCorner") - Button { - actions.answerWallEnd(turnsCorner: false) - } label: { - Label("Something blocks it", systemImage: "xmark.octagon") - .frame(maxWidth: .infinity) - } - .buttonStyle(.secondaryProminent) - .accessibilityHint("A fence, gate, or your neighbor's yard") - .accessibilityIdentifier("action.endBlocked") - // A garden wall or a fence can just stop, with no corner and nothing in the way, - // and neither answer above fits it (#70). Ends the wall as "Something blocks it" - // does: the usable wall stops here. - Button { - actions.answerWallEnd(turnsCorner: false) - } label: { - Label("The wall just ends", systemImage: "arrow.right.to.line") - .frame(maxWidth: .infinity) - } - .buttonStyle(.secondaryProminent) - .accessibilityHint("No corner and nothing in the way. The wall stops here.") - .accessibilityIdentifier("action.endEnds") - } - .transition(.opacity) + EndQuestionAnswers(actions: actions) + .transition(.opacity) case .overheadQuestion: OverheadAnswers(actions: actions) .transition(.opacity) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Guidance/WalkedEnd.swift b/ios/HouseScanKit/Sources/HouseScanKit/Guidance/WalkedEnd.swift index cad0bfea1..32a616ecd 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Guidance/WalkedEnd.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Guidance/WalkedEnd.swift @@ -174,6 +174,16 @@ extension CoverageMap { return rightEnd - leftEnd < WallFrame.minWallLength } + /// True when an end on `side` at `s` would leave the wall shorter than + /// `WallFrame.minWallLength` from the other end, or from the meter while the other side has + /// none: the same policy as `endsTooClose`, asked before the end is set (B-12, a past-end + /// request's end marked again nearer). The other end is never moved. + public func endWouldLeaveTooLittle(_ side: WalkSide, at s: Float) -> Bool { + let other: Float = side == .left ? (rightEnd ?? 0) : (leftEnd ?? 0) + let length = side == .left ? other - s : s - other + return length < WallFrame.minWallLength + } + /// The unexplored end the camera stands beyond while its view shows nothing between the marked /// ends, or nil. A photo taken there adds nothing: cells past an end are never observed /// (`isWithinEnds`). Past a limit end it is nil, since the ground seen there still counts diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/PacketMarks.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/PacketMarks.swift index deb42bab3..64f097e99 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Packet/PacketMarks.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/PacketMarks.swift @@ -54,17 +54,20 @@ public struct PacketMark: Codable, Sendable, Equatable { public var kind: Kind /// Meter frame, meters. public var points: [SIMD3] - /// Uptime when it was marked. + /// Uptime when the homeowner marked it. An inferred wall end has none (`inferred`). public var t: Double? public var photoIDs: [String]? public var side: Side? public var endKind: EndKind? /// `attrs.operable` of an opening; nil when the homeowner was not asked. public var operable: Bool? + /// `attrs.inferred` of a wall end the app placed from the walk rather than one the homeowner + /// marked (`WallEndSource`). Written only when true. + public var inferred: Bool private init( id: String, kind: Kind, points: [SIMD3], t: Double?, photoIDs: [String]?, - side: Side? = nil, endKind: EndKind? = nil, operable: Bool? = nil + side: Side? = nil, endKind: EndKind? = nil, operable: Bool? = nil, inferred: Bool = false ) { self.id = id self.kind = kind @@ -74,6 +77,7 @@ public struct PacketMark: Codable, Sendable, Equatable { self.side = side self.endKind = endKind self.operable = operable + self.inferred = inferred } /// The meter itself: the meter frame's origin. @@ -81,13 +85,16 @@ public struct PacketMark: Codable, Sendable, Equatable { PacketMark(id: id, kind: .meter, points: [.zero], t: t, photoIDs: photoIDs) } - /// Where the wall was marked as ending, at `s` meters along `wall`'s chain. The point is on - /// the wall face at the meter's height, as S4's synthetic fixture places wall ends. + /// Where the wall ends, at `s` meters along `wall`'s chain. The point is on the wall face at + /// the meter's height, as S4's synthetic fixture places wall ends. `stamp` says whether the + /// homeowner marked it, with the time, or the app inferred it from the walk, with no time. public static func wallEnd( - id: String, side: Side, endKind: EndKind, s: Float, wall: SceneWall, frame: MeterFrame, t: Double? = nil + id: String, side: Side, endKind: EndKind, s: Float, wall: SceneWall, frame: MeterFrame, stamp: WallEndStamp ) -> PacketMark { let point = frame.point(on: wall, s: s, height: wall.meter.y - wall.groundY, out: 0) - return PacketMark(id: id, kind: .wallEnd, points: [point], t: t, photoIDs: nil, side: side, endKind: endKind) + return PacketMark( + id: id, kind: .wallEnd, points: [point], t: stamp.markedAt, photoIDs: nil, side: side, endKind: endKind, + inferred: stamp.isInferred) } /// A door, window or garage door: the bottom-left and top-right corners on the wall face, @@ -147,6 +154,8 @@ public struct PacketMark: Codable, Sendable, Equatable { if points.count != kind.pointCount { return "\(points.count) points, a \(kind.rawValue) has \(kind.pointCount)" } if !points.allSatisfy(PacketNumber.isFinite) { return "points must be finite" } if kind == .wallEnd, side == nil || endKind == nil { return "a wall end needs side and end_kind" } + if inferred, kind != .wallEnd { return "only a wall end can be inferred" } + if inferred, t != nil { return "an inferred wall end has no mark time: nobody marked it" } return nil } @@ -158,6 +167,7 @@ public struct PacketMark: Codable, Sendable, Equatable { private struct Attrs: Codable { var operable: Bool? + var inferred: Bool? } public init(from decoder: any Decoder) throws { @@ -174,7 +184,9 @@ public struct PacketMark: Codable, Sendable, Equatable { photoIDs = try c.decodeIfPresent([String].self, forKey: .photoIDs) side = try c.decodeIfPresent(Side.self, forKey: .side) endKind = try c.decodeIfPresent(EndKind.self, forKey: .endKind) - operable = try c.decodeIfPresent(Attrs.self, forKey: .attrs)?.operable + let attrs = try c.decodeIfPresent(Attrs.self, forKey: .attrs) + operable = attrs?.operable + inferred = attrs?.inferred ?? false } public func encode(to encoder: any Encoder) throws { @@ -186,7 +198,9 @@ public struct PacketMark: Codable, Sendable, Equatable { try c.encodeIfPresent(photoIDs, forKey: .photoIDs) try c.encodeIfPresent(side, forKey: .side) try c.encodeIfPresent(endKind, forKey: .endKind) - try c.encodeIfPresent(operable.map { Attrs(operable: $0) }, forKey: .attrs) + if operable != nil || inferred { + try c.encode(Attrs(operable: operable, inferred: inferred ? true : nil), forKey: .attrs) + } } } diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/WallEndProvenance.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/WallEndProvenance.swift new file mode 100644 index 000000000..d3c3001a7 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/WallEndProvenance.swift @@ -0,0 +1,64 @@ +import Foundation + +/// Whether the homeowner put a wall end where it is, or the app inferred it from the walk. +/// +/// Before, every end carried a mark time, so the packet listed ends the app placed as ends the +/// homeowner marked: "Can't get there" and "The wall keeps going" put the end where the walk +/// reached (`WalkedEnd`), and a past-end request met by walking past moved it on to the far edge +/// of what the request showed (`GapPlanner.endAfterPastEnd`). Field analysts read those as marks +/// (run3-2055, A41-2055-06). Either kind of end keeps its place and kind in scene.json and the +/// packet; only the packet says which it is (packet/README.md, `attrs.inferred`). +public enum WallEndSource: Sendable, Equatable { + /// "Wall ends here", at the circle or where the homeowner stands. + case homeowner + /// Placed from the walk: where it reached, or past what a past-end request showed. + case inferred +} + +/// A wall end's source and, for a homeowner's mark, when it was made. +public struct WallEndStamp: Sendable, Equatable { + public let source: WallEndSource + /// Uptime of the homeowner's mark. Nil for an inferred end, which nobody marked, and nil for + /// a mark made before the packet's clock started; absence never stands for a homeowner. + public let markedAt: Double? + + private init(source: WallEndSource, markedAt: Double?) { + self.source = source + self.markedAt = markedAt + } + + /// The homeowner marked the end at uptime `time`. + public static func marked(at time: Double?) -> WallEndStamp { + WallEndStamp(source: .homeowner, markedAt: time) + } + + /// The app placed the end from the walk; it has no mark time. + public static let inferred = WallEndStamp(source: .inferred, markedAt: nil) + + public var isInferred: Bool { source == .inferred } + + /// How a "mark the end" request closes once this end is set: met only when the homeowner + /// marked it. An end the walk inferred answers a different request ("Can't get there", which + /// closes as `cannot_reach` before the end is placed), so it never counts as the mark. + public var markEndOutcome: PacketGuidanceEntry.Outcome { + source == .homeowner ? .met : .superseded + } +} + +/// What becomes of the end a past-end request cleared when the request leaves the screen. +public enum PastEndSettlement: Sendable, Equatable { + /// The homeowner marked the end during the request: that mark stands, nearer or farther than + /// the cleared end, since the cleared end only said how far the walk had seen. + case keepMarked + /// The request was met by views and nobody marked the end: it moves on past the ground the + /// request showed, still unexplored, as an inferred end (`GapPlanner.endAfterPastEnd`). + case moveOn + /// The homeowner said "I can't get there": the cleared end comes back with its place, kind, + /// source and mark time, and nothing past it counts as seen. + case restore + + public static func when(endMarkedDuringRequest: Bool, met: Bool) -> PastEndSettlement { + if endMarkedDuringRequest { return .keepMarked } + return met ? .moveOn : .restore + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift index 0904fda17..ab6b59e8b 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift @@ -6,13 +6,15 @@ import ImageIO import simd import Testing -/// The capture packet's manifest schema, version 1.1, byte for byte from t3/packet at d5439cf -/// (packet/manifest.schema.json). `vendoredManifestSchemaIsTheRecordedRevision` fails if the copy -/// is edited by hand. There is no live copy to compare with: t3/packet retired 1.1 at 6a12700 in -/// favour of the server team's packet 0.4, which the writer moves to next. +/// The capture packet's manifest schema, version 1.1: t3/packet at d5439cf +/// (packet/manifest.schema.json, sha256 44c9c9beā€¦), plus one additive change made here, the +/// optional `attrs.inferred` on a wall end and `t`'s description (B-12, packet/README.md). +/// `vendoredManifestSchemaIsTheRecordedRevision` fails if the copy changes without this record. +/// There is no live copy to compare with: t3/packet retired 1.1 at 6a12700 in favour of the server +/// team's packet 0.4, which the writer moves to next. enum PacketSchema { static let name = "manifest.schema.json" - static let sha256 = "44c9c9beb1a95b85d069d9abc52c193008a50aa75e1dff52f5d47431f10c5c75" + static let sha256 = "54d2a0d104162801a35599c5300acff09b2c2061aa8e88291f2621c04398bc27" static func validator() throws -> JSONSchemaValidator { try JSONSchemaValidator(schema: SceneSchemas.data(name)) } } @@ -167,8 +169,9 @@ struct SyntheticPacket { id: "ground", alignment: .horizontal, classification: nil, pose: frame.pose(matrix_identity_float4x4), extent: SIMD2(6, 3))) try writer.setMarks([ .meter(id: "m1", t: 100.1, photoIDs: ["p00001"]), - .wallEnd(id: "m2", side: .left, endKind: .unexplored, s: -3, wall: wall, frame: frame, t: 100.2), - .wallEnd(id: "m3", side: .right, endKind: .limit, s: 3, wall: wall, frame: frame, t: 101.9), + // Placed from the walk ("Can't get there"): no mark time, attrs.inferred. + .wallEnd(id: "m2", side: .left, endKind: .unexplored, s: -3, wall: wall, frame: frame, stamp: .inferred), + .wallEnd(id: "m3", side: .right, endKind: .limit, s: 3, wall: wall, frame: frame, stamp: .marked(at: 101.9)), try .from(.opening(kind: .window, span: -2 ... -1.25, bottom: 1, top: 2, operable: true), id: "m4", wall: wall, frame: frame, t: 100.4), try .from(.pointObject(kind: .gasMeter, tap: SIMD3(1.5, 1, 0.1), bottom: nil, top: nil), id: "m5", wall: wall, frame: frame), try .from(.driveway(edge: [SIMD3(2, 0, 0.5), SIMD3(2, 0, 4)]), id: "m6", wall: wall, frame: frame, t: 101.5), @@ -320,6 +323,8 @@ struct SyntheticPacket { // Marks and guidance come back as written. let marks = try #require(manifest.marks) #expect(marks.map(\.kind) == [.meter, .wallEnd, .wallEnd, .window, .gasMeter, .driveEdge]) + #expect(marks[1].inferred && marks[1].t == nil, "the inferred left end") + #expect(!marks[2].inferred && marks[2].t == 101.9, "the marked right end") let window: [SIMD3] = [SIMD3(-2, -0.5, 0), SIMD3(-1.25, 0.5, 0)] let drive: [SIMD3] = [SIMD3(2, -1.5, 0.5), SIMD3(2, -1.5, 4)] #expect(marks[3].points == window) @@ -767,7 +772,7 @@ private func photoJPEG() throws -> URL { let fence = try PacketMark.from(.fence(foot: [SIMD3(-1, 0, 3), SIMD3(1, 0, 3)]), id: "f", wall: wall, frame: frame) #expect(fence.kind == .fence && fence.points == [SIMD3(-1, -1.5, 3), SIMD3(1, -1.5, 3)]) #expect(throws: PacketError.self) { try PacketMark.from(.driveway(edge: [SIMD3(0, 0, 1)]), id: "x", wall: wall, frame: frame) } - let end = PacketMark.wallEnd(id: "e", side: .right, endKind: .limit, s: 2.5, wall: wall, frame: frame) + let end = PacketMark.wallEnd(id: "e", side: .right, endKind: .limit, s: 2.5, wall: wall, frame: frame, stamp: .marked(at: nil)) #expect(end.points == [SIMD3(2.5, 0, 0)] && end.side == .right && end.endKind == .limit) #expect(PacketMark.meter(id: "m").points == [.zero]) } @@ -796,7 +801,7 @@ private func photoJPEG() throws -> URL { @Suite struct PacketSchemaTests { @Test func vendoredManifestSchemaIsTheRecordedRevision() throws { let digest = SHA256.hash(data: try SceneSchemas.data(PacketSchema.name)).map { String(format: "%02x", $0) }.joined() - #expect(digest == PacketSchema.sha256, "Schemas/\(PacketSchema.name) is not the copy taken from t3/packet d5439cf") + #expect(digest == PacketSchema.sha256, "Schemas/\(PacketSchema.name) is not t3/packet d5439cf plus attrs.inferred") } /// The schema is not a rubber stamp: a manifest missing required fields or breaking a pattern diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/Schemas/manifest.schema.json b/ios/HouseScanKit/Tests/HouseScanKitTests/Schemas/manifest.schema.json index dd235455b..deebb59b0 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/Schemas/manifest.schema.json +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/Schemas/manifest.schema.json @@ -535,6 +535,7 @@ } }, "t": { + "description": "When the homeowner made the mark. Absent on a wall end the app inferred (attrs.inferred); an absent t never means the homeowner marked it.", "$ref": "#/$defs/time" }, "photo_ids": { @@ -561,6 +562,10 @@ "properties": { "operable": { "type": "boolean" + }, + "inferred": { + "description": "On a wall end only, and only true: the app placed this end from the walk (where it reached, or past what a past_end request showed), not the homeowner. Such an end has no t. Its point and end_kind are as for a marked end.", + "type": "boolean" } } } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/WallEndProvenanceTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/WallEndProvenanceTests.swift new file mode 100644 index 000000000..2c83fded3 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/WallEndProvenanceTests.swift @@ -0,0 +1,198 @@ +import Foundation +import HouseScanKit +import simd +import Testing + +// B-12: a wall end the homeowner marked and one the app inferred from the walk keep the same +// place and kind; only the packet's `t` and `attrs.inferred` tell them apart. The wall is z = 0 +// facing +z with the meter 1.5 m up, synthetic values chosen for the tests. + +@Suite struct WallEndProvenanceTests { + static let wall = SceneWall(meter: SIMD3(0, 1.5, 0), outward: SIMD3(0, 0, 1), groundY: 0) + + static func end(_ stamp: WallEndStamp, kind: PacketMark.EndKind = .unexplored) throws -> PacketMark { + let frame = try #require(MeterFrame(wall: wall)) + return .wallEnd(id: "wall_end_right", side: .right, endKind: kind, s: 2.5, wall: wall, frame: frame, stamp: stamp) + } + + static func json(_ mark: PacketMark) throws -> [String: Any] { + try #require(JSONSerialization.jsonObject(with: JSONEncoder().encode(mark)) as? [String: Any]) + } + + // MARK: Stamps + + @Test func aHomeownerMarkKeepsItsTimeAndMeetsTheRequest() { + let stamp = WallEndStamp.marked(at: 12.5) + #expect(stamp.source == .homeowner && stamp.markedAt == 12.5 && !stamp.isInferred) + #expect(stamp.markEndOutcome == .met) + } + + @Test func anInferredEndHasNoTimeAndNeverMeetsAMarkRequest() { + let stamp = WallEndStamp.inferred + #expect(stamp.source == .inferred && stamp.markedAt == nil && stamp.isInferred) + #expect(stamp.markEndOutcome == .superseded) + } + + /// A homeowner mark made before the packet's clock started has no time but is still the + /// homeowner's: absence of a time is never read as either source. + @Test func aMarkWithoutAClockIsStillTheHomeowners() { + let stamp = WallEndStamp.marked(at: nil) + #expect(stamp.source == .homeowner && stamp.markedAt == nil) + #expect(stamp.markEndOutcome == .met) + #expect(stamp != .inferred) + } + + // MARK: Past-end requests + + @Test(arguments: [ + // A mark during the request stands, met or not: it may be nearer than the cleared end. + (true, true, PastEndSettlement.keepMarked), + (true, false, PastEndSettlement.keepMarked), + // Met by views with nobody marking: the end moves on, inferred. + (false, true, PastEndSettlement.moveOn), + // "I can't get there": the cleared end comes back. + (false, false, PastEndSettlement.restore), + ]) + func pastEndSettlement(marked: Bool, met: Bool, expected: PastEndSettlement) { + #expect(PastEndSettlement.when(endMarkedDuringRequest: marked, met: met) == expected) + } + + // MARK: Packet encoding + + @Test func aMarkedEndCarriesItsTimeAndNoFlag() throws { + let object = try Self.json(try Self.end(.marked(at: 12.5))) + #expect(object["t"] as? Double == 12.5) + #expect(object["attrs"] == nil) + } + + @Test func anInferredEndCarriesTheFlagAndNoTime() throws { + let object = try Self.json(try Self.end(.inferred)) + #expect(object["t"] == nil) + #expect((object["attrs"] as? [String: Any])?["inferred"] as? Bool == true) + } + + /// Provenance changes nothing about where the end is or what it is. + @Test(arguments: [PacketMark.EndKind.unexplored, .limit]) + func provenanceKeepsPlaceAndKind(kind: PacketMark.EndKind) throws { + let marked = try Self.end(.marked(at: 3), kind: kind) + let inferred = try Self.end(.inferred, kind: kind) + #expect(marked.points == inferred.points && marked.points == [SIMD3(2.5, 0, 0)]) + #expect(marked.side == inferred.side && marked.endKind == kind && inferred.endKind == kind) + #expect(marked.kind == .wallEnd && inferred.kind == .wallEnd) + } + + @Test func bothRoundTripThroughJSON() throws { + for mark in [try Self.end(.marked(at: 7)), try Self.end(.inferred), try Self.end(.marked(at: nil))] { + let decoded = try JSONDecoder().decode(PacketMark.self, from: JSONEncoder().encode(mark)) + #expect(decoded == mark) + } + } + + /// An older packet's wall end, without attrs, decodes as not inferred, its time kept. + @Test func anEndWithoutAttrsIsNotInferred() throws { + let json = #"{"id": "e", "kind": "wall_end", "points": [[2.5, 0, 0]], "t": 4, "side": "right", "end_kind": "limit"}"# + let mark = try JSONDecoder().decode(PacketMark.self, from: Data(json.utf8)) + #expect(!mark.inferred && mark.t == 4) + } + + /// The flag sits beside an opening's `operable` without disturbing it. + @Test func operableStaysAlone() throws { + let frame = try #require(MeterFrame(wall: Self.wall)) + let window = PacketMark.opening(.window, id: "w", span: 0 ... 1, bottom: 1, top: 2, operable: true, wall: Self.wall, frame: frame) + let attrs = try #require(try Self.json(window)["attrs"] as? [String: Any]) + #expect(attrs.keys.sorted() == ["operable"]) + } + + /// The written synthetic packet carries an inferred left end and a marked right end, and its + /// manifest validates (`syntheticPacketIsComplete`). Here: the schema reads the flag as a + /// boolean, so a manifest with any other value fails at that mark. + @Test func theSchemaRejectsANonBooleanFlag() throws { + let folder = URL(fileURLWithPath: NSTemporaryDirectory()).appending(path: "hs-provenance-\(UUID())") + let jpegs = URL(fileURLWithPath: NSTemporaryDirectory()).appending(path: "hs-provenance-jpegs-\(UUID())") + try FileManager.default.createDirectory(at: jpegs, withIntermediateDirectories: true) + defer { + try? FileManager.default.removeItem(at: folder) + try? FileManager.default.removeItem(at: jpegs) + } + _ = try SyntheticPacket().write(to: folder, jpegs: jpegs) + let data = try Data(contentsOf: folder.appendingPathComponent("manifest.json")) + let validator = try PacketSchema.validator() + #expect(try validator.validate(data) == []) + var manifest = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any]) + var marks = try #require(manifest["marks"] as? [[String: Any]]) + #expect((marks[1]["attrs"] as? [String: Any])?["inferred"] as? Bool == true && marks[1]["t"] == nil) + marks[1]["attrs"] = ["inferred": "yes"] + manifest["marks"] = marks + let errors = try validator.validate(JSONSerialization.data(withJSONObject: manifest)) + #expect(errors.contains { $0.contains("marks") && $0.contains("inferred") }, "\(errors)") + } + + /// The writer's own check: an inferred end with a time, or the flag on another kind of mark, + /// is refused before anything is written. + @Test func theWriterRefusesAnInferredEndWithATime() throws { + var mark = try Self.end(.inferred) + mark.t = 3 + let folder = URL(fileURLWithPath: NSTemporaryDirectory()).appending(path: "hs-provenance-\(UUID())") + defer { try? FileManager.default.removeItem(at: folder) } + var writer = try PacketWriter(folder: folder, session: try SyntheticPacket().session) + #expect(throws: PacketError.self) { try writer.setMarks([mark]) } + var meter = PacketMark.meter(id: "m") + meter.inferred = true + #expect(throws: PacketError.self) { try writer.setMarks([meter]) } + #expect(throws: Never.self) { try writer.setMarks([try Self.end(.inferred), try Self.end(.marked(at: 3)).renamed("wall_end_left")]) } + } +} + +private extension PacketMark { + func renamed(_ id: String) -> PacketMark { + var mark = self + mark.id = id + return mark + } +} + +// MARK: Shorter bounds + +/// An end the homeowner marks nearer than the end a past-end request cleared bounds the export: +/// nothing seen past it is reported, on either band, and the other side is untouched. +@Suite struct NearerEndBoundsTests { + @Test func aNearerEndClipsWhatIsReportedAndLeavesTheOtherSide() throws { + var map = CoverageMap(wall: standardWall()) + for (i, c) in stride(from: Float(-3), through: 3, by: 0.25).enumerated() { + map.observe(wallCamera(s: c), trackingNormal: true, time: Double(i)) + map.observe(groundCamera(s: c), trackingNormal: true, time: Double(i) + 0.5) + } + map.setEnd(.left, at: -2.5) + map.setEnd(.right, at: 2.5) + let before = try #require(map.wallSeenSpans().map(\.span.upperBound).max()) + #expect(before > 2 && before <= 2.5 + 1e-4) + #expect((map.coveredIntervals(.ground).map(\.upperBound).max() ?? 0) > 2) + // The wall really stops at 1.5 m. + map.setEnd(.right, at: 1.5) + #expect(map.rightEnd == 1.5 && map.leftEnd == -2.5) + for span in map.wallSeenSpans() + map.groundDepthSpans() { + #expect(span.span.upperBound <= 1.5 + 1e-4, "\(span.span) runs past the nearer end") + } + #expect(map.coveredIntervals(.wall).allSatisfy { $0.upperBound <= 1.5 + 1e-4 }) + #expect(map.coveredIntervals(.ground).allSatisfy { $0.upperBound <= 1.5 + 1e-4 }) + // The left side is untouched. + #expect((map.wallSeenSpans().map(\.span.lowerBound).min() ?? 0) < -2) + } + + /// A nearer end keeps the walk's minimum (`WallFrame.minWallLength`), measured from the other + /// end, or from the meter while that side has none, and never moves the other end. + @Test func aNearerEndKeepsTheMinimumWallLength() { + var map = CoverageMap(wall: standardWall()) + map.setEnd(.left, at: -1) + let least = WallFrame.minWallLength + #expect(map.endWouldLeaveTooLittle(.right, at: -1 + least - 0.01)) + #expect(!map.endWouldLeaveTooLittle(.right, at: -1 + least + 0.01)) + // No right end: measured from the meter. + #expect(map.endWouldLeaveTooLittle(.left, at: -(least - 0.01))) + #expect(!map.endWouldLeaveTooLittle(.left, at: -(least + 0.01))) + map.setEnd(.right, at: 0.2) + #expect(map.endWouldLeaveTooLittle(.left, at: -0.1), "a 0.3 m wall") + #expect(!map.endWouldLeaveTooLittle(.left, at: 0.2 - least - 0.01)) + #expect(map.leftEnd == -1 && map.rightEnd == 0.2, "asking moved an end") + } +} diff --git a/ios/HouseScanUITests/PastEndRecoveryUITests.swift b/ios/HouseScanUITests/PastEndRecoveryUITests.swift new file mode 100644 index 000000000..63c89d14e --- /dev/null +++ b/ios/HouseScanUITests/PastEndRecoveryUITests.swift @@ -0,0 +1,351 @@ +import CryptoKit +import Foundation +import XCTest + +/// B-12: a past_end request (the walk stopped at an end, and the check asks to walk on past it) +/// offers the walk's own "Wall ends here" for that end, with the circle, the refusal when the +/// circle isn't on the wall, and the end question; "I can't get there" stays the honest decline. +/// +/// Two kinds of test, which prove different things. The frozen demo (`-uiDemoGap pastEnd`) shows +/// the screen's states for the captures and checks what is offered, at the default and the +/// largest text size. The replay runs the real engine with the test answering its uploads +/// (`-answersFromGate`), and checks what marking again or declining does to the scene the next +/// upload sends and to the packet's marks and guidance. There the autopilot marks the end with a +/// tap where the replay shows the place (`-autopilotMarkPastEnd`), through the same `EndAim` check +/// as the circle, but no camera is aimed: aiming a phone at a real wall is not tested here. +final class PastEndRecoveryUITests: XCTestCase { + private static let largestText = ["-UIPreferredContentSizeCategoryName", "UICTContentSizeCategoryAccessibilityXXXL"] + + override func setUp() { + continueAfterFailure = false + } + + // MARK: The screen (demo engine) + + @MainActor + func testAPastEndRequestOffersWallEndsHere() throws { + try offersWallEndsHere(textSize: [], name: "gapRequest-pastEnd") + } + + @MainActor + func testAPastEndRequestOffersWallEndsHereAtLargestTextSize() throws { + try offersWallEndsHere(textSize: Self.largestText, name: "gapRequest-pastEnd-AX5") + } + + @MainActor + private func offersWallEndsHere(textSize: [String], name: String) throws { + let app = launch(["-uiDemoGap", "pastEnd"] + textSize) + let card = label(app, "instruction") + XCTAssertTrue(card.contains("Keep walking past the right end"), "the request's own words: \(card)") + XCTAssertTrue(card.contains("If the wall stops sooner, aim where it stops and tap Wall ends here."), "card reads: \(card)") + XCTAssertTrue(element(app, "action.skipGap").exists, "\"I can't get there\" stays the decline") + let mark = app.buttons["action.markEnd"] + XCTAssertTrue(mark.waitForExistence(timeout: 5), "a past-end request must offer Wall ends here") + XCTAssertEqual(mark.label, "Wall ends here") + scrollIntoView(mark, in: app) + XCTAssertTrue(mark.isHittable, "Wall ends here can't be reached") + snap(app, name) + } + + /// The circle off the wall: the card says why and what to do, the decline stays, and pressing + /// again with the circle still off the wall refuses again rather than marking anything. + @MainActor + func testARefusedEndOnARequestSaysWhatToDo() throws { + try refusal(textSize: [], name: "gapRequest-pastEnd-refused") + } + + @MainActor + func testARefusedEndOnARequestSaysWhatToDoAtLargestTextSize() throws { + try refusal(textSize: Self.largestText, name: "gapRequest-pastEnd-refused-AX5") + } + + @MainActor + private func refusal(textSize: [String], name: String) throws { + let app = launch(["-uiDemoGap", "pastEnd", "-uiDemoEndMarkRefusal"] + textSize) + var card = label(app, "instruction") + XCTAssertTrue(card.contains("The circle isn't on the wall"), "card reads: \(card)") + XCTAssertTrue(card.contains("Aim it at the wall where it stops or turns"), "card reads: \(card)") + XCTAssertTrue(element(app, "action.skipGap").exists) + snap(app, name) + let mark = app.buttons["action.markEnd"] + scrollIntoView(mark, in: app) + tapWhenReady(mark) + card = label(app, "instruction") + XCTAssertTrue(card.contains("The circle isn't on the wall"), "a second press marked something: \(card)") + XCTAssertFalse(element(app, "action.endBlocked").exists, "no end was marked, so nothing to ask") + } + + /// "Wall ends here" asks the walk's end question; it replaces the request and its decline + /// until answered, and the answer settles the request, which goes on to the check. + @MainActor + func testMarkingTheEndAsksWhatIsThereAndSettlesTheRequest() throws { + try question(textSize: [], name: "gapRequest-pastEnd-question") + } + + @MainActor + func testMarkingTheEndAsksWhatIsThereAtLargestTextSize() throws { + try question(textSize: Self.largestText, name: "gapRequest-pastEnd-question-AX5") + } + + @MainActor + private func question(textSize: [String], name: String) throws { + let app = launch(["-uiDemoGap", "pastEnd"] + textSize) + let mark = app.buttons["action.markEnd"] + scrollIntoView(mark, in: app) + tapWhenReady(mark) + let asked = app.descendants(matching: .any) + .matching(NSPredicate(format: "identifier == 'instruction' AND label CONTAINS %@", "What's at the right end?")).firstMatch + XCTAssertTrue(asked.waitForExistence(timeout: 5), "marking the end must ask what is there") + XCTAssertFalse(element(app, "action.skipGap").exists, "the question replaces the decline until answered") + XCTAssertFalse(element(app, "action.markEnd").exists) + for answer in ["action.endCorner", "action.endBlocked", "action.endEnds"] { + XCTAssertTrue(app.buttons[answer].exists, "\(answer) missing") + } + snap(app, name) + let blocked = app.buttons["action.endBlocked"] + scrollIntoView(blocked, in: app) + tapWhenReady(blocked) + XCTAssertTrue(element(app, "screen.uploading").waitForExistence(timeout: 10), "the answer must settle the request") + } + + /// Only a past-end request offers the end: the phone's own request and the server's other + /// kinds keep their own controls. + @MainActor + func testOtherRequestsDoNotOfferTheEnd() throws { + for gap in [[], ["-uiDemoGap", "groundOut"], ["-uiDemoGap", "walkOut"], ["-uiDemoGap", "overhead"]] { + let app = launch(gap) + XCTAssertFalse(app.buttons["action.markEnd"].exists, "\(gap) offers Wall ends here") + XCTAssertFalse(label(app, "instruction").contains("Wall ends here"), "\(gap) mentions Wall ends here") + app.terminate() + } + } + + // MARK: The real engine (replay, answer gate) + + /// The check asks to walk past the left end. The autopilot marks it again nearer than before + /// and answers "Something blocks it". The next upload's scene has the shorter left bound as a + /// limit; the request's guidance is met and the walk's own "mark the end" stays as it was; the + /// packet's left end is the homeowner's mark, with its time and no inferred flag. + @MainActor + func testMarkingThePastEndAgainShortensTheWallOnTheRealEngine() throws { + let run = try runReplay(markPastEnd: true) + let mark = try XCTUnwrap(run.pastEndMark, "the autopilot left no past-end-mark.json") + let marked = try XCTUnwrap(mark["s"] as? Double), cleared = try XCTUnwrap(mark["cleared_s"] as? Double) + XCTAssertEqual(mark["side"] as? String, "left") + XCTAssertGreaterThan(marked, cleared, "the end was meant to come nearer than the cleared one") + + let first = try XCTUnwrap(run.scenes.first), last = try XCTUnwrap(run.scenes.last) + XCTAssertGreaterThanOrEqual(run.scenes.count, 2) + XCTAssertEqual(try Self.endKind(last, "left"), "limit") + XCTAssertEqual(try Self.leftEndFeet(last), marked * Self.feetPerMeter, accuracy: 0.2, "the scene's left bound is the mark") + XCTAssertGreaterThan(try Self.leftEndFeet(last), try Self.leftEndFeet(first) + 1, "the left bound came nearer") + XCTAssertEqual(try Self.endKind(last, "right"), try Self.endKind(first, "right"), "the right side changed") + // Wall seen past the nearer end is not reported. Ground past it is, now that it is a limit + // (server README, "Ends and corners"): that is ground seen, not wall claimed. + XCTAssertTrue(try Self.observedLeftEdges(last, band: "wall").allSatisfy { $0 >= marked * Self.feetPerMeter - 0.05 }, "wall reported past the nearer end") + + let packet = try XCTUnwrap(run.packet, "no packet-marks-guidance.json") + let left = try XCTUnwrap(Self.wallEnd(packet, "left")) + XCTAssertNotNil(left["t"], "a homeowner's mark keeps its time") + XCTAssertNil((left["attrs"] as? [String: Any])?["inferred"], "a homeowner's mark is not inferred") + XCTAssertEqual(left["end_kind"] as? String, "limit") + let pastEnd = Self.guidance(packet, kind: "gap_past_end") + XCTAssertEqual(pastEnd.last?["outcome"] as? String, "met", "\(pastEnd)") + attach(run) + } + + /// The check asks to walk past the left end; the replay can't show it, and the autopilot says + /// "I can't get there". The next upload's scene has the left end back where it was, with its + /// kind, and claims nothing past it; the request is cannot_reach, and the packet's left end + /// keeps the source and time it had. + @MainActor + func testDecliningThePastEndRestoresTheEndOnTheRealEngine() throws { + let run = try runReplay(markPastEnd: false) + XCTAssertNil(run.pastEndMark) + XCTAssertGreaterThanOrEqual(run.scenes.count, 2) + let first = try XCTUnwrap(run.scenes.first), last = try XCTUnwrap(run.scenes.last) + XCTAssertEqual(try Self.leftEndFeet(last), try Self.leftEndFeet(first), accuracy: 0.01, "the left end moved") + XCTAssertEqual(try Self.endKind(last, "left"), try Self.endKind(first, "left")) + let bound = try Self.leftEndFeet(last) + // Past an unexplored end nothing is reported; past a limit only ground is (the autopilot's + // fallback marks a limit when no frame shows the end). + let pastBand: String? = try Self.endKind(last, "left") == "limit" ? "wall" : nil + XCTAssertTrue(try Self.observedLeftEdges(last, band: pastBand).allSatisfy { $0 >= bound - 0.05 }, "coverage claimed past the restored end") + + let packet = try XCTUnwrap(run.packet, "no packet-marks-guidance.json") + let left = try XCTUnwrap(Self.wallEnd(packet, "left")) + let firstPacketLeft = try XCTUnwrap(run.firstPacketLeftEnd, "the first upload's packet had no left end") + XCTAssertEqual(left["t"] as? Double, firstPacketLeft["t"] as? Double, "the restored end lost or changed its time") + XCTAssertEqual((left["attrs"] as? [String: Any])?["inferred"] as? Bool, (firstPacketLeft["attrs"] as? [String: Any])?["inferred"] as? Bool) + let pastEnd = Self.guidance(packet, kind: "gap_past_end") + XCTAssertEqual(pastEnd.last?["outcome"] as? String, "cannot_reach", "\(pastEnd)") + attach(run) + } + + private struct ReplayRun { + var scenes: [[String: Any]] + var packet: [String: Any]? + var firstPacketLeftEnd: [String: Any]? + var pastEndMark: [String: Any]? + var log: String + } + + /// Plays the synthetic wall with the test as the placement server. The first answer is the + /// bundled sample, whose missing evidence asks to walk past the left end; later answers drop + /// that item, so the request is raised once. Returns every scene uploaded, in order, and the + /// packet projection and mark record the autopilot leaves at the result. + @MainActor + private func runReplay(markPastEnd: Bool) throws -> ReplayRun { + let sample = String(decoding: try Data(contentsOf: UploadRecoveryUITests.sampleResult), as: UTF8.self) + let zeros = String(repeating: "0", count: 64) + XCTAssertTrue(sample.contains("\"input_sha256\": \"\(zeros)\"")) + // The first answer asks only to walk past the left end, so that request is the one raised; + // the sample's ground request ahead of it would take the first upload. Later answers ask + // for nothing, so the request is raised once. + var answer = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(sample.utf8)) as? [String: Any]) + let missing = (answer["missing_evidence"] as? [[String: Any]]) ?? [] + let pastEnd = missing.filter { $0["kind"] as? String == "past_end" && $0["side"] as? String == "left" } + XCTAssertEqual(pastEnd.count, 1, "the sample no longer asks to walk past the left end") + answer["missing_evidence"] = pastEnd + let onlyPastEnd = String(decoding: try JSONSerialization.data(withJSONObject: answer, options: [.sortedKeys]), as: UTF8.self) + answer["missing_evidence"] = [[String: Any]]() + let nothingMissing = String(decoding: try JSONSerialization.data(withJSONObject: answer, options: [.sortedKeys]), as: UTF8.self) + XCTAssertTrue(onlyPastEnd.contains("\"input_sha256\":\"\(zeros)\""), "the binding placeholder moved") + + let files = FileManager.default + let gate = URL(fileURLWithPath: NSTemporaryDirectory()).appending(path: "housescan-gate-\(UUID().uuidString)", directoryHint: .isDirectory) + try files.createDirectory(at: gate, withIntermediateDirectories: true) + defer { try? files.removeItem(at: gate) } + // Every screen up to the result is let through; the result's stays shut, so the autopilot + // stops there with its files written. + for phase in ["onboarding", "findMeter", "meterCloseUp", "wallWalk", "markFeatures", "gapRequest", "uploading", "spotConfirm"] { + try Data().write(to: gate.appending(path: phase)) + } + let server = GateServer(gate: gate) { body, index in + let template = index == 0 ? onlyPastEnd : nothingMissing + return Data(template.replacingOccurrences(of: zeros, with: UploadRecoveryUITests.sha256(body)).utf8) + } + defer { server.stop() } + + let app = XCUIApplication() + app.launchArguments = [ + "-replay", FullFlowUITests.fixture, "-autopilot", "-autopilotHold", "1.0", "-autopilotGate", gate.path, + "-practiceMeter", "NO", "-serverURL", "http://placement.invalid", "-answersFromGate", + ] + (markPastEnd ? ["-autopilotMarkPastEnd"] : []) + app.launch() + defer { app.terminate() } + let any = app.descendants(matching: .any) + XCTAssertTrue(any["screen.result"].waitForExistence(timeout: 600), "the flow never reached the result") + let sceneFile = gate.appending(path: "scene.json") + let deadline = Date().addingTimeInterval(30) + while !files.fileExists(atPath: sceneFile.path), Date() < deadline { Thread.sleep(forTimeInterval: 0.5) } + let requests = server.requests.filter { $0.target == "POST /v1/placements" } + let scenes = try requests.map { try XCTUnwrap(JSONSerialization.jsonObject(with: $0.body) as? [String: Any]) } + func json(_ name: String) -> [String: Any]? { + (try? Data(contentsOf: gate.appending(path: name))).flatMap { try? JSONSerialization.jsonObject(with: $0) as? [String: Any] } + } + // The packet of the upload that raised the past-end request, which the autopilot leaves as + // `packet-first.json` when the request shows: its left end is the walk's own. + let first = json("packet-first.json").flatMap { Self.wallEnd($0, "left") } + return ReplayRun( + scenes: scenes, packet: json("packet-marks-guidance.json"), firstPacketLeftEnd: first, + pastEndMark: json("past-end-mark.json"), log: requests.map(\.target).joined(separator: "\n")) + } + + // MARK: Reading scenes and packets + + private static let feetPerMeter = 3.280839895 + + /// The left end of the first wall's baseline, in feet along it from the meter (negative left). + private static func leftEndFeet(_ scene: [String: Any]) throws -> Double { + let walls = try XCTUnwrap(scene["walls"] as? [[String: Any]]) + let baseline = try XCTUnwrap(walls.first?["baseline"] as? [[Double]]) + let meter = try XCTUnwrap((scene["meter"] as? [String: Any])?["pos"] as? [Double]) + let first = try XCTUnwrap(baseline.first), last = try XCTUnwrap(baseline.last) + let length = hypot(last[0] - first[0], last[1] - first[1]) + return ((first[0] - meter[0]) * (last[0] - first[0]) + (first[1] - meter[2]) * (last[1] - first[1])) / length + } + + private static func endKind(_ scene: [String: Any], _ side: String) throws -> String? { + let coverage = try XCTUnwrap(scene["coverage"] as? [String: Any]) + return (coverage["ends"] as? [String: [String: String]])?[side]?["kind"] + } + + /// The lower edge of every observed span of `band` (every band when nil), in feet. + private static func observedLeftEdges(_ scene: [String: Any], band: String?) throws -> [Double] { + let coverage = try XCTUnwrap(scene["coverage"] as? [String: Any]) + let observed = ((coverage["observed"] as? [[String: Any]]) ?? []).filter { band == nil || $0["band"] as? String == band } + return observed.compactMap { ($0["span_ft"] as? [Double]).map { min($0[0], $0[1]) } } + } + + private static func wallEnd(_ packet: [String: Any], _ side: String) -> [String: Any]? { + ((packet["marks"] as? [[String: Any]]) ?? []).first { $0["kind"] as? String == "wall_end" && $0["side"] as? String == side } + } + + private static func guidance(_ packet: [String: Any], kind: String) -> [[String: Any]] { + ((packet["guidance"] as? [[String: Any]]) ?? []).filter { $0["kind"] as? String == kind } + } + + // MARK: Helpers + + @MainActor + private func attach(_ run: ReplayRun) { + let text = "uploads: \(run.scenes.count)\npast-end mark: \(run.pastEndMark ?? [:])\npacket: \(run.packet ?? [:])" + let attachment = XCTAttachment(string: text) + attachment.name = "pastEnd-replay" + attachment.lifetime = .keepAlways + add(attachment) + } + + @MainActor + private func launch(_ arguments: [String]) -> XCUIApplication { + let app = XCUIApplication() + app.launchArguments = ["-practiceMeter", "NO", "-uiDemo", "-uiDemoFreeze", "-uiDemoPhase", "gapRequest"] + arguments + app.launch() + XCTAssertTrue(element(app, "screen.gapRequest").waitForExistence(timeout: 15), "screen.gapRequest never appeared") + XCTAssertTrue(element(app, "instruction").waitForExistence(timeout: 5)) + return app + } + + /// Scrolls in measured steps until the whole element is in the window: at the largest text + /// sizes the actions sit below the card, reached by scrolling. + @MainActor + private func scrollIntoView(_ target: XCUIElement, in app: XCUIApplication) { + let window = app.windows.firstMatch.frame + let scroll = app.scrollViews.firstMatch + guard scroll.exists else { return } + let start = scroll.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.6)) + for _ in 0..<12 where !window.contains(target.frame) { + let frame = target.frame + let shift: CGFloat = frame.maxY > window.maxY + ? -min(frame.maxY - window.maxY + 24, window.height / 3) + : min(window.minY - frame.minY + 24, window.height / 3) + start.press(forDuration: 0.05, thenDragTo: start.withOffset(CGVector(dx: 0, dy: shift))) + } + } + + @MainActor + private func snap(_ app: XCUIApplication, _ name: String) { + let shot = XCTAttachment(screenshot: app.screenshot()) + shot.name = name + shot.lifetime = .keepAlways + add(shot) + } + + @MainActor + private func element(_ app: XCUIApplication, _ identifier: String) -> XCUIElement { + app.descendants(matching: .any)[identifier].firstMatch + } + + @MainActor + private func label(_ app: XCUIApplication, _ identifier: String) -> String { + ElementRead.snapshot(element(app, identifier))?.label ?? "" + } + + @MainActor + private func tapWhenReady(_ target: XCUIElement) { + let ready = XCTNSPredicateExpectation(predicate: NSPredicate(format: "isEnabled == true AND isHittable == true"), object: target) + XCTAssertEqual(XCTWaiter().wait(for: [ready], timeout: 10), .completed, "\(target) never took taps") + target.tap() + } +} diff --git a/packet/README.md b/packet/README.md index 98fa39619..c2f680004 100644 --- a/packet/README.md +++ b/packet/README.md @@ -39,18 +39,33 @@ performance. homeowner gave about it: - whether a wall end is a real end (something blocks it) or unexplored (it turns a corner, or no answer), which decides whether the server may reject for want of space; + - whether the homeowner marked a wall end ("Wall ends here") or the app inferred it from the + walk: where the walk reached after "Can't get there" or "The wall keeps going", or past the + ground a `past_end` request showed. An inferred end has the same place and kind as a marked + one, but nobody pointed at it, so it says where the scan stopped, not where the wall does; - whether a window opens, which the rules may use (whether a fixed window is exempt from the opening clearance is the rules' choice, not the app's); - which corner taps belong to the same door or window. - Proposal: `taps[].attrs` with `endKind` (`limit`, `unexplored`) and `operable`, and - `taps[].group` for taps that outline one object. + Proposal: `taps[].attrs` with `endKind` (`limit`, `unexplored`), `operable` and `inferred`, + and `taps[].group` for taps that outline one object. + + The app's own packet 1.1 (the `scan.zip` behind Share scan) already writes the provenance: a + wall end the homeowner marked has its mark time `t`; an inferred one has no `t` and + `attrs.inferred: true`. A missing `t` alone never means the homeowner marked it. 4. **The guidance log.** Every request the homeowner was shown (walk, tilt to the ground, mark an end, close-up, a gap the phone or the server asked for), when, and its outcome: `met`, `skipped`, `cannot_reach`, `superseded` or `unresolved`. Today "I can't get there" never reaches the server, so the server can ask for the same unreachable view again. Proposal: `guidance[]` with `id`, `kind`, `origin` (`phone`, `server`), `message`, optional `band` and `span`, `tShown`, `tResolved` and `outcome`. + + For the wall's ends, as the app's packet 1.1 records them: `mark_end` is `met` only when the + homeowner marked that end. "The wall keeps going" and "Can't get there" close it as + `cannot_reach`, because the end lay beyond where the homeowner got, and the end placed there + is inferred. `gap_past_end` is `met` both when views past the end settle it and when the + homeowner marks the end again; the wall end's provenance tells them apart. "I can't get + there" on it is `cannot_reach`, and the end it had cleared comes back as it was. 5. **Depth between keyframes** (from S6's live map, draft PR #21). Keyframes are kept about every 0.5 m, but the phone fuses depth at about 10 Hz to decide what was seen. With keyframe depth alone the server cannot rebuild that decision. Proposal: optional `depthFrames[]` with `t`, From f4c3e177aef2f628df3b4b5b1d2bb070736b60d3 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 19:21:36 -0500 Subject: [PATCH 11/71] ios: the too-short end is refused by the aim check the preview shares; its fade stays scoped UNVALIDATED: not compiled; no package, app or UI tests have run. Root's review of a7c65bd2: `markWallEnd` refused an end that left too little wall only after `aimedEnd` had accepted it, while the tape's preview went through `circleEnd` without that check, so the preview showed an end the button refused, and `publishEndPreview` cleared the refusal on the next frame. The past-end-only minimum (`CoverageMap.endWouldLeaveTooLittle`, `WallFrame.minWallLength`) is now part of `aimedEnd`, which the button, the preview and the refusal clearing all use. The walk is unchanged: its minimum still applies at "Done with this wall". Whether 0.79 m is the right minimum stays open. The refusal reads "House Scan can't use a wall that short" / "If this is where the wall ends, tap I can't get there to continue.": an app limit, with no claim about what a battery needs and no nudge to mark a farther end. Motion (prep-for-prod review): the circle stays mounted on a past-end request and fades with "Wall ends here" on the same `Motion.settle` spring, so the pair no longer parts. The end controls fade in their own container; before, the whole bottom stack animated with them, sliding the progress and the tape, and still moved under Reduce Motion. Now the rows below move into place at once, and under Reduce Motion the controls swap without animation and the circle only changes opacity briefly. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/Contract/ScanContract.swift | 5 +-- .../Runtime/ScanEngine+Actions.swift | 6 ---- ios/HouseScan/Runtime/ScanEngine+Ends.swift | 10 +++++- ios/HouseScan/UI/Copy/ScanCopy.swift | 6 +++- .../UI/Screens/GapRequestScreen.swift | 31 +++++++++++++------ 5 files changed, 38 insertions(+), 20 deletions(-) diff --git a/ios/HouseScan/Contract/ScanContract.swift b/ios/HouseScan/Contract/ScanContract.swift index e70e8ec97..cd3ffdfdb 100644 --- a/ios/HouseScan/Contract/ScanContract.swift +++ b/ios/HouseScan/Contract/ScanContract.swift @@ -567,8 +567,9 @@ enum EndMarkRefusal: Error, Equatable, Sendable { case otherSide(WallSide) /// Tracking isn't normal, as for a feature mark (`MarkRefusal.trackingNotReady`). case trackingNotReady - /// During a past_end request: an end there would leave less wall than a battery needs - /// (`CoverageMap.endWouldLeaveTooLittle`), the walk's own minimum (`WallFrame.minWallLength`). + /// During a past_end request: an end there would leave less wall than the walk's minimum + /// (`CoverageMap.endWouldLeaveTooLittle`, `WallFrame.minWallLength`), which the app can't use. + /// Decided in `aimedEnd`, so the preview and the button agree. case tooLittleWall } diff --git a/ios/HouseScan/Runtime/ScanEngine+Actions.swift b/ios/HouseScan/Runtime/ScanEngine+Actions.swift index 1f970742d..af878768d 100644 --- a/ios/HouseScan/Runtime/ScanEngine+Actions.swift +++ b/ios/HouseScan/Runtime/ScanEngine+Actions.swift @@ -164,12 +164,6 @@ extension ScanEngine: ScanActions { hit = found } let side: WallSide = hit.s < 0 ? .left : .right - // The walk checks the wall's length when it finishes ("Done with this wall"); a request - // settles without that step, so its end is checked here against the other end, which - // stays where it is. - if state.phase == .gapRequest, map.endWouldLeaveTooLittle(side.walk, at: hit.s) { - return refuseEndMark(.tooLittleWall) - } state.endMarkRefusal = nil // Unexplored until the homeowner says something blocks the wall there: an unanswered // question must not tell the server the usable wall stops at this point. diff --git a/ios/HouseScan/Runtime/ScanEngine+Ends.swift b/ios/HouseScan/Runtime/ScanEngine+Ends.swift index c3f749f85..3395eee80 100644 --- a/ios/HouseScan/Runtime/ScanEngine+Ends.swift +++ b/ios/HouseScan/Runtime/ScanEngine+Ends.swift @@ -130,6 +130,12 @@ extension ScanEngine { /// `circleEnd` for the ray through `pixel` of the sensor image. A tap at a point (the /// autopilot's) during a past_end request goes through the same check as the circle. + /// + /// During a past_end request only, an end that would leave less wall than the walk's minimum + /// (`WallFrame.minWallLength`, `CoverageMap.endWouldLeaveTooLittle`) is refused here, so the + /// button, its refusal and the tape's preview agree. The walk applies that minimum when it + /// finishes ("Done with this wall") and keeps doing so; a request settles without that step. + /// Whether 0.79 m is the right minimum is an open product question, not settled here. func aimedEnd(asked: WallSide, pixel: SIMD2, frame: SourceFrame, map: CoverageMap) -> Result { let hit = map.wall.intersectWall(frame.camera.ray(throughPixel: pixel)) switch EndAim.verdict( @@ -138,7 +144,9 @@ extension ScanEngine { // without replacing the sampled frame whose ray this is. groundError: map.heightError, askedLeft: asked == .left, trackingNormal: frame.tracking == .normal && state.tracking == .normal ) { - case .end(let point): return .success(point) + case .end(let point): + if state.phase == .gapRequest, map.endWouldLeaveTooLittle(asked.walk, at: point.s) { return .failure(.tooLittleWall) } + return .success(point) case .trackingLimited: return .failure(.trackingNotReady) case .offWall: return .failure(.noWall) case .otherSide: return .failure(.otherSide(asked == .left ? .right : .left)) diff --git a/ios/HouseScan/UI/Copy/ScanCopy.swift b/ios/HouseScan/UI/Copy/ScanCopy.swift index fcfcf3995..b981b38c6 100644 --- a/ios/HouseScan/UI/Copy/ScanCopy.swift +++ b/ios/HouseScan/UI/Copy/ScanCopy.swift @@ -398,7 +398,11 @@ enum ScanCopy { case .trackingNotReady: Instruction(title: "One moment, your phone is still finding its place", detail: "Then aim at the \(side.rawValue) end.") case .tooLittleWall: - Instruction(title: "That leaves too little wall for a battery", detail: "Aim where the wall really stops, or tap I can't get there.") + // Says what the app can't do, not where to aim: the homeowner must not be nudged to + // mark an end farther than the wall really goes. "I can't get there" stays the way on. + // The app's own minimum (`WallFrame.minWallLength`), not a measured installation rule: + // no reason is given for it, and rule distances belong to the server. + Instruction(title: "House Scan can't use a wall that short", detail: "If this is where the wall ends, tap I can't get there to continue.") } } diff --git a/ios/HouseScan/UI/Screens/GapRequestScreen.swift b/ios/HouseScan/UI/Screens/GapRequestScreen.swift index bc94f342b..fb2394ac8 100644 --- a/ios/HouseScan/UI/Screens/GapRequestScreen.swift +++ b/ios/HouseScan/UI/Screens/GapRequestScreen.swift @@ -32,12 +32,16 @@ struct GapRequestScreen: View { SavedMeterPhoto(image: meterPhoto) .transition(.opacity) } - // The end lands under the circle, as on the walk's "Is this the right end?". - if offersEndMark { + // The end lands under the circle, as on the walk's "Is this the right end?". It fades + // with "Wall ends here" on the same spring, so the pair comes and goes together; + // scoped to the circle, so nothing else on the camera animates with it. Under Reduce + // Motion only its opacity changes, briefly: nothing moves. + if state.gap?.pastEndSide != nil { Reticle(diameter: 56) .frame(maxWidth: .infinity, maxHeight: .infinity) .ignoresSafeArea() - .transition(.opacity) + .opacity(offersEndMark ? 1 : 0) + .animation(reduceMotion ? .easeOut(duration: 0.15) : Motion.settle, value: offersEndMark) } if state.gap?.isSatisfied == true { SuccessBadge() @@ -70,13 +74,20 @@ struct GapRequestScreen: View { OverheadAnswers(actions: actions) .transition(.opacity) } - if askingEnd { - EndQuestionAnswers(actions: actions) - .transition(.opacity) - } - if offersEndMark { - markEndButton + // The end's controls fade on their own: the rows below them move into place at + // once rather than sliding with the stack, and under Reduce Motion nothing + // animates. + VStack(spacing: 10) { + if askingEnd { + EndQuestionAnswers(actions: actions) + .transition(.opacity) + } + if offersEndMark { + markEndButton + } } + .animation(reduceMotion ? nil : Motion.settle, value: askingEnd) + .animation(reduceMotion ? nil : Motion.settle, value: offersEndMark) if !typeSize.isAccessibilitySize { showResult } @@ -99,7 +110,7 @@ struct GapRequestScreen: View { } } .animation(reduceMotion ? .easeOut(duration: 0.15) : Motion.settle, value: asking) - .animation(reduceMotion ? .easeOut(duration: 0.15) : Motion.settle, value: askingEnd) + } } .animation(reduceMotion ? .easeOut(duration: 0.15) : Motion.pin, value: state.gap?.isSatisfied) From a76ba4d47a25fbdbcb30ae21e314e93df6dcd898 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 19:31:37 -0500 Subject: [PATCH 12/71] ios: the B44 cleanup tests allow five minutes before a hang fails them The one-minute limit sat close to what these tests reported in the concurrent package suite: up to 82.5 s in hosted run 37161498639, which had no limit. The limit only turns a gate that is never released into a failure; the gates and assertions are unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Tests/HouseScanKitTests/KeptScansTests.swift | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/KeptScansTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/KeptScansTests.swift index 5e6ca08d1..9b21146c7 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/KeptScansTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/KeptScansTests.swift @@ -220,6 +220,11 @@ import Testing } // MARK: A bundle still being written when Start over lists its folder (B44) + // + // The async tests below carry a five-minute time limit, only so that a gate never released + // fails instead of hanging; the gates and assertions decide pass or fail. In the concurrent + // package suite these tests reported up to 82.5 s (hosted run 37161498639, which had no + // limit), so a one-minute limit was too tight to rely on. /// A scan folder whose bundle is being written: the photos, stamp and raw streams the packet /// is built from and the half-assembled `packet/`, but no `scan.zip` yet. @@ -292,7 +297,7 @@ import Testing /// with no zip yet it is listed as never packaged. The deletion waits for the write; a zip /// written to the end keeps the folder, one cut short keeps it without a place (as any /// partial bundle), and a write that failed before zipping leaves it to be deleted. - @Test(.timeLimit(.minutes(1)), arguments: WriteEnd.allCases) + @Test(.timeLimit(.minutes(5)), arguments: WriteEnd.allCases) func aBundleStillBeingWrittenIsJudgedOnlyAfterItsWrite(end: WriteEnd) async throws { let root = try Self.makeRoot() defer { try? FileManager.default.removeItem(at: root) } @@ -388,7 +393,7 @@ import Testing /// its inputs in place, and finds them intact. The first scan's bundle is kept, the second's /// only if its write ran (one queued for a scan already started over is skipped), and the /// newest folders are never listed. - @Test(.timeLimit(.minutes(1)), arguments: [true, false]) + @Test(.timeLimit(.minutes(5)), arguments: [true, false]) func rapidStartOversWaitForTheWholeChain(secondScanWrites: Bool) async throws { let root = try Self.makeRoot() defer { try? FileManager.default.removeItem(at: root) } From 63bf4e7a61f0313b2955512eac223e8aea7d0ce3 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 19:32:27 -0500 Subject: [PATCH 13/71] ios: a tracking change republishes the end preview, so a pose-only frame can't leave a stale end on the tape Since a7c65bd2 the circle's end also checks the phone's tracking now (state.tracking), which a pose-only frame updates before ingest returns early, without the guidance update that republishes the preview. The tape then showed an end that "Wall ends here" refused (Greptile, #213). Unvalidated: pose-only frames come only from live capture, so no replay or UI test drives this path. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/Runtime/ScanEngine.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index 6901aaaf3..012dfd3b5 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -563,6 +563,10 @@ final class ScanEngine { if state.tracking == .normal { breakWalkedPath(because: "tracking left normal") } state.tracking = frame.tracking live?.setResultVisible(frame.tracking == .normal) + // The circle's end checks this tracking (`aimedEnd`), and a pose-only frame returns + // below before guidance republishes the preview, so the tape would keep an end that + // "Wall ends here" now refuses (review of #213). + publishEndPreview() } noteMeterAnchor(frame) // A frame made before the meter was anchored again carries the old anchor's pose. First, From 448c45a788e5aedec43db43f98ba36cf6c73c3ed Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 19:36:48 -0500 Subject: [PATCH 14/71] ios: the provenance tests expect the inferred end's shape and refuse for the reason they name CI run 37164872574 failed two expectations of mine, not the code under test: manifestJSONShape still expected the synthetic packet's left end to carry t, which it lost when that end became inferred; and theWriterRefusesAnInferredEndWithATime marked its homeowner end at uptime 3, before the synthetic capture's start at 100, so the writer rightly refused it. The refusals are now pinned to their exact errors, so the inferred-with-a-time case can't pass on the earlier time check instead. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Tests/HouseScanKitTests/PacketWriterTests.swift | 4 +++- .../HouseScanKitTests/WallEndProvenanceTests.swift | 12 ++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift index ab6b59e8b..4768300df 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift @@ -367,7 +367,9 @@ struct SyntheticPacket { #expect(Set(planes[1].keys) == ["id", "alignment", "pose", "extent_m"]) let marks = try #require(json["marks"] as? [[String: Any]]) #expect(Set(marks[0].keys) == ["id", "kind", "points", "t", "photo_ids"]) - #expect(Set(marks[1].keys) == ["id", "kind", "points", "t", "side", "end_kind"]) + // The left end is inferred: the flag, no mark time. The right end is the homeowner's. + #expect(Set(marks[1].keys) == ["id", "kind", "points", "side", "end_kind", "attrs"]) + #expect(Set(marks[2].keys) == ["id", "kind", "points", "t", "side", "end_kind"]) #expect((marks[3]["attrs"] as? [String: Bool]) == ["operable": true]) let guidance = try #require(json["guidance"] as? [[String: Any]]) #expect(Set(guidance[1].keys) == ["id", "kind", "origin", "message", "band", "span_m", "t_shown", "t_resolved", "outcome"]) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/WallEndProvenanceTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/WallEndProvenanceTests.swift index 2c83fded3..2dc72dae7 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/WallEndProvenanceTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/WallEndProvenanceTests.swift @@ -130,16 +130,20 @@ import Testing /// The writer's own check: an inferred end with a time, or the flag on another kind of mark, /// is refused before anything is written. @Test func theWriterRefusesAnInferredEndWithATime() throws { + // Times after the synthetic capture's start, so only the rule under test can refuse. + let t = SyntheticPacket.start + 3 var mark = try Self.end(.inferred) - mark.t = 3 + mark.t = t let folder = URL(fileURLWithPath: NSTemporaryDirectory()).appending(path: "hs-provenance-\(UUID())") defer { try? FileManager.default.removeItem(at: folder) } var writer = try PacketWriter(folder: folder, session: try SyntheticPacket().session) - #expect(throws: PacketError.self) { try writer.setMarks([mark]) } + #expect(throws: PacketError.invalidMark(id: "wall_end_right", reason: "an inferred wall end has no mark time: nobody marked it")) { + try writer.setMarks([mark]) + } var meter = PacketMark.meter(id: "m") meter.inferred = true - #expect(throws: PacketError.self) { try writer.setMarks([meter]) } - #expect(throws: Never.self) { try writer.setMarks([try Self.end(.inferred), try Self.end(.marked(at: 3)).renamed("wall_end_left")]) } + #expect(throws: PacketError.invalidMark(id: "m", reason: "only a wall end can be inferred")) { try writer.setMarks([meter]) } + #expect(throws: Never.self) { try writer.setMarks([try Self.end(.inferred), try Self.end(.marked(at: t)).renamed("wall_end_left")]) } } } From 15cf658e6e4c8af07f1bd4e9eae28a4ceebf7fcb Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 19:42:48 -0500 Subject: [PATCH 15/71] ios: a corner marked on a past-end request closes it cannot_reach, and its repeat isn't raised "It turns a corner" on a past_end request left the end unexplored and logged the request met (Codex P1, #213). The server settles an unexplored end only with views past it (server/README.md, "unexplored"), so the next answer could ask again, and a request can't follow a corner (`markNextWall` runs only on the walk). The answer now goes through `skipCurrentGap`: the request closes cannot_reach with no check mark, and the corner stays the homeowner's mark, unexplored (`PastEndSettlement.keepMarked`), not the cleared end restored. The skipped plan was planned from the cleared end, and the repeat is planned from the corner, so exact-span matching missed it whenever the mark moved the end. `GapPlanner.pastEndPlan(side:end:)` now owns the 2 m past-end extent for `plan(for:)`, and the corner answer also records the request planned from the corner among the skipped. That request was never shown: no guidance entry and no cells for it. A past_end from a different end later on that side, and every other request, can still be raised. Docs: packet/README.md says up front that the app's packet 1.1 is the local Share scan archive, not intake and not an approved 0.4 revision, and that `gap_past_end` is met by views or an end the wall stops at, not by a corner. Tests: ServerGapTests covers the 2 m owner, plus nearer and farther corners on both sides: repeat dropped, later end raised, ground in the same answer raised. A demo UI test shows no "Got it, thanks". A real-engine replay through the answer gate (`-autopilotPastEndCorner`) checks one past_end (cannot_reach), one server ground request, and the homeowner's unexplored corner with its time and nothing reported past it. Not run locally. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/Runtime/Autopilot.swift | 8 ++- ios/HouseScan/Runtime/LaunchOptions.swift | 5 ++ .../Runtime/ScanEngine+Actions.swift | 19 ++++- ios/HouseScan/Runtime/ScanEngine.swift | 22 ++++-- ios/HouseScan/UI/Preview/DemoEngine.swift | 6 ++ .../UI/Screens/GapRequestScreen.swift | 3 +- .../HouseScanKit/Guidance/GapPlanner.swift | 19 +++-- .../HouseScanKitTests/ServerGapTests.swift | 34 +++++++++ .../PastEndRecoveryUITests.swift | 69 +++++++++++++++++-- packet/README.md | 28 +++++--- 10 files changed, 183 insertions(+), 30 deletions(-) diff --git a/ios/HouseScan/Runtime/Autopilot.swift b/ios/HouseScan/Runtime/Autopilot.swift index e92ced56b..eb6809c0a 100644 --- a/ios/HouseScan/Runtime/Autopilot.swift +++ b/ios/HouseScan/Runtime/Autopilot.swift @@ -478,7 +478,8 @@ final class Autopilot { /// `-autopilotMarkPastEnd`: marks a past_end request's end again, 60% of the way from the meter /// to the end the request cleared, and answers "Something blocks it", so the wall comes out - /// shorter and its end a limit. The mark is a tap where the replay shows that place on the + /// shorter and its end a limit; with `-autopilotPastEndCorner`, "It turns a corner", which + /// leaves that end unexplored and the request cannot_reach. The mark is a tap where the replay shows that place on the /// wall, not the circle in the middle of the view: it goes through the same `EndAim` check /// from that point (`ScanEngine.markWallEnd`), but no camera was aimed. Where it marked goes /// to the gate folder as `past-end-mark.json` (side, s in meters), for the UI test. @@ -509,8 +510,9 @@ final class Autopilot { try? JSONSerialization.data(withJSONObject: record).write(to: gate.appending(path: "past-end-mark.json")) } await pause(hold) - engine.answerWallEnd(turnsCorner: false) - log("answered the \(side.rawValue) end: something blocks it") + let corner = engine.options.autopilotPastEndCorner + engine.answerWallEnd(turnsCorner: corner) + log("answered the \(side.rawValue) end: \(corner ? "it turns a corner" : "something blocks it")") } /// Waits while the engine plays the replay's frames for the current request. A replay shows diff --git a/ios/HouseScan/Runtime/LaunchOptions.swift b/ios/HouseScan/Runtime/LaunchOptions.swift index 665b1b400..b19340cdb 100644 --- a/ios/HouseScan/Runtime/LaunchOptions.swift +++ b/ios/HouseScan/Runtime/LaunchOptions.swift @@ -26,6 +26,8 @@ import OSLog /// nearer than the end the request cleared, with a tap where the replay shows that place, and /// answers "Something blocks it" (`Autopilot.markPastEnd`). Without it the autopilot plays the /// request's frames and says "I can't get there" when they don't settle it. +/// - `-autopilotPastEndCorner`: with `-autopilotMarkPastEnd`, the autopilot answers "It turns a +/// corner" instead, which a request can't follow (`ScanEngine.answerWallEnd`). /// - `-autopilotCannotCheck`: the autopilot answers the first spot check "I can't check this /// area" instead of "It's clear". It can't be combined with `-autopilotSomethingThere`. /// - `-sampleResultAfterSpotAnswer ` (debug builds only): with the bundled sample, the @@ -59,6 +61,7 @@ struct LaunchOptions: Equatable { var autopilotSomethingThere = false var autopilotCannotCheck = false var autopilotMarkPastEnd = false + var autopilotPastEndCorner = false var sampleResultAfterSpotAnswer: URL? var simulateAppStore = false var injectGroundRise: Float? @@ -81,6 +84,8 @@ struct LaunchOptions: Equatable { autopilotSomethingThere = arguments.contains("-autopilotSomethingThere") autopilotCannotCheck = arguments.contains("-autopilotCannotCheck") autopilotMarkPastEnd = arguments.contains("-autopilotMarkPastEnd") + autopilotPastEndCorner = arguments.contains("-autopilotPastEndCorner") + precondition(!autopilotPastEndCorner || autopilotMarkPastEnd, "-autopilotPastEndCorner answers the end -autopilotMarkPastEnd marks; pass both") precondition(!(autopilotSomethingThere && autopilotCannotCheck), "-autopilotSomethingThere and -autopilotCannotCheck each choose the first spot answer; pass one") #if DEBUG sampleResultAfterSpotAnswer = value(after: "-sampleResultAfterSpotAnswer").map { URL(fileURLWithPath: $0) } diff --git a/ios/HouseScan/Runtime/ScanEngine+Actions.swift b/ios/HouseScan/Runtime/ScanEngine+Actions.swift index af878768d..c6cf6cbcf 100644 --- a/ios/HouseScan/Runtime/ScanEngine+Actions.swift +++ b/ios/HouseScan/Runtime/ScanEngine+Actions.swift @@ -193,7 +193,24 @@ extension ScanEngine: ScanActions { nextWallSide = side nextWallRefusal = nil } - if state.phase == .gapRequest, side == pastEndSide { settlePastEnd() } + if state.phase == .gapRequest, side == pastEndSide { + if turnsCorner { + // The wall goes on round the corner, so its end stays unexplored, and the server + // settles an unexplored end only by views past it (server/README.md, "unexplored"). + // This request can't follow a corner; the walk does that (`markNextWall`). So it + // closes as cannot_reach, never met: the homeowner's corner stays where they + // marked it (`PastEndSettlement.keepMarked`), and nothing past it counts as seen. + // The next answer can ask again to walk past this side, now planned from the + // corner: the same request this one couldn't serve, so it is recorded as skipped + // too. Only from this end; a past_end from a different end later is a new view. + let end = side == .left ? coverage?.leftEnd : coverage?.rightEnd + skipCurrentGap( + because: "the wall turns a corner at the homeowner's end, and this request can't survey past a corner", + deferring: end.map { gapPlanner.pastEndPlan(side: side.walk, end: $0) }) + } else { + settlePastEnd() + } + } if state.phase == .wallWalk, let frame = currentFrame { resetGuidanceAfterSkip(camera: frame.camera, time: frame.timestamp) } diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index 012dfd3b5..0129d4989 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -128,10 +128,11 @@ final class ScanEngine { /// new one (`keepOverheadView`); counting keyframes would not do, since the walk keeps them too. private var overheadViewsAtGapStart = 0 /// Requests the homeowner skipped or answered with something overhead: they go to installer - /// review, and the result doesn't offer them as captures again. + /// review, and the result doesn't offer them as captures again. With them, the past_end + /// request from a corner a request couldn't follow (`skipCurrentGap(deferring:)`). private(set) var skippedGaps: [GapPlan] = [] /// The side of a server past_end request being captured: that end was cleared, and marking - /// it again settles the request (see `markWallEnd`). + /// it again as where the wall stops settles the request (see `markWallEnd`, `answerWallEnd`). var pastEndSide: WallSide? /// The end the past_end request cleared: where it was, its kind and its stamp (marked, with /// when, or inferred), put back or moved on when the request ends without it marked again @@ -1393,7 +1394,7 @@ final class ScanEngine { store.keyframes.count > keyframesAtGapStart } // Not while the end question is up: the homeowner marked the end and is saying what is - // there, and that answer settles the request (`settlePastEnd`). + // there, and that answer closes the request (`answerWallEnd`). let satisfied = gapPlanner.isSatisfied(plan, map) && fresh && state.endQuestion == nil state.guidance = .gap let center = (plan.span.lowerBound + plan.span.upperBound) / 2 @@ -1988,8 +1989,9 @@ final class ScanEngine { startUpload() } - /// The past_end request's end was marked again and its question answered: the request is - /// settled, so the scan goes to the upload like a closed gap. + /// The past_end request's end was marked again and the homeowner said the wall stops there: + /// the request is settled, so the scan goes to the upload like a closed gap. A corner doesn't + /// settle it (`answerWallEnd`). func settlePastEnd() { guard state.phase == .gapRequest, var request = state.gap, !request.isSatisfied else { return } resolveGuidance(.met) @@ -2037,7 +2039,14 @@ final class ScanEngine { /// overhead, or "Show my result"): recorded for installer review, then on to the upload. The /// answer that follows raises the next item it lists, never this one again /// (`automaticGapQueue`); only "Show my result" (`stopGapRequests`) ends the requests. - func skipCurrentGap(because reason: String = "the homeowner can't get there", refused: Bool = true) { + /// + /// `deferring` is a request this one stands for that nobody was shown: the past_end request + /// from the corner the homeowner just marked (`answerWallEnd`), which this request can't + /// follow. Recorded with the skipped requests only, so the next answer doesn't raise it again + /// (`GapPlan.asksForSameView`); it logs no guidance and marks no cells, since no view was + /// asked for or taken there. A past_end from a different end later, and other requests, can + /// still be raised. + func skipCurrentGap(because reason: String = "the homeowner can't get there", refused: Bool = true, deferring: GapPlan? = nil) { guard let plan = gapPlan else { return } // Something overhead is an answer: the request goes to review without its view. resolveGuidance(refused ? .cannotReach : .skipped) @@ -2045,6 +2054,7 @@ final class ScanEngine { // nothing about the band the strip draws. if plan.need == .cells { coverage?.markSkipped(plan.band, plan.span) } skippedGaps.append(plan) + if let deferring { skippedGaps.append(deferring) } publishCoverage() RuntimeLog.engine.info("gap \(self.gapCounter) left for installer review: \(reason, privacy: .public)") afterGapResolved() diff --git a/ios/HouseScan/UI/Preview/DemoEngine.swift b/ios/HouseScan/UI/Preview/DemoEngine.swift index ab27474d3..77d39f6db 100644 --- a/ios/HouseScan/UI/Preview/DemoEngine.swift +++ b/ios/HouseScan/UI/Preview/DemoEngine.swift @@ -932,6 +932,12 @@ final class DemoEngine: ScanActions { state.endQuestionLeavesOut = nil state.endQuestionLeavesOutSeen = false if state.phase == .gapRequest, var gap = state.gap, gap.pastEndSide == side { + // A corner the request can't follow: no check mark, straight on to the upload, as + // `ScanEngine.skipCurrentGap` does. + if turnsCorner { + enterUpload() + return + } // The end marked again settles the request, as `ScanEngine.settlePastEnd` does; the // check runs again after the request's check mark has been seen. gap.isSatisfied = true diff --git a/ios/HouseScan/UI/Screens/GapRequestScreen.swift b/ios/HouseScan/UI/Screens/GapRequestScreen.swift index fb2394ac8..39d635e00 100644 --- a/ios/HouseScan/UI/Screens/GapRequestScreen.swift +++ b/ios/HouseScan/UI/Screens/GapRequestScreen.swift @@ -146,7 +146,8 @@ struct GapRequestScreen: View { private var asking: Bool { state.overheadQuestion && state.gap?.isSatisfied != true } /// The end question after "Wall ends here" on a past_end request: it replaces the request, - /// "I can't get there" and "Show my result" until answered, and the answer settles the request. + /// "I can't get there" and "Show my result" until answered. An end the wall stops at settles the + /// request; a corner, which the request can't follow, moves on without settling it. private var askingEnd: Bool { guard let side = state.endQuestion, let gap = state.gap else { return false } return gap.pastEndSide == side && !gap.isSatisfied diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Guidance/GapPlanner.swift b/ios/HouseScanKit/Sources/HouseScanKit/Guidance/GapPlanner.swift index c90e98838..972cb7fd1 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Guidance/GapPlanner.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Guidance/GapPlanner.swift @@ -374,18 +374,25 @@ extension GapPlanner { case .pastEnd: guard let side = item.side else { return nil } switch side { - case .left: - let end = leftEnd ?? 0 - return GapPlan(band: .ground, span: (end - 2)...end, reason: .server) - case .right: - let end = rightEnd ?? 0 - return GapPlan(band: .ground, span: end...(end + 2), reason: .server) + case .left: return pastEndPlan(side: .left, end: leftEnd ?? 0) + case .right: return pastEndPlan(side: .right, end: rightEnd ?? 0) } case .unknown: return nil } } + /// The request a past_end item makes from the end at `end` on `side`: the ground 2 m beyond + /// it (`plan(for:leftEnd:rightEnd:limitEnds:)`). The one place that extent is decided, so a + /// request recorded against an end the homeowner marked (`ScanEngine.skipCurrentGap`) is the + /// same view the next answer's past_end item plans from that end. + public func pastEndPlan(side: WalkSide, end: Float) -> GapPlan { + switch side { + case .left: GapPlan(band: .ground, span: (end - 2)...end, reason: .server) + case .right: GapPlan(band: .ground, span: end...(end + 2), reason: .server) + } + } + /// Where the end a past_end request cleared goes once the request is met without the end /// marked again: past the stretch the request showed, at its far edge (never back inside /// `clearedAt`), so the wall runs on over ground that was seen and a next past_end request diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift index d479ca93b..be8adf4a4 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/ServerGapTests.swift @@ -243,6 +243,40 @@ import Testing #expect(left.band == .ground && left.span == -5 ... -3) let right = try #require(GapPlanner().plan(for: item(#"{"kind":"past_end","side":"right","message":"m"}"#), leftEnd: -3, rightEnd: 4)) #expect(right.span == 4...6) + // One owner of the 2 m: the item's request is the planner's past-end request from that end. + #expect(left == GapPlanner().pastEndPlan(side: .left, end: -3)) + #expect(right == GapPlanner().pastEndPlan(side: .right, end: 4)) + } + + /// "It turns a corner" on a past_end request: the request can't follow the corner, so it is + /// skipped, and the request planned from the homeowner's corner is recorded with it + /// (`ScanEngine.skipCurrentGap(deferring:)`). The next answer's repeat of that side is not + /// raised; a past_end from a different end on that side, or another request, still is. Ends + /// at -3 and 4 m; the corner nearer or farther than the cleared end, on either side. + @Test(arguments: [ + (WalkSide.left, Float(-2), Float(-1)), (.left, -4, -5), + (.right, 3, 2), (.right, 5, 6), + ]) + func aCornerIsNotAskedForAgainFromThatEnd(side: WalkSide, corner: Float, laterEnd: Float) throws { + let planner = GapPlanner() + let pastEnd = try item(#"{"kind":"past_end","side":"\#(side.rawValue)","message":"m"}"#) + let ground = try item(#"{"kind":"band","band":"ground","span_ft":[3.0,5.5],"message":"m"}"#) + func ends(_ s: Float) -> (left: Float, right: Float) { side == .left ? (s, 4) : (-3, s) } + let cleared = ends(side == .left ? -3 : 4) + let raised = try #require(planner.plan(for: pastEnd, leftEnd: cleared.left, rightEnd: cleared.right)) + let deferred = planner.pastEndPlan(side: side, end: corner) + func next(at end: (left: Float, right: Float), skipped: [GapPlan]) -> [PlacementMissingEvidence] { + planner.serverRequests( + in: [pastEnd, ground], leftEnd: end.left, rightEnd: end.right, limitEnds: [], + asked: [raised], skipped: skipped, limit: 5 + ).map(\.item) + } + // The skipped request alone misses the repeat: the corner moved the end. + #expect(next(at: ends(corner), skipped: [raised]) == [pastEnd, ground]) + // With the corner's request recorded, the repeat is not raised; the ground request is. + #expect(next(at: ends(corner), skipped: [raised, deferred]) == [ground]) + // The end moved on from the corner later: walking past it is a new view. + #expect(next(at: ends(laterEnd), skipped: [raised, deferred]) == [pastEnd, ground]) } /// Issue #39: the answer asks for two views and the homeowner can't get to the first. The diff --git a/ios/HouseScanUITests/PastEndRecoveryUITests.swift b/ios/HouseScanUITests/PastEndRecoveryUITests.swift index 63c89d14e..ccd9a7894 100644 --- a/ios/HouseScanUITests/PastEndRecoveryUITests.swift +++ b/ios/HouseScanUITests/PastEndRecoveryUITests.swift @@ -76,7 +76,7 @@ final class PastEndRecoveryUITests: XCTestCase { } /// "Wall ends here" asks the walk's end question; it replaces the request and its decline - /// until answered, and the answer settles the request, which goes on to the check. + /// until answered, and "Something blocks it" settles the request, which goes on to the check. @MainActor func testMarkingTheEndAsksWhatIsThereAndSettlesTheRequest() throws { try question(textSize: [], name: "gapRequest-pastEnd-question") @@ -108,6 +108,22 @@ final class PastEndRecoveryUITests: XCTestCase { XCTAssertTrue(element(app, "screen.uploading").waitForExistence(timeout: 10), "the answer must settle the request") } + /// "It turns a corner" can't settle a past-end request, which can't follow the corner: no + /// check mark, no "Got it, thanks", straight on to the check, as "I can't get there" goes. + @MainActor + func testACornerAtThePastEndGoesOnWithoutSettlingIt() throws { + let app = launch(["-uiDemoGap", "pastEnd"]) + let mark = app.buttons["action.markEnd"] + scrollIntoView(mark, in: app) + tapWhenReady(mark) + let corner = app.buttons["action.endCorner"] + XCTAssertTrue(corner.waitForExistence(timeout: 5)) + scrollIntoView(corner, in: app) + tapWhenReady(corner) + XCTAssertFalse(label(app, "instruction").contains("Got it, thanks"), "a corner settled the request") + XCTAssertTrue(element(app, "screen.uploading").waitForExistence(timeout: 10), "the corner must move on to the check") + } + /// Only a past-end request offers the end: the phone's own request and the server's other /// kinds keep their own controls. @MainActor @@ -182,6 +198,40 @@ final class PastEndRecoveryUITests: XCTestCase { attach(run) } + /// The check asks to walk past the left end. The autopilot marks it again nearer and answers + /// "It turns a corner", which the request can't follow. The next answer asks again to walk + /// past the left end, now from the corner, and also for ground on the right. The repeat is not + /// raised; the ground request is. The corner stays the homeowner's unexplored end, nothing past + /// it is reported, and the request closed cannot_reach, not met. + @MainActor + func testACornerAtThePastEndIsNotAskedForAgainOnTheRealEngine() throws { + let run = try runReplay(markPastEnd: true, corner: true) + let mark = try XCTUnwrap(run.pastEndMark, "the autopilot left no past-end-mark.json") + let marked = try XCTUnwrap(mark["s"] as? Double), cleared = try XCTUnwrap(mark["cleared_s"] as? Double) + XCTAssertEqual(mark["side"] as? String, "left") + XCTAssertGreaterThan(marked, cleared, "the end was meant to come nearer than the cleared one") + + // The first upload, one after the corner, one after the ground request. + XCTAssertGreaterThanOrEqual(run.scenes.count, 3, run.log) + let last = try XCTUnwrap(run.scenes.last) + XCTAssertEqual(try Self.endKind(last, "left"), "unexplored") + XCTAssertEqual(try Self.leftEndFeet(last), marked * Self.feetPerMeter, accuracy: 0.2, "the scene's left bound is the corner") + XCTAssertTrue(try Self.observedLeftEdges(last, band: nil).allSatisfy { $0 >= marked * Self.feetPerMeter - 0.05 }, "coverage claimed past the corner") + + let packet = try XCTUnwrap(run.packet, "no packet-marks-guidance.json") + let left = try XCTUnwrap(Self.wallEnd(packet, "left")) + XCTAssertNotNil(left["t"], "the corner is the homeowner's mark, with its time") + XCTAssertNil((left["attrs"] as? [String: Any])?["inferred"], "the corner is not inferred") + XCTAssertEqual(left["end_kind"] as? String, "unexplored") + let pastEnd = Self.guidance(packet, kind: "gap_past_end") + XCTAssertEqual(pastEnd.count, 1, "the repeated past_end was raised again: \(pastEnd)") + XCTAssertEqual(pastEnd.first?["outcome"] as? String, "cannot_reach", "\(pastEnd)") + // The phone's own requests are gap_band too; the server's are the ones in its answers. + let serverBand = Self.guidance(packet, kind: "gap_band").filter { $0["origin"] as? String == "server" } + XCTAssertEqual(serverBand.count, 1, "the ground request in the same answer was not raised: \(serverBand)") + attach(run) + } + private struct ReplayRun { var scenes: [[String: Any]] var packet: [String: Any]? @@ -194,8 +244,11 @@ final class PastEndRecoveryUITests: XCTestCase { /// bundled sample, whose missing evidence asks to walk past the left end; later answers drop /// that item, so the request is raised once. Returns every scene uploaded, in order, and the /// packet projection and mark record the autopilot leaves at the result. + /// + /// With `corner`, the autopilot answers "It turns a corner", and the second answer asks again + /// to walk past the left end together with the sample's ground request on the right. @MainActor - private func runReplay(markPastEnd: Bool) throws -> ReplayRun { + private func runReplay(markPastEnd: Bool, corner: Bool = false) throws -> ReplayRun { let sample = String(decoding: try Data(contentsOf: UploadRecoveryUITests.sampleResult), as: UTF8.self) let zeros = String(repeating: "0", count: 64) XCTAssertTrue(sample.contains("\"input_sha256\": \"\(zeros)\"")) @@ -208,6 +261,10 @@ final class PastEndRecoveryUITests: XCTestCase { XCTAssertEqual(pastEnd.count, 1, "the sample no longer asks to walk past the left end") answer["missing_evidence"] = pastEnd let onlyPastEnd = String(decoding: try JSONSerialization.data(withJSONObject: answer, options: [.sortedKeys]), as: UTF8.self) + let ground = missing.filter { $0["kind"] as? String == "band" && $0["band"] as? String == "ground" } + XCTAssertEqual(ground.count, 1, "the sample no longer asks for ground") + answer["missing_evidence"] = pastEnd + ground + let pastEndAgain = String(decoding: try JSONSerialization.data(withJSONObject: answer, options: [.sortedKeys]), as: UTF8.self) answer["missing_evidence"] = [[String: Any]]() let nothingMissing = String(decoding: try JSONSerialization.data(withJSONObject: answer, options: [.sortedKeys]), as: UTF8.self) XCTAssertTrue(onlyPastEnd.contains("\"input_sha256\":\"\(zeros)\""), "the binding placeholder moved") @@ -222,7 +279,11 @@ final class PastEndRecoveryUITests: XCTestCase { try Data().write(to: gate.appending(path: phase)) } let server = GateServer(gate: gate) { body, index in - let template = index == 0 ? onlyPastEnd : nothingMissing + let template = switch index { + case 0: onlyPastEnd + case 1 where corner: pastEndAgain + default: nothingMissing + } return Data(template.replacingOccurrences(of: zeros, with: UploadRecoveryUITests.sha256(body)).utf8) } defer { server.stop() } @@ -231,7 +292,7 @@ final class PastEndRecoveryUITests: XCTestCase { app.launchArguments = [ "-replay", FullFlowUITests.fixture, "-autopilot", "-autopilotHold", "1.0", "-autopilotGate", gate.path, "-practiceMeter", "NO", "-serverURL", "http://placement.invalid", "-answersFromGate", - ] + (markPastEnd ? ["-autopilotMarkPastEnd"] : []) + ] + (markPastEnd ? ["-autopilotMarkPastEnd"] : []) + (corner ? ["-autopilotPastEndCorner"] : []) app.launch() defer { app.terminate() } let any = app.descendants(matching: .any) diff --git a/packet/README.md b/packet/README.md index c2f680004..541ce1982 100644 --- a/packet/README.md +++ b/packet/README.md @@ -9,7 +9,13 @@ a format of its own. This folder holds the app side's proposals for the next revision (0.5): what the app records that 0.4 has no field for, and three arrival checks for the server's validator ([validator-checks.md](validator-checks.md)). Packet 1.1, which this folder used to define, is -retired; it is still readable at `t3/packet` commit `d5439cf`. +retired as a contract proposal; that version is still readable at `t3/packet` commit `d5439cf`. + +The app still writes a packet 1.1 for its own use: the `scan.zip` that "Share scan" offers, +for looking into a scan by hand. Nothing uploads it, the server's intake does not accept it, +and it is not an approved revision of 0.4. Fields it carries that 0.4 lacks, such as the end +provenance and guidance log below, are additive local diagnostics. Each is also listed here as +a proposal for 0.5, which the contract's owners may adopt, change or decline. ## Proposals for 0.5 @@ -50,9 +56,9 @@ performance. Proposal: `taps[].attrs` with `endKind` (`limit`, `unexplored`), `operable` and `inferred`, and `taps[].group` for taps that outline one object. - The app's own packet 1.1 (the `scan.zip` behind Share scan) already writes the provenance: a - wall end the homeowner marked has its mark time `t`; an inferred one has no `t` and - `attrs.inferred: true`. A missing `t` alone never means the homeowner marked it. + The app's local packet 1.1 (above) already writes the provenance: a wall end the homeowner + marked has its mark time `t`; an inferred one has no `t` and `attrs.inferred: true`. A + missing `t` alone never means the homeowner marked it. 4. **The guidance log.** Every request the homeowner was shown (walk, tilt to the ground, mark an end, close-up, a gap the phone or the server asked for), when, and its outcome: `met`, `skipped`, `cannot_reach`, `superseded` or `unresolved`. Today "I can't get there" never @@ -60,12 +66,16 @@ performance. `guidance[]` with `id`, `kind`, `origin` (`phone`, `server`), `message`, optional `band` and `span`, `tShown`, `tResolved` and `outcome`. - For the wall's ends, as the app's packet 1.1 records them: `mark_end` is `met` only when the - homeowner marked that end. "The wall keeps going" and "Can't get there" close it as + For the wall's ends, as the app's local packet 1.1 records them: `mark_end` is `met` only + when the homeowner marked that end. "The wall keeps going" and "Can't get there" close it as `cannot_reach`, because the end lay beyond where the homeowner got, and the end placed there - is inferred. `gap_past_end` is `met` both when views past the end settle it and when the - homeowner marks the end again; the wall end's provenance tells them apart. "I can't get - there" on it is `cannot_reach`, and the end it had cleared comes back as it was. + is inferred. `gap_past_end` is `met` when views past the end settle it, or when the homeowner + marks the end again and says the wall stops there (something blocks it, or it just ends); the + wall end's provenance tells those apart. A corner the homeowner marks during the request is + not `met`: the wall goes on round it, the request can't follow it there, and the server + settles an unexplored end only with views past it. It closes as `cannot_reach`, and the + corner stays where the homeowner marked it, unexplored. "I can't get there" is `cannot_reach` + too, and the end the request had cleared comes back as it was. 5. **Depth between keyframes** (from S6's live map, draft PR #21). Keyframes are kept about every 0.5 m, but the phone fuses depth at about 10 Hz to decide what was seen. With keyframe depth alone the server cannot rebuild that decision. Proposal: optional `depthFrames[]` with `t`, From c9295df2172cb0533c607f499bacb5a61467c6c1 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 20:00:26 -0500 Subject: [PATCH 16/71] Take the first match for each identifier in the provisional-placement UI tests Three demo-answer tests failed at the scroll helper with "Multiple matching elements found" (Actions run 37162669527): Details' DisclosureGroup puts its identifier, result.details, on more than one element, and reading a frame or tapping needs exactly one. Every lookup in the file now takes the first match, as ResultCardUITests' element helper does. The engine tests, the reject test and every other suite in that run passed. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../ProvisionalPlacementUITests.swift | 62 ++++++++++--------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/ios/HouseScanUITests/ProvisionalPlacementUITests.swift b/ios/HouseScanUITests/ProvisionalPlacementUITests.swift index 5ed0e8fab..f5ce8ea0a 100644 --- a/ios/HouseScanUITests/ProvisionalPlacementUITests.swift +++ b/ios/HouseScanUITests/ProvisionalPlacementUITests.swift @@ -37,17 +37,17 @@ final class ProvisionalPlacementUITests: XCTestCase { let tryAgain = app.buttons["action.retryUpload"] XCTAssertTrue(tryAgain.waitForExistence(timeout: 300), "the unbound answer never offered Try again") tryAgain.tap() - XCTAssertTrue(any["screen.result"].waitForExistence(timeout: 120), "the passing answer never reached the result") + XCTAssertTrue(any["screen.result"].firstMatch.waitForExistence(timeout: 120), "the passing answer never reached the result") XCTAssertEqual(server.requests.count, 2) - XCTAssertEqual(any["result.headline"].label, Self.candidate) - XCTAssertFalse(any["result.spotRefused"].exists) + XCTAssertEqual(any["result.headline"].firstMatch.label, Self.candidate) + XCTAssertFalse(any["result.spotRefused"].firstMatch.exists) assertCandidateCard(app) attach(app, name: "result-candidate-engine") // The autopilot opens the AR view once the result's gate opens. try Data().write(to: gate.appending(path: "result")) - let instruction = any["instruction"] - XCTAssertTrue(any["screen.resultAR"].waitForExistence(timeout: 20)) + let instruction = any["instruction"].firstMatch + XCTAssertTrue(any["screen.resultAR"].firstMatch.waitForExistence(timeout: 20)) XCTAssertTrue(instruction.waitForExistence(timeout: 10)) XCTAssertTrue(instruction.label.contains(Self.candidate), instruction.label) XCTAssertTrue(instruction.label.contains("An installer needs to check the fit on site."), instruction.label) @@ -72,18 +72,18 @@ final class ProvisionalPlacementUITests: XCTestCase { defer { app.terminate() } let any = app.descendants(matching: .any) - XCTAssertTrue(any["screen.result"].waitForExistence(timeout: 420), "the second passing answer never reached the result") + XCTAssertTrue(any["screen.result"].firstMatch.waitForExistence(timeout: 420), "the second passing answer never reached the result") XCTAssertEqual(server.requests.count, 2, "the spot check's answer should send the scan once more") - XCTAssertEqual(any["result.headline"].label, Self.candidate) - XCTAssertTrue(any["result.spotRefused"].exists, "the settled answer's notice is missing") + XCTAssertEqual(any["result.headline"].firstMatch.label, Self.candidate) + XCTAssertTrue(any["result.spotRefused"].firstMatch.exists, "the settled answer's notice is missing") assertCandidateCard(app) attach(app, name: "result-candidate-settled-AX5") - scroll(to: any["result.candidateNote"], in: app) + scroll(to: any["result.candidateNote"].firstMatch, in: app) attach(app, name: "result-candidate-settled-AX5-note") try Data().write(to: gate.appending(path: "result")) - let instruction = any["instruction"] - XCTAssertTrue(any["screen.resultAR"].waitForExistence(timeout: 20)) + let instruction = any["instruction"].firstMatch + XCTAssertTrue(any["screen.resultAR"].firstMatch.waitForExistence(timeout: 20)) XCTAssertTrue(instruction.waitForExistence(timeout: 10)) XCTAssertTrue(instruction.label.contains(Self.candidate), instruction.label) XCTAssertTrue(instruction.label.contains("An installer needs to check the fit on site."), instruction.label) @@ -113,12 +113,12 @@ final class ProvisionalPlacementUITests: XCTestCase { app.launchArguments = ["-practiceMeter", "NO", "-uiDemo", "-uiDemoFreeze", "-uiDemoPhase", "result"] + extra app.launch() let any = app.descendants(matching: .any) - XCTAssertTrue(any["screen.result"].waitForExistence(timeout: 15)) - XCTAssertNotEqual(any["result.headline"].label, Self.candidate, "\(extra)") - let details = any["result.details"] + XCTAssertTrue(any["screen.result"].firstMatch.waitForExistence(timeout: 15)) + XCTAssertNotEqual(any["result.headline"].firstMatch.label, Self.candidate, "\(extra)") + let details = any["result.details"].firstMatch scroll(to: details, in: app) details.tap() - let summary = any["result.summary"] + let summary = any["result.summary"].firstMatch XCTAssertTrue(summary.waitForExistence(timeout: 10), "\(extra): a review answer's summary is missing") assertNoFitWords(summary.label) XCTAssertFalse(summary.label.lowercased().contains("fits"), summary.label) @@ -132,9 +132,9 @@ final class ProvisionalPlacementUITests: XCTestCase { let app = Self.launchFixture("reject-nearest", textSize: []) defer { app.terminate() } let any = app.descendants(matching: .any) - XCTAssertEqual(any["result.headline"].label, "Not on this wall") - XCTAssertFalse(any["result.candidateNote"].exists) - XCTAssertTrue(any["result.nearest"].label.hasPrefix("The closest spot")) + XCTAssertEqual(any["result.headline"].firstMatch.label, "Not on this wall") + XCTAssertFalse(any["result.candidateNote"].firstMatch.exists) + XCTAssertTrue(any["result.nearest"].firstMatch.label.hasPrefix("The closest spot")) attach(app, name: "result-reject") } @@ -143,7 +143,7 @@ final class ProvisionalPlacementUITests: XCTestCase { let app = Self.launchFixture("pass", textSize: textSize) defer { app.terminate() } let any = app.descendants(matching: .any) - XCTAssertEqual(any["result.headline"].label, Self.candidate) + XCTAssertEqual(any["result.headline"].firstMatch.label, Self.candidate) assertCandidateCard(app) let model = any.matching(NSPredicate(format: "label == '3D view of your wall'")).firstMatch XCTAssertTrue(model.waitForExistence(timeout: 10)) @@ -153,13 +153,13 @@ final class ProvisionalPlacementUITests: XCTestCase { // Details: the checks as calculations on the recorded scan, a pass without the server's // positive reason, and no server summary that says the spot fits. - let details = any["result.details"] + let details = any["result.details"].firstMatch scroll(to: details, in: app) details.tap() - let note = any["result.calculatedNote"] + let note = any["result.calculatedNote"].firstMatch XCTAssertTrue(note.waitForExistence(timeout: 10)) - XCTAssertFalse(any["result.summary"].exists, "a possible spot's summary is the server's pass") - let row = any["detail.check.gas_clearance"] + XCTAssertFalse(any["result.summary"].firstMatch.exists, "a possible spot's summary is the server's pass") + let row = any["detail.check.gas_clearance"].firstMatch XCTAssertTrue(row.exists) XCTAssertEqual(row.label, "Distance from gas equipment: Passes on recorded data", row.label) let rowValue = row.value as? String ?? "" @@ -169,12 +169,12 @@ final class ProvisionalPlacementUITests: XCTestCase { // The AR view draws the proposed spot. The demo's answer is a sample, so the AR title // says so (`ResultARScreen`); the drawing names the spot as proposed. - scroll(to: any["action.showAR"], in: app) + scroll(to: any["action.showAR"].firstMatch, in: app) app.buttons["action.showAR"].tap() - let overlay = any["ar.overlay"] + let overlay = any["ar.overlay"].firstMatch XCTAssertTrue(overlay.waitForExistence(timeout: 10)) XCTAssertEqual(overlay.label, "A proposed battery spot, drawn on your wall") - assertNoFitWords(any["instruction"].label) + assertNoFitWords(any["instruction"].firstMatch.label) attach(app, name: "\(name)-AR") } @@ -185,13 +185,13 @@ final class ProvisionalPlacementUITests: XCTestCase { @MainActor private func assertCandidateCard(_ app: XCUIApplication) { let any = app.descendants(matching: .any) - XCTAssertTrue(any["result.placement"].exists) - let note = any["result.candidateNote"] + XCTAssertTrue(any["result.placement"].firstMatch.exists) + let note = any["result.candidateNote"].firstMatch XCTAssertTrue(note.exists, "the candidate note is missing") XCTAssertTrue(note.label.contains("couldn't confirm all the space a battery needs"), note.label) let lines = any.matching(NSPredicate(format: "identifier BEGINSWITH 'check.'")) XCTAssertEqual(lines.count, 0, "a passing check reached the card") - XCTAssertFalse(any["result.installerConfirms"].exists) + XCTAssertFalse(any["result.installerConfirms"].firstMatch.exists) XCTAssertEqual(app.buttons["action.showAR"].label, "See this spot on your wall") // Details' scope note says, truthfully, that a pass is "not ... confirmed clear": the sweep // leaves that one qualified sentence out, whether or not Details is open. @@ -255,11 +255,13 @@ final class ProvisionalPlacementUITests: XCTestCase { app.launchArguments = ["-practiceMeter", "NO", "-uiDemo", "-uiDemoFreeze", "-uiDemoPhase", "result", "-uiDemoResultFile", file] + textSize app.launch() - XCTAssertTrue(app.descendants(matching: .any)["screen.result"].waitForExistence(timeout: 15)) + XCTAssertTrue(app.descendants(matching: .any)["screen.result"].firstMatch.waitForExistence(timeout: 15)) return app } /// Scrolls until the element is inside the window, a third of the window at most per drag. + /// Lookups here take the first match: a DisclosureGroup's identifier ("result.details") + /// matches more than one element, and reading a frame needs exactly one. @MainActor private func scroll(to target: XCUIElement, in app: XCUIApplication) { let window = app.windows.firstMatch.frame From a7b4ec30068f41fe8b647a998e89080b56ec905a Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 20:03:29 -0500 Subject: [PATCH 17/71] Fold the AR card at the largest text sizes while the spot is in view At AX-XXXL the AR card held the title, the placement and the on-site caveat in full and covered nearly the whole camera (run 37162669527, resultAR-candidate-settled-AX5), so "See this spot on your wall" showed little of the spot. While the phone tracks and the result has a spot, the AR screen now sets CameraChrome's existing `aims`: at the accessibility sizes the card keeps its title ("A possible battery spot", or the sample's "Example spot, not your result") and folds the rest under Details, leaving at least the chrome's 180 pt camera window. "Point at your meter" while tracking is lost keeps every word in view. CameraChrome itself is unchanged. The settled engine test, at AX-XXXL, checks the folded card, measures the open camera between it and Done, opens Details to read the placement and caveat, folds it again, measures again, and returns with Done, with screenshots of each state. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/UI/Screens/ResultARScreen.swift | 13 +++++- .../ProvisionalPlacementUITests.swift | 44 +++++++++++++++++-- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/ios/HouseScan/UI/Screens/ResultARScreen.swift b/ios/HouseScan/UI/Screens/ResultARScreen.swift index 14bb00bbc..fa37ca3b5 100644 --- a/ios/HouseScan/UI/Screens/ResultARScreen.swift +++ b/ios/HouseScan/UI/Screens/ResultARScreen.swift @@ -41,7 +41,8 @@ struct ResultARScreen: View { CameraChrome( instruction: instruction, isReplay: state.isReplay, - isAutopilot: state.isAutopilot + isAutopilot: state.isAutopilot, + aims: showsSpot ) { Button("Done") { actions.closeAR() } .buttonStyle(.primary) @@ -53,6 +54,16 @@ struct ResultARScreen: View { } } + /// Whether the spot is on the wall in view: the phone tracks and the result has a spot. Then + /// the homeowner is looking at the spot, so at the accessibility text sizes the card folds + /// its detail (the placement, and what the fit still needs) under Details and leaves the + /// camera open (`CameraChrome.aims`). At AX-XXXL the full card covered nearly the whole + /// camera (run 37162669527, resultAR-candidate-settled-AX5). "Point at your meter" while + /// tracking is lost keeps every word in view: it is what to do, and nothing is drawn yet. + private var showsSpot: Bool { + state.tracking == .normal && state.result?.spot != nil + } + private static func overlayLabel(_ result: ResultPresentation) -> String { guard result.spot != nil else { return "The cable run and clearances, drawn on your wall" } return result.spotIsClean diff --git a/ios/HouseScanUITests/ProvisionalPlacementUITests.swift b/ios/HouseScanUITests/ProvisionalPlacementUITests.swift index f5ce8ea0a..c5e72094d 100644 --- a/ios/HouseScanUITests/ProvisionalPlacementUITests.swift +++ b/ios/HouseScanUITests/ProvisionalPlacementUITests.swift @@ -81,14 +81,50 @@ final class ProvisionalPlacementUITests: XCTestCase { scroll(to: any["result.candidateNote"].firstMatch, in: app) attach(app, name: "result-candidate-settled-AX5-note") + // The AR view at AX-XXXL: the card folds to its title and Details, so the camera stays + // open on the spot; Details holds the placement and what the fit still needs. try Data().write(to: gate.appending(path: "result")) let instruction = any["instruction"].firstMatch + let details = any["instruction.details"].firstMatch + let detail = any["instruction.detail"].firstMatch + let done = app.buttons["action.closeAR"] XCTAssertTrue(any["screen.resultAR"].firstMatch.waitForExistence(timeout: 20)) XCTAssertTrue(instruction.waitForExistence(timeout: 10)) - XCTAssertTrue(instruction.label.contains(Self.candidate), instruction.label) - XCTAssertTrue(instruction.label.contains("An installer needs to check the fit on site."), instruction.label) - XCTAssertTrue(app.buttons["action.closeAR"].isHittable, "Done is out of reach under the AR instruction") - attach(app, name: "resultAR-candidate-settled-AX5") + XCTAssertTrue(details.waitForExistence(timeout: 10), "the AR card didn't fold at the largest text size") + XCTAssertEqual(instruction.label, Self.candidate) + XCTAssertFalse(detail.exists) + assertOpenCamera(below: details, above: done, in: app, "folded") + attach(app, name: "resultAR-candidate-settled-AX5-folded") + + details.tap() + XCTAssertTrue(detail.waitForExistence(timeout: 5), "Details didn't open the placement and caveat") + XCTAssertTrue(detail.label.contains("right of your meter"), detail.label) + XCTAssertTrue(detail.label.contains("An installer needs to check the fit on site."), detail.label) + assertNoFitWords(detail.label) + attach(app, name: "resultAR-candidate-settled-AX5-details") + + details.tap() + XCTAssertTrue(detail.waitForNonExistence(timeout: 5), "Details didn't close") + assertOpenCamera(below: details, above: done, in: app, "folded again") + attach(app, name: "resultAR-candidate-settled-AX5-refolded") + XCTAssertTrue(done.isHittable) + done.tap() + XCTAssertTrue(any["screen.result"].firstMatch.waitForExistence(timeout: 10), "Done didn't return to the result") + } + + /// The camera between the folded card and Done is at least the chrome's least open window + /// (`CameraChromeLayout.minCameraWindow`, 180 pt), measured from the elements' frames, not + /// inferred from Done being hittable. + @MainActor + private func assertOpenCamera(below card: XCUIElement, above done: XCUIElement, in app: XCUIApplication, _ state: String, + file: StaticString = #filePath, line: UInt = #line) { + let open = done.frame.minY - card.frame.maxY + XCTAssertGreaterThanOrEqual(open, 180, "\(state): \(open) pt of camera between the card (\(card.frame)) and Done (\(done.frame))", + file: file, line: line) + let note = XCTAttachment(string: "\(state): open camera \(open) pt, card bottom \(card.frame.maxY), Done top \(done.frame.minY), window \(app.windows.firstMatch.frame)") + note.name = "resultAR-open-camera-\(state)" + note.lifetime = .keepAlways + add(note) } // MARK: Server answers in Fixtures/results From 0fb7380ce4a9da5190aca1412719962f573d25b1 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 20:06:21 -0500 Subject: [PATCH 18/71] Measure the AR camera window from the card's bottom, clipped to the screen The open-camera check measured from the Details label, which leaves out the card's padding under it, and could count space off screen (Sol review of a7b4ec30). It now adds that padding, clips both edges to the window, and requires Done to be hittable before measuring. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../ProvisionalPlacementUITests.swift | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/ios/HouseScanUITests/ProvisionalPlacementUITests.swift b/ios/HouseScanUITests/ProvisionalPlacementUITests.swift index c5e72094d..5e84bef0c 100644 --- a/ios/HouseScanUITests/ProvisionalPlacementUITests.swift +++ b/ios/HouseScanUITests/ProvisionalPlacementUITests.swift @@ -112,16 +112,22 @@ final class ProvisionalPlacementUITests: XCTestCase { XCTAssertTrue(any["screen.result"].firstMatch.waitForExistence(timeout: 10), "Done didn't return to the result") } - /// The camera between the folded card and Done is at least the chrome's least open window - /// (`CameraChromeLayout.minCameraWindow`, 180 pt), measured from the elements' frames, not - /// inferred from Done being hittable. + /// The open camera between the folded card and Done, measured from the elements' frames and + /// clipped to the window, is at least the chrome's least open camera window + /// (`CameraChromeLayout.minCameraWindow`, 180 pt). The card's bottom is its Details label + /// plus the card's padding under it (8 pt, `InstructionCard.details`), so the area isn't + /// overstated; Done must be on screen and hittable first. @MainActor - private func assertOpenCamera(below card: XCUIElement, above done: XCUIElement, in app: XCUIApplication, _ state: String, + private func assertOpenCamera(below details: XCUIElement, above done: XCUIElement, in app: XCUIApplication, _ state: String, file: StaticString = #filePath, line: UInt = #line) { - let open = done.frame.minY - card.frame.maxY - XCTAssertGreaterThanOrEqual(open, 180, "\(state): \(open) pt of camera between the card (\(card.frame)) and Done (\(done.frame))", + XCTAssertTrue(done.isHittable, "\(state): Done isn't reachable", file: file, line: line) + let window = app.windows.firstMatch.frame + let top = max(details.frame.maxY + 8, window.minY) + let bottom = min(done.frame.minY, window.maxY) + let open = bottom - top + XCTAssertGreaterThanOrEqual(open, 180, "\(state): \(open) pt of camera between the card (bottom \(top)) and Done (top \(bottom))", file: file, line: line) - let note = XCTAttachment(string: "\(state): open camera \(open) pt, card bottom \(card.frame.maxY), Done top \(done.frame.minY), window \(app.windows.firstMatch.frame)") + let note = XCTAttachment(string: "\(state): open camera \(open) pt, card bottom \(top), Done top \(bottom), window \(window)") note.name = "resultAR-open-camera-\(state)" note.lifetime = .keepAlways add(note) From 21ac61ec5776bae4081d2031c649d394318ee57b Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 21:17:08 -0500 Subject: [PATCH 19/71] ios: at the largest text sizes, the wall-end card folds so the circle stays in view UNVALIDATED: not compiled; no package, app or UI tests have run. CI run 37165788075 on 15cf658e (iPhone 17) showed the card covering the circle in the middle of the camera at AX5: on the walk's "Is this the right end of the wall?" (`wallWalk-markEnd-AX5-stacked`) and on a past_end request's offer and refusal (`gapRequest-pastEnd-AX5`, `-refused-AX5`). "Wall ends here" was reachable, but the circle it marks at was hidden. Folding alone didn't fix it: in the folded walk frame a three-line title, Details and the card's reply still ended at 490 pt, past the circle's top at 409 pt (874 pt screen, 56 pt circle at its middle). So, on those two steps at the accessibility sizes, using each screen's existing `aims` and `Instruction.folded`, with no change to CameraChrome or InstructionCard: - The card leads with two lines: "Aim at the right end"; "Keep walking right" on a past_end request; and a refusal's correction: "Aim at the wall", "Turn to the right end", "Wait a moment", "Too short to use". Every word of the unfolded card is under Details (`ScanCopy.endAimFold`). - The reply ("The wall keeps going", "I can't get there") moves from the card to the actions under "Wall ends here". A past_end request's "One more view to finish" leads the words under Details. - Refusals fold too: before, they kept every word and covered the circle. - The default sizes are unchanged. The circle is an accessibility element ("Aiming circle", `aim.circle`), at the call site rather than in `Reticle`. UI tests read its real frame and check that it is the middle of the window, where `circleEnd`'s ray goes; that it is wholly on screen; and that it is clear of the card and every action wherever "Wall ends here" can be pressed. They cover default and AX5 on the walk's end and the past_end offer and refusal, a left past_end request (`-uiDemoGap pastEndLeft`), Details opening every word and closing back to an open circle, the decline's new place, and the end question keeping its three answers in order with no circle. The demo's past_end request now carries its answer and eyebrow, as the real one does. Not tested: Reduce Motion (no hook can force it in a UI test; the moved decline joins the end controls' existing animation, which is off under Reduce Motion), VoiceOver speech, physical aiming, and screens other than the iPhone 17 at AX5. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../UI/Components/EndAimCircle.swift | 16 +++ ios/HouseScan/UI/Copy/ScanCopy.swift | 44 ++++++-- ios/HouseScan/UI/Preview/DemoEngine.swift | 31 ++++-- .../UI/Screens/GapRequestScreen.swift | 51 ++++++++- ios/HouseScan/UI/Screens/WallWalkScreen.swift | 37 +++++-- .../EndAimCircleAssertions.swift | 36 +++++++ .../PastEndRecoveryUITests.swift | 101 ++++++++++++++++-- .../WalkRecoveryUITests.swift | 85 +++++++++++++++ 8 files changed, 367 insertions(+), 34 deletions(-) create mode 100644 ios/HouseScan/UI/Components/EndAimCircle.swift create mode 100644 ios/HouseScanUITests/EndAimCircleAssertions.swift diff --git a/ios/HouseScan/UI/Components/EndAimCircle.swift b/ios/HouseScan/UI/Components/EndAimCircle.swift new file mode 100644 index 000000000..810649c08 --- /dev/null +++ b/ios/HouseScan/UI/Components/EndAimCircle.swift @@ -0,0 +1,16 @@ +import SwiftUI + +extension View { + /// The circle a mark lands under (`Reticle`), as one accessibility element: "Aiming circle". + /// The reticle itself is hidden from VoiceOver as decoration. Here it says where "Wall ends + /// here" and the other marks land, and UI tests read its real frame to check that nothing + /// covers it (`PastEndRecoveryUITests`, `ScreenStatesUITests`). Apply it before the reticle + /// is stretched to the screen, so the element's frame is the circle's. `shown` false while + /// the circle is faded out, so VoiceOver doesn't stop on what isn't there. + func endAimCircle(shown: Bool = true) -> some View { + accessibilityElement(children: .ignore) + .accessibilityLabel("Aiming circle") + .accessibilityIdentifier("aim.circle") + .accessibilityHidden(!shown) + } +} diff --git a/ios/HouseScan/UI/Copy/ScanCopy.swift b/ios/HouseScan/UI/Copy/ScanCopy.swift index b981b38c6..a2dea8332 100644 --- a/ios/HouseScan/UI/Copy/ScanCopy.swift +++ b/ios/HouseScan/UI/Copy/ScanCopy.swift @@ -71,7 +71,8 @@ enum ScanCopy { case .markEnd(let side): Instruction( title: "Is this the \(side.rawValue) end of the wall?", - detail: "Aim where it stops or turns a corner and tap Wall ends here. If it goes on, tap The wall keeps going." + detail: "Aim where it stops or turns a corner and tap Wall ends here. If it goes on, tap The wall keeps going.", + folded: endAimFold("Aim at the \(side.rawValue) end", "Is this the \(side.rawValue) end of the wall? Aim where it stops or turns a corner and tap Wall ends here. If it goes on, tap The wall keeps going.") ) case .aimAtGround(let s): // A cell counts once seen from two places at least 0.25 m apart (`coveringBaseline`), @@ -389,23 +390,46 @@ enum ScanCopy { static let wallKeepsGoing = "The wall keeps going" /// Why "Wall ends here" marked nothing, and what to do instead (`EndMarkRefusal`). + /// Folded at the largest text sizes, each refusal leads with what to do about it, and its + /// reason and the full words go under Details (`endAimFold`). static func endMarkRefusal(_ refusal: EndMarkRefusal, asked side: WallSide) -> Instruction { switch refusal { case .noWall: - Instruction(title: "The circle isn't on the wall", detail: "Aim it at the wall where it stops or turns, then tap Wall ends here.") + Instruction( + title: "The circle isn't on the wall", detail: "Aim it at the wall where it stops or turns, then tap Wall ends here.", + folded: endAimFold("Aim at the wall", "The circle isn't on the wall. Aim it at the wall where it stops or turns, then tap Wall ends here.")) case .otherSide(let landed): - Instruction(title: "That's the \(landed.rawValue) side of your meter", detail: "Turn to the \(side.rawValue) end, then tap Wall ends here.") + Instruction( + title: "That's the \(landed.rawValue) side of your meter", detail: "Turn to the \(side.rawValue) end, then tap Wall ends here.", + folded: endAimFold("Turn to the \(side.rawValue) end", "That's the \(landed.rawValue) side of your meter. Turn to the \(side.rawValue) end, then tap Wall ends here.")) case .trackingNotReady: - Instruction(title: "One moment, your phone is still finding its place", detail: "Then aim at the \(side.rawValue) end.") + Instruction( + title: "One moment, your phone is still finding its place", detail: "Then aim at the \(side.rawValue) end.", + folded: endAimFold("Wait a moment", "Your phone is still finding its place. Then aim at the \(side.rawValue) end.")) case .tooLittleWall: // Says what the app can't do, not where to aim: the homeowner must not be nudged to // mark an end farther than the wall really goes. "I can't get there" stays the way on. // The app's own minimum (`WallFrame.minWallLength`), not a measured installation rule: - // no reason is given for it, and rule distances belong to the server. - Instruction(title: "House Scan can't use a wall that short", detail: "If this is where the wall ends, tap I can't get there to continue.") + // no reason is given for it, and rule distances belong to the server. Folded, the lead + // says only that; aiming elsewhere is right only if the circle was off the real end, + // so it doesn't lead. + Instruction( + title: "House Scan can't use a wall that short", detail: "If this is where the wall ends, tap I can't get there to continue.", + folded: endAimFold("Too short to use", "House Scan can't use a wall that short. If this is where the wall ends, tap I can't get there to continue.")) } } + /// The words of a step that marks the wall's end at the circle, folded at the largest text + /// sizes (`InstructionCard.foldsDetail`): a lead short enough to stay two lines, even beside + /// the refusal triangle, and every word of the unfolded card under Details. Unfolded, the + /// card covered the circle in the middle of the camera at AX5 (CI run 37165788075, iPhone + /// 17): the circle's top is 28 pt above the middle, and a two-line card without its reply + /// ends above it, where one with a third line or the reply doesn't (`GapRequestScreen`, + /// `WallWalkScreen` move the reply to their actions at those sizes). + static func endAimFold(_ lead: String, _ everything: String) -> Instruction.Folded { + Instruction.Folded(title: lead, detail: everything) + } + static func nextWallRefusal(_ refusal: NextWallRefusal) -> Instruction { switch refusal { case .noSurface: Instruction(title: "No wall under the circle", detail: "Step closer and aim at the next wall.") @@ -462,8 +486,12 @@ enum ScanCopy { let title = gap.band == .ground ? "Show the ground \(stretch)" : "Show the wall \(stretch)" // A past_end request asks to walk on past an end; the wall may really stop before // that, and the screen offers the walk's own "Wall ends here" for it (B-12). - guard gap.pastEndSide != nil else { return Instruction(title: title, detail: detail) } - return Instruction(title: title, detail: "\(detail) \(pastEndAlternative)") + guard let side = gap.pastEndSide else { return Instruction(title: title, detail: detail) } + // Folded, walking on leads, as the request asks; marking the end is under Details, + // with the circle and "Wall ends here" in view. + return Instruction( + title: title, detail: "\(detail) \(pastEndAlternative)", + folded: endAimFold("Keep walking \(side.rawValue)", "\(title). \(detail) \(pastEndAlternative)")) case .groundOut(let out): return Instruction( title: "Show the ground out to about \(Distance.feetAtLeast(out)) from the wall", diff --git a/ios/HouseScan/UI/Preview/DemoEngine.swift b/ios/HouseScan/UI/Preview/DemoEngine.swift index 77d39f6db..2a943ef2e 100644 --- a/ios/HouseScan/UI/Preview/DemoEngine.swift +++ b/ios/HouseScan/UI/Preview/DemoEngine.swift @@ -26,7 +26,8 @@ final class DemoEngine: ScanActions { /// `-uiDemoUnusableAnswer `: how many answers in a row House Scan can't use; 0 when off. private let unusableAnswers: Int /// Which request the gap screen shows (`-uiDemoGap`); the phone's ground request by default. - /// `pastEnd` is a server past_end request on the right, with "Wall ends here" (B-12). + /// `pastEnd` is a server past_end request on the right, with "Wall ends here" (B-12); + /// `pastEndLeft` the same on the left. private let gapKind: String? /// `-uiDemoEndMarkRefusal`: "Wall ends here" was pressed with the circle off the wall, and the /// demo's circle stays off it, so pressing again refuses again. @@ -422,25 +423,33 @@ final class DemoEngine: ScanActions { state.gap?.band = .wall state.target = DemoScene.wall.world(s: 1.7, height: 3.0) state.path = [] - case "pastEnd": - // The check asks to walk on past the right end, which it clears, as the real engine's + case "pastEnd", "pastEndLeft": + // The check asks to walk on past an end, which it clears, as the real engine's // `beginServerGap` does; the wall may stop sooner, so "Wall ends here" is offered. - let end = demoRightEnd - let past: ClosedRange = end...(end + 0.6) - state.wall?.rightEnd = nil + let side: WallSide = gapKind == "pastEndLeft" ? .left : .right + let sign: Float = side == .right ? 1 : -1 + let end = side == .right ? demoRightEnd : demoLeftEnd + let past: ClosedRange = side == .right ? end...(end + 0.6) : (end - 0.6)...end + if side == .right { state.wall?.rightEnd = nil } else { state.wall?.leftEnd = nil } state.gap?.origin = .server - state.gap?.reason = .server(detail: "Keep walking past the right end of the scan (14 ft 1 in right of the meter): a spot within reach may be there.") + state.gap?.reason = .server(detail: side == .right + ? "Keep walking past the right end of the scan (14 ft 1 in right of the meter): a spot within reach may be there." + : "Keep walking past the left end of the scan (9 ft 6 in left of the meter): a spot within reach may be there.") state.gap?.span = past - state.gap?.pastEndSide = .right - state.target = DemoScene.wall.world(s: end + 0.3, height: 0, out: 0.5) - state.path = DemoScene.path(toward: end + 0.3) + state.gap?.pastEndSide = side + // The check sends a past_end request with its answer, so the card says "One more view + // to finish" over it, as the real engine's does (`GapRequestScreen.followUps`). + state.result = sample + state.followUps = 1 + state.target = DemoScene.wall.world(s: end + sign * 0.3, height: 0, out: 0.5) + state.path = DemoScene.path(toward: end + sign * 0.3) if endMarkRefused { // As in the real engine, a circle off the wall has no end preview (`circleEnd`). state.endMarkRefusal = .noWall state.endPreview = nil } else { // The circle on the wall a little before the old end: where the wall really stops. - state.endPreview = EndPreview(side: .right, s: end - 0.4, atReticle: true) + state.endPreview = EndPreview(side: side, s: end - sign * 0.4, atReticle: true) } return default: diff --git a/ios/HouseScan/UI/Screens/GapRequestScreen.swift b/ios/HouseScan/UI/Screens/GapRequestScreen.swift index 39d635e00..d0aa2b79e 100644 --- a/ios/HouseScan/UI/Screens/GapRequestScreen.swift +++ b/ios/HouseScan/UI/Screens/GapRequestScreen.swift @@ -38,6 +38,7 @@ struct GapRequestScreen: View { // Motion only its opacity changes, briefly: nothing moves. if state.gap?.pastEndSide != nil { Reticle(diameter: 56) + .endAimCircle(shown: offersEndMark) .frame(maxWidth: .infinity, maxHeight: .infinity) .ignoresSafeArea() .opacity(offersEndMark ? 1 : 0) @@ -52,7 +53,7 @@ struct GapRequestScreen: View { CameraChrome( instruction: instruction, tone: tone, - reply: state.gap?.isSatisfied == true || asking || askingEnd ? nil : InstructionCard.Reply( + reply: state.gap?.isSatisfied == true || asking || askingEnd || endAimFolds ? nil : InstructionCard.Reply( title: "I can't get there", identifier: "action.skipGap", hint: skipHint, @@ -61,7 +62,7 @@ struct GapRequestScreen: View { // as the phone moves: neither locks the reply again (InstructionCard.replyLock). task: state.gap.map { ScanCopy.gapTask($0) } ), - eyebrow: followUps > 0 && state.gap?.isSatisfied != true ? ScanCopy.followUp(remaining: followUps) : nil, + eyebrow: followUps > 0 && state.gap?.isSatisfied != true && !endAimFolds ? ScanCopy.followUp(remaining: followUps) : nil, photoCount: state.captureCount, lastCaptureID: state.lastCapture?.id, isReplay: state.isReplay, @@ -85,6 +86,9 @@ struct GapRequestScreen: View { if offersEndMark { markEndButton } + if endAimFolds { + declineButton + } } .animation(reduceMotion ? nil : Motion.settle, value: askingEnd) .animation(reduceMotion ? nil : Motion.settle, value: offersEndMark) @@ -174,6 +178,32 @@ struct GapRequestScreen: View { .transition(.opacity) } + /// "I can't get there" among the actions, under "Wall ends here", while the card folds around + /// the circle (`endAimFolds`): the same words, hint and action as the card's reply. + private var declineButton: some View { + Button { + actions.skipGap() + } label: { + Text("I can't get there") + .frame(maxWidth: .infinity) + } + .buttonStyle(.secondary) + .accessibilityHint(skipHint) + .accessibilityIdentifier("action.skipGap") + .transition(.opacity) + } + + /// At the largest text sizes while "Wall ends here" is offered, the card folds to a two-line + /// lead, as on the walk's end, so the circle in the middle of the camera stays open: the + /// request's words and its "One more view to finish" go under Details, and "I can't get + /// there" moves from the card to the actions. With either still on the card it reached past + /// the circle (`ScanCopy.endAimFold`). Not while coaching replaces the request: then the card + /// says why nothing can be aimed yet. + private var endAimFolds: Bool { + guard offersEndMark, typeSize.isAccessibilitySize else { return false } + return !(coaching.map(ScanCopy.coachingReplacesTask) ?? false) + } + /// A refusal on "Wall ends here" keeps its red triangle, as on the walk; otherwise coaching /// marks the card. private var tone: InstructionCard.Tone { @@ -212,9 +242,13 @@ struct GapRequestScreen: View { /// needs" reading or "tap I can't get there", tilting up to the roof or sky, and a server's /// own words, which this screen can't judge. Nor the overhead question, a request already /// seen, or coaching that replaces the request. + /// + /// A past_end request offering "Wall ends here" aims too, refusal or not (`endAimFolds`). private var aims: Bool { - guard let gap = state.gap, !asking, !askingEnd, refusal == nil, !gap.isSatisfied else { return false } + guard let gap = state.gap, !asking, !askingEnd, !gap.isSatisfied else { return false } if let coaching, ScanCopy.coachingReplacesTask(coaching) { return false } + if offersEndMark { return true } + guard refusal == nil else { return false } if case .groundNearCandidate = gap.reason { return true } return false } @@ -224,6 +258,17 @@ struct GapRequestScreen: View { /// can stay up for a whole night request, and replacing the card hid the request and "Got it, /// thanks" for as long as it did (field test 4.1, run 3). private var instruction: Instruction { + var card = requestCard + // Folded around the circle, "One more view to finish" leaves the card's top (`endAimFolds`) + // and leads the words under Details. + if endAimFolds, followUps > 0, let folded = card.folded { + card.folded?.detail = "\(ScanCopy.followUp(remaining: followUps)). \(folded.detail)" + } + return card + } + + /// The card's words before folding around the circle moves the eyebrow (`instruction`). + private var requestCard: Instruction { if asking { return ScanCopy.overheadQuestion } if askingEnd, let side = state.endQuestion { return ScanCopy.endQuestion(side) } if let coaching, ScanCopy.coachingReplacesTask(coaching) { return ScanCopy.coaching(coaching, meterPhoto: meterPhoto != nil) } diff --git a/ios/HouseScan/UI/Screens/WallWalkScreen.swift b/ios/HouseScan/UI/Screens/WallWalkScreen.swift index ae42499a6..7c29e652f 100644 --- a/ios/HouseScan/UI/Screens/WallWalkScreen.swift +++ b/ios/HouseScan/UI/Screens/WallWalkScreen.swift @@ -34,6 +34,7 @@ struct WallWalkScreen: View { // asks for it) lands under the circle. if state.marking != nil || controlsKey == .nextWall || controlsKey == .markEnd { Reticle(diameter: 56) + .endAimCircle() .frame(maxWidth: .infinity, maxHeight: .infinity) .ignoresSafeArea() TapRipple(ripples: $taps) @@ -136,21 +137,32 @@ struct WallWalkScreen: View { /// place, and a step to the side moves into the title when it is needed (`ScanCopy.aim`). /// Looking past an obstruction folds the other way: its action ("Look around it") leads and /// the situation and how go under Details (`Instruction.folded`), since unfolded at AX5 it - /// covered the camera and the spot it asks about. Not where the second line is the action - /// and nothing leads with it: a step back added to an aim step, the wall's end, whose title - /// only asks ("Aim where the wall stops"), marking the next wall. Nor a question, a mark, the - /// tray, coaching that replaces the task, or a refusal. + /// covered the camera and the spot it asks about. The wall's end folds the same way, around + /// the circle (`endAimFolds`). Not where the second line is the action and nothing leads with + /// it: a step back added to an aim step, marking the next wall. Nor a question, a mark, the + /// tray, coaching that replaces the task, or a refusal other than the end's. private var aims: Bool { switch controlsKey { case .walking, .finish: if let coaching, ScanCopy.coachingReplacesTask(coaching) { return false } if state.guidanceHint?.stepBack == true { return false } return tone != .refusal - case .markEnd, .nextWall, .marking, .endScanQuestion, .endQuestion, .overheadQuestion, .nextWallConfirm, .tray: + case .markEnd: + // Folds around the circle, refusal or not (`endAimFolds`). + if let coaching, ScanCopy.coachingReplacesTask(coaching) { return false } + return true + case .nextWall, .marking, .endScanQuestion, .endQuestion, .overheadQuestion, .nextWallConfirm, .tray: return false } } + /// At the largest text sizes on "Is this the right end of the wall?", the card folds to a + /// two-line lead ("Aim at the right end", or a refusal's correction) so the circle in the + /// middle of the camera stays open, and "The wall keeps going" moves from the card to the + /// actions under "Wall ends here": with it on the card, or the unfolded words, the card + /// reached past the circle (`ScanCopy.endAimFold`). As on a past_end request. + private var endAimFolds: Bool { controlsKey == .markEnd && aims && typeSize.isAccessibilitySize } + // MARK: Controls private enum ControlsKey: Hashable { @@ -338,6 +350,19 @@ struct WallWalkScreen: View { .accessibilityHint("Marks the end of the wall at the circle in the middle of the screen") .accessibilityIdentifier("action.markEnd") .transition(.opacity) + if endAimFolds, let copy = ScanCopy.reply(for: state.guidance) { + // The card's reply, moved here while the card folds around the circle. + Button { + actions.cannotAccessArea() + } label: { + Text(copy.title) + .spansStack(stacksActions) + } + .buttonStyle(.secondary) + .accessibilityHint(copy.hint) + .accessibilityIdentifier("action.cannotAccess") + .transition(.opacity) + } case .walking where offersEndHere: Button { actions.endWallHere() @@ -403,7 +428,7 @@ struct WallWalkScreen: View { /// every step. private var reply: InstructionCard.Reply? { guard state.marking == nil, !state.endScanQuestion, state.endQuestion == nil, !state.overheadQuestion, state.nextWallConfirm == nil, !coachingHidesReply, !trayOpen, - let copy = ScanCopy.reply(for: state.guidance) else { return nil } + !endAimFolds, let copy = ScanCopy.reply(for: state.guidance) else { return nil } return InstructionCard.Reply( title: copy.title, identifier: "action.cannotAccess", diff --git a/ios/HouseScanUITests/EndAimCircleAssertions.swift b/ios/HouseScanUITests/EndAimCircleAssertions.swift new file mode 100644 index 000000000..28d02767c --- /dev/null +++ b/ios/HouseScanUITests/EndAimCircleAssertions.swift @@ -0,0 +1,36 @@ +import XCTest + +/// The circle "Wall ends here" marks at (`endAimCircle`, identifier `aim.circle`), read from the +/// production layout: the app's own view, as VoiceOver sees it, not a test-only overlay. +extension XCTestCase { + /// Checks the circle is drawn in the middle of the window, where the ray "Wall ends here" uses + /// goes through (`ScanEngine.circleEnd`: the middle of the sensor image, which the camera view + /// fills and centres), is wholly on screen, and is under none of `covers`: the card's words, + /// its Details, and the actions. The card's scrim runs `cardPadding` below the last of its + /// parts, so the circle must clear that too. + @MainActor + func assertEndCircleOpen( + _ app: XCUIApplication, covers: [String], _ context: String, + file: StaticString = #filePath, line: UInt = #line + ) { + let window = app.windows.firstMatch.frame + let circle = app.descendants(matching: .any)["aim.circle"].firstMatch + XCTAssertTrue(circle.waitForExistence(timeout: 5), "\(context): no aiming circle", file: file, line: line) + let frame = circle.frame + XCTAssertEqual(frame.midX, window.midX, accuracy: 1, "\(context): the circle \(frame) must be the middle of the camera", file: file, line: line) + XCTAssertEqual(frame.midY, window.midY, accuracy: 1, "\(context): the circle \(frame) must be the middle of the camera", file: file, line: line) + XCTAssertTrue(window.contains(frame), "\(context): the circle \(frame) is not wholly on screen", file: file, line: line) + let card = ["instruction", "instruction.details"].map { app.descendants(matching: .any)[$0].firstMatch }.filter(\.exists) + if let cardBottom = card.map(\.frame.maxY).max() { + XCTAssertGreaterThanOrEqual(frame.minY, cardBottom + Self.cardPadding, "\(context): the circle \(frame) is under the card, which ends at \(cardBottom + Self.cardPadding)", file: file, line: line) + } + for identifier in covers { + let cover = app.descendants(matching: .any)[identifier].firstMatch + guard cover.exists else { continue } + XCTAssertFalse(frame.intersects(cover.frame), "\(context): the circle \(frame) is under \(identifier) \(cover.frame)", file: file, line: line) + } + } + + /// The card's scrim below Details without a reply (`InstructionCard.details`, bottom padding). + static var cardPadding: CGFloat { 8 } +} diff --git a/ios/HouseScanUITests/PastEndRecoveryUITests.swift b/ios/HouseScanUITests/PastEndRecoveryUITests.swift index ccd9a7894..c683efba1 100644 --- a/ios/HouseScanUITests/PastEndRecoveryUITests.swift +++ b/ios/HouseScanUITests/PastEndRecoveryUITests.swift @@ -32,21 +32,74 @@ final class PastEndRecoveryUITests: XCTestCase { try offersWallEndsHere(textSize: Self.largestText, name: "gapRequest-pastEnd-AX5") } + @MainActor + func testAPastEndRequestOnTheLeftKeepsTheCircleOpenAtLargestTextSize() throws { + let app = launch(["-uiDemoGap", "pastEndLeft"] + Self.largestText) + let card = label(app, "instruction") + XCTAssertTrue(card.contains("Keep walking left"), "the folded lead names the side: \(card)") + let mark = app.buttons["action.markEnd"] + XCTAssertTrue(mark.waitForExistence(timeout: 5)) + scrollIntoView(mark, in: app) + XCTAssertTrue(mark.isHittable, "Wall ends here can't be reached") + assertEndCircleOpen(app, covers: Self.endCovers, "left past-end, AX5") + snap(app, "gapRequest-pastEndLeft-AX5") + } + @MainActor private func offersWallEndsHere(textSize: [String], name: String) throws { + let folded = !textSize.isEmpty let app = launch(["-uiDemoGap", "pastEnd"] + textSize) let card = label(app, "instruction") - XCTAssertTrue(card.contains("Keep walking past the right end"), "the request's own words: \(card)") - XCTAssertTrue(card.contains("If the wall stops sooner, aim where it stops and tap Wall ends here."), "card reads: \(card)") + if folded { + // Folded around the circle: walking on leads; the request's words, the eyebrow and + // the way to mark the end are under Details. + XCTAssertTrue(card.contains("Keep walking right"), "card reads: \(card)") + XCTAssertFalse(card.contains("If the wall stops sooner"), "the how-to words must fold under Details: \(card)") + XCTAssertFalse(card.contains("One more view to finish"), "the eyebrow must fold under Details: \(card)") + } else { + XCTAssertTrue(card.contains("One more view to finish"), "card reads: \(card)") + XCTAssertTrue(card.contains("Keep walking past the right end"), "the request's own words: \(card)") + XCTAssertTrue(card.contains("If the wall stops sooner, aim where it stops and tap Wall ends here."), "card reads: \(card)") + } XCTAssertTrue(element(app, "action.skipGap").exists, "\"I can't get there\" stays the decline") let mark = app.buttons["action.markEnd"] XCTAssertTrue(mark.waitForExistence(timeout: 5), "a past-end request must offer Wall ends here") XCTAssertEqual(mark.label, "Wall ends here") scrollIntoView(mark, in: app) XCTAssertTrue(mark.isHittable, "Wall ends here can't be reached") + // Where "Wall ends here" can be pressed, the circle it marks at is in view. + assertEndCircleOpen(app, covers: Self.endCovers, name) snap(app, name) + guard folded else { return } + + // Details opens every word of the unfolded card, and closes back to the open circle. + scrollToTop(app) + let details = app.buttons["instruction.details"] + XCTAssertTrue(details.waitForExistence(timeout: 5), "the folded card needs Details") + tapWhenReady(details) + let detail = element(app, "instruction.detail") + XCTAssertTrue(detail.waitForExistence(timeout: 5), "Details must open the folded words") + for words in ["One more view to finish.", "Show the ground", "Keep walking past the right end", "If the wall stops sooner, aim where it stops and tap Wall ends here."] { + XCTAssertTrue(detail.label.contains(words), "Details must hold \"\(words)\": \(detail.label)") + } + snap(app, "\(name)-details") + tapWhenReady(details) + XCTAssertTrue(waitForAbsence(detail), "Details must close again") + scrollIntoView(mark, in: app) + XCTAssertTrue(mark.isHittable) + assertEndCircleOpen(app, covers: Self.endCovers, "\(name), Details closed") + // The decline moved from the card to the actions, and stays within reach. + let decline = app.buttons["action.skipGap"] + scrollIntoView(decline, in: app) + XCTAssertTrue(decline.isHittable, "I can't get there can't be reached") + XCTAssertEqual(decline.label, "I can't get there") + XCTAssertGreaterThan(decline.frame.minY, mark.frame.maxY - 0.5, "the decline sits under Wall ends here") } + /// What may cover the circle: the actions and the card's reply. The card itself is checked by + /// its bottom edge (`assertEndCircleOpen`). + private static let endCovers = ["action.markEnd", "action.skipGap", "action.showResult", "action.endCorner", "action.endBlocked", "action.endEnds"] + /// The circle off the wall: the card says why and what to do, the decline stays, and pressing /// again with the circle still off the wall refuses again rather than marking anything. @MainActor @@ -61,18 +114,33 @@ final class PastEndRecoveryUITests: XCTestCase { @MainActor private func refusal(textSize: [String], name: String) throws { + let folded = !textSize.isEmpty let app = launch(["-uiDemoGap", "pastEnd", "-uiDemoEndMarkRefusal"] + textSize) + // Folded, the correction leads and the reason goes under Details. + let says = folded ? "Aim at the wall" : "The circle isn't on the wall" var card = label(app, "instruction") - XCTAssertTrue(card.contains("The circle isn't on the wall"), "card reads: \(card)") - XCTAssertTrue(card.contains("Aim it at the wall where it stops or turns"), "card reads: \(card)") + XCTAssertTrue(card.contains(says), "card reads: \(card)") + if folded { + XCTAssertFalse(card.contains("The circle isn't on the wall"), "the reason must fold under Details: \(card)") + } else { + XCTAssertTrue(card.contains("Aim it at the wall where it stops or turns"), "card reads: \(card)") + } XCTAssertTrue(element(app, "action.skipGap").exists) - snap(app, name) let mark = app.buttons["action.markEnd"] scrollIntoView(mark, in: app) + XCTAssertTrue(mark.isHittable) + assertEndCircleOpen(app, covers: Self.endCovers, name) + snap(app, name) tapWhenReady(mark) card = label(app, "instruction") - XCTAssertTrue(card.contains("The circle isn't on the wall"), "a second press marked something: \(card)") + XCTAssertTrue(card.contains(says), "a second press marked something: \(card)") XCTAssertFalse(element(app, "action.endBlocked").exists, "no end was marked, so nothing to ask") + guard folded else { return } + scrollToTop(app) + tapWhenReady(app.buttons["instruction.details"]) + let detail = element(app, "instruction.detail") + XCTAssertTrue(detail.waitForExistence(timeout: 5)) + XCTAssertTrue(detail.label.contains("The circle isn't on the wall. Aim it at the wall where it stops or turns, then tap Wall ends here."), "Details reads: \(detail.label)") } /// "Wall ends here" asks the walk's end question; it replaces the request and its decline @@ -101,6 +169,10 @@ final class PastEndRecoveryUITests: XCTestCase { for answer in ["action.endCorner", "action.endBlocked", "action.endEnds"] { XCTAssertTrue(app.buttons[answer].exists, "\(answer) missing") } + // In their order, and the question needs no circle. + let tops = ["action.endCorner", "action.endBlocked", "action.endEnds"].map { app.buttons[$0].frame.minY } + XCTAssertEqual(tops, tops.sorted(), "the answers moved out of order: \(tops)") + XCTAssertFalse(element(app, "aim.circle").exists, "the question shows no circle") snap(app, name) let blocked = app.buttons["action.endBlocked"] scrollIntoView(blocked, in: app) @@ -385,6 +457,23 @@ final class PastEndRecoveryUITests: XCTestCase { } } + /// Drags the screen down until its top shows. + @MainActor + private func scrollToTop(_ app: XCUIApplication) { + let scroll = app.scrollViews.firstMatch + guard scroll.exists else { return } + let start = scroll.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.3)) + for _ in 0..<4 { + start.press(forDuration: 0.05, thenDragTo: start.withOffset(CGVector(dx: 0, dy: app.windows.firstMatch.frame.height * 0.4))) + } + } + + @MainActor + private func waitForAbsence(_ target: XCUIElement) -> Bool { + let gone = XCTNSPredicateExpectation(predicate: NSPredicate(format: "exists == false"), object: target) + return XCTWaiter().wait(for: [gone], timeout: 5) == .completed + } + @MainActor private func snap(_ app: XCUIApplication, _ name: String) { let shot = XCTAttachment(screenshot: app.screenshot()) diff --git a/ios/HouseScanUITests/WalkRecoveryUITests.swift b/ios/HouseScanUITests/WalkRecoveryUITests.swift index 3f2b45f07..9e7278af7 100644 --- a/ios/HouseScanUITests/WalkRecoveryUITests.swift +++ b/ios/HouseScanUITests/WalkRecoveryUITests.swift @@ -16,6 +16,7 @@ final class WalkRecoveryUITests: XCTestCase { func testTheEndCardAnswersThatTheWallKeepsGoing() throws { let app = launch(["-uiDemoPhase", "wallWalk", "-uiDemoMarkEnd"]) XCTAssertTrue(label(app, "instruction").contains("Is this the right end of the wall?")) + assertEndCircleOpen(app, covers: Self.endCovers, "walk mark-end") snap(app, "wallWalk-markEnd-keepsGoing") XCTAssertTrue(element(app, "action.markEnd").exists, "Wall ends here must stay on offer") let keepsGoing = reply(app, "The wall keeps going") @@ -39,6 +40,90 @@ final class WalkRecoveryUITests: XCTestCase { XCTAssertTrue(card.contains("Aim it at the wall where it stops or turns"), "card reads: \(card)") XCTAssertTrue(element(app, "action.markEnd").exists) XCTAssertTrue(reply(app, "The wall keeps going").waitForExistence(timeout: 5)) + assertEndCircleOpen(app, covers: Self.endCovers, "walk mark-end refused") + } + + /// At AX5 the card covered the circle "Wall ends here" marks at (CI run 37165788075, + /// `wallWalk-markEnd-AX5-stacked`). Folded, it leads with where to aim, every word is under + /// Details, and "The wall keeps going" moves to the actions under "Wall ends here", so with + /// "Wall ends here" in reach the circle is in view. + @MainActor + func testTheEndCircleStaysOpenAtLargestTextSize() throws { + let app = launch(["-uiDemoPhase", "wallWalk", "-uiDemoMarkEnd"] + Self.largestText) + let card = label(app, "instruction") + XCTAssertTrue(card.contains("Aim at the right end"), "card reads: \(card)") + XCTAssertFalse(card.contains("Is this the right end of the wall?"), "the question folds under Details: \(card)") + let mark = element(app, "action.markEnd") + scrollIntoView(mark, in: app) + XCTAssertTrue(mark.isHittable, "Wall ends here can't be reached") + assertEndCircleOpen(app, covers: Self.endCovers, "walk mark-end, AX5") + snap(app, "wallWalk-markEnd-AX5") + + scrollToTop(app) + let details = element(app, "instruction.details") + tapWhenReady(details) + let detail = element(app, "instruction.detail") + XCTAssertTrue(detail.waitForExistence(timeout: 5), "Details must open the folded words") + XCTAssertTrue( + detail.label.contains("Is this the right end of the wall? Aim where it stops or turns a corner and tap Wall ends here. If it goes on, tap The wall keeps going."), + "Details reads: \(detail.label)") + snap(app, "wallWalk-markEnd-AX5-details") + tapWhenReady(details) + let gone = XCTNSPredicateExpectation(predicate: NSPredicate(format: "exists == false"), object: detail) + XCTAssertEqual(XCTWaiter().wait(for: [gone], timeout: 5), .completed, "Details must close again") + scrollIntoView(mark, in: app) + assertEndCircleOpen(app, covers: Self.endCovers, "walk mark-end, AX5, Details closed") + + let keepsGoing = reply(app, "The wall keeps going") + XCTAssertTrue(keepsGoing.waitForExistence(timeout: 5), "the wall that goes on still needs its answer") + scrollIntoView(keepsGoing, in: app) + XCTAssertTrue(keepsGoing.isHittable, "The wall keeps going can't be reached") + XCTAssertGreaterThan(keepsGoing.frame.minY, mark.frame.maxY - 0.5, "the answer sits under Wall ends here") + } + + /// Folded, a refusal leads with what to do, and keeps the circle in view. + @MainActor + func testARefusedWallEndKeepsTheCircleOpenAtLargestTextSize() throws { + let app = launch(["-uiDemoPhase", "wallWalk", "-uiDemoMarkEnd", "-uiDemoEndMarkRefusal"] + Self.largestText) + let card = label(app, "instruction") + XCTAssertTrue(card.contains("Aim at the wall"), "card reads: \(card)") + let mark = element(app, "action.markEnd") + scrollIntoView(mark, in: app) + XCTAssertTrue(mark.isHittable) + assertEndCircleOpen(app, covers: Self.endCovers, "walk mark-end refused, AX5") + snap(app, "wallWalk-markEnd-refused-AX5") + } + + private static let largestText = ["-UIPreferredContentSizeCategoryName", "UICTContentSizeCategoryAccessibilityXXXL"] + /// What may cover the circle on the walk's end: its actions and the card's reply. + private static let endCovers = ["action.markEnd", "action.markSomething", "action.cannotAccess"] + + /// Scrolls in measured steps until the whole element is in the window: at the largest text + /// sizes the actions sit below the card, reached by scrolling. + @MainActor + private func scrollIntoView(_ target: XCUIElement, in app: XCUIApplication) { + let window = app.windows.firstMatch.frame + let scroll = app.scrollViews.firstMatch + guard scroll.exists else { return } + let start = scroll.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.6)) + for _ in 0..<12 where !window.contains(target.frame) { + let frame = target.frame + let shift: CGFloat = frame.maxY > window.maxY + ? -min(frame.maxY - window.maxY + 24, window.height / 3) + : min(window.minY - frame.minY + 24, window.height / 3) + start.press(forDuration: 0.05, thenDragTo: start.withOffset(CGVector(dx: 0, dy: shift))) + } + } + + /// Drags the screen down until its top shows. + @MainActor + private func scrollToTop(_ app: XCUIApplication) { + let scroll = app.scrollViews.firstMatch + guard scroll.exists else { return } + let start = scroll.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.3)) + for _ in 0..<4 { + start.press(forDuration: 0.05, thenDragTo: start.withOffset(CGVector(dx: 0, dy: app.windows.firstMatch.frame.height * 0.4))) + } } /// B-23: "Point at the meter like this." needs the photo it points to. The follow-up view From 254a4566d3443e140fff8847bc304ed231f2db38 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 21:27:18 -0500 Subject: [PATCH 20/71] ios: folded end cards keep coaching off the circle, and the moved reply keeps its lock UNVALIDATED: not compiled; no package, app or UI tests have run. Fixes from the prep-for-prod review of 21ac61ec: - Coaching riding along kept its note under the folded lead, and the dark note runs several lines at AX5. By the frames' measurements, even one line uses the room left above the circle. Folded around the circle, the coaching now leads in a few words beside its symbol ("It's dark here", `ScanCopy.coachingLead`), and its whole note opens Details (`ScanCopy.foldedAroundCircle`). With a refusal, the correction keeps the lead and the coaching opens Details. - The reply moved to the actions lost the card's 600 ms input lock (#82). `MovedReplyButton` keeps it, keyed to the same task as the card's reply. - Under Reduce Motion the walk's controls eased their swap over 0.15 s, which moved the map as the end step's added reply grew the stack. They now swap at once. - The test helper measured the card's end by its words and Details with 8 pt, but unfolded the card ends at its reply plus 12 pt. It now takes whether the card is folded, and it also checks the circle is the 56 pt reticle. - "Turn to the right end" would likely break to three lines beside the refusal triangle; it is now "Face the right end". Tests added: coached folded cards on both screens (`-uiDemoCoaching tooDark`), and the other-side refusal on both sides (`-uiDemoEndMarkOtherSide`, new in the demo). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../UI/Components/MovedReplyButton.swift | 37 +++++++++++++ ios/HouseScan/UI/Copy/ScanCopy.swift | 31 ++++++++++- ios/HouseScan/UI/Preview/DemoEngine.swift | 24 ++++++--- .../UI/Screens/GapRequestScreen.swift | 23 ++++---- ios/HouseScan/UI/Screens/WallWalkScreen.swift | 25 +++++---- .../EndAimCircleAssertions.swift | 33 +++++++----- .../PastEndRecoveryUITests.swift | 52 +++++++++++++++++-- .../WalkRecoveryUITests.swift | 25 +++++++-- 8 files changed, 197 insertions(+), 53 deletions(-) create mode 100644 ios/HouseScan/UI/Components/MovedReplyButton.swift diff --git a/ios/HouseScan/UI/Components/MovedReplyButton.swift b/ios/HouseScan/UI/Components/MovedReplyButton.swift new file mode 100644 index 000000000..312cc0d48 --- /dev/null +++ b/ios/HouseScan/UI/Components/MovedReplyButton.swift @@ -0,0 +1,37 @@ +import SwiftUI + +/// A card's reply ("I can't get there", "The wall keeps going") shown among the actions instead, +/// while the card folds around the circle at the largest text sizes (`GapRequestScreen`, +/// `WallWalkScreen`). It keeps the card reply's input lock: after its task changes it takes no +/// taps for `InstructionCard.replyLock`, so a tap meant for the step that just left can't answer +/// this one (#82). Coaching or a refusal within the same task keeps it unlocked, as on the card. +/// The secondary style doesn't read `isEnabled`, so the lock doesn't change how it looks; +/// VoiceOver and UI tests read it as dimmed. +struct MovedReplyButton: View { + let title: String + let hint: String + let identifier: String + /// The step the reply answers; a new one locks the button again. + let task: Instruction + let perform: () -> Void + + @State private var unlockedTask: Instruction? + + var body: some View { + Button(action: perform) { + Text(title) + .frame(maxWidth: .infinity) + } + .buttonStyle(.secondary) + .disabled(unlockedTask != task) + .accessibilityHint(hint) + .accessibilityIdentifier(identifier) + .task(id: task) { + // Cleared first, so a task that comes back within the lock is locked again. + let shown = task + unlockedTask = nil + do { try await Task.sleep(for: InstructionCard.replyLock) } catch { return } + unlockedTask = shown + } + } +} diff --git a/ios/HouseScan/UI/Copy/ScanCopy.swift b/ios/HouseScan/UI/Copy/ScanCopy.swift index a2dea8332..7f017fddc 100644 --- a/ios/HouseScan/UI/Copy/ScanCopy.swift +++ b/ios/HouseScan/UI/Copy/ScanCopy.swift @@ -401,7 +401,7 @@ enum ScanCopy { case .otherSide(let landed): Instruction( title: "That's the \(landed.rawValue) side of your meter", detail: "Turn to the \(side.rawValue) end, then tap Wall ends here.", - folded: endAimFold("Turn to the \(side.rawValue) end", "That's the \(landed.rawValue) side of your meter. Turn to the \(side.rawValue) end, then tap Wall ends here.")) + folded: endAimFold("Face the \(side.rawValue) end", "That's the \(landed.rawValue) side of your meter. Turn to the \(side.rawValue) end, then tap Wall ends here.")) case .trackingNotReady: Instruction( title: "One moment, your phone is still finding its place", detail: "Then aim at the \(side.rawValue) end.", @@ -430,6 +430,35 @@ enum ScanCopy { Instruction.Folded(title: lead, detail: everything) } + /// A folded end-aim card with coaching riding along (`withCoaching`). Its note under the lead + /// would reach the circle again: the long dark note is several lines at AX5, and even one + /// line uses the room above the circle. So the coaching leads instead, in a few words beside + /// its symbol, and its whole note opens Details. With a refusal, the refusal's correction + /// keeps the lead and the coaching opens Details. Only for a card that folds; coaching that + /// replaces the task isn't folded at all. + static func foldedAroundCircle(_ card: Instruction, coaching: Coaching?, refused: Bool) -> Instruction { + guard let coaching, !coachingReplacesTask(coaching), var folded = card.folded else { return card } + var out = card + out.note = nil + folded.detail = "\(coachingNote(coaching)) \(folded.detail)" + if !refused { folded.title = coachingLead(coaching) } + out.folded = folded + return out + } + + /// Coaching in a few words, for a lead that must stay two lines beside its symbol at AX5. + static func coachingLead(_ coaching: Coaching) -> String { + switch coaching { + case .slowDown: "Slow down" + case .needsTexture: "Find more texture" + case .tooDark: "It's dark here" + case .tooDarkToMeasure: "Too dark to measure" + case .holdSteady: "Hold steady" + case .turnSlowly: "Turn more slowly" + case .initializing, .relocalizing, .trackingLost, .pastWallEnd: ScanCopy.coaching(coaching).title + } + } + static func nextWallRefusal(_ refusal: NextWallRefusal) -> Instruction { switch refusal { case .noSurface: Instruction(title: "No wall under the circle", detail: "Step closer and aim at the next wall.") diff --git a/ios/HouseScan/UI/Preview/DemoEngine.swift b/ios/HouseScan/UI/Preview/DemoEngine.swift index 2a943ef2e..0ca546ba1 100644 --- a/ios/HouseScan/UI/Preview/DemoEngine.swift +++ b/ios/HouseScan/UI/Preview/DemoEngine.swift @@ -32,6 +32,9 @@ final class DemoEngine: ScanActions { /// `-uiDemoEndMarkRefusal`: "Wall ends here" was pressed with the circle off the wall, and the /// demo's circle stays off it, so pressing again refuses again. private let endMarkRefused: Bool + /// `-uiDemoEndMarkOtherSide`: as `-uiDemoEndMarkRefusal`, but the circle is on the wall on + /// the meter's other side (`EndMarkRefusal.otherSide`). + private let endMarkOtherSide: Bool /// The tilt-up step was answered or skipped. private var tiltUpSettled = false /// Walk-script ticks spent on the tilt-up step, standing in for the phone being tilted up. @@ -89,7 +92,8 @@ final class DemoEngine: ScanActions { rejectUpload = arguments.contains("-uiDemoRejected") unusableAnswers = arguments.contains("-uiDemoUnusableAnswer") ? max(value("-uiDemoUnusableAnswer").flatMap(Int.init) ?? 1, 1) : 0 gapKind = value("-uiDemoGap") - endMarkRefused = arguments.contains("-uiDemoEndMarkRefusal") + endMarkOtherSide = arguments.contains("-uiDemoEndMarkOtherSide") + endMarkRefused = arguments.contains("-uiDemoEndMarkRefusal") || endMarkOtherSide state.feed = DemoScene.image.map(CameraFeed.still) ?? .none state.isReplay = true state.tracking = .normal @@ -129,9 +133,9 @@ final class DemoEngine: ScanActions { reachedRight = demoRightEnd refreshCoverage() refreshGuidance() - if arguments.contains("-uiDemoEndMarkRefusal") { - // As in the real engine, a circle off the wall has no end preview (`circleEnd`). - state.endMarkRefusal = .noWall + if endMarkRefused { + // As in the real engine, a refused circle has no end preview (`circleEnd`). + state.endMarkRefusal = demoRefusal(asked: .right) state.endPreview = nil } } @@ -444,8 +448,8 @@ final class DemoEngine: ScanActions { state.target = DemoScene.wall.world(s: end + sign * 0.3, height: 0, out: 0.5) state.path = DemoScene.path(toward: end + sign * 0.3) if endMarkRefused { - // As in the real engine, a circle off the wall has no end preview (`circleEnd`). - state.endMarkRefusal = .noWall + // As in the real engine, a refused circle has no end preview (`circleEnd`). + state.endMarkRefusal = demoRefusal(asked: side) state.endPreview = nil } else { // The circle on the wall a little before the old end: where the wall really stops. @@ -889,10 +893,16 @@ final class DemoEngine: ScanActions { } } + /// Why the demo's "Wall ends here" marks nothing: the circle off the wall, or on the wall on + /// the other side of the meter from the end `asked` (`-uiDemoEndMarkOtherSide`). + private func demoRefusal(asked: WallSide) -> EndMarkRefusal { + endMarkOtherSide ? .otherSide(asked == .right ? .left : .right) : .noWall + } + func markWallEnd(at point: CGPoint?, viewSize: CGSize) { if state.phase == .gapRequest, let side = state.gap?.pastEndSide, state.gap?.isSatisfied == false, state.endQuestion == nil { if endMarkRefused { - state.endMarkRefusal = .noWall + state.endMarkRefusal = demoRefusal(asked: side) state.endPreview = nil return } diff --git a/ios/HouseScan/UI/Screens/GapRequestScreen.swift b/ios/HouseScan/UI/Screens/GapRequestScreen.swift index d0aa2b79e..e251018f9 100644 --- a/ios/HouseScan/UI/Screens/GapRequestScreen.swift +++ b/ios/HouseScan/UI/Screens/GapRequestScreen.swift @@ -180,17 +180,13 @@ struct GapRequestScreen: View { /// "I can't get there" among the actions, under "Wall ends here", while the card folds around /// the circle (`endAimFolds`): the same words, hint and action as the card's reply. - private var declineButton: some View { - Button { - actions.skipGap() - } label: { - Text("I can't get there") - .frame(maxWidth: .infinity) + @ViewBuilder private var declineButton: some View { + if let gap = state.gap { + MovedReplyButton( + title: "I can't get there", hint: skipHint, identifier: "action.skipGap", + task: ScanCopy.gapTask(gap), perform: { actions.skipGap() }) + .transition(.opacity) } - .buttonStyle(.secondary) - .accessibilityHint(skipHint) - .accessibilityIdentifier("action.skipGap") - .transition(.opacity) } /// At the largest text sizes while "Wall ends here" is offered, the card folds to a two-line @@ -259,9 +255,12 @@ struct GapRequestScreen: View { /// thanks" for as long as it did (field test 4.1, run 3). private var instruction: Instruction { var card = requestCard - // Folded around the circle, "One more view to finish" leaves the card's top (`endAimFolds`) + guard endAimFolds else { return card } + // Folded around the circle, ride-along coaching leads or opens Details + // (`ScanCopy.foldedAroundCircle`), and "One more view to finish" leaves the card's top // and leads the words under Details. - if endAimFolds, followUps > 0, let folded = card.folded { + card = ScanCopy.foldedAroundCircle(card, coaching: coaching, refused: refusal != nil) + if followUps > 0, let folded = card.folded { card.folded?.detail = "\(ScanCopy.followUp(remaining: followUps)). \(folded.detail)" } return card diff --git a/ios/HouseScan/UI/Screens/WallWalkScreen.swift b/ios/HouseScan/UI/Screens/WallWalkScreen.swift index 7c29e652f..7ff4b685e 100644 --- a/ios/HouseScan/UI/Screens/WallWalkScreen.swift +++ b/ios/HouseScan/UI/Screens/WallWalkScreen.swift @@ -69,7 +69,10 @@ struct WallWalkScreen: View { ) } } - .animation(reduceMotion ? .easeOut(duration: 0.15) : Motion.settle, value: controlsKey) + // Under Reduce Motion the controls swap at once: the swap changes the stack's height + // (the end's step adds its reply at the largest sizes), and an eased resize still + // moves the map under it. + .animation(reduceMotion ? nil : Motion.settle, value: controlsKey) } } .animation(.easeOut(duration: 0.2), value: showsMeterPhoto) @@ -81,6 +84,14 @@ struct WallWalkScreen: View { // MARK: Instruction private var instruction: Instruction { + // Folded around the circle, ride-along coaching leads or opens Details instead of sitting + // under the lead (`ScanCopy.foldedAroundCircle`). + guard endAimFolds else { return stepCard } + return ScanCopy.foldedAroundCircle(stepCard, coaching: coaching, refused: state.endMarkRefusal != nil) + } + + /// The card's words for the step, before folding around the circle (`instruction`). + private var stepCard: Instruction { if let marking = state.marking { let prompt = ScanCopy.markingPrompt(marking) if let refusal = marking.refusal { @@ -352,15 +363,9 @@ struct WallWalkScreen: View { .transition(.opacity) if endAimFolds, let copy = ScanCopy.reply(for: state.guidance) { // The card's reply, moved here while the card folds around the circle. - Button { - actions.cannotAccessArea() - } label: { - Text(copy.title) - .spansStack(stacksActions) - } - .buttonStyle(.secondary) - .accessibilityHint(copy.hint) - .accessibilityIdentifier("action.cannotAccess") + MovedReplyButton( + title: copy.title, hint: copy.hint, identifier: "action.cannotAccess", + task: replyTask, perform: { actions.cannotAccessArea() }) .transition(.opacity) } case .walking where offersEndHere: diff --git a/ios/HouseScanUITests/EndAimCircleAssertions.swift b/ios/HouseScanUITests/EndAimCircleAssertions.swift index 28d02767c..01c9a1e0a 100644 --- a/ios/HouseScanUITests/EndAimCircleAssertions.swift +++ b/ios/HouseScanUITests/EndAimCircleAssertions.swift @@ -3,34 +3,39 @@ import XCTest /// The circle "Wall ends here" marks at (`endAimCircle`, identifier `aim.circle`), read from the /// production layout: the app's own view, as VoiceOver sees it, not a test-only overlay. extension XCTestCase { - /// Checks the circle is drawn in the middle of the window, where the ray "Wall ends here" uses - /// goes through (`ScanEngine.circleEnd`: the middle of the sensor image, which the camera view - /// fills and centres), is wholly on screen, and is under none of `covers`: the card's words, - /// its Details, and the actions. The card's scrim runs `cardPadding` below the last of its - /// parts, so the circle must clear that too. + /// Checks the circle is the 56 pt reticle drawn in the middle of the window, where the ray + /// "Wall ends here" uses goes through (`ScanEngine.circleEnd`: the middle of the sensor image, + /// which the camera view fills and centres), is wholly on screen, and is under neither the + /// card nor any of `covers`. + /// + /// The card ends at its last part plus the scrim's padding under it (`InstructionCard`): + /// unfolded, the reply (`reply`), 12 pt above the scrim's edge; folded around the circle, + /// Details, 8 pt above it, since the reply has moved to the actions. @MainActor func assertEndCircleOpen( - _ app: XCUIApplication, covers: [String], _ context: String, + _ app: XCUIApplication, folded: Bool, reply: String, covers: [String], _ context: String, file: StaticString = #filePath, line: UInt = #line ) { + func element(_ identifier: String) -> XCUIElement { app.descendants(matching: .any)[identifier].firstMatch } let window = app.windows.firstMatch.frame - let circle = app.descendants(matching: .any)["aim.circle"].firstMatch + let circle = element("aim.circle") XCTAssertTrue(circle.waitForExistence(timeout: 5), "\(context): no aiming circle", file: file, line: line) let frame = circle.frame + XCTAssertEqual(frame.width, 56, accuracy: 1, "\(context): the aiming circle is \(frame.size), not the 56 pt reticle", file: file, line: line) + XCTAssertEqual(frame.height, 56, accuracy: 1, "\(context): the aiming circle is \(frame.size), not the 56 pt reticle", file: file, line: line) XCTAssertEqual(frame.midX, window.midX, accuracy: 1, "\(context): the circle \(frame) must be the middle of the camera", file: file, line: line) XCTAssertEqual(frame.midY, window.midY, accuracy: 1, "\(context): the circle \(frame) must be the middle of the camera", file: file, line: line) XCTAssertTrue(window.contains(frame), "\(context): the circle \(frame) is not wholly on screen", file: file, line: line) - let card = ["instruction", "instruction.details"].map { app.descendants(matching: .any)[$0].firstMatch }.filter(\.exists) - if let cardBottom = card.map(\.frame.maxY).max() { - XCTAssertGreaterThanOrEqual(frame.minY, cardBottom + Self.cardPadding, "\(context): the circle \(frame) is under the card, which ends at \(cardBottom + Self.cardPadding)", file: file, line: line) + + let parts = (folded ? ["instruction", "instruction.details"] : ["instruction", reply]).map(element).filter(\.exists) + let padding: CGFloat = folded ? 8 : 12 + if let last = parts.map(\.frame.maxY).max() { + XCTAssertGreaterThanOrEqual(frame.minY, last + padding, "\(context): the circle \(frame) is under the card, which ends at \(last + padding)", file: file, line: line) } for identifier in covers { - let cover = app.descendants(matching: .any)[identifier].firstMatch + let cover = element(identifier) guard cover.exists else { continue } XCTAssertFalse(frame.intersects(cover.frame), "\(context): the circle \(frame) is under \(identifier) \(cover.frame)", file: file, line: line) } } - - /// The card's scrim below Details without a reply (`InstructionCard.details`, bottom padding). - static var cardPadding: CGFloat { 8 } } diff --git a/ios/HouseScanUITests/PastEndRecoveryUITests.swift b/ios/HouseScanUITests/PastEndRecoveryUITests.swift index c683efba1..0ec6a0685 100644 --- a/ios/HouseScanUITests/PastEndRecoveryUITests.swift +++ b/ios/HouseScanUITests/PastEndRecoveryUITests.swift @@ -41,7 +41,7 @@ final class PastEndRecoveryUITests: XCTestCase { XCTAssertTrue(mark.waitForExistence(timeout: 5)) scrollIntoView(mark, in: app) XCTAssertTrue(mark.isHittable, "Wall ends here can't be reached") - assertEndCircleOpen(app, covers: Self.endCovers, "left past-end, AX5") + assertEndCircleOpen(app, folded: true, reply: "action.skipGap", covers: Self.endCovers, "left past-end, AX5") snap(app, "gapRequest-pastEndLeft-AX5") } @@ -68,7 +68,7 @@ final class PastEndRecoveryUITests: XCTestCase { scrollIntoView(mark, in: app) XCTAssertTrue(mark.isHittable, "Wall ends here can't be reached") // Where "Wall ends here" can be pressed, the circle it marks at is in view. - assertEndCircleOpen(app, covers: Self.endCovers, name) + assertEndCircleOpen(app, folded: folded, reply: "action.skipGap", covers: Self.endCovers, name) snap(app, name) guard folded else { return } @@ -87,7 +87,7 @@ final class PastEndRecoveryUITests: XCTestCase { XCTAssertTrue(waitForAbsence(detail), "Details must close again") scrollIntoView(mark, in: app) XCTAssertTrue(mark.isHittable) - assertEndCircleOpen(app, covers: Self.endCovers, "\(name), Details closed") + assertEndCircleOpen(app, folded: true, reply: "action.skipGap", covers: Self.endCovers, "\(name), Details closed") // The decline moved from the card to the actions, and stays within reach. let decline = app.buttons["action.skipGap"] scrollIntoView(decline, in: app) @@ -96,6 +96,50 @@ final class PastEndRecoveryUITests: XCTestCase { XCTAssertGreaterThan(decline.frame.minY, mark.frame.maxY - 0.5, "the decline sits under Wall ends here") } + /// Coaching riding along leads the folded card in a few words, and its whole note opens + /// Details: under the lead, the dark note's lines reached the circle. + @MainActor + func testCoachingOnAFoldedPastEndCardKeepsTheCircleOpen() throws { + let app = launch(["-uiDemoGap", "pastEnd", "-uiDemoCoaching", "tooDark"] + Self.largestText) + let card = label(app, "instruction") + XCTAssertTrue(card.contains("It's dark here"), "card reads: \(card)") + XCTAssertFalse(card.contains("flashlight"), "the coaching's note must fold under Details: \(card)") + let mark = app.buttons["action.markEnd"] + scrollIntoView(mark, in: app) + XCTAssertTrue(mark.isHittable) + assertEndCircleOpen(app, folded: true, reply: "action.skipGap", covers: Self.endCovers, "coached past-end, AX5") + snap(app, "gapRequest-pastEnd-tooDark-AX5") + scrollToTop(app) + tapWhenReady(app.buttons["instruction.details"]) + let detail = element(app, "instruction.detail") + XCTAssertTrue(detail.waitForExistence(timeout: 5)) + for words in ["Try your phone's flashlight", "Keep walking past the right end", "If the wall stops sooner"] { + XCTAssertTrue(detail.label.contains(words), "Details must hold \"\(words)\": \(detail.label)") + } + } + + /// The circle on the other side of the meter, on either side: the folded card leads with + /// which end to face, and the circle stays open. + @MainActor + func testTheOtherSideRefusalFoldsOnBothSides() throws { + for (gap, asked, landed) in [("pastEnd", "right", "left"), ("pastEndLeft", "left", "right")] { + let app = launch(["-uiDemoGap", gap, "-uiDemoEndMarkOtherSide"] + Self.largestText) + let card = label(app, "instruction") + XCTAssertTrue(card.contains("Face the \(asked) end"), "\(gap): card reads: \(card)") + let mark = app.buttons["action.markEnd"] + scrollIntoView(mark, in: app) + XCTAssertTrue(mark.isHittable) + assertEndCircleOpen(app, folded: true, reply: "action.skipGap", covers: Self.endCovers, "\(gap) other side, AX5") + snap(app, "gapRequest-\(gap)-otherSide-AX5") + scrollToTop(app) + tapWhenReady(app.buttons["instruction.details"]) + let detail = element(app, "instruction.detail") + XCTAssertTrue(detail.waitForExistence(timeout: 5)) + XCTAssertTrue(detail.label.contains("That's the \(landed) side of your meter. Turn to the \(asked) end"), "\(gap): Details reads: \(detail.label)") + app.terminate() + } + } + /// What may cover the circle: the actions and the card's reply. The card itself is checked by /// its bottom edge (`assertEndCircleOpen`). private static let endCovers = ["action.markEnd", "action.skipGap", "action.showResult", "action.endCorner", "action.endBlocked", "action.endEnds"] @@ -129,7 +173,7 @@ final class PastEndRecoveryUITests: XCTestCase { let mark = app.buttons["action.markEnd"] scrollIntoView(mark, in: app) XCTAssertTrue(mark.isHittable) - assertEndCircleOpen(app, covers: Self.endCovers, name) + assertEndCircleOpen(app, folded: folded, reply: "action.skipGap", covers: Self.endCovers, name) snap(app, name) tapWhenReady(mark) card = label(app, "instruction") diff --git a/ios/HouseScanUITests/WalkRecoveryUITests.swift b/ios/HouseScanUITests/WalkRecoveryUITests.swift index 9e7278af7..03888733a 100644 --- a/ios/HouseScanUITests/WalkRecoveryUITests.swift +++ b/ios/HouseScanUITests/WalkRecoveryUITests.swift @@ -16,7 +16,7 @@ final class WalkRecoveryUITests: XCTestCase { func testTheEndCardAnswersThatTheWallKeepsGoing() throws { let app = launch(["-uiDemoPhase", "wallWalk", "-uiDemoMarkEnd"]) XCTAssertTrue(label(app, "instruction").contains("Is this the right end of the wall?")) - assertEndCircleOpen(app, covers: Self.endCovers, "walk mark-end") + assertEndCircleOpen(app, folded: false, reply: "action.cannotAccess", covers: Self.endCovers, "walk mark-end") snap(app, "wallWalk-markEnd-keepsGoing") XCTAssertTrue(element(app, "action.markEnd").exists, "Wall ends here must stay on offer") let keepsGoing = reply(app, "The wall keeps going") @@ -40,7 +40,7 @@ final class WalkRecoveryUITests: XCTestCase { XCTAssertTrue(card.contains("Aim it at the wall where it stops or turns"), "card reads: \(card)") XCTAssertTrue(element(app, "action.markEnd").exists) XCTAssertTrue(reply(app, "The wall keeps going").waitForExistence(timeout: 5)) - assertEndCircleOpen(app, covers: Self.endCovers, "walk mark-end refused") + assertEndCircleOpen(app, folded: false, reply: "action.cannotAccess", covers: Self.endCovers, "walk mark-end refused") } /// At AX5 the card covered the circle "Wall ends here" marks at (CI run 37165788075, @@ -56,7 +56,7 @@ final class WalkRecoveryUITests: XCTestCase { let mark = element(app, "action.markEnd") scrollIntoView(mark, in: app) XCTAssertTrue(mark.isHittable, "Wall ends here can't be reached") - assertEndCircleOpen(app, covers: Self.endCovers, "walk mark-end, AX5") + assertEndCircleOpen(app, folded: true, reply: "action.cannotAccess", covers: Self.endCovers, "walk mark-end, AX5") snap(app, "wallWalk-markEnd-AX5") scrollToTop(app) @@ -72,7 +72,7 @@ final class WalkRecoveryUITests: XCTestCase { let gone = XCTNSPredicateExpectation(predicate: NSPredicate(format: "exists == false"), object: detail) XCTAssertEqual(XCTWaiter().wait(for: [gone], timeout: 5), .completed, "Details must close again") scrollIntoView(mark, in: app) - assertEndCircleOpen(app, covers: Self.endCovers, "walk mark-end, AX5, Details closed") + assertEndCircleOpen(app, folded: true, reply: "action.cannotAccess", covers: Self.endCovers, "walk mark-end, AX5, Details closed") let keepsGoing = reply(app, "The wall keeps going") XCTAssertTrue(keepsGoing.waitForExistence(timeout: 5), "the wall that goes on still needs its answer") @@ -90,10 +90,25 @@ final class WalkRecoveryUITests: XCTestCase { let mark = element(app, "action.markEnd") scrollIntoView(mark, in: app) XCTAssertTrue(mark.isHittable) - assertEndCircleOpen(app, covers: Self.endCovers, "walk mark-end refused, AX5") + assertEndCircleOpen(app, folded: true, reply: "action.cannotAccess", covers: Self.endCovers, "walk mark-end refused, AX5") snap(app, "wallWalk-markEnd-refused-AX5") } + /// The walk's end with coaching riding along: the coaching leads in a few words, its note + /// opens Details, and the circle stays open. + @MainActor + func testCoachingOnTheFoldedEndCardKeepsTheCircleOpen() throws { + let app = launch(["-uiDemoPhase", "wallWalk", "-uiDemoMarkEnd", "-uiDemoCoaching", "tooDark"] + Self.largestText) + let card = label(app, "instruction") + XCTAssertTrue(card.contains("It's dark here"), "card reads: \(card)") + XCTAssertFalse(card.contains("flashlight"), "the coaching's note must fold under Details: \(card)") + let mark = element(app, "action.markEnd") + scrollIntoView(mark, in: app) + XCTAssertTrue(mark.isHittable) + assertEndCircleOpen(app, folded: true, reply: "action.cannotAccess", covers: Self.endCovers, "coached walk mark-end, AX5") + snap(app, "wallWalk-markEnd-tooDark-AX5") + } + private static let largestText = ["-UIPreferredContentSizeCategoryName", "UICTContentSizeCategoryAccessibilityXXXL"] /// What may cover the circle on the walk's end: its actions and the card's reply. private static let endCovers = ["action.markEnd", "action.markSomething", "action.cannotAccess"] From b1dc25251c3ba10ddc18cb836c17d990529f3849 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 21:37:33 -0500 Subject: [PATCH 21/71] ios: folded end cards lead with the remedy for darkness, and with the step for a too-short wall UNVALIDATED: not compiled; not run. Greptile on #214: folded around the circle, darkness coaching led with "It's dark here" and left the remedy under Details, though darkness can last a whole walk. The coaching leads are now what to do ("Try your flashlight", "Try in daylight"); the moon symbol already says what is wrong, and the whole note still opens Details. The walk can ask for the end again after too-close ends were cleared, with `wallTooShort` still set. Folded, its card kept the full title, "The ends were too close. Walk along the wall first.", several lines at AX5 beside the triangle, which would cover the circle again. It now leads with "Walk the wall first", and the rest goes under Details. Coaching then opens Details too, as with a refusal. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/UI/Copy/ScanCopy.swift | 10 +++++++--- ios/HouseScan/UI/Screens/WallWalkScreen.swift | 8 ++++++-- ios/HouseScanUITests/PastEndRecoveryUITests.swift | 4 ++-- ios/HouseScanUITests/WalkRecoveryUITests.swift | 2 +- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/ios/HouseScan/UI/Copy/ScanCopy.swift b/ios/HouseScan/UI/Copy/ScanCopy.swift index 7f017fddc..d4cff3aad 100644 --- a/ios/HouseScan/UI/Copy/ScanCopy.swift +++ b/ios/HouseScan/UI/Copy/ScanCopy.swift @@ -252,6 +252,9 @@ enum ScanCopy { /// Over the walk's own prompt after "Done with this wall" was refused and the ends cleared /// (`ScanViewState.wallTooShort`). static let wallTooShort = "The ends were too close. Walk along the wall first." + /// `wallTooShort` on a card folded around the circle, as the walk's end can be when both ends + /// were cleared: two lines beside the refusal triangle, the rest under Details. + static let wallTooShortLead = "Walk the wall first" // MARK: Close-up @@ -446,13 +449,14 @@ enum ScanCopy { return out } - /// Coaching in a few words, for a lead that must stay two lines beside its symbol at AX5. + /// Coaching in a few words, for a lead that must stay two lines beside its symbol at AX5: what + /// to do, since the symbol already says what is wrong and darkness can last a whole walk. static func coachingLead(_ coaching: Coaching) -> String { switch coaching { case .slowDown: "Slow down" case .needsTexture: "Find more texture" - case .tooDark: "It's dark here" - case .tooDarkToMeasure: "Too dark to measure" + case .tooDark: "Try your flashlight" + case .tooDarkToMeasure: "Try in daylight" case .holdSteady: "Hold steady" case .turnSlowly: "Turn more slowly" case .initializing, .relocalizing, .trackingLost, .pastWallEnd: ScanCopy.coaching(coaching).title diff --git a/ios/HouseScan/UI/Screens/WallWalkScreen.swift b/ios/HouseScan/UI/Screens/WallWalkScreen.swift index 7ff4b685e..699259dd0 100644 --- a/ios/HouseScan/UI/Screens/WallWalkScreen.swift +++ b/ios/HouseScan/UI/Screens/WallWalkScreen.swift @@ -87,7 +87,7 @@ struct WallWalkScreen: View { // Folded around the circle, ride-along coaching leads or opens Details instead of sitting // under the lead (`ScanCopy.foldedAroundCircle`). guard endAimFolds else { return stepCard } - return ScanCopy.foldedAroundCircle(stepCard, coaching: coaching, refused: state.endMarkRefusal != nil) + return ScanCopy.foldedAroundCircle(stepCard, coaching: coaching, refused: state.endMarkRefusal != nil || state.wallTooShort) } /// The card's words for the step, before folding around the circle (`instruction`). @@ -111,7 +111,11 @@ struct WallWalkScreen: View { // Coaching about how the photos come out (the capture gate's, and too little texture) // rides along with the task (`ScanCopy.withCoaching`), and its symbol marks the card (`tone`). if state.wallTooShort { - return ScanCopy.withCoaching(Instruction(title: ScanCopy.wallTooShort, detail: ScanCopy.guidance(state.guidance).title), coaching) + let step = ScanCopy.guidance(state.guidance).title + // Folded around the circle (the walk can ask for the end again after the too-close + // ends were cleared), the long title would cover it: the correction leads instead. + let folded = Instruction.Folded(title: ScanCopy.wallTooShortLead, detail: "\(ScanCopy.wallTooShort) \(step)") + return ScanCopy.withCoaching(Instruction(title: ScanCopy.wallTooShort, detail: step, folded: folded), coaching) } return ScanCopy.withCoaching(ScanCopy.guidance(state.guidance, hint: state.guidanceHint), coaching) } diff --git a/ios/HouseScanUITests/PastEndRecoveryUITests.swift b/ios/HouseScanUITests/PastEndRecoveryUITests.swift index 0ec6a0685..87b568069 100644 --- a/ios/HouseScanUITests/PastEndRecoveryUITests.swift +++ b/ios/HouseScanUITests/PastEndRecoveryUITests.swift @@ -102,7 +102,7 @@ final class PastEndRecoveryUITests: XCTestCase { func testCoachingOnAFoldedPastEndCardKeepsTheCircleOpen() throws { let app = launch(["-uiDemoGap", "pastEnd", "-uiDemoCoaching", "tooDark"] + Self.largestText) let card = label(app, "instruction") - XCTAssertTrue(card.contains("It's dark here"), "card reads: \(card)") + XCTAssertTrue(card.contains("Try your flashlight"), "card reads: \(card)") XCTAssertFalse(card.contains("flashlight"), "the coaching's note must fold under Details: \(card)") let mark = app.buttons["action.markEnd"] scrollIntoView(mark, in: app) @@ -113,7 +113,7 @@ final class PastEndRecoveryUITests: XCTestCase { tapWhenReady(app.buttons["instruction.details"]) let detail = element(app, "instruction.detail") XCTAssertTrue(detail.waitForExistence(timeout: 5)) - for words in ["Try your phone's flashlight", "Keep walking past the right end", "If the wall stops sooner"] { + for words in ["It's dark here. Try your phone's flashlight", "Keep walking past the right end", "If the wall stops sooner"] { XCTAssertTrue(detail.label.contains(words), "Details must hold \"\(words)\": \(detail.label)") } } diff --git a/ios/HouseScanUITests/WalkRecoveryUITests.swift b/ios/HouseScanUITests/WalkRecoveryUITests.swift index 03888733a..1d802449a 100644 --- a/ios/HouseScanUITests/WalkRecoveryUITests.swift +++ b/ios/HouseScanUITests/WalkRecoveryUITests.swift @@ -100,7 +100,7 @@ final class WalkRecoveryUITests: XCTestCase { func testCoachingOnTheFoldedEndCardKeepsTheCircleOpen() throws { let app = launch(["-uiDemoPhase", "wallWalk", "-uiDemoMarkEnd", "-uiDemoCoaching", "tooDark"] + Self.largestText) let card = label(app, "instruction") - XCTAssertTrue(card.contains("It's dark here"), "card reads: \(card)") + XCTAssertTrue(card.contains("Try your flashlight"), "card reads: \(card)") XCTAssertFalse(card.contains("flashlight"), "the coaching's note must fold under Details: \(card)") let mark = element(app, "action.markEnd") scrollIntoView(mark, in: app) From 55449e33254d3069a4e8603804fe4c39e1925fe0 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 23:01:20 -0500 Subject: [PATCH 22/71] ios: the past-end circle is in the view only while offered, so VoiceOver can't stop on it under the question CI run 37171342665 on 254a4566: the circle's geometry held in every new test. Each one measured it 56x56, at the window's exact centre, and clear of the card and the actions: walk and past-end, default and AX5, the left side, coached and other-side refusals. But testMarkingTheEndAsksWhatIsThere(AndSettlesTheRequest|AtLargestTextSize) failed on "the question shows no circle". The recording shows the circle faded out under the question, yet its `aim.circle` element was still in the tree: `.accessibilityHidden` on the element `.accessibilityElement(children: .ignore)` made didn't remove it. So VoiceOver could stop on an "Aiming circle" that isn't on screen. The circle is now inserted and removed with an opacity transition instead of kept at zero opacity. The fade is unchanged: Motion.settle with "Wall ends here", and a brief ease under Reduce Motion. The test's assertion is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../UI/Components/EndAimCircle.swift | 10 +++++----- .../UI/Screens/GapRequestScreen.swift | 20 ++++++++++++------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/ios/HouseScan/UI/Components/EndAimCircle.swift b/ios/HouseScan/UI/Components/EndAimCircle.swift index 810649c08..7cef095aa 100644 --- a/ios/HouseScan/UI/Components/EndAimCircle.swift +++ b/ios/HouseScan/UI/Components/EndAimCircle.swift @@ -4,13 +4,13 @@ extension View { /// The circle a mark lands under (`Reticle`), as one accessibility element: "Aiming circle". /// The reticle itself is hidden from VoiceOver as decoration. Here it says where "Wall ends /// here" and the other marks land, and UI tests read its real frame to check that nothing - /// covers it (`PastEndRecoveryUITests`, `ScreenStatesUITests`). Apply it before the reticle - /// is stretched to the screen, so the element's frame is the circle's. `shown` false while - /// the circle is faded out, so VoiceOver doesn't stop on what isn't there. - func endAimCircle(shown: Bool = true) -> some View { + /// covers it (`PastEndRecoveryUITests`, `WalkRecoveryUITests`). Apply it before the reticle + /// is stretched to the screen, so the element's frame is the circle's. Show the circle only + /// while it is offered: `.accessibilityHidden` on this element left it in the tree at zero + /// opacity (CI run 37171342665), so VoiceOver would stop on a circle that isn't there. + func endAimCircle() -> some View { accessibilityElement(children: .ignore) .accessibilityLabel("Aiming circle") .accessibilityIdentifier("aim.circle") - .accessibilityHidden(!shown) } } diff --git a/ios/HouseScan/UI/Screens/GapRequestScreen.swift b/ios/HouseScan/UI/Screens/GapRequestScreen.swift index e251018f9..63577b486 100644 --- a/ios/HouseScan/UI/Screens/GapRequestScreen.swift +++ b/ios/HouseScan/UI/Screens/GapRequestScreen.swift @@ -35,14 +35,20 @@ struct GapRequestScreen: View { // The end lands under the circle, as on the walk's "Is this the right end?". It fades // with "Wall ends here" on the same spring, so the pair comes and goes together; // scoped to the circle, so nothing else on the camera animates with it. Under Reduce - // Motion only its opacity changes, briefly: nothing moves. + // Motion only its opacity changes, briefly: nothing moves. It is there only while + // offered, rather than kept at zero opacity: kept, its "Aiming circle" stayed in the + // accessibility tree under the end question, hidden or not (CI run 37171342665). if state.gap?.pastEndSide != nil { - Reticle(diameter: 56) - .endAimCircle(shown: offersEndMark) - .frame(maxWidth: .infinity, maxHeight: .infinity) - .ignoresSafeArea() - .opacity(offersEndMark ? 1 : 0) - .animation(reduceMotion ? .easeOut(duration: 0.15) : Motion.settle, value: offersEndMark) + ZStack { + if offersEndMark { + Reticle(diameter: 56) + .endAimCircle() + .transition(.opacity) + } + } + .frame(maxWidth: .infinity, maxHeight: .infinity) + .ignoresSafeArea() + .animation(reduceMotion ? .easeOut(duration: 0.15) : Motion.settle, value: offersEndMark) } if state.gap?.isSatisfied == true { SuccessBadge() From 6a5b24018fce045061dd4a7a584a4a3566be71c4 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sat, 3 Oct 2026 23:05:17 -0500 Subject: [PATCH 23/71] ios: the coached fold tests look for the note's words, not the lead's Greptile on #214: since b1dc2525 the folded lead is "Try your flashlight", and the tests asserted both that the card shows it and that the card has no "flashlight". They now check that the note's own words, "Try your phone's flashlight", are under Details, not on the card. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScanUITests/PastEndRecoveryUITests.swift | 2 +- ios/HouseScanUITests/WalkRecoveryUITests.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ios/HouseScanUITests/PastEndRecoveryUITests.swift b/ios/HouseScanUITests/PastEndRecoveryUITests.swift index 87b568069..14d2250ab 100644 --- a/ios/HouseScanUITests/PastEndRecoveryUITests.swift +++ b/ios/HouseScanUITests/PastEndRecoveryUITests.swift @@ -103,7 +103,7 @@ final class PastEndRecoveryUITests: XCTestCase { let app = launch(["-uiDemoGap", "pastEnd", "-uiDemoCoaching", "tooDark"] + Self.largestText) let card = label(app, "instruction") XCTAssertTrue(card.contains("Try your flashlight"), "card reads: \(card)") - XCTAssertFalse(card.contains("flashlight"), "the coaching's note must fold under Details: \(card)") + XCTAssertFalse(card.contains("Try your phone's flashlight"), "the coaching's note must fold under Details: \(card)") let mark = app.buttons["action.markEnd"] scrollIntoView(mark, in: app) XCTAssertTrue(mark.isHittable) diff --git a/ios/HouseScanUITests/WalkRecoveryUITests.swift b/ios/HouseScanUITests/WalkRecoveryUITests.swift index 1d802449a..e13626a39 100644 --- a/ios/HouseScanUITests/WalkRecoveryUITests.swift +++ b/ios/HouseScanUITests/WalkRecoveryUITests.swift @@ -101,7 +101,7 @@ final class WalkRecoveryUITests: XCTestCase { let app = launch(["-uiDemoPhase", "wallWalk", "-uiDemoMarkEnd", "-uiDemoCoaching", "tooDark"] + Self.largestText) let card = label(app, "instruction") XCTAssertTrue(card.contains("Try your flashlight"), "card reads: \(card)") - XCTAssertFalse(card.contains("flashlight"), "the coaching's note must fold under Details: \(card)") + XCTAssertFalse(card.contains("Try your phone's flashlight"), "the coaching's note must fold under Details: \(card)") let mark = element(app, "action.markEnd") scrollIntoView(mark, in: app) XCTAssertTrue(mark.isHittable) From 87acda43d43aa45b90890f88ff3410394d652071 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 00:30:20 -0500 Subject: [PATCH 24/71] Import PR #192's capture packet 0.4, uploader and result client Copies the files from #192 at c4645771 byte for byte: CaptureResult/*, Packet/Gzip.swift, Packet/Packet04*.swift and Upload/Capture*.swift, with their tests (CaptureFolderOwnershipTests, CaptureResultSummaryTests, CaptureResultTests, CaptureSessionCoordinatorTests, CaptureUploaderTests, LoopbackCaptureAPI, Packet04Tests) and #192's two-line PacketWriterTests change, which adds the `shift` argument to makeJPEG. Nothing in the app uses them yet. Validated: every imported file's git blob equals c4645771's, and `swift test` in ios/HouseScanKit, run through heavy-admit, passed 818 tests in 105 suites. The app was not built. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../CaptureResult/CaptureResult.swift | 166 ++++ .../CaptureResultPlacement.swift | 182 +++++ .../CaptureResult/CaptureResultSummary.swift | 51 ++ .../Sources/HouseScanKit/Packet/Gzip.swift | 19 + .../HouseScanKit/Packet/Packet04.swift | 306 ++++++++ .../HouseScanKit/Packet/Packet04Check.swift | 115 +++ .../Packet/Packet04Producer.swift | 385 +++++++++ .../HouseScanKit/Packet/Packet04Streams.swift | 147 ++++ .../HouseScanKit/Upload/CaptureAPI.swift | 227 ++++++ .../Upload/CaptureSessionCoordinator.swift | 502 ++++++++++++ .../Upload/CaptureUploadGate.swift | 72 ++ .../Upload/CaptureUploadState.swift | 197 +++++ .../HouseScanKit/Upload/CaptureUploader.swift | 734 ++++++++++++++++++ .../CaptureFolderOwnershipTests.swift | 224 ++++++ .../CaptureResultSummaryTests.swift | 142 ++++ .../CaptureResultTests.swift | 320 ++++++++ .../CaptureSessionCoordinatorTests.swift | 711 +++++++++++++++++ .../CaptureUploaderTests.swift | 602 ++++++++++++++ .../LoopbackCaptureAPI.swift | 391 ++++++++++ .../HouseScanKitTests/Packet04Tests.swift | 277 +++++++ .../HouseScanKitTests/PacketWriterTests.swift | 4 +- 21 files changed, 5772 insertions(+), 2 deletions(-) create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResult.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultPlacement.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultSummary.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Packet/Gzip.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Check.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Producer.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Streams.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadGate.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/CaptureFolderOwnershipTests.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultSummaryTests.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultTests.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/Packet04Tests.swift diff --git a/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResult.swift b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResult.swift new file mode 100644 index 000000000..e04617f41 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResult.swift @@ -0,0 +1,166 @@ +import Foundation + +/// The capture API's device result, `GET /v1/captures/{id}/result`, written from the server's +/// documented wire contract. The whole body is camelCase. +/// +/// Only the fields the app shows or gates on are decoded. The verdict and its URL are left out on +/// purpose: the server masks a reject as manual review for the device, and the app must not read +/// around that mask. Unknown keys are ignored, because the server serves a result even when it has +/// drifted from its own schema. A status or outcome kind the app doesn't know decodes as +/// `.unknown`, so a new server value never hides the message and views that came with it. +public enum CaptureResult { + public struct Response: Decodable, Sendable, Equatable { + /// Nil before the server has started a run for the capture. + public var runId: String? + public var status: Status + public var viewsNeeded: [ViewNeeded] + public var memberActions: [String] + /// Nil until the run writes its result: while uploading, awaiting files or processing. + public var outcome: Outcome? + /// Nil when the server sent no criteria. The app shows none rather than inventing rows. + public var criteria: [Criterion]? + /// A signed URL that expires; nil when the run made no preview. + public var previewUrl: String? + /// A path on the API, not a full URL. + public var reviewUrl: String? + } + + public enum Status: Sendable, Equatable, Decodable { + case uploading, awaitingFiles, processing, needsViews, complete, manualReview, failed, expired + case unknown(String) + + public init(rawValue: String) { + self = switch rawValue { + case "uploading": .uploading + case "awaiting_files": .awaitingFiles + case "processing": .processing + case "needs_views": .needsViews + case "complete": .complete + case "manual_review": .manualReview + case "failed": .failed + case "expired": .expired + default: .unknown(rawValue) + } + } + + public init(from decoder: any Decoder) throws { + self.init(rawValue: try decoder.singleValueContainer().decode(String.self)) + } + } + + /// The decision shown to the homeowner. It is already masked for the device. + public struct Outcome: Decodable, Sendable, Equatable { + public var kind: OutcomeKind + public var profile: String + /// Text for the homeowner, shown as written. + public var message: String + public var viewsNeeded: [ViewNeeded] + /// Empty when the server shows a reject as manual review. + public var reasons: [Reason] + /// The ARKit world the spatial fields are in. Nil when the scene was not built in one. + public var arkitEpoch: String? + public var recommendedPlacement: RecommendedPlacement? + } + + public enum OutcomeKind: Sendable, Equatable, Decodable { + case eligible, needsMorePhotos, notEligible, manualReview + case unknown(String) + + public init(rawValue: String) { + self = switch rawValue { + case "eligible": .eligible + case "needs_more_photos": .needsMorePhotos + case "not_eligible": .notEligible + case "manual_review": .manualReview + default: .unknown(rawValue) + } + } + + public init(from decoder: any Decoder) throws { + self.init(rawValue: try decoder.singleValueContainer().decode(String.self)) + } + } + + public struct RecommendedPlacement: Decodable, Sendable, Equatable { + public var wallId: String + public var startSM: Double + public var confidence: Double + /// Nil when the scene was not built in an ARKit world. + public var boxArkitWorld: BoxArkitWorld? + } + + /// The box in ARKit world coordinates, as sent. The numbers are checked only when the app asks + /// for an AR placement (`Record.placement`), so a malformed box withholds AR and leaves the + /// rest of the result readable. + public struct BoxArkitWorld: Decodable, Sendable, Equatable { + /// Box centre to world, 16 numbers column by column. Local +x runs along the wall (left to + /// right seen from outside), +y up, +z out of the wall. + public var pose: [Double] + /// Width, height and depth in ARKit metres. + public var size: [Double] + } + + public struct ViewNeeded: Decodable, Sendable, Equatable { + public var id: String + /// The server's view kind, kept as text so a new kind still shows its prompt. + public var kind: String + public var criteria: [String] + public var why: String + public var promptId: String + /// Rendered text; show it verbatim. + public var prompt: Prompt + public var band: String? + public var wallId: String? + public var target: String? + /// Set in the photo flow instead of ARKit anchors. + public var referenceCrop: ReferenceCrop? + } + + public struct Prompt: Decodable, Sendable, Equatable { + public var title: String + public var body: String + } + + public struct ReferenceCrop: Decodable, Sendable, Equatable { + public var image: String + /// Pixel box in `image`: x0, y0, x1, y1. + public var boxPx: [Double] + } + + public struct Reason: Decodable, Sendable, Equatable { + public var criterion: String + public var text: String + public var measuredFt: Double? + public var plusMinusFt: Double? + public var thresholdFt: Double? + public var comparison: String? + } + + public struct Criterion: Decodable, Sendable, Equatable { + public var id: String + public var outcome: CriterionOutcome + public var unsureCause: String? + public var measuredFt: Double? + public var plusMinusFt: Double? + public var thresholdFt: Double? + public var coverage: String? + } + + public enum CriterionOutcome: Sendable, Equatable, Decodable { + case pass, fail, unsure + case unknown(String) + + public init(rawValue: String) { + self = switch rawValue { + case "pass": .pass + case "fail": .fail + case "unsure": .unsure + default: .unknown(rawValue) + } + } + + public init(from decoder: any Decoder) throws { + self.init(rawValue: try decoder.singleValueContainer().decode(String.self)) + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultPlacement.swift b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultPlacement.swift new file mode 100644 index 000000000..118c246b2 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultPlacement.swift @@ -0,0 +1,182 @@ +import simd + +extension CaptureResult { + /// The local identities a result belongs to. The response carries a run id and an ARKit epoch + /// but not the session or capture, and epochs repeat ("e1" starts every session), so only the + /// caller can say which scan a result is for. + public struct Association: Sendable, Equatable, Hashable { + /// The app's id for one run of the ARKit session. A new session gets a new id, even when it + /// reuses the epoch string. + public var sessionID: String + public var captureID: String + /// The run that finalize started for the capture. + public var runID: String + /// The ARKit epoch the packet was captured in. + public var epoch: String + + public init(sessionID: String, captureID: String, runID: String, epoch: String) { + self.sessionID = sessionID + self.captureID = captureID + self.runID = runID + self.epoch = epoch + } + } + + /// Whether the caller knows the server ran real analysis for this result. A stubbed server + /// returns a result that passes the contract without reconstructing anything, and nothing in + /// the JSON tells the two apart, so only the caller can grant `.verified`. + public enum Analysis: Sendable, Equatable { + case unverified + case verified + } + + /// A decoded result kept with the local scan it was fetched for. + public struct Record: Sendable, Equatable { + public let response: Response + public let association: Association + public let analysis: Analysis + + public init(response: Response, association: Association, analysis: Analysis = .unverified) { + self.response = response + self.association = association + self.analysis = analysis + } + + /// The recommended box in the meter anchor's frame, or the reason there is none. + /// + /// - Parameters: + /// - world: the session, capture, finalized run and epoch on screen now. + /// - meterAnchor: the meter anchor's anchor-to-world pose, in the same world coordinates + /// as the uploaded packet. Attaching the returned box to the anchor then carries it + /// through ARKit's later corrections. + public func placement(in world: Association, meterAnchor: simd_float4x4) -> Placement { + guard let outcome = response.outcome else { return .unavailable(.noOutcome(response.status)) } + guard outcome.kind == .eligible else { return .unavailable(.notEligible(outcome.kind)) } + guard response.status == .complete else { return .unavailable(.statusNotComplete(response.status)) } + guard analysis == .verified else { return .unavailable(.analysisUnverified) } + + guard world.sessionID == association.sessionID else { return .unavailable(.mismatch(.session)) } + guard world.captureID == association.captureID else { return .unavailable(.mismatch(.capture)) } + guard world.runID == association.runID, response.runId == association.runID else { + return .unavailable(.mismatch(.run)) + } + guard let epoch = outcome.arkitEpoch, + let box = outcome.recommendedPlacement?.boxArkitWorld + else { + return .unavailable(outcome.recommendedPlacement == nil ? .noPlacement : .notInARKitWorld) + } + guard world.epoch == association.epoch, epoch == association.epoch else { + return .unavailable(.mismatch(.epoch)) + } + + guard box.pose.count == 16 else { return .unavailable(.malformedBox(.poseCount(box.pose.count))) } + guard box.size.count == 3 else { return .unavailable(.malformedBox(.sizeCount(box.size.count))) } + guard (box.pose + box.size).allSatisfy(\.isFinite) else { return .unavailable(.malformedBox(.nonFinite)) } + guard box.size.allSatisfy({ $0 > 0 }) else { return .unavailable(.malformedBox(.nonPositiveSize)) } + let worldFromBox = Self.matrix(columnMajor: box.pose) + // The server divides the world transform's scale out of the pose's rotation and + // multiplies it into `size` instead. A pose that still carries a scale was built some + // other way, and using it would scale the box a second time. + guard Self.isRigid(worldFromBox) else { return .unavailable(.malformedBox(.notRigid)) } + + let worldFromAnchor = Self.double(meterAnchor) + guard Self.isRigid(worldFromAnchor) else { return .unavailable(.invalidMeterAnchor) } + + let anchorFromBox = Self.float(worldFromAnchor.inverse * worldFromBox) + let size = SIMD3(Float(box.size[0]), Float(box.size[1]), Float(box.size[2])) + // Finite Doubles can overflow Float, and positive sizes can round to zero. + let columns = [anchorFromBox.columns.0, anchorFromBox.columns.1, anchorFromBox.columns.2, anchorFromBox.columns.3] + guard columns.allSatisfy({ c in (0..<4).allSatisfy { c[$0].isFinite } }), + (0..<3).allSatisfy({ size[$0].isFinite }) + else { return .unavailable(.malformedBox(.nonFinite)) } + guard (0..<3).allSatisfy({ size[$0] > 0 }) else { return .unavailable(.malformedBox(.nonPositiveSize)) } + return .box(AnchoredBox(anchorFromBox: anchorFromBox, size: size)) + } + + private static func matrix(columnMajor v: [Double]) -> simd_double4x4 { + simd_double4x4( + SIMD4(v[0], v[1], v[2], v[3]), + SIMD4(v[4], v[5], v[6], v[7]), + SIMD4(v[8], v[9], v[10], v[11]), + SIMD4(v[12], v[13], v[14], v[15])) + } + + private static func double(_ m: simd_float4x4) -> simd_double4x4 { + simd_double4x4( + SIMD4(m.columns.0), SIMD4(m.columns.1), + SIMD4(m.columns.2), SIMD4(m.columns.3)) + } + + private static func float(_ m: simd_double4x4) -> simd_float4x4 { + simd_float4x4( + SIMD4(m.columns.0), SIMD4(m.columns.1), + SIMD4(m.columns.2), SIMD4(m.columns.3)) + } + + /// A finite rotation with no scale or shear, a translation, and a bottom row of 0 0 0 1. + /// The server writes each entry to six decimals, so its rotations are orthonormal to about + /// 1e-6; 1e-3 accepts that and ARKit's float poses, and rejects a 0.1% scale (whose + /// squared column length is off by 2e-3). + private static func isRigid(_ m: simd_double4x4) -> Bool { + let tolerance = 1e-3 + let columns = [m.columns.0, m.columns.1, m.columns.2, m.columns.3] + guard columns.allSatisfy({ c in (0..<4).allSatisfy { c[$0].isFinite } }) else { return false } + let bottom = SIMD4(m.columns.0.w, m.columns.1.w, m.columns.2.w, m.columns.3.w) + guard simd_reduce_max(simd_abs(bottom - SIMD4(0, 0, 0, 1))) <= tolerance else { return false } + let r = simd_double3x3( + SIMD3(m.columns.0.x, m.columns.0.y, m.columns.0.z), + SIMD3(m.columns.1.x, m.columns.1.y, m.columns.1.z), + SIMD3(m.columns.2.x, m.columns.2.y, m.columns.2.z)) + let gram = r.transpose * r + let identity = matrix_identity_double3x3 + for i in 0..<3 where simd_reduce_max(simd_abs(gram[i] - identity[i])) > tolerance { return false } + return r.determinant > 0 + } + } + + public enum Placement: Sendable, Equatable { + case box(AnchoredBox) + case unavailable(Unavailable) + } + + public struct AnchoredBox: Sendable, Equatable { + /// Box centre to meter anchor, rigid. Local +x along the wall, +y up, +z out of the wall. + public var anchorFromBox: simd_float4x4 + /// Width, height and depth in metres, as the server stated them. The server has already + /// applied the scene-to-ARKit scale, so the app uses these as they are. + public var size: SIMD3 + } + + /// Why a result has no AR placement. The rest of the result stays readable in every case. + public enum Unavailable: Sendable, Equatable { + /// The run has not written a result yet. + case noOutcome(Status) + /// Manual review, more photos, not eligible, or a kind this app doesn't know. + case notEligible(OutcomeKind) + case statusNotComplete(Status) + /// The caller has not confirmed that real analysis produced the result. + case analysisUnverified + /// The result belongs to another session, capture, run or ARKit world than the one on + /// screen. + case mismatch(IdentityField) + /// An eligible outcome without a recommended placement, which breaks the contract. + case noPlacement + /// The scene was not built in an ARKit world, so there is no box to anchor. + case notInARKitWorld + case malformedBox(BoxProblem) + /// The caller's meter anchor pose is not finite and rigid. + case invalidMeterAnchor + } + + public enum IdentityField: Sendable, Equatable, CaseIterable { + case session, capture, run, epoch + } + + public enum BoxProblem: Sendable, Equatable { + case poseCount(Int) + case sizeCount(Int) + case nonFinite + case nonPositiveSize + case notRigid + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultSummary.swift b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultSummary.swift new file mode 100644 index 000000000..65e156101 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultSummary.swift @@ -0,0 +1,51 @@ +import Foundation +import simd + +/// What the integration build shows of a capture-API result: where it came from, what the server +/// decided, its own words, the views it asks for, and why there is no AR box. Built from the +/// decoded response only; nothing is inferred, and an absent part stays absent. +public struct CaptureResultSummary: Sendable, Equatable { + public enum State: Sendable, Equatable { + /// The run has no outcome yet. + case working(CaptureResult.Status) + case decided(CaptureResult.OutcomeKind) + /// The capture is over without an outcome: it failed or expired. + case ended(CaptureResult.Status) + } + + public var state: State + /// The server's message for the homeowner, verbatim; nil before an outcome. + public var message: String? + /// Each requested view's prompt, verbatim, title then body. + public var prompts: [CaptureResult.Prompt] + /// The criteria rows the server sent; nil when it sent none. Never filled in by the app. + public var criteria: [CaptureResult.Criterion]? + /// False until someone has verified the server ran real analysis for this result. + public var analysisVerified: Bool + /// Why AR shows nothing for this result on the scan now on screen. + public var arUnavailable: CaptureResult.Unavailable? + + /// `world` is the scan on screen now; `meterAnchor` its meter anchor's pose in that world, nil + /// when the scan has none to place a box on (AR then says so, after the other checks). + public init(_ record: CaptureResult.Record, world: CaptureResult.Association, meterAnchor: simd_float4x4?) { + let response = record.response + state = if let outcome = response.outcome { + .decided(outcome.kind) + } else { + switch response.status { + case .failed, .expired: .ended(response.status) + default: .working(response.status) + } + } + message = response.outcome?.message + let views = response.viewsNeeded.isEmpty ? response.outcome?.viewsNeeded ?? [] : response.viewsNeeded + prompts = views.map(\.prompt) + criteria = response.criteria + analysisVerified = record.analysis == .verified + // A zero matrix is not a pose, so without an anchor the gate ends on `.invalidMeterAnchor`. + arUnavailable = switch record.placement(in: world, meterAnchor: meterAnchor ?? simd_float4x4()) { + case .box: nil + case .unavailable(let reason): reason + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Gzip.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Gzip.swift new file mode 100644 index 000000000..e4cd01dff --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Gzip.swift @@ -0,0 +1,19 @@ +import Foundation + +/// gzip (RFC 1952) around the raw DEFLATE stream Apple's `.zlib` compression writes. The packet's +/// streams travel as `.csv.gz`, and Foundation has no gzip writer of its own. +public enum Gzip { + public static func compress(_ data: Data) throws -> Data { + // NSData's .zlib is raw DEFLATE (RFC 1951) with no zlib header, which is what gzip wraps. + let deflated = try (data as NSData).compressed(using: .zlib) as Data + var out = Data([0x1F, 0x8B, 0x08, 0x00, 0, 0, 0, 0, 0x00, 0xFF]) + out.append(deflated) + appendLE(ZipCRC32.checksum(data), to: &out) + appendLE(UInt32(truncatingIfNeeded: data.count), to: &out) + return out + } + + private static func appendLE(_ value: UInt32, to data: inout Data) { + withUnsafeBytes(of: value.littleEndian) { data.append(contentsOf: $0) } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04.swift new file mode 100644 index 000000000..f7ce75d4d --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04.swift @@ -0,0 +1,306 @@ +import Foundation + +/// The capture packet 0.4 that the capture API's finalize call takes as its body: `packet.json`, +/// camelCase, in the ARKit world frame of one epoch. This is the subset House Scan produces, +/// written from the published wire contract, not copied from the server's models. Optional +/// fields are left out when nil, which the server reads as absent. +/// +/// Unlike the 1.1 packet (`PacketManifest`), poses here are the raw ARKit camera-to-world +/// transforms, never moved into the meter frame. +public enum Packet04 { + public static let format = "house-capture-packet" + public static let formatVersion = "0.4" + + public struct Packet: Codable, Sendable, Equatable { + public var format = Packet04.format + public var formatVersion = Packet04.formatVersion + public var packetId: String + public var createdAt: String + public var units = Units() + public var conventions = Conventions() + public var source: Source + public var session: Session + public var epochs: [Epoch] + public var keyframes: [Keyframe] + public var stills: [Still]? + public var taps: [Tap]? + public var tracking: [TrackingEvent] + public var streams: Streams + public var scaleReference: ScaleReference + public var files: [FileEntry] + public var ext: [String: String]? + } + + public struct Units: Codable, Sendable, Equatable { + public var length = "meters" + public var angle = "degrees" + public var time = "seconds of device uptime (ARFrame.timestamp and Core Motion timestamp)" + public var image = "pixels of the saved, unrotated landscape JPEG" + public var rotationRate = "radians per second" + public var acceleration = "g (9.80665 m/s^2)" + } + + public struct Conventions: Codable, Sendable, Equatable { + public var world = "ARKit world, worldAlignment .gravity: right-handed, +y up, origin where the epoch started" + public var camera = "ARKit camera: +x right, +y up in the unrotated landscape image, looking along -z" + public var pose = "camera-to-world 4x4, 16 numbers column by column (simd_float4x4 layout), meters" + public var intrinsics = "[fx, fy, cx, cy] in pixels of the saved, unrotated landscape image" + public var pixel = "[u, v] continuous: (0, 0) is the top-left corner of the top-left pixel, v grows down" + } + + public struct Source: Codable, Sendable, Equatable { + public enum Kind: String, Codable, Sendable { case device, replay, synthetic } + public var kind: Kind + public var appBuild: String? + + public init(kind: Kind, appBuild: String?) { + self.kind = kind + self.appBuild = appBuild + } + } + + public enum Tier: String, Codable, Sendable { case arkit, arkitLidar = "arkit_lidar" } + + public struct VideoFormat: Codable, Sendable, Equatable { + public var imageResolution: [Int] + public var framesPerSecond: Double? + + public init(width: Int, height: Int, framesPerSecond: Double?) { + imageResolution = [width, height] + self.framesPerSecond = framesPerSecond + } + } + + public struct Session: Codable, Sendable, Equatable { + public var id: String + public var startedAt: String + public var startedAtUptime: Double + public var endedAtUptime: Double? + public var timeZone: String? + public var appVersion: String + public var deviceModel: String + public var systemVersion: String + public var tier: Tier + public var lidarAvailable: Bool + public var meshReconstructionSupported: Bool? + public var sceneDepthEnabled: Bool + public var sceneReconstruction: String? + public var worldAlignment = "gravity" + public var planeDetection: [String]? + public var videoFormat: VideoFormat + public var motionReferenceFrame: String? + } + + public struct Epoch: Codable, Sendable, Equatable { + public var id: String + public var startTime: Double + public var reason: String + + public init(id: String, startTime: Double, reason: String) { + self.id = id + self.startTime = startTime + self.reason = reason + } + } + + public struct Exposure: Codable, Sendable, Equatable { + public var duration: Double? + public var offset: Double? + public var iso: Double? + public var fNumber: Double? + + public init(duration: Double?, offset: Double?, iso: Double?, fNumber: Double?) { + self.duration = duration + self.offset = offset + self.iso = iso + self.fNumber = fNumber + } + } + + public struct DepthRef: Codable, Sendable, Equatable { + public var file: String + public var confidenceFile: String? + public var w: Int + public var h: Int + } + + public struct Keyframe: Codable, Sendable, Equatable { + public var id: String + public var img: String + public var w: Int + public var h: Int + public var intrinsics: [Double] + public var pose: [Double] + public var timestamp: Double + public var tracking: String + public var epoch: String + public var reason: String + public var exposure: Exposure? + public var depth: DepthRef? + public var purpose: String? + } + + public struct Still: Codable, Sendable, Equatable { + public var id: String + public var purpose: String + public var img: String + public var w: Int + public var h: Int + public var timestamp: Double + /// EXIF orientation of the stored pixels. House Scan stores the sensor image unrotated. + public var orientation: Int + public var keyframe: String? + public var intrinsics: [Double]? + public var intrinsicsSource: String? + } + + public struct TapHit: Codable, Sendable, Equatable { + public var position: [Double] + public var target: String + public var alignment: String? + public var distance: Double? + + public init(position: [Double], target: String, alignment: String?, distance: Double?) { + self.position = position + self.target = target + self.alignment = alignment + self.distance = distance + } + } + + public struct Tap: Codable, Sendable, Equatable { + public var id: String + public var time: Double + public var label: String + public var keyframe: String + public var pixel: [Double] + public var rayOrigin: [Double] + public var rayDirection: [Double] + public var hit: TapHit? + public var epoch: String? + } + + public struct TrackingEvent: Codable, Sendable, Equatable { + public var time: Double + public var state: String + public var epoch: String? + + public init(time: Double, state: String, epoch: String?) { + self.time = time + self.state = state + self.epoch = epoch + } + } + + public struct StreamRef: Codable, Sendable, Equatable { + public var file: String + public var columns: [String] + public var rateHz: Double + } + + public struct Streams: Codable, Sendable, Equatable { + public var imuRaw: StreamRef + public var arkitPoses: StreamRef + } + + public struct CloseUp: Codable, Sendable, Equatable { + public var captured: Bool + public var still: String? + public var arkitDistanceM: Double? + public var side: String? + + public init(captured: Bool, still: String?, arkitDistanceM: Double?, side: String?) { + self.captured = captured + self.still = still + self.arkitDistanceM = arkitDistanceM + self.side = side + } + + public static let notCaptured = CloseUp(captured: false, still: nil, arkitDistanceM: nil, side: nil) + } + + public struct ScaleExt: Codable, Sendable, Equatable { + public var method = "a9-meter-model" + public var obliqueCloseUp: CloseUp + /// Head-on views of doors on the meter side. House Scan does not detect doors yet, so this + /// is empty, which the contract allows. + public var doorViews: [String] = [] + } + + public struct ScaleReference: Codable, Sendable, Equatable { + public var meterCloseUp: CloseUp + public var ext: ScaleExt + } + + public enum Role: String, Codable, Sendable, CaseIterable { + case keyframe, depth, confidence, features, still, stream, other + } + + public struct FileEntry: Codable, Sendable, Equatable { + public var path: String + public var bytes: Int + public var sha256: String + public var role: Role + } + + /// The `meta` a register call sends with an image: exactly one record, the same one the final + /// packet lists. + public enum ImageRecord: Codable, Sendable, Equatable { + case keyframe(Keyframe) + case still(Still) + + public var path: String { + switch self { + case .keyframe(let k): k.img + case .still(let s): s.img + } + } + + private enum CodingKeys: String, CodingKey { case keyframe, still } + + public func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .keyframe(let k): try container.encode(k, forKey: .keyframe) + case .still(let s): try container.encode(s, forKey: .still) + } + } + + public init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + if let k = try container.decodeIfPresent(Keyframe.self, forKey: .keyframe) { + self = .keyframe(k) + } else { + self = .still(try container.decode(Still.self, forKey: .still)) + } + } + } + + // MARK: Id and path patterns the server enforces + + /// `^[A-Za-z0-9_-]{8,64}$`: packetId. + public static func isStorageID(_ s: String) -> Bool { + (8...64).contains(s.count) && s.unicodeScalars.allSatisfy { isAlnum($0) || $0 == "_" || $0 == "-" } + } + + /// `^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$`: keyframe and still ids. + public static func isSegmentID(_ s: String) -> Bool { + guard let first = s.unicodeScalars.first, isAlnum(first), s.count <= 128 else { return false } + return s.unicodeScalars.allSatisfy { isAlnum($0) || $0 == "_" || $0 == "-" } + } + + /// Every segment starts with an alphanumeric, and a dot always starts a new alphanumeric token. + public static func isRelPath(_ s: String) -> Bool { + guard !s.isEmpty else { return false } + return s.split(separator: "/", omittingEmptySubsequences: false).allSatisfy { segment in + segment.split(separator: ".", omittingEmptySubsequences: false).allSatisfy { token in + guard let first = token.unicodeScalars.first, isAlnum(first) else { return false } + return token.unicodeScalars.allSatisfy { isAlnum($0) || $0 == "_" || $0 == "-" } + } + } + } + + private static func isAlnum(_ c: Unicode.Scalar) -> Bool { + ("a"..."z").contains(c) || ("A"..."Z").contains(c) || ("0"..."9").contains(c) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Check.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Check.swift new file mode 100644 index 000000000..3ea995149 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Check.swift @@ -0,0 +1,115 @@ +import Foundation +import simd + +/// House Scan's own checks of a 0.4 packet against the rules the capture API documents as +/// rejections: id and path patterns, the ARKit tier's required sections, listed files that exist +/// with their stated size and SHA-256, rigid poses, plausible intrinsics, still-to-keyframe links +/// and the tap replay. A packet that passes can still fail a server check this does not repeat. +public enum Packet04Check { + /// The server's tap replay tolerance on origin and direction. + public static let tapReplayTolerance = 1e-3 + + public static func problems(_ p: Packet04.Packet, folder: URL?) -> [String] { + var out: [String] = [] + if !Packet04.isStorageID(p.packetId) { out.append("packetId \(p.packetId) is not a StorageId") } + if p.formatVersion != Packet04.formatVersion { out.append("formatVersion \(p.formatVersion)") } + if p.keyframes.isEmpty { out.append("ARKit tier needs at least one keyframe") } + if p.session.tier == .arkitLidar, !p.session.lidarAvailable { out.append("arkit_lidar needs lidarAvailable") } + if p.session.worldAlignment != "gravity" { out.append("worldAlignment must be gravity") } + if p.streams.imuRaw.rateHz < Packet04Streams.minimumIMUHz { out.append("imuRaw rateHz \(p.streams.imuRaw.rateHz) < 50") } + if p.streams.arkitPoses.rateHz < Packet04Streams.minimumPoseHz { out.append("arkitPoses rateHz \(p.streams.arkitPoses.rateHz) < 10") } + + let epochs = Set(p.epochs.map(\.id)) + var listed: [String: Packet04.FileEntry] = [:] + for entry in p.files { + if !Packet04.isRelPath(entry.path) { out.append("file path \(entry.path) is not a RelPath") } + if listed.updateValue(entry, forKey: entry.path) != nil { out.append("file \(entry.path) listed twice") } + } + func requireListed(_ path: String, _ role: Packet04.Role, _ place: String) { + guard let entry = listed[path] else { return out.append("\(place) \(path) is not in files") } + if entry.role != role { out.append("\(place) \(path) has role \(entry.role.rawValue), not \(role.rawValue)") } + } + + var frames: [String: Packet04.Keyframe] = [:] + var byTime: [Double: Packet04.Keyframe] = [:] + for k in p.keyframes { + if !Packet04.isSegmentID(k.id) { out.append("keyframe id \(k.id) is not a SegmentId") } + if frames.updateValue(k, forKey: k.id) != nil { out.append("keyframe \(k.id) listed twice") } + if !epochs.contains(k.epoch) { out.append("keyframe \(k.id) names unknown epoch \(k.epoch)") } + requireListed(k.img, .keyframe, "keyframe \(k.id) image") + if let depth = k.depth { + requireListed(depth.file, .depth, "keyframe \(k.id) depth") + if let confidence = depth.confidenceFile { requireListed(confidence, .confidence, "keyframe \(k.id) confidence") } + if let entry = listed[depth.file], entry.bytes != depth.w * depth.h * 4 { out.append("keyframe \(k.id) depth size") } + } + guard let pose = matrix(k.pose) else { out.append("keyframe \(k.id) pose is not 16 finite numbers"); continue } + if !PacketPose.isRigid(pose) { out.append("keyframe \(k.id) pose is not rigid") } + if k.intrinsics.count != 4 { + out.append("keyframe \(k.id) intrinsics") + } else if let problem = PacketWriter.intrinsicsProblem( + SIMD4(k.intrinsics.map(Float.init)), width: k.w, height: k.h) { + out.append("keyframe \(k.id) intrinsics: \(problem)") + } + if let other = byTime[k.timestamp], other.epoch == k.epoch, other.pose != k.pose { + out.append("keyframes \(other.id) and \(k.id) share timestamp \(k.timestamp) with different poses") + } + byTime[k.timestamp] = k + } + var stillIDs = Set() + for s in p.stills ?? [] { + if !Packet04.isSegmentID(s.id) { out.append("still id \(s.id) is not a SegmentId") } + stillIDs.insert(s.id) + requireListed(s.img, .still, "still \(s.id) image") + if let link = s.keyframe { + guard let frame = frames[link] else { out.append("still \(s.id) names unknown keyframe \(link)"); continue } + if frame.w != s.w || frame.h != s.h { out.append("still \(s.id) and keyframe \(link) sizes differ") } + } + } + for tap in p.taps ?? [] { + guard let frame = frames[tap.keyframe] else { out.append("tap \(tap.id) names unknown keyframe \(tap.keyframe)"); continue } + if tap.pixel.count != 2 || !(0...Double(frame.w)).contains(tap.pixel[0]) || !(0...Double(frame.h)).contains(tap.pixel[1]) { + out.append("tap \(tap.id) pixel is outside keyframe \(frame.id)") + continue + } + if let epoch = tap.epoch, epoch != frame.epoch { out.append("tap \(tap.id) epoch differs from its keyframe's") } + // Compared pairwise below, a missing or extra component, or a NaN, would go unseen. + guard tap.rayOrigin.count == 3, tap.rayDirection.count == 3, (tap.rayOrigin + tap.rayDirection).allSatisfy(\.isFinite) else { + out.append("tap \(tap.id) ray is not two finite 3-vectors") + continue + } + let replay = ray(pose: frame.pose, intrinsics: frame.intrinsics, pixel: tap.pixel) + let error = zip(replay.origin + replay.direction, tap.rayOrigin + tap.rayDirection).map { abs($0 - $1) }.max() ?? .infinity + if !(error <= tapReplayTolerance) { out.append("tap \(tap.id) ray replays with error \(error)") } + } + for (name, close) in [("meterCloseUp", p.scaleReference.meterCloseUp), ("obliqueCloseUp", p.scaleReference.ext.obliqueCloseUp)] { + if close.captured, !(close.still.map(stillIDs.contains) ?? false) { out.append("\(name) names a still that is not in the packet") } + } + requireListed(p.streams.imuRaw.file, .stream, "imuRaw") + requireListed(p.streams.arkitPoses.file, .stream, "arkitPoses") + + if let folder { + for entry in p.files { + guard let data = try? Data(contentsOf: folder.appending(path: entry.path)) else { out.append("file \(entry.path) is missing"); continue } + if data.count != entry.bytes { out.append("file \(entry.path) is \(data.count) bytes, listed \(entry.bytes)") } + if PacketFiles.sha256(data) != entry.sha256 { out.append("file \(entry.path) SHA-256 differs from its listing") } + } + } + return out + } + + /// The ray through `pixel` as the server replays it: origin the camera position, direction + /// the unit vector of R Ā· ((u āˆ’ cx)/fx, āˆ’(v āˆ’ cy)/fy, āˆ’1). + public static func ray(pose: [Double], intrinsics k: [Double], pixel: [Double]) -> (origin: [Double], direction: [Double]) { + guard pose.count == 16, k.count == 4, pixel.count == 2 else { return ([], []) } + let d = SIMD3((pixel[0] - k[2]) / k[0], -(pixel[1] - k[3]) / k[1], -1) + let c0 = SIMD3(pose[0], pose[1], pose[2]), c1 = SIMD3(pose[4], pose[5], pose[6]), c2 = SIMD3(pose[8], pose[9], pose[10]) + let world = simd_normalize(c0 * d.x + c1 * d.y + c2 * d.z) + return ([pose[12], pose[13], pose[14]], [world.x, world.y, world.z]) + } + + static func matrix(_ values: [Double]) -> simd_float4x4? { + guard values.count == 16, values.allSatisfy(\.isFinite) else { return nil } + let f = values.map(Float.init) + return simd_float4x4(SIMD4(f[0], f[1], f[2], f[3]), SIMD4(f[4], f[5], f[6], f[7]), SIMD4(f[8], f[9], f[10], f[11]), SIMD4(f[12], f[13], f[14], f[15])) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Producer.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Producer.swift new file mode 100644 index 000000000..7691ebe16 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Producer.swift @@ -0,0 +1,385 @@ +import CryptoKit +import Foundation +import ImageIO +import simd + +/// A file of the 0.4 packet that is on disk and will never change: its bytes, digests and, for +/// an image, the record that describes it. The same value registers the file with the capture API +/// and lists it in the final packet, so the two can't disagree. +public struct SealedFile: Codable, Sendable, Equatable { + public var path: String + public var bytes: Int + public var sha256: String + /// Base64 of the 16-byte MD5, as the register call and the storage PUT's Content-MD5 want it. + public var md5: String + public var role: Packet04.Role + public var contentType: String + public var meta: Packet04.ImageRecord? + /// Lower goes first: the meter close-ups before keyframes before streams. + public var priority: Int + + public var entry: Packet04.FileEntry { .init(path: path, bytes: bytes, sha256: sha256, role: role) } + + static func seal(_ data: Data, path: String, role: Packet04.Role, contentType: String, meta: Packet04.ImageRecord?, priority: Int) -> SealedFile { + SealedFile( + path: path, bytes: data.count, sha256: PacketFiles.sha256(data), + md5: Data(Insecure.MD5.hash(data: data)).base64EncodedString(), role: role, contentType: contentType, meta: meta, + priority: priority) + } +} + +/// What the capture observed for one saved camera image, read from its own ARFrame when the +/// pixels were taken: the raw ARKit camera-to-world pose, never one corrected into the meter frame. +public struct Packet04Observation: Sendable, Equatable { + public var t: Double + public var cameraToWorld: simd_float4x4 + /// [fx, fy, cx, cy] in pixels of the stored JPEG. + public var intrinsics: SIMD4 + public var width: Int + public var height: Int + public var tracking: PacketTracking + public var exposure: Packet04.Exposure? + + public init(t: Double, cameraToWorld: simd_float4x4, intrinsics: SIMD4, width: Int, height: Int, tracking: PacketTracking, exposure: Packet04.Exposure? = nil) { + self.t = t + self.cameraToWorld = cameraToWorld + self.intrinsics = intrinsics + self.width = width + self.height = height + self.tracking = tracking + self.exposure = exposure + } +} + +/// ARKit depth for one keyframe: Float32 meters and UInt8 confidence, row by row. +public struct Packet04Depth: Sendable { + public var meters: [Float] + public var confidence: [UInt8] + public var width: Int + public var height: Int + + public init(meters: [Float], confidence: [UInt8], width: Int, height: Int) { + self.meters = meters + self.confidence = confidence + self.width = width + self.height = height + } +} + +/// Fixed facts about the capture, known when the ARKit session starts and the homeowner consents, +/// so the create request can be sent before the first photo. +public struct Packet04SessionInfo: Codable, Sendable, Equatable { + public var packetID: String + public var sessionID: String + public var source: Packet04.Source.Kind + public var appVersion: String + public var deviceModel: String + public var systemVersion: String + public var lidarAvailable: Bool + public var sceneDepthEnabled: Bool + public var meshReconstructionSupported: Bool? + public var sceneReconstruction: String? + public var planeDetection: [String]? + public var videoWidth: Int + public var videoHeight: Int + public var framesPerSecond: Double? + public var timeZone: String? + + public init( + packetID: String, sessionID: String, source: Packet04.Source.Kind, appVersion: String, deviceModel: String, systemVersion: String, + lidarAvailable: Bool, sceneDepthEnabled: Bool, meshReconstructionSupported: Bool?, sceneReconstruction: String?, + planeDetection: [String]?, videoWidth: Int, videoHeight: Int, framesPerSecond: Double?, timeZone: String? + ) { + self.packetID = packetID + self.sessionID = sessionID + self.source = source + self.appVersion = appVersion + self.deviceModel = deviceModel + self.systemVersion = systemVersion + self.lidarAvailable = lidarAvailable + self.sceneDepthEnabled = sceneDepthEnabled + self.meshReconstructionSupported = meshReconstructionSupported + self.sceneReconstruction = sceneReconstruction + self.planeDetection = planeDetection + self.videoWidth = videoWidth + self.videoHeight = videoHeight + self.framesPerSecond = framesPerSecond + self.timeZone = timeZone + } + + /// `arkit_lidar` needs depth the session actually recorded, not a phone that could have. + public var tier: Packet04.Tier { lidarAvailable && sceneDepthEnabled ? .arkitLidar : .arkit } +} + +public enum Packet04Error: Error, Equatable, CustomStringConvertible { + case invalidID(String) + case sealedAfterFinish(String) + case duplicate(String) + case invalidImage(id: String, reason: String) + case notRigid(String) + case unknownKeyframe(String) + case streamTooSlow(stream: String, measuredHz: Double, minimumHz: Double) + case streamEmpty(String) + case noKeyframes + case invalidPacket([String]) + + public var description: String { + switch self { + case .invalidID(let id): "\(id) does not fit the server's id pattern" + case .sealedAfterFinish(let what): "\(what) arrived after the packet was finished" + case .duplicate(let id): "\(id) was sealed twice" + case .invalidImage(let id, let reason): "image \(id): \(reason)" + case .notRigid(let place): "\(place): pose is not a rotation plus a translation" + case .unknownKeyframe(let id): "no keyframe \(id)" + case .streamTooSlow(let stream, let hz, let minimum): "\(stream) measured \(hz) Hz; the contract requires at least \(minimum) Hz" + case .streamEmpty(let stream): "\(stream) has no rows" + case .noKeyframes: "an ARKit packet needs at least one keyframe" + case .invalidPacket(let problems): "packet fails its own checks: \(problems.joined(separator: "; "))" + } + } +} + +/// Builds one 0.4 packet while the capture runs. Each image is written into `folder` and sealed +/// the moment it is observed, so it can be uploaded during the capture; `finish` adds the streams +/// and returns the exact `packet.json` bytes, which are frozen from then on. +/// +/// One producer is one ARKit epoch. A world reset ends the packet: the app starts a new producer +/// with a new packet id rather than mixing two worlds under one epoch. +public actor Packet04Producer { + public nonisolated let folder: URL + public nonisolated let info: Packet04SessionInfo + public nonisolated let epoch = "e1" + private let startedAtUptime: Double + private let startedAt: Date + private var keyframes: [Packet04.Keyframe] = [] + private var stills: [Packet04.Still] = [] + private var taps: [Packet04.Tap] = [] + private var files: [SealedFile] = [] + private var finished: Data? + + /// `startedAtUptime` is the uptime of the session's first ARFrame and `startedAt` the wall clock + /// at that instant. + public init(folder: URL, info: Packet04SessionInfo, startedAtUptime: Double, startedAt: Date) throws { + guard Packet04.isStorageID(info.packetID) else { throw Packet04Error.invalidID(info.packetID) } + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + self.folder = folder + self.info = info + self.startedAtUptime = startedAtUptime + self.startedAt = startedAt + } + + public var sealedFiles: [SealedFile] { files } + public var keyframeCount: Int { keyframes.count } + public var isFinished: Bool { finished != nil } + + /// The keyframe sealed from the frame at `t`, if there is one: a still taken from a frame that + /// was already kept links to it instead of repeating it. + public func keyframeID(at t: Double) -> String? { keyframes.first { $0.timestamp == t }?.id } + + /// "k00001" for 1. + public static func keyframeID(_ number: Int) -> String { + let digits = String(number) + return "k" + String(repeating: "0", count: max(0, 5 - digits.count)) + digits + } + + // MARK: Images + + /// Writes the JPEG as keyframes/.jpg with, when given, its depth and confidence, and seals + /// them. `reason` is the contract's keyframe reason (motion, tap, still, request, auto). + public func sealKeyframe( + jpeg: Data, observation o: Packet04Observation, reason: String, purpose: String? = nil, depth: Packet04Depth? = nil + ) throws -> [SealedFile] { + guard finished == nil else { throw Packet04Error.sealedAfterFinish("keyframe") } + let id = Self.keyframeID(keyframes.count + 1) + try check(o, id: id, jpeg: jpeg) + guard let tracking = Self.trackingText(o.tracking) else { + throw Packet04Error.invalidImage(id: id, reason: "tracking is limited for a reason the contract has no name for") + } + var sealed: [SealedFile] = [] + var depthRef: Packet04.DepthRef? + if let depth, depth.meters.count == depth.width * depth.height, depth.confidence.count == depth.meters.count, depth.width > 0 { + let map = try write(PacketFiles.depthData(meters: depth.meters), "keyframes/\(id).depth.f32", role: .depth, type: "application/octet-stream", priority: 20) + let confidence = try write(Data(depth.confidence), "keyframes/\(id).confidence.u8", role: .confidence, type: "application/octet-stream", priority: 20) + sealed += [map, confidence] + depthRef = .init(file: map.path, confidenceFile: confidence.path, w: depth.width, h: depth.height) + } + let record = Packet04.Keyframe( + id: id, img: "keyframes/\(id).jpg", w: o.width, h: o.height, intrinsics: Self.numbers(o.intrinsics), + pose: PacketPose.columnMajor(o.cameraToWorld), timestamp: o.t, tracking: tracking, epoch: epoch, + reason: reason, exposure: o.exposure, depth: depthRef, purpose: purpose) + let image = try write(jpeg, record.img, role: .keyframe, type: "image/jpeg", meta: .keyframe(record), priority: 10) + keyframes.append(record) + return [image] + sealed + } + + /// A checklist still taken from an ARKit keyframe already sealed: the same pixels under + /// stills/.jpg, linked to the keyframe so it keeps its pose. Stills go first. + /// + /// A retake of the same purpose is a new still ("meter_close-2"): sealed files never change, + /// and `finish` says which one the scale reference names. + public func sealStill(purpose: String, keyframe keyframeID: String) throws -> SealedFile { + guard finished == nil else { throw Packet04Error.sealedAfterFinish("still") } + guard Packet04.isSegmentID(purpose) else { throw Packet04Error.invalidID(purpose) } + guard let frame = keyframes.first(where: { $0.id == keyframeID }) else { throw Packet04Error.unknownKeyframe(keyframeID) } + if let same = stills.last(where: { $0.purpose == purpose }), same.keyframe == keyframeID { + throw Packet04Error.duplicate(same.id) + } + let taken = stills.filter { $0.purpose == purpose }.count + let id = taken == 0 ? purpose : "\(purpose)-\(taken + 1)" + let jpeg = try Data(contentsOf: folder.appending(path: frame.img)) + let still = Packet04.Still( + id: id, purpose: purpose, img: "stills/\(id).jpg", w: frame.w, h: frame.h, timestamp: frame.timestamp, + orientation: 1, keyframe: frame.id, intrinsics: frame.intrinsics, intrinsicsSource: "arkit") + let file = try write(jpeg, still.img, role: .still, type: "image/jpeg", meta: .still(still), priority: 0) + stills.append(still) + return file + } + + /// A tap on a sealed keyframe's pixel, with the ray rebuilt from that keyframe's own pose and + /// intrinsics, so the server's replay reproduces it exactly. + public func addTap(id: String, label: String, keyframe keyframeID: String, pixel: SIMD2, time: Double, hit: Packet04.TapHit?) throws { + guard finished == nil else { throw Packet04Error.sealedAfterFinish("tap") } + guard let frame = keyframes.first(where: { $0.id == keyframeID }) else { throw Packet04Error.unknownKeyframe(keyframeID) } + let ray = Packet04Check.ray(pose: frame.pose, intrinsics: frame.intrinsics, pixel: [pixel.x, pixel.y]) + taps.append(.init( + id: id, time: time, label: label, keyframe: keyframeID, pixel: [pixel.x, pixel.y], rayOrigin: ray.origin, + rayDirection: ray.direction, hit: hit, epoch: epoch)) + } + + /// The frame the homeowner tapped, kept as a keyframe (reason `tap`) unless it already is one, + /// and the tap on it: `pixel` in that frame's stored image, the ray rebuilt from its own pose + /// and intrinsics, and what the phone's raycast hit. + public func sealTap( + id: String, label: String, jpeg: Data, observation: Packet04Observation, pixel: SIMD2, hit: Packet04.TapHit? + ) throws -> [SealedFile] { + var files: [SealedFile] = [] + if keyframeID(at: observation.t) == nil { + files = try sealKeyframe(jpeg: jpeg, observation: observation, reason: "tap") + } + guard let keyframe = keyframeID(at: observation.t) else { throw Packet04Error.unknownKeyframe("at \(observation.t)") } + try addTap(id: id, label: label, keyframe: keyframe, pixel: pixel, time: observation.t, hit: hit) + return files + } + + // MARK: Finish + + /// Writes the two required streams, then returns their sealed files and the exact packet + /// bytes. The packet is checked against the contract's rules first; a failure names each + /// problem instead of sending a packet the server would reject. + /// `acceptedCloseUpAt` is the frame time of the head-on close-up the scan accepted, nil when + /// none was (skipped, or every shot refused): only that still is the scale reference. + public func finish( + poses: [Packet04Streams.PoseRow], accelerometer: [Packet04Streams.MotionRow], gyroscope: [Packet04Streams.MotionRow], + endedAtUptime: Double, acceptedCloseUpAt: Double?, closeUpDistanceM: Double? = nil, createdAt: Date = Date() + ) throws -> (streams: [SealedFile], packet: Data) { + if let finished { return (files.filter { $0.role == .stream }, finished) } + guard !keyframes.isEmpty else { throw Packet04Error.noKeyframes } + let posesCSV = try Packet04Streams.arkitPoses(poses, epoch: epoch) + let imu = try Packet04Streams.imuRaw(accelerometer: accelerometer, gyroscope: gyroscope) + let poseFile = try write(try Gzip.compress(posesCSV.csv), "streams/arkit_poses.csv.gz", role: .stream, type: "application/gzip", priority: 30) + let imuFile = try write(try Gzip.compress(imu.csv), "streams/imu_raw.csv.gz", role: .stream, type: "application/gzip", priority: 30) + // Each sensor as Core Motion timed it, so the server can re-pair them its own way. + let accelFile = try write( + try Gzip.compress(try Packet04Streams.rawMotionCSV(accelerometer, columns: ["t", "ax", "ay", "az"])), + "streams/accelerometer_raw.csv.gz", role: .stream, type: "application/gzip", priority: 31) + let gyroFile = try write( + try Gzip.compress(try Packet04Streams.rawMotionCSV(gyroscope, columns: ["t", "gx", "gy", "gz"])), + "streams/gyroscope_raw.csv.gz", role: .stream, type: "application/gzip", priority: 31) + var ext = imu.pairingNote + ext["rawAccelerometerFile"] = accelFile.path + ext["rawGyroscopeFile"] = gyroFile.path + + let close = acceptedCloseUpAt.flatMap { t in stills.last { $0.purpose == "meter_close" && $0.timestamp == t } } + // The app takes no oblique close-up yet, so none is ever accepted. + let oblique: Packet04.Still? = nil + let packet = Packet04.Packet( + packetId: info.packetID, createdAt: PacketWriter.iso8601(createdAt), + source: .init(kind: info.source, appBuild: info.appVersion), + session: .init( + id: info.sessionID, startedAt: PacketWriter.iso8601(startedAt), startedAtUptime: startedAtUptime, + endedAtUptime: max(endedAtUptime, startedAtUptime), timeZone: info.timeZone, appVersion: info.appVersion, + deviceModel: info.deviceModel, systemVersion: info.systemVersion, tier: info.tier, lidarAvailable: info.lidarAvailable, + meshReconstructionSupported: info.meshReconstructionSupported, sceneDepthEnabled: info.sceneDepthEnabled, + sceneReconstruction: info.sceneReconstruction, planeDetection: info.planeDetection, + videoFormat: .init(width: info.videoWidth, height: info.videoHeight, framesPerSecond: info.framesPerSecond), + motionReferenceFrame: nil), + epochs: [.init(id: epoch, startTime: startedAtUptime, reason: "sessionStart")], + keyframes: keyframes, stills: stills.isEmpty ? nil : stills, taps: taps.isEmpty ? nil : taps, + tracking: Packet04Streams.trackingChanges(poses, epoch: epoch), + streams: .init( + imuRaw: .init(file: imuFile.path, columns: Packet04Streams.imuColumns, rateHz: imu.rateHz), + arkitPoses: .init(file: poseFile.path, columns: Packet04Streams.poseColumns, rateHz: posesCSV.rateHz)), + scaleReference: .init( + meterCloseUp: close.map { .init(captured: true, still: $0.id, arkitDistanceM: closeUpDistanceM, side: nil) } ?? .notCaptured, + ext: .init(obliqueCloseUp: oblique.map { .init(captured: true, still: $0.id, arkitDistanceM: nil, side: nil) } ?? .notCaptured)), + files: files.map(\.entry), + ext: ext) + let problems = Packet04Check.problems(packet, folder: folder) + guard problems.isEmpty else { throw Packet04Error.invalidPacket(problems) } + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys, .withoutEscapingSlashes] + let data = try encoder.encode(packet) + try data.write(to: folder.appending(path: "packet.json"), options: .atomic) + finished = data + return ([poseFile, imuFile, accelFile, gyroFile], data) + } + + // MARK: Helpers + + private func check(_ o: Packet04Observation, id: String, jpeg: Data) throws { + guard PacketPose.isRigid(o.cameraToWorld) else { throw Packet04Error.notRigid("keyframe \(id)") } + guard o.t.isFinite, o.t >= 0 else { throw Packet04Error.invalidImage(id: id, reason: "timestamp \(o.t)") } + if let problem = PacketWriter.intrinsicsProblem(o.intrinsics, width: o.width, height: o.height) { + throw Packet04Error.invalidImage(id: id, reason: problem) + } + if let problem = Self.jpegProblem(jpeg, width: o.width, height: o.height) { + throw Packet04Error.invalidImage(id: id, reason: problem) + } + if let same = keyframes.first(where: { $0.timestamp == o.t }) { + throw Packet04Error.invalidImage(id: id, reason: "keyframe \(same.id) already has timestamp \(o.t)") + } + } + + private func write(_ data: Data, _ path: String, role: Packet04.Role, type: String, meta: Packet04.ImageRecord? = nil, priority: Int) throws -> SealedFile { + guard Packet04.isRelPath(path) else { throw Packet04Error.invalidID(path) } + guard !files.contains(where: { $0.path == path }) else { throw Packet04Error.duplicate(path) } + let url = folder.appending(path: path) + try FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true) + try data.write(to: url, options: .atomic) + let file = SealedFile.seal(data, path: path, role: role, contentType: type, meta: meta, priority: priority) + files.append(file) + return file + } + + /// The bytes must be a JPEG of the stated size, stored unrotated. + static func jpegProblem(_ jpeg: Data, width: Int, height: Int) -> String? { + guard let source = CGImageSourceCreateWithData(jpeg as CFData, nil), CGImageSourceGetType(source) as String? == "public.jpeg", + let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any] + else { return "not a readable JPEG" } + let w = properties[kCGImagePropertyPixelWidth] as? Int, h = properties[kCGImagePropertyPixelHeight] as? Int + guard w == width, h == height else { return "the JPEG is \(w ?? 0)x\(h ?? 0), not \(width)x\(height)" } + if let orientation = properties[kCGImagePropertyOrientation] as? Int, orientation != 1 { + return "EXIF orientation \(orientation); store the unrotated sensor image" + } + return nil + } + + static func numbers(_ k: SIMD4) -> [Double] { [k.x, k.y, k.z, k.w].map(PacketNumber.double) } + + /// The contract's tracking state. Nil for a limited state whose reason the contract has no + /// name for (a reason added after iOS 26): no value there would be an observation. + public static func trackingText(_ tracking: PacketTracking) -> String? { + switch tracking { + case .normal: "normal" + case .notAvailable: "notAvailable" + case .limited(let why): + switch why { + case .initializing?: "limited.initializing" + case .excessiveMotion?: "limited.excessiveMotion" + case .insufficientFeatures?: "limited.insufficientFeatures" + case .relocalizing?: "limited.relocalizing" + case nil: nil + } + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Streams.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Streams.swift new file mode 100644 index 000000000..be5cabecc --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Streams.swift @@ -0,0 +1,147 @@ +import Foundation +import simd + +/// The 0.4 packet's two required streams as CSV, and the tracking changes read from the poses. +public enum Packet04Streams { + public static let poseColumns = ["t", "epoch", "tracking", "tx", "ty", "tz", "qx", "qy", "qz", "qw", "fx", "fy", "cx", "cy"] + public static let imuColumns = ["t", "ax", "ay", "az", "gx", "gy", "gz"] + /// The contract's floors for `rateHz`. + public static let minimumPoseHz = 10.0 + public static let minimumIMUHz = 50.0 + + /// One ARFrame: raw camera-to-world and that frame's intrinsics at the video format's size. + public struct PoseRow: Sendable, Equatable { + public var t: Double + public var tracking: PacketTracking + public var cameraToWorld: simd_float4x4 + public var intrinsics: SIMD4 + + public init(t: Double, tracking: PacketTracking, cameraToWorld: simd_float4x4, intrinsics: SIMD4) { + self.t = t + self.tracking = tracking + self.cameraToWorld = cameraToWorld + self.intrinsics = intrinsics + } + } + + /// One Core Motion sample with its own timestamp: g for the accelerometer, rad/s for the gyro. + public struct MotionRow: Sendable, Equatable { + public var t: Double + public var value: SIMD3 + + public init(t: Double, value: SIMD3) { + self.t = t + self.value = value + } + } + + public static func arkitPoses(_ rows: [PoseRow], epoch: String) throws -> (csv: Data, rateHz: Double) { + guard !rows.isEmpty else { throw Packet04Error.streamEmpty("arkitPoses") } + var text = poseColumns.joined(separator: ",") + "\n" + var times: [Double] = [] + for row in rows where row.t > (times.last ?? -.infinity) { + guard PacketPose.isRigid(row.cameraToWorld) else { continue } + let p = PacketPose.translation(row.cameraToWorld) + let q = PacketPose.quaternion(PacketPose.rotation(row.cameraToWorld)) + let k = row.intrinsics + // The column is free text: a limited state whose reason the contract can't name is + // still limited, and the pose it came with is still a measurement. + let cells = [row.t.description, epoch, Packet04Producer.trackingText(row.tracking) ?? "limited"] + + [p.x, p.y, p.z].map { PacketNumber.double($0).description } + + [q.x, q.y, q.z, q.w].map(\.description) + + [k.x, k.y, k.z, k.w].map { PacketNumber.double($0).description } + text += cells.joined(separator: ",") + "\n" + times.append(row.t) + } + let rate = measuredRate(times) + guard rate >= minimumPoseHz else { throw Packet04Error.streamTooSlow(stream: "arkitPoses", measuredHz: rate, minimumHz: minimumPoseHz) } + return (Data(text.utf8), rate) + } + + /// Accelerometer and gyro rows for `imuRaw`. Core Motion times the two sensors separately, and + /// the contract's row holds both. Each row is an accelerometer sample paired with the gyro + /// sample nearest in time, only when that gyro sample is within `maxPairingOffset`; an + /// accelerometer sample with no gyro sample that close is left out rather than given one. + /// How the server wants the two clocks joined is an open question to the server team, so the + /// rule and its measured worst offset travel in the packet's `ext`. + public static func imuRaw( + accelerometer: [MotionRow], gyroscope: [MotionRow], maxPairingOffset: Double = 0.005 + ) throws -> (csv: Data, rateHz: Double, pairingNote: [String: String]) { + let accel = accelerometer.sorted { $0.t < $1.t } + let gyro = gyroscope.sorted { $0.t < $1.t } + guard !accel.isEmpty, !gyro.isEmpty else { throw Packet04Error.streamEmpty("imuRaw") } + var text = imuColumns.joined(separator: ",") + "\n" + var times: [Double] = [] + var worst = 0.0 + var g = 0 + for a in accel where a.t > (times.last ?? -.infinity) { + while g + 1 < gyro.count, abs(gyro[g + 1].t - a.t) <= abs(gyro[g].t - a.t) { g += 1 } + let offset = abs(gyro[g].t - a.t) + guard offset <= maxPairingOffset else { continue } + worst = max(worst, offset) + let cells = [a.t, a.value.x, a.value.y, a.value.z, gyro[g].value.x, gyro[g].value.y, gyro[g].value.z] + text += cells.map(\.description).joined(separator: ",") + "\n" + times.append(a.t) + } + guard !times.isEmpty else { throw Packet04Error.streamEmpty("imuRaw") } + let rate = measuredRate(times) + guard rate >= minimumIMUHz else { throw Packet04Error.streamTooSlow(stream: "imuRaw", measuredHz: rate, minimumHz: minimumIMUHz) } + let note = [ + "imuRawRowClock": "accelerometer", + "imuRawGyroPairing": "nearest gyro sample within \(maxPairingOffset) s; accelerometer samples without one are left out", + "imuRawGyroOffsetMaxS": worst.description, + "imuRawRowsKept": "\(times.count) of \(accel.count)", + "imuRawPairingStatus": "proposal from the phone; the server has not agreed how accelerometer and gyro samples join into one row", + ] + return (Data(text.utf8), rate, note) + } + + /// The first state and every change. A limited state with no name in the contract is not + /// listed; the rows around it still are. + public static func trackingChanges(_ rows: [PoseRow], epoch: String) -> [Packet04.TrackingEvent] { + var events: [Packet04.TrackingEvent] = [] + for row in rows { + guard let state = Packet04Producer.trackingText(row.tracking), state != events.last?.state else { continue } + events.append(.init(time: row.t, state: state, epoch: epoch)) + } + return events + } + + /// One sensor's samples exactly as Core Motion timed them, `t,x,y,z`: the raw evidence behind + /// `imuRaw`, kept because the pairing rule there is a proposal the server has not agreed to. + public static func rawMotionCSV(_ rows: [MotionRow], columns: [String]) throws -> Data { + guard !rows.isEmpty else { throw Packet04Error.streamEmpty(columns.joined(separator: ",")) } + var text = columns.joined(separator: ",") + "\n" + for row in rows.sorted(by: { $0.t < $1.t }) { + text += [row.t, row.value.x, row.value.y, row.value.z].map(\.description).joined(separator: ",") + "\n" + } + return Data(text.utf8) + } + + /// Pose rows from the recorder's raw rows: trajectory rows `[t, tracking state, tracking + /// reason, 16 camera-to-world values column by column]` and intrinsics rows `[t, fx, fy, cx, + /// cy, width, height]` written for the same frame at the same t. A pose with no intrinsics row + /// at its t is left out rather than given another frame's calibration. + public static func poseRows(trajectory: [[Double]], intrinsics: [[Double]], tracking: (Int, Int) -> PacketTracking) -> [PoseRow] { + var calibration: [Double: SIMD4] = [:] + for row in intrinsics where row.count >= 5 { calibration[row[0]] = SIMD4(Float(row[1]), Float(row[2]), Float(row[3]), Float(row[4])) } + return trajectory.compactMap { row in + guard row.count == 19, let k = calibration[row[0]] else { return nil } + let f = row[3...].map(Float.init) + let m = simd_float4x4(SIMD4(f[0], f[1], f[2], f[3]), SIMD4(f[4], f[5], f[6], f[7]), SIMD4(f[8], f[9], f[10], f[11]), SIMD4(f[12], f[13], f[14], f[15])) + return PoseRow(t: row[0], tracking: tracking(Int(row[1]), Int(row[2])), cameraToWorld: m, intrinsics: k) + } + } + + /// Motion rows from raw `[t, x, y, z]` rows. + public static func motionRows(_ rows: [[Double]]) -> [MotionRow] { + rows.filter { $0.count == 4 }.map { MotionRow(t: $0[0], value: SIMD3($0[1], $0[2], $0[3])) } + } + + /// 1 / the median interval between rows, the rate actually recorded. 0 with fewer than two. + public static func measuredRate(_ times: [Double]) -> Double { + let deltas = zip(times.dropFirst(), times).map { $0 - $1 }.filter { $0 > 0 }.sorted() + guard !deltas.isEmpty else { return 0 } + return 1 / deltas[deltas.count / 2] + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift new file mode 100644 index 000000000..dc33d562d --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift @@ -0,0 +1,227 @@ +import Foundation + +/// The capture API's request and response bodies that House Scan sends and reads (`/v1/captures` +/// and below), written from the documented wire contract. Unknown response fields are ignored. +public enum CaptureAPI { + public struct Device: Codable, Sendable, Equatable { + public var model: String + public var systemVersion: String + public var appVersion: String + + public init(model: String, systemVersion: String, appVersion: String) { + self.model = model + self.systemVersion = systemVersion + self.appVersion = appVersion + } + } + + /// The create body. Expected counts are left out: they are optional, and a count that grows as + /// photos arrive would change the body the idempotency key is bound to. + public struct CreateRequest: Codable, Sendable, Equatable { + public var packetId: String + public var formatVersion = Packet04.formatVersion + public var tier: Packet04.Tier + public var flow = "guided" + public var device: Device + + public init(packetId: String, tier: Packet04.Tier, device: Device) { + self.packetId = packetId + self.tier = tier + self.device = device + } + } + + public struct Limits: Codable, Sendable, Equatable { + public var maxSinglePutBytes: Int? + public var urlTtlS: Int? + public var maxFiles: Int? + public var maxBatch: Int? + } + + public struct CreateResponse: Codable, Sendable, Equatable { + public var captureId: String + public var status: String + public var upload: Limits? + public var finalizeBy: String? + } + + public struct RegisterFile: Codable, Sendable, Equatable { + public var path: String + public var bytes: Int + public var sha256: String + public var md5: String + public var role: Packet04.Role + public var contentType: String + public var meta: Packet04.ImageRecord? + + public init(_ file: SealedFile) { + path = file.path + bytes = file.bytes + sha256 = file.sha256 + md5 = file.md5 + role = file.role + contentType = file.contentType + meta = file.meta + } + } + + public struct RegisterRequest: Codable, Sendable, Equatable { + public var files: [RegisterFile] + } + + public struct UploadTarget: Codable, Sendable, Equatable { + public var method: String + public var url: String? + public var headers: [String: String]? + public var expiresAt: String? + } + + public struct RegisteredFile: Codable, Sendable, Equatable { + public var path: String + public var state: String + public var upload: UploadTarget? + } + + public struct RegisterResponse: Codable, Sendable, Equatable { + public var files: [RegisteredFile] + } + + public struct CommitFile: Codable, Sendable, Equatable { + public var path: String + public var sha256: String + } + + public struct CommitRequest: Codable, Sendable, Equatable { + public var files: [CommitFile] + } + + public struct CommitResponse: Codable, Sendable, Equatable { + public var committed: [String] + public var notFound: [String] + public var mismatch: [String] + } + + public struct FinalizeResponse: Codable, Sendable, Equatable { + public var status: String + public var missing: [String] + public var runId: String + public var etaS: Int? + } + + public struct Event: Codable, Sendable, Equatable { + public var seq: Int + public var type: String + public var at: String + /// The fields the app reads. Others, which may carry text meant for the homeowner, are + /// not kept. + public var data: EventData? + } + + public struct EventData: Codable, Sendable, Equatable { + public var code: String? + public var kind: String? + public var stage: String? + public var status: String? + public var runId: String? + public var next: String? + } + + public struct EventsResponse: Codable, Sendable, Equatable { + public var status: String + public var next: Int + public var events: [Event] + } + + public struct ErrorItem: Codable, Sendable, Equatable { + public var code: String + public var pointer: String? + public var message: String? + } + + public struct ErrorBody: Codable, Sendable, Equatable { + public var errors: [ErrorItem] + } + + public static func errorCodes(_ body: Data) -> [String] { + (try? JSONDecoder().decode(ErrorBody.self, from: body))?.errors.map(\.code) ?? [] + } + + /// Statuses after which the capture changes no more. + public static let terminalStatuses: Set = ["complete", "manual_review", "failed", "expired", "needs_views"] + + /// Terminal statuses that can end a capture with no answer, so a result read carrying one and + /// a null outcome is final. The others come with the run's outcome. + public static let statusesWithoutOutcome: Set = ["failed", "expired"] +} + +/// An HTTP answer: status, body and the headers the uploader reads. +public struct HTTPReply: Sendable, Equatable { + public var status: Int + public var body: Data + public var retryAfter: Double? + + public init(status: Int, body: Data, retryAfter: Double? = nil) { + self.status = status + self.body = body + self.retryAfter = retryAfter + } +} + +/// How the uploader reaches the network. HTTP error statuses come back as replies; only transport +/// failures throw. An implementation must not follow redirects: the uploader checks each URL +/// before it sends a photo there, and a redirect would send it somewhere unchecked. A redirect +/// comes back as its own 3xx reply. +public protocol CaptureHTTP: Sendable { + func send(_ request: URLRequest) async throws -> HTTPReply + /// `file` as the whole body of `request`, streamed from disk. + func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply +} + +/// `CaptureHTTP` on one ordinary `URLSession`. +public struct URLSessionCaptureHTTP: CaptureHTTP { + public let session: URLSession + + public init(session: URLSession) { + self.session = session + } + + /// An ephemeral session: nothing cached or stored in cookies, and no credentials offered to + /// the signed storage URLs. + public static func ephemeral(timeout: Double = 60) -> URLSessionCaptureHTTP { + let configuration = URLSessionConfiguration.ephemeral + configuration.timeoutIntervalForRequest = timeout + configuration.httpCookieStorage = nil + configuration.urlCredentialStorage = nil + configuration.httpMaximumConnectionsPerHost = 4 + return URLSessionCaptureHTTP(session: URLSession(configuration: configuration)) + } + + public func send(_ request: URLRequest) async throws -> HTTPReply { + let (data, response) = try await session.data(for: request, delegate: NoRedirects.shared) + return Self.reply(data, response) + } + + public func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + let (data, response) = try await session.upload(for: request, fromFile: file, delegate: NoRedirects.shared) + return Self.reply(data, response) + } + + /// Refuses every redirect, so the task returns the 3xx response itself. URLSession follows + /// redirects by default, and App Transport Security alone would still allow one to another + /// https host, or to plain http on the local network, which the app's Info.plist permits. + final class NoRedirects: NSObject, URLSessionTaskDelegate, Sendable { + static let shared = NoRedirects() + + func urlSession( + _ session: URLSession, task: URLSessionTask, willPerformHTTPRedirection response: HTTPURLResponse, newRequest request: URLRequest + ) async -> URLRequest? { + nil + } + } + + static func reply(_ data: Data, _ response: URLResponse) -> HTTPReply { + let http = response as? HTTPURLResponse + return HTTPReply( + status: http?.statusCode ?? 0, body: data, retryAfter: http?.value(forHTTPHeaderField: "Retry-After").flatMap(Double.init)) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift new file mode 100644 index 000000000..d35fea631 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift @@ -0,0 +1,502 @@ +import Foundation +import simd + +/// A photo the scan kept, as the app stores it: the JPEG on disk and the facts of the ARFrame +/// whose pixels it holds, in that frame's own terms (raw world pose, the camera image's +/// intrinsics and size). +public struct KeptPhoto: Sendable { + public var t: Double + public var cameraToWorld: simd_float4x4 + /// [fx, fy, cx, cy] in pixels of the camera image (`cameraImageSize`). + public var cameraIntrinsics: SIMD4 + public var cameraImageSize: SIMD2 + /// Pixels of the stored JPEG. + public var width: Int + public var height: Int + public var tracking: PacketTracking + public var exposure: Packet04.Exposure? + public var jpeg: URL + /// The still's purpose ("meter_close"); nil for a keyframe. + public var purpose: String? + /// Reads the photo's ARKit depth. Called once, when the photo is kept, because the app writes a + /// retaken close-up's depth over the same files. + public var depth: @Sendable () -> Packet04Depth? + + public init( + t: Double, cameraToWorld: simd_float4x4, cameraIntrinsics: SIMD4, cameraImageSize: SIMD2, width: Int, height: Int, + tracking: PacketTracking, exposure: Packet04.Exposure?, jpeg: URL, purpose: String?, depth: @escaping @Sendable () -> Packet04Depth? = { nil } + ) { + self.t = t + self.cameraToWorld = cameraToWorld + self.cameraIntrinsics = cameraIntrinsics + self.cameraImageSize = cameraImageSize + self.width = width + self.height = height + self.tracking = tracking + self.exposure = exposure + self.jpeg = jpeg + self.purpose = purpose + self.depth = depth + } + + /// The observation in the stored JPEG's pixels: intrinsics scaled when the JPEG is not the + /// camera image's size. + public var observation: Packet04Observation { + let scale = SIMD2(Float(width), Float(height)) / cameraImageSize + let k = cameraIntrinsics + return Packet04Observation( + t: t, cameraToWorld: cameraToWorld, intrinsics: SIMD4(k.x * scale.x, k.y * scale.y, k.z * scale.x, k.w * scale.y), + width: width, height: height, tracking: tracking, exposure: exposure) + } +} + +/// The frame under the homeowner's tap, read when the tap happened. +public struct TapObservation: Sendable { + public var t: Double + public var cameraToWorld: simd_float4x4 + /// [fx, fy, cx, cy] in pixels of the camera image, which the JPEG stores unscaled. + public var intrinsics: SIMD4 + public var width: Int + public var height: Int + public var tracking: PacketTracking + /// The tap in the sensor image's pixels, (0, 0) its top-left corner. + public var pixel: SIMD2 + /// Encodes the frame's image; called off the main actor. + public var jpeg: @Sendable () -> Data? + + /// The same frame with the tap moved to where `world` (the raycast's hit) lands in its image, + /// so pixel, ray and hit agree in this one frame even when it is a frame or two newer than + /// the one the raycast used. Nil when the point is behind the camera or outside the image. + public func pointing(at world: SIMD3) -> TapObservation? { + let p = cameraToWorld.inverse * SIMD4(world, 1) + guard p.z < -0.05 else { return nil } + let u = Double(intrinsics.z + intrinsics.x * p.x / -p.z), v = Double(intrinsics.w - intrinsics.y * p.y / -p.z) + guard (0...Double(width)).contains(u), (0...Double(height)).contains(v) else { return nil } + var aimed = self + aimed.pixel = SIMD2(u, v) + return aimed + } + + public init( + t: Double, cameraToWorld: simd_float4x4, intrinsics: SIMD4, width: Int, height: Int, tracking: PacketTracking, + pixel: SIMD2, jpeg: @escaping @Sendable () -> Data? + ) { + self.t = t + self.cameraToWorld = cameraToWorld + self.intrinsics = intrinsics + self.width = width + self.height = height + self.tracking = tracking + self.pixel = pixel + self.jpeg = jpeg + } +} + +/// Where the coordinator reads the world's recorded streams: `CaptureRecorder` in the app. +public struct RecordingSource: Sendable { + /// Uptime and wall clock of the world's first recorded frame. + public var start: @Sendable () -> (uptime: Double, date: Date)? + /// Every row so far; called off the main actor. + public var rows: @Sendable () -> RecorderRows + + public init(start: @escaping @Sendable () -> (uptime: Double, date: Date)?, rows: @escaping @Sendable () -> RecorderRows) { + self.start = start + self.rows = rows + } +} + +/// The recorder's raw rows: trajectory `[t, state, reason, 16 pose values]`, intrinsics +/// `[t, fx, fy, cx, cy, w, h]`, and Core Motion's `[t, x, y, z]`, each sensor at its own times. +public struct RecorderRows: Sendable { + public var trajectory: [[Double]] + public var intrinsics: [[Double]] + public var accelerometer: [[Double]] + public var gyroscope: [[Double]] + + public init(trajectory: [[Double]], intrinsics: [[Double]], accelerometer: [[Double]], gyroscope: [[Double]]) { + self.trajectory = trajectory + self.intrinsics = intrinsics + self.accelerometer = accelerometer + self.gyroscope = gyroscope + } +} + +/// What the integration build does with a scan: one 0.4 packet per ARKit world, each kept photo +/// sealed into it on the phone as it is kept, the packet frozen when the scan is sent for +/// placement. Sending is separate: only a build allowed to send device data (`sends`) sends, and +/// only after the homeowner's yes for this scan, asked before it starts. From the yes on, every +/// sealed file goes up, and later photos go up as they are kept. A new scan asks again; a world +/// reset within the scan keeps its answer for the scan's next packet. +/// +/// Work for one world runs in order on that world's chain; a world reset or start over ends the +/// session, and nothing queued for it reaches the next one. +@MainActor +public final class CaptureSessionCoordinator { + public struct Environment: Sendable { + public var endpoint: URL + /// Whether device captures may go to `endpoint` at all (`CaptureIntegrationMode.send`). + public var sends: Bool + public var http: any CaptureHTTP + /// Each session writes to `/`. + public var capturesFolder: URL + public var sessionInfo: @Sendable (_ packetID: String, _ video: SIMD2) -> Packet04SessionInfo + public var device: CaptureAPI.Device + public var tier: Packet04.Tier + public var policy: CaptureUploader.Policy + public var log: @Sendable (String) -> Void + + public init( + endpoint: URL, sends: Bool, http: any CaptureHTTP, capturesFolder: URL, + sessionInfo: @escaping @Sendable (_ packetID: String, _ video: SIMD2) -> Packet04SessionInfo, + device: CaptureAPI.Device, tier: Packet04.Tier, policy: CaptureUploader.Policy = .init(), log: @escaping @Sendable (String) -> Void = { _ in } + ) { + self.endpoint = endpoint + self.sends = sends + self.http = http + self.capturesFolder = capturesFolder + self.sessionInfo = sessionInfo + self.device = device + self.tier = tier + self.policy = policy + self.log = log + } + } + + /// Checklist purposes the packet's `stills` take; any other still stays a keyframe only. + public static let stillPurposes: Set = ["meter_close", "meter_oblique"] + + @MainActor + public final class Session { + /// This app's id for this session, new for every session even when two packets both call + /// their ARKit epoch "e1": a result is bound to it, so a late result can't land on a newer scan. + public let localID = UUID().uuidString + public let packetID: String + public let folder: URL + /// Nil until the homeowner says yes on a build that sends. + public fileprivate(set) var uploader: CaptureUploader? + public private(set) var producer: Packet04Producer? + let recording: RecordingSource + var chain: Task? + var sealing = false + var ended = false + /// The homeowner said no after a yes: this capture is never sent again. + var consentWithdrawn = false + var withdrawalFailure: ConsentWithdrawalError? + /// packet.json as frozen, once the scan was sent for placement. + var frozen: Data? + /// The server's result for this session's capture, decoded and bound to it. + public fileprivate(set) var result: CaptureResult.Record? + + init(packetID: String, folder: URL, recording: RecordingSource) { + self.packetID = packetID + self.folder = folder + self.recording = recording + } + + /// Runs `work` after everything queued before it, unless the session has ended by then. + /// `cleanup` runs either way, after `work` when it ran. + func enqueue(_ work: @escaping @MainActor (Session) async -> Void, cleanup: (@MainActor @Sendable () -> Void)? = nil) { + let previous = chain + chain = Task { + await previous?.value + defer { cleanup?() } + guard !self.ended else { return } + await work(self) + } + } + + func producer(first photo: (t: Double, video: SIMD2), info: (String, SIMD2) -> Packet04SessionInfo) throws -> Packet04Producer { + if let producer { return producer } + // The packet starts at the world's first recorded frame; a photo can't be older. + let start = recording.start() ?? (photo.t, Date(timeIntervalSinceNow: photo.t - ProcessInfo.processInfo.systemUptime)) + let made = try Packet04Producer(folder: folder, info: info(packetID, photo.video), startedAtUptime: min(start.uptime, photo.t), startedAt: start.date) + producer = made + return made + } + } + + public let environment: Environment? + /// The homeowner's answer for this scan: nil before they were asked. + public private(set) var consent: Bool? + private var consentedAt: Date? + public private(set) var session: Session? + private var recording: RecordingSource? + /// The current session's status; nil when there is none or nothing is being sent. + public var onStatus: (@MainActor (CaptureUploadStatus?) -> Void)? + /// The current session's result, once the server has one. Never called for an ended session. + public var onResult: (@MainActor (CaptureResult.Record) -> Void)? + /// The packet's ARKit epoch: one packet is one world. + public static let epoch = "e1" + + public init(environment: Environment?) { + self.environment = environment + } + + /// Whether the homeowner should be asked now: a build that sends, a scan under way, and no + /// answer for it yet. + public var needsConsent: Bool { + environment?.sends == true && consent == nil && session != nil + } + + public enum ConsentWithdrawalError: Error, Sendable, Equatable { + /// Sending stopped in this process, but a relaunch may still find the old saved yes. + case notRecorded(String) + } + + /// A no after a yes stops this capture's upload for good: nothing kept later is sent, and a + /// second yes in the same scan does not start it again. What the server already has stays there. + /// Failure means sending stopped here but the withdrawal could not be saved for a relaunch. + @discardableResult + public func answerConsent(_ yes: Bool, at time: Date = Date()) -> Result { + consent = yes + consentedAt = yes ? time : nil + guard let session else { return .success(()) } + if let failure = session.withdrawalFailure { return .failure(failure) } + if yes { + startUploading(session) + } else if let uploader = session.uploader { + session.consentWithdrawn = true + session.uploader = nil + // The result distinguishes durable withdrawal from stopping only this live process. + switch uploader.withdrawConsent() { + case .marked: break + case .savedStateRemoved: + environment?.log("capture upload: the withdrawal marker could not be written; the saved upload was removed instead") + case .notRecorded(let error): + session.withdrawalFailure = .notRecorded(error) + environment?.log("capture upload: the withdrawal could not be saved (\(error)); sending stopped in this process, but a relaunch could still find the saved consent") + } + Task { await uploader.abandon(CaptureUploader.withdrawnReason) } + } + if let failure = session.withdrawalFailure { return .failure(failure) } + return .success(()) + } + + // MARK: Scan events + + /// The live source started recording a world. + public func begin(recording: RecordingSource) { + self.recording = recording + if environment != nil, session == nil { startSession() } + } + + /// The ARKit world was thrown away, or (`newScan`) the scan started over: this packet can't + /// be finished, and the next one belongs to `recording`'s new world. A new scan asks again. + public func newWorld(_ reason: String, recording: RecordingSource, newScan: Bool) { + endSession(reason) + if newScan { + consent = nil + consentedAt = nil + } + self.recording = recording + if environment != nil { startSession() } + } + + public func kept(_ photo: KeptPhoto) { + guard let session, !session.sealing, let environment else { return } + // The app writes a retaken close-up (JPEG and depth) over the same files, so both are taken + // now, the JPEG as an APFS clone, not when the queued work gets to them. + let depth = photo.depth() + let staged = environment.capturesFolder.appending(path: "staging/\(UUID().uuidString).jpg") + do { + try FileManager.default.createDirectory(at: staged.deletingLastPathComponent(), withIntermediateDirectories: true) + try FileManager.default.copyItem(at: photo.jpeg, to: staged) + } catch { + environment.log("capture packet: a kept photo could not be copied: \(error)") + return + } + // The copy is deleted even when a reset ends the session before this work runs. + session.enqueue({ session in + do { + let producer = try session.producer(first: (photo.t, photo.cameraImageSize), info: environment.sessionInfo) + let data = try await Task.detached(priority: .utility) { try Data(contentsOf: staged) }.value + var files: [SealedFile] = [] + if await producer.keyframeID(at: photo.t) == nil { + files += try await producer.sealKeyframe( + jpeg: data, observation: photo.observation, reason: photo.purpose == nil ? "auto" : "still", purpose: photo.purpose, depth: depth) + } + if let purpose = photo.purpose, Self.stillPurposes.contains(purpose), let keyframe = await producer.keyframeID(at: photo.t) { + files.append(try await producer.sealStill(purpose: purpose, keyframe: keyframe)) + } + await session.uploader?.add(files) + } catch { + environment.log("capture packet: a kept photo was not sealed: \(error)") + } + }, cleanup: { try? FileManager.default.removeItem(at: staged) }) + } + + /// The homeowner marked the meter: its frame becomes a keyframe and the tap goes on it. + public func meterTapped(_ tap: TapObservation, hit: Packet04.TapHit?) { + guard let session, !session.sealing, let environment else { return } + session.enqueue { session in + do { + let producer = try session.producer(first: (tap.t, SIMD2(Float(tap.width), Float(tap.height))), info: environment.sessionInfo) + guard let jpeg = await Task.detached(priority: .userInitiated, operation: { tap.jpeg() }).value else { + environment.log("capture packet: the meter tap's frame could not be encoded; no tap") + return + } + let observation = Packet04Observation( + t: tap.t, cameraToWorld: tap.cameraToWorld, intrinsics: tap.intrinsics, width: tap.width, height: tap.height, tracking: tap.tracking) + let files = try await producer.sealTap(id: "meter", label: "meter", jpeg: jpeg, observation: observation, pixel: tap.pixel, hit: hit) + await session.uploader?.add(files) + } catch { + environment.log("capture packet: the meter tap was not recorded: \(error)") + } + } + } + + /// The scan was sent for placement: freeze the packet with the streams recorded so far. + /// `acceptedCloseUpAt` is the frame time of the close-up the scan accepted, nil after a skip. + /// Later sends of the same scan (a retry, one more view) change nothing. + public func captureEnded(acceptedCloseUpAt: Double?) { + guard let session, !session.sealing, let environment else { return } + session.sealing = true + session.enqueue { session in + guard let producer = session.producer else { + await session.uploader?.abandon("no photos were kept") + return + } + let recording = session.recording + let rows = await Task.detached(priority: .utility) { recording.rows() }.value + let poses = Packet04Streams.poseRows(trajectory: rows.trajectory, intrinsics: rows.intrinsics, tracking: PacketTracking.init(recorderState:reason:)) + do { + let finished = try await producer.finish( + poses: poses, accelerometer: Packet04Streams.motionRows(rows.accelerometer), + gyroscope: Packet04Streams.motionRows(rows.gyroscope), endedAtUptime: poses.last?.t ?? 0, acceptedCloseUpAt: acceptedCloseUpAt) + session.frozen = finished.packet + await session.uploader?.seal(packet: finished.packet, files: await producer.sealedFiles) + } catch { + environment.log("capture packet not finished: \(error)") + await session.uploader?.abandon("packet refused on the phone") + } + } + } + + /// Waits until the current session's queued work and uploads are idle. For tests and the + /// evidence run. + public func settle() async { + guard let session else { return } + while let chain = session.chain { + await chain.value + if session.chain == chain { break } + } + await session.uploader?.settled() + } + + /// After a relaunch: captures frozen before the app quit finish uploading, each only toward the + /// endpoint it was created on and with the yes it recorded for that endpoint. Returns them. + public func resumeSealedCaptures() -> [CaptureUploader] { + guard let environment, environment.sends, + let folders = try? FileManager.default.contentsOfDirectory(at: environment.capturesFolder, includingPropertiesForKeys: nil) + else { return [] } + var resumed: [CaptureUploader] = [] + for folder in folders { + guard let saved = try? CaptureUploadState.load(from: CaptureUploader.stateURL(in: folder)), saved.end == nil, saved.packet != nil, + let uploader = try? CaptureUploader.resume(folder: folder, base: environment.endpoint, http: environment.http, policy: environment.policy) + else { continue } + resumed.append(uploader) + Task { await uploader.kick() } + } + return resumed + } + + // MARK: Sessions + + private func startSession() { + guard let environment, let recording else { return } + let packetID = UUID().uuidString + let session = Session(packetID: packetID, folder: environment.capturesFolder.appending(path: packetID, directoryHint: .isDirectory), recording: recording) + self.session = session + environment.log("capture packet: new packet for this world") + if consent == true { startUploading(session) } + } + + /// Opens the capture on the server and queues everything sealed so far, then the frozen + /// packet if the scan was already sent. Photos kept later go up as they are sealed. + private func startUploading(_ session: Session) { + guard let environment, environment.sends, consent == true, let consentedAt, session.uploader == nil, !session.ended, + !session.consentWithdrawn else { return } + do { + let uploader = try CaptureUploader.start( + folder: session.folder, base: environment.endpoint, http: environment.http, + create: .init(packetId: session.packetID, tier: environment.tier, device: environment.device), consentedAt: consentedAt, + policy: environment.policy) + session.uploader = uploader + // Status and results from an ended session's uploader never reach the screen. + let publish: @Sendable (CaptureUploadStatus) -> Void = { [weak self, weak session] status in + Task { @MainActor in + guard let self, let session, self.session === session else { return } + self.onStatus?(status) + if status.resultAvailable, session.result == nil { await self.bindResult(of: session) } + } + } + let log = environment.log + Task { + await uploader.observe(publish, log: log) + await uploader.kick() + } + session.enqueue { session in + guard let producer = session.producer else { return } + let files = await producer.sealedFiles + if let frozen = session.frozen { + await uploader.seal(packet: frozen, files: files) + } else { + await uploader.add(files) + } + } + environment.log("capture upload: sending this world's packet") + } catch { + environment.log("capture upload not started: \(error)") + } + } + + /// Decodes the uploader's result and binds it to the session, capture and run that asked for + /// it. The analysis stays unverified: nothing in the response proves the server reconstructed + /// anything, so the spatial result is withheld (`CaptureResult.Record.placement`). + private func bindResult(of session: Session) async { + guard let uploader = session.uploader else { return } + let state = await uploader.snapshot + guard self.session === session, session.result == nil, let body = state.result, let captureID = state.captureID, + let runID = state.finalized?.runID else { return } + do { + let response = try JSONDecoder().decode(CaptureResult.Response.self, from: body) + let record = CaptureResult.Record( + response: response, + association: .init(sessionID: session.localID, captureID: captureID, runID: runID, epoch: Self.epoch)) + session.result = record + onResult?(record) + } catch { + environment?.log("capture result could not be read: \(error)") + } + } + + private func endSession(_ reason: String) { + guard let session else { return } + self.session = nil + session.ended = true + onStatus?(nil) + if let uploader = session.uploader { + uploader.stopSending() + Task { await uploader.abandon(reason) } + } + } +} + +extension PacketTracking { + /// The recorder's codes: state 0 normal, 1 limited, 2 not available; reason 0 none or + /// unknown, 1 initializing, 2 relocalizing, 3 excessive motion, 4 insufficient features. + public init(recorderState state: Int, reason: Int) { + let why: Reason? = switch reason { + case 1: .initializing + case 2: .relocalizing + case 3: .excessiveMotion + case 4: .insufficientFeatures + default: nil + } + self = switch state { + case 0: .normal + case 1: .limited(why) + default: .notAvailable + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadGate.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadGate.swift new file mode 100644 index 000000000..068ee46e2 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadGate.swift @@ -0,0 +1,72 @@ +import Foundation + +/// Whether this build and this homeowner allow a capture upload. No endpoint means nothing is +/// offered and nothing is sent; an endpoint without the homeowner's yes sends nothing either. +public enum CaptureUploadGate { + public enum Decision: Sendable, Equatable { + case off(String) + case on(URL) + } + + /// `endpoint` is the API base including `/v1`, from the build's Info.plist. An empty build + /// setting, an unexpanded `$(...)`, or anything but https (http only to this machine, for + /// tests) is off. + public static func decide(endpoint: String?, consented: Bool) -> Decision { + let text = endpoint?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard !text.isEmpty, !text.hasPrefix("$(") else { return .off("no capture endpoint in this build") } + guard let url = URL(string: text), let scheme = url.scheme?.lowercased(), let host = url.host(), url.user == nil, url.password == nil else { + return .off("capture endpoint is not a URL") + } + let local = host == "127.0.0.1" || host == "localhost" + guard scheme == "https" || (scheme == "http" && local) else { return .off("capture endpoint must be https") } + guard consented else { return .off("the homeowner has not agreed to send the capture") } + return .on(url) + } +} + +/// What the app does with its captures, from three build settings read once at launch. Only the +/// integration build records a 0.4 capture at all; only a build that also names an endpoint and +/// says device data may go there sends one, and then only after the homeowner's yes. +public enum CaptureIntegrationMode: Sendable, Equatable { + /// The client build, or no usable endpoint: nothing is recorded for the capture API or sent. + case off(String) + /// The capture is recorded on the phone and never sent: the endpoint may not receive device + /// data (an unauthenticated test server, unless the build says otherwise). + case recordOnly(URL) + /// Recorded, and sent after the homeowner's yes. + case send(URL) + + public var endpoint: URL? { + switch self { + case .off: nil + case .recordOnly(let url), .send(let url): url + } + } + + /// Where the scan's pictures come from. + public enum Source: Sendable, Equatable { + /// The phone's camera: sending needs the build's device-data switch. + case device + /// A recorded session played back (`-replay`). Its pictures can be anyone's, so it is sent + /// only to a receiver on this machine, and only when the run asks for that + /// (`sendToLocalReceiver`, the UI tests' switch); never to a remote endpoint. + case replay(sendToLocalReceiver: Bool) + } + + /// `integrationBuild` and `sendDeviceData` are Info.plist values that must read exactly "YES". + /// A launch argument can name `endpoint`, but can't turn a client build into an integration + /// build or allow device data. + public static func resolve(integrationBuild: String?, endpoint: String?, sendDeviceData: String?, source: Source = .device) -> CaptureIntegrationMode { + guard integrationBuild == "YES" else { return .off("not the integration build") } + switch CaptureUploadGate.decide(endpoint: endpoint, consented: true) { + case .off(let reason): return .off(reason) + case .on(let url): + switch source { + case .device: return sendDeviceData == "YES" ? .send(url) : .recordOnly(url) + case .replay(let local): + let loopback = url.host() == "127.0.0.1" || url.host() == "localhost" + return local && loopback ? .send(url) : .recordOnly(url) + } + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift new file mode 100644 index 000000000..f27754c2c --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift @@ -0,0 +1,197 @@ +import Foundation + +/// One capture's upload, saved in its folder after every change so a relaunch resumes the same +/// capture with the same identity. The request bodies that idempotency keys are bound to (create +/// and finalize) are saved as the exact bytes first sent, never re-encoded. +public struct CaptureUploadState: Codable, Sendable, Equatable { + public static let fileName = "capture-upload.json" + /// Written into a capture's folder when the homeowner takes back their yes. `CaptureUploader.resume` + /// never resumes a folder that has it. + public static let withdrawnFileName = "consent-withdrawn" + + public static func withdrawnURL(in folder: URL) -> URL { folder.appending(path: withdrawnFileName) } + + /// Called with the uploader's folder registry locked so no initial save or reply can restore + /// a yes between these writes. Neither failing proves anything durable after a process exit. + static func recordWithdrawal(in folder: URL) -> CaptureUploader.WithdrawalRecord { + do { + try Data().write(to: withdrawnURL(in: folder), options: .atomic) + return .marked + } catch { + let markerError = CaptureUploader.describe(error) + do { + try FileManager.default.removeItem(at: CaptureUploader.stateURL(in: folder)) + return .savedStateRemoved + } catch let removal as NSError where removal.domain == NSCocoaErrorDomain && removal.code == NSFileNoSuchFileError { + return .savedStateRemoved + } catch { + return .notRecorded("marker: \(markerError); state removal: \(CaptureUploader.describe(error))") + } + } + } + + public enum FilePhase: Codable, Sendable, Equatable { + /// Sealed on the phone, not yet registered (or its URL must be fetched again). + case queued + case registered(url: String, headers: [String: String], expiresAt: Date?) + /// The storage PUT answered 2xx. Not yet received: only a commit says that. + case uploaded + /// Acknowledged by files:commit. + case committed + } + + public struct File: Codable, Sendable, Equatable { + public var sealed: SealedFile + public var phase: FilePhase = .queued + public var attempts = 0 + /// Register answers that left this file out, plus commit answers that didn't acknowledge + /// it. Nil in state saved before this was counted. + public var unacknowledged: Int? + /// Order in which the file was sealed, the tie-break after priority. + public var sequence: Int + + public init(sealed: SealedFile, phase: FilePhase = .queued, attempts: Int = 0, sequence: Int) { + self.sealed = sealed + self.phase = phase + self.attempts = attempts + self.sequence = sequence + } + } + + public enum End: Codable, Sendable, Equatable { + /// The ARKit world was reset or the scan started over: this packet will never be finished. + case abandoned(String) + /// The server refused something a retry can't change. + case failed(step: String, codes: [String], status: Int) + /// The capture reached a status after which it changes no more. + case finished(status: String) + } + + public struct Finalized: Codable, Sendable, Equatable { + public var status: String + public var missing: [String] + public var runID: String + + public init(status: String, missing: [String], runID: String) { + self.status = status + self.missing = missing + self.runID = runID + } + } + + /// Local identity of this upload. Answers that arrive for another attempt are dropped. + public var attemptID: String + public var packetID: String + /// The API base the capture was created on. A resume goes only there: its capture id and + /// signed URLs mean nothing to another server. + public var destination: String? + /// The homeowner's yes for this capture and this destination. A capture without one is + /// never resumed. + public var consent: Consent? + + public struct Consent: Codable, Sendable, Equatable { + public var grantedAt: Date + public var destination: String + + public init(grantedAt: Date, destination: String) { + self.grantedAt = grantedAt + self.destination = destination + } + } + /// The create body exactly as first sent. + public var createBody: Data + public var captureID: String? + public var maxBatch = 50 + public var files: [String: File] = [:] + public var nextSequence = 0 + /// packet.json exactly as frozen when the capture stopped. + public var packet: Data? + public var finalized: Finalized? + public var eventCursor = 0 + /// `retry_finalize` answers followed so far. + public var finalizeRetries = 0 + /// Result reads that found no answer yet. Saved so a relaunch continues the same limit; nil in + /// state saved before this was counted. + public var notReadyReads: Int? + public var backendStatus: String? + public var lastEvent: String? + public var result: Data? + public var end: End? + /// Wall-clock marks of each stage, for latency: no ids, URLs or photos. + public var marks: [String: Date] = [:] + + public init(attemptID: String, packetID: String, createBody: Data) { + self.attemptID = attemptID + self.packetID = packetID + self.createBody = createBody + } + + public var committedCount: Int { files.values.filter { $0.phase == .committed }.count } + public var committedBytes: Int { files.values.filter { $0.phase == .committed }.reduce(0) { $0 + $1.sealed.bytes } } + public var totalBytes: Int { files.values.reduce(0) { $0 + $1.sealed.bytes } } + + public func save(to url: URL) throws { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + try encoder.encode(self).write(to: url, options: .atomic) + } + + public static func load(from url: URL) throws -> CaptureUploadState { + try JSONDecoder().decode(CaptureUploadState.self, from: Data(contentsOf: url)) + } +} + +/// What the experience layer shows: separate facts, never one percentage. +public struct CaptureUploadStatus: Sendable, Equatable { + public enum Phase: Sendable, Equatable { + case creating, uploading, processing, finished, failed, abandoned + } + + public var phase: Phase + /// Files sealed on the phone for this capture. + public var retained: Int + /// Files the server acknowledged with a commit. + public var committed: Int + public var committedBytes: Int + public var totalBytes: Int + /// Whether packet.json is frozen (the capture stopped). + public var sealed: Bool + /// The capture's status as the server last reported it. + public var backendStatus: String? + public var lastEvent: String? + public var eventCursor: Int + public var resultAvailable: Bool + /// The server's id for the capture, once created: what an operator copies to find this scan. + public var captureID: String? + /// The run finalize started, once accepted. + public var runID: String? + /// Set while the uploader waits out a retryable failure. + public var retryingAt: Date? + public var detail: String? + + public init(_ state: CaptureUploadState, retryingAt: Date? = nil, detail: String? = nil) { + phase = switch state.end { + case .abandoned?: .abandoned + case .failed?: .failed + case .finished?: .finished + case nil: state.captureID == nil ? .creating : state.finalized == nil ? .uploading : .processing + } + retained = state.files.count + committed = state.committedCount + committedBytes = state.committedBytes + totalBytes = state.totalBytes + sealed = state.packet != nil + backendStatus = state.backendStatus + lastEvent = state.lastEvent + eventCursor = state.eventCursor + resultAvailable = state.result != nil + captureID = state.captureID + runID = state.finalized?.runID + self.retryingAt = retryingAt + if case .failed(let step, let codes, let status) = state.end { + self.detail = "\(step) \(status) \(codes.joined(separator: ","))" + } else { + self.detail = detail + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift new file mode 100644 index 000000000..6fbf75319 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -0,0 +1,734 @@ +import Foundation +import Synchronization + +/// Sends one capture to the capture API while it is being taken: create once, then register, PUT +/// and commit each sealed file as it arrives, then finalize with the frozen packet (possibly +/// before the last commits), then follow the events to a result. +/// +/// Every answer is applied only while the attempt that asked is still current, so a reply that +/// arrives after a world reset or a new capture changes nothing. The saved state +/// (`CaptureUploadState`) is written after every change, which is what resumes a capture after a +/// relaunch with the same packet id, capture id and frozen request bodies. +public actor CaptureUploader { + public struct Policy: Sendable { + public var maxConcurrentPuts = 3 + /// Files per register and commit call: the contract's "about every 10 files". + public var batch = 10 + public var firstDelay = 1.0 + public var maxDelay = 30.0 + /// Seconds the server may hold an events poll. + public var eventsWait = 20 + /// A signed URL this close to expiry is fetched again before the PUT. + public var expiryMargin = 60.0 + /// PUTs of one file the storage refuses (not transient) before the upload stops. + public var maxRefusedPuts = 6 + /// `retry_finalize` answers followed before the upload stops. + public var maxFinalizeRetries = 3 + /// Result reads that find the run's answer not readable yet before the upload stops. Each + /// waits the retry delay first, so the default allows about three minutes. No measurement + /// of how long a finished run's answer takes to become readable sets this number; it only + /// keeps an answer that never appears from holding the upload open. + public var maxNotReadyResults = 10 + /// Retries after a register answer leaves a file out or a commit doesn't acknowledge it, + /// counted per file per server capture. Each retry waits the retry delay; the upload stops + /// on the next unacknowledged answer, so a file is sent at most `maxUnacknowledged + 1` + /// times. No measurement sets this; it keeps a server that never takes a file from getting + /// the same photo forever. + public var maxUnacknowledged = 5 + /// Seconds between events polls that come back at once with nothing new and the capture + /// still running, for a server or proxy that doesn't hold the poll. Not measured. + public var minimumPollInterval = 2.0 + + public init() {} + + public func delay(afterFailure n: Int, retryAfter: Double?) -> Double { + if let retryAfter, retryAfter > 0 { return min(retryAfter, 120) } + return min(maxDelay, firstDelay * pow(2, Double(max(n, 1) - 1))) + } + } + + /// A failure a later try can fix. + struct Transient: Error { + var step: String + var detail: String + var retryAfter: Double? + } + + /// The homeowner took back their yes: nothing more is sent. + struct Withdrawn: Error {} + + /// The server refused something a retry can't change. + struct Refused: Error { + var step: String + var status: Int + var codes: [String] + } + + public nonisolated let folder: URL + private let base: URL + private let http: any CaptureHTTP + private let policy: Policy + private let now: @Sendable () -> Date + private let sleep: @Sendable (Double) async throws -> Void + private var state: CaptureUploadState + private var loop: Task? + private var failures = 0 + private var retryingAt: Date? + private var digestRetried: Set = [] + /// The registry covers public start/resume as well as coordinator discovery. An idle live + /// uploader still owns its folder, since it can receive more sealed files. Weak ownership + /// releases only after the uploader and its running task are gone. Withdrawn entries stay + /// for this process's lifetime, including when every disk write failed. + private struct FolderEntry { + weak var owner: CaptureUploader? + var withdrawn = false + var stopped = false + var running: Task? + } + private static let folders = Mutex<[String: FolderEntry]>([:]) + private nonisolated let folderKey: String + + public enum OwnershipError: Error, Sendable, Equatable { + case alreadyOwned + case consentWithdrawn + /// Use resume for an existing saved upload; start must never overwrite its consent. + case savedUploadExists + case folderIdentityUnavailable + } + /// Set when a state change could not be saved: the loop stops rather than send a request the + /// saved state doesn't know about. + private var saveFailed = false + private var observer: (@Sendable (CaptureUploadStatus) -> Void)? + private var log: (@Sendable (String) -> Void)? + + public nonisolated static func stateURL(in folder: URL) -> URL { folder.appending(path: CaptureUploadState.fileName) } + + /// Resolve symbolic links before asking for the filesystem's case-correct path. The + /// canonical-path resource alone preserves a final-component symlink on macOS. + /// Refuse an unknown identity instead of allowing a second registry key for one folder. + private static func canonicalFolder(_ folder: URL) throws -> URL { + let fresh = URL(fileURLWithPath: folder.path, isDirectory: true).standardizedFileURL.resolvingSymlinksInPath() + guard let path = try fresh.resourceValues(forKeys: [.canonicalPathKey]).canonicalPath else { + throw OwnershipError.folderIdentityUnavailable + } + return URL(fileURLWithPath: path, isDirectory: true) + } + + private init( + folder: URL, base: URL, http: any CaptureHTTP, state: CaptureUploadState, policy: Policy, + now: @escaping @Sendable () -> Date, sleep: @escaping @Sendable (Double) async throws -> Void + ) { + self.folder = folder + self.folderKey = folder.path + self.base = base + self.http = http + self.state = state + self.policy = policy + self.now = now + self.sleep = sleep + } + + /// A new capture. The create body is encoded once here and kept as those bytes for every retry. + /// `consentedAt` is when the homeowner said yes to sending this capture to `base`. + public static func start( + folder: URL, base: URL, http: any CaptureHTTP, create: CaptureAPI.CreateRequest, consentedAt: Date, policy: Policy = .init(), + now: @escaping @Sendable () -> Date = { Date() }, + sleep: @escaping @Sendable (Double) async throws -> Void = { try await Task.sleep(for: .seconds($0)) } + ) throws -> CaptureUploader { + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + let folder = try canonicalFolder(folder) + return try folders.withLock { entries in + entries = entries.filter { $0.value.owner != nil || $0.value.withdrawn } + guard entries[folder.path]?.withdrawn != true, + !FileManager.default.fileExists(atPath: CaptureUploadState.withdrawnURL(in: folder).path) + else { throw OwnershipError.consentWithdrawn } + guard entries[folder.path]?.owner == nil else { throw OwnershipError.alreadyOwned } + guard !FileManager.default.fileExists(atPath: stateURL(in: folder).path) else { throw OwnershipError.savedUploadExists } + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + var state = CaptureUploadState(attemptID: UUID().uuidString, packetID: create.packetId, createBody: try encoder.encode(create)) + state.destination = base.absoluteString + state.consent = .init(grantedAt: consentedAt, destination: base.absoluteString) + state.marks["started"] = now() + try state.save(to: stateURL(in: folder)) + let uploader = CaptureUploader(folder: folder, base: base, http: http, state: state, policy: policy, now: now, sleep: sleep) + entries[folder.path] = FolderEntry(owner: uploader) + return uploader + } + } + + /// Resumes a saved upload with its recorded consent and endpoint. Returns nil when absent, + /// ended or withdrawn; throws `alreadyOwned` while another live uploader owns the folder. + /// The new attempt id and ownership are saved together, before any request can start. + public static func resume( + folder: URL, base: URL, http: any CaptureHTTP, policy: Policy = .init(), + now: @escaping @Sendable () -> Date = { Date() }, + sleep: @escaping @Sendable (Double) async throws -> Void = { try await Task.sleep(for: .seconds($0)) } + ) throws -> CaptureUploader? { + guard FileManager.default.fileExists(atPath: folder.path) else { return nil } + let folder = try canonicalFolder(folder) + return try folders.withLock { entries in + entries = entries.filter { $0.value.owner != nil || $0.value.withdrawn } + let url = stateURL(in: folder) + guard entries[folder.path]?.withdrawn != true, + FileManager.default.fileExists(atPath: url.path), + !FileManager.default.fileExists(atPath: CaptureUploadState.withdrawnURL(in: folder).path) + else { return nil } + guard entries[folder.path]?.owner == nil else { throw OwnershipError.alreadyOwned } + var state = try CaptureUploadState.load(from: url) + guard state.end == nil, state.destination == base.absoluteString, state.consent?.destination == base.absoluteString else { return nil } + state.attemptID = UUID().uuidString + try state.save(to: url) + let uploader = CaptureUploader(folder: folder, base: base, http: http, state: state, policy: policy, now: now, sleep: sleep) + entries[folder.path] = FolderEntry(owner: uploader) + return uploader + } + } + + public func observe(_ observer: @escaping @Sendable (CaptureUploadStatus) -> Void, log: (@Sendable (String) -> Void)? = nil) { + self.observer = observer + self.log = log + observer(status) + } + + public var status: CaptureUploadStatus { + CaptureUploadStatus(state, retryingAt: retryingAt, detail: saveFailed ? "upload state could not be saved" : nil) + } + public var snapshot: CaptureUploadState { state } + + // MARK: Input + + /// Sealed files, uploaded as soon as the capture exists. A path already known is ignored: + /// sealed files never change. + public func add(_ files: [SealedFile]) { + guard state.end == nil, state.packet == nil else { return } + var added = false + for file in files where state.files[file.path] == nil { + state.files[file.path] = .init(sealed: file, sequence: state.nextSequence) + state.nextSequence += 1 + added = true + } + guard added else { return } + mark("firstRetained") + persist() + kick() + } + + /// The capture stopped: `packet` is frozen as these bytes, and `files` is every file it lists. + /// Any the uploader has not seen yet (an `add` still on its way) is queued here, so the + /// server never waits for a listed file that nobody sends. + public func seal(packet: Data, files: [SealedFile]) { + guard state.end == nil, state.packet == nil else { return } + for file in files where state.files[file.path] == nil { + state.files[file.path] = .init(sealed: file, sequence: state.nextSequence) + state.nextSequence += 1 + } + state.packet = packet + mark("sealed") + persist() + kick() + } + + public static let withdrawnReason = "consent withdrawn" + + /// How a withdrawal was recorded on disk. + public enum WithdrawalRecord: Sendable, Equatable { + /// The marker `resume` refuses was written. + case marked + /// The marker couldn't be written, so the saved state was removed instead. + case savedStateRemoved + /// Neither the marker nor the removal succeeded: a relaunch could still find the saved + /// yes until a later save records the upload as abandoned. + case notRecorded(String) + } + + /// Revokes the folder before returning and cancels its running task. A request already + /// admitted to a transport that ignores cancellation may finish; its reply cannot admit + /// another request or save resumable consent. `.notRecorded` means only the live process + /// is protected: after a crash the old saved yes may still exist. + @discardableResult + public nonisolated func withdrawConsent() -> WithdrawalRecord { + let (record, task) = Self.folders.withLock { entries in + entries[folderKey, default: FolderEntry()].withdrawn = true + let record = CaptureUploadState.recordWithdrawal(in: folder) + return (record, entries[folderKey]?.running) + } + // Cancellation handlers belong to the transport; never call them while holding our lock. + task?.cancel() + return record + } + + /// Closes admission synchronously when a coordinator ends a world, before its queued actor + /// work can kick the upload. Ownership stays until the old uploader and its requests die. + nonisolated func stopSending() { + let task = Self.folders.withLock { entries in + entries[folderKey]?.stopped = true + return entries[folderKey]?.running + } + task?.cancel() + } + + /// Ends this upload for good: a world reset or starting over. Replies still in flight are + /// dropped when they arrive. + public func abandon(_ reason: String) { + stopSending() + guard state.end == nil else { return } + state.end = .abandoned(reason) + state.attemptID = UUID().uuidString + loop?.cancel() + persist() + } + + /// Starts the loop if it is idle: after a relaunch, or when the network is back. + public func kick() { + guard loop == nil, state.end == nil, !saveFailed else { return } + Self.folders.withLock { entries in + guard let entry = entries[folderKey], entry.owner === self, !entry.withdrawn, !entry.stopped else { return } + let task = Task { await self.drive() } + loop = task + entries[folderKey]?.running = task + } + } + + /// Waits for the loop to go idle: nothing to send until more input, or ended. + public func settled() async { + while let running = loop { await running.value; if loop == running { loop = nil } } + } + + // MARK: Loop + + private func drive() async { + while !Task.isCancelled, state.end == nil, !saveFailed { + // A withdrawal sends nothing more; the save records the upload as abandoned. + if isWithdrawn { persist(); break } + let attempt = state.attemptID + do { + let progressed = try await step() + guard attempt == state.attemptID, state.end == nil else { break } + failures = 0 + if retryingAt != nil { retryingAt = nil; publish() } + if !progressed { break } + } catch let error as Transient { + guard attempt == state.attemptID else { break } + failures += 1 + let delay = policy.delay(afterFailure: failures, retryAfter: error.retryAfter) + retryingAt = now().addingTimeInterval(delay) + log?("capture-upload retry step=\(error.step) in=\(String(format: "%.1f", delay))s detail=\(error.detail)") + publish() + do { try await sleep(delay) } catch { break } + } catch is Withdrawn { + persist() + break + } catch let error as Refused { + guard attempt == state.attemptID else { break } + state.end = .failed(step: error.step, codes: error.codes, status: error.status) + log?("capture-upload failed step=\(error.step) status=\(error.status) codes=\(error.codes.joined(separator: ","))") + persist() + } catch { + break + } + } + loop = nil + Self.folders.withLock { $0[folderKey]?.running = nil } + } + + /// One unit of work. False when there is nothing to do until more input arrives. + private func step() async throws -> Bool { + if state.captureID == nil { try await create(); return true } + // The frozen packet goes as soon as the capture stops, before the files still uploading: + // the server answers awaiting_files and starts the run on the last commit. + if state.packet != nil, state.finalized == nil { try await finalize(); return true } + requeueExpiring() + let queued = ordered(where: { $0.phase == .queued }) + if !queued.isEmpty { try await register(Array(queued.prefix(min(policy.batch, state.maxBatch)))); return true } + // Each round of PUTs is committed before the next starts, so the first photos count as + // received after one round trip of uploads rather than a whole register batch. + let uploaded = ordered(where: { $0.phase == .uploaded }) + if !uploaded.isEmpty { try await commit(Array(uploaded.prefix(min(policy.batch, state.maxBatch)))); return true } + let registered = ordered(where: { if case .registered = $0.phase { true } else { false } }) + if !registered.isEmpty { try await put(Array(registered.prefix(policy.maxConcurrentPuts))); return true } + if state.finalized != nil, state.end == nil { + if state.result != nil || CaptureAPI.terminalStatuses.contains(state.backendStatus ?? "") { + try await fetchResult() + return true + } + try await pollEvents() + return true + } + return false + } + + private func ordered(where include: (CaptureUploadState.File) -> Bool) -> [CaptureUploadState.File] { + state.files.values.filter(include).sorted { ($0.sealed.priority, $0.sequence) < ($1.sealed.priority, $1.sequence) } + } + + // MARK: Calls + + private func create() async throws { + var request = jsonRequest("captures", method: "POST", body: state.createBody) + request.setValue(state.packetID, forHTTPHeaderField: "Idempotency-Key") + let (reply, attempt) = try await call("create", request) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let body = try? JSONDecoder().decode(CaptureAPI.CreateResponse.self, from: reply.body) else { + throw Refused(step: "create", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + state.captureID = body.captureId + state.backendStatus = body.status + if let batch = body.upload?.maxBatch, batch > 0 { state.maxBatch = batch } + mark("created") + persist() + } + + private func register(_ files: [CaptureUploadState.File]) async throws { + let body = try JSONEncoder().encode(CaptureAPI.RegisterRequest(files: files.map { .init($0.sealed) })) + let (reply, attempt) = try await call("register", jsonRequest("captures/\(captureID)/files", method: "POST", body: body)) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let answer = try? JSONDecoder().decode(CaptureAPI.RegisterResponse.self, from: reply.body) else { + if try captureGone(reply) { return } + throw Refused(step: "register", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + for file in answer.files where state.files[file.path] != nil { + if file.state == "committed" { + setPhase(file.path, .committed) + } else if let target = file.upload, target.method == "PUT", let url = target.url { + // Photos go only to an https URL, or http to this machine for tests, and only to a + // URL the PUT can use: a URL that doesn't parse would leave the file registered + // forever. This checks the URL as given; `CaptureHTTP` refuses redirects, so a + // storage server can't send the photo on to a URL this never checked. + if let problem = Self.uploadURLProblem(url) { throw Refused(step: "register", status: reply.status, codes: [problem]) } + setPhase(file.path, .registered(url: url, headers: target.headers ?? [:], expiresAt: target.expiresAt.flatMap(Self.parseDate))) + } else { + throw Refused(step: "register", status: reply.status, codes: ["upload_method_\(file.upload?.method ?? "none")"]) + } + } + let answered = Set(answer.files.map(\.path)) + try unacknowledged(files.map(\.sealed.path).filter { !answered.contains($0) }, step: "register", status: reply.status) + } + + /// Counts files the server didn't take, saves, and backs off (or stops once one has been left + /// out more than `maxUnacknowledged` times). Saves and returns when there are none. + private func unacknowledged(_ paths: [String], step: String, status: Int) throws { + for path in paths { + let count = (state.files[path]?.unacknowledged ?? 0) + 1 + state.files[path]?.unacknowledged = count + } + persist() + guard !paths.isEmpty else { return } + if paths.contains(where: { (state.files[$0]?.unacknowledged ?? 0) > policy.maxUnacknowledged }) { + throw Refused(step: step, status: status, codes: ["not_acknowledged"]) + } + throw Transient(step: step, detail: "\(paths.count) not acknowledged", retryAfter: nil) + } + + /// Why `text` can't receive a photo: nil when it is an https URL, or http to this machine. + static func uploadURLProblem(_ text: String) -> String? { + guard let url = URL(string: text), let scheme = url.scheme?.lowercased(), let host = url.host(), !host.isEmpty else { + return "upload_url_invalid" + } + let local = host == "127.0.0.1" || host == "localhost" + return scheme == "https" || (scheme == "http" && local) ? nil : "upload_url_insecure" + } + + private func put(_ files: [CaptureUploadState.File]) async throws { + let attempt = state.attemptID + var transient: Transient? + var index = 0 + while index < files.count { + // Each batch of PUTs is admitted only while the homeowner's yes stands. + if !canSend || Task.isCancelled { throw Withdrawn() } + let chunk = files[index..).self) { group in + for file in chunk { + guard case .registered(let url, let headers, _) = file.phase, Self.uploadURLProblem(url) == nil, let target = URL(string: url) else { continue } + var request = URLRequest(url: target) + request.httpMethod = "PUT" + // Exactly the headers the server signed; never the API's own credentials. + for (name, value) in headers { request.setValue(value, forHTTPHeaderField: name) } + let local = folder.appending(path: file.sealed.path) + let http = self.http + group.addTask { + if !self.canSend || Task.isCancelled { return (file.sealed.path, .failure(Withdrawn())) } + do { return (file.sealed.path, .success(try await http.upload(request, file: local))) } catch { return (file.sealed.path, .failure(error)) } + } + } + var out: [(String, Result)] = [] + for await result in group { out.append(result) } + return out + } + if !canSend || Task.isCancelled { throw Withdrawn() } + guard attempt == state.attemptID, state.end == nil else { return } + for (path, result) in results { + state.files[path]?.attempts += 1 + switch result { + case .success(let reply) where (200..<300).contains(reply.status): + setPhase(path, .uploaded) + case .success(let reply) where reply.status == 400 && !digestRetried.contains(path): + // BadDigest: the bytes on disk are sealed, so sending them once more is the retry. + digestRetried.insert(path) + case .success(let reply) where reply.status == 400: + // The same sealed bytes refused twice: the file on disk no longer matches its digest. + throw Refused(step: "put", status: 400, codes: ["bad_digest"]) + case .success(let reply) where Self.isTransient(reply.status): + transient = Transient(step: "put", detail: "status \(reply.status)", retryAfter: reply.retryAfter) + case .success(let reply): + // 403 expired signature or a spent token: a fresh URL, a bounded number of times. + guard (state.files[path]?.attempts ?? 0) < policy.maxRefusedPuts else { + throw Refused(step: "put", status: reply.status, codes: ["storage_refused"]) + } + setPhase(path, .queued) + case .failure(let error): + transient = Transient(step: "put", detail: Self.describe(error), retryAfter: nil) + } + } + persist() + } + if let transient { throw transient } + } + + private func commit(_ files: [CaptureUploadState.File]) async throws { + let body = try JSONEncoder().encode(CaptureAPI.CommitRequest(files: files.map { .init(path: $0.sealed.path, sha256: $0.sealed.sha256) })) + let (reply, attempt) = try await call("commit", jsonRequest("captures/\(captureID)/files:commit", method: "POST", body: body)) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let answer = try? JSONDecoder().decode(CaptureAPI.CommitResponse.self, from: reply.body) else { + if try captureGone(reply) { return } + throw Refused(step: "commit", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + let committed = Set(answer.committed) + for file in files { + // Only a commit acknowledges a file; notFound, mismatch or no mention sends it again. + setPhase(file.sealed.path, committed.contains(file.sealed.path) ? .committed : .queued) + } + if !committed.isEmpty { mark("firstCommit") } + if state.packet != nil, state.files.values.allSatisfy({ $0.phase == .committed }) { mark("lastCommit") } + try unacknowledged(files.map(\.sealed.path).filter { !committed.contains($0) }, step: "commit", status: reply.status) + } + + private func finalize() async throws { + guard let packet = state.packet else { return } + var request = jsonRequest("captures/\(captureID)/finalize", method: "POST", body: packet) + request.setValue(state.packetID, forHTTPHeaderField: "Idempotency-Key") + request.setValue(PacketFiles.sha256(packet), forHTTPHeaderField: "X-Packet-Sha256") + let (reply, attempt) = try await call("finalize", request) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let answer = try? JSONDecoder().decode(CaptureAPI.FinalizeResponse.self, from: reply.body) else { + if try captureGone(reply) { return } + throw Refused(step: "finalize", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + state.finalized = .init(status: answer.status, missing: answer.missing, runID: answer.runId) + state.backendStatus = answer.status + mark("finalizeAccepted") + persist() + } + + private func pollEvents() async throws { + var components = URLComponents(url: base.appending(path: "captures/\(captureID)/events"), resolvingAgainstBaseURL: false)! + components.queryItems = [.init(name: "after", value: String(state.eventCursor)), .init(name: "wait", value: String(policy.eventsWait))] + var request = URLRequest(url: components.url!) + request.setValue("application/json", forHTTPHeaderField: "Accept") + request.timeoutInterval = Double(policy.eventsWait) + 30 + let sent = now() + let (reply, attempt) = try await call("events", request) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let answer = try? JSONDecoder().decode(CaptureAPI.EventsResponse.self, from: reply.body) else { + if try captureGone(reply) { return } + throw Refused(step: "events", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + state.backendStatus = answer.status + state.eventCursor = max(state.eventCursor, answer.next) + for event in answer.events { + mark("firstEvent") + state.lastEvent = [event.type, event.data?.stage, event.data?.status, event.data?.kind, event.data?.code].compactMap { $0 }.joined(separator: " ") + // A verdict for another run is not this capture's result. + if event.type == "verdict_ready", event.data?.runId == nil || event.data?.runId == state.finalized?.runID { mark("verdictReady") } + if event.type == "failed", event.data?.next == "retry_finalize" { + // The server lost the finalize to a transient storage failure: send the same frozen + // packet again. + guard state.finalizeRetries < policy.maxFinalizeRetries else { + throw Refused(step: "finalize", status: 0, codes: [event.data?.code ?? "retry_finalize"]) + } + state.finalizeRetries += 1 + state.finalized = nil + } + } + if !answer.events.isEmpty { log?("capture-upload events status=\(answer.status) last=\(state.lastEvent ?? "")") } + persist() + // An answer that came back at once with nothing new means the server didn't hold the + // poll; waiting here keeps the loop from asking again immediately. + let waited = now().timeIntervalSince(sent) + if answer.events.isEmpty, !CaptureAPI.terminalStatuses.contains(answer.status), waited < policy.minimumPollInterval { + try await sleep(policy.minimumPollInterval - waited) + } + } + + private func fetchResult() async throws { + var request = URLRequest(url: base.appending(path: "captures/\(captureID)/result")) + request.setValue("application/json", forHTTPHeaderField: "Accept") + let (reply, attempt) = try await call("result", request) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status) else { + if try captureGone(reply) { return } + // The run reads as done before its answer is stored. + if reply.status == 404, CaptureAPI.errorCodes(reply.body).contains("result_not_ready") { try notReady(reply) } + throw Refused(step: "result", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + // The whole body must decode as the result the app reads, or a finished upload could hold + // an answer nothing can show. + let decoder = JSONDecoder() + guard let response = try? decoder.decode(CaptureResult.Response.self, from: reply.body), + let head = try? decoder.decode(ResultHead.self, from: reply.body) + else { + throw Refused(step: "result", status: reply.status, codes: ["result_unreadable"]) + } + if let runID = head.runId, runID != state.finalized?.runID { + throw Refused(step: "result", status: reply.status, codes: ["result_for_another_run"]) + } + let status = head.status ?? state.backendStatus ?? "unknown" + // Until the run has written its answer the API serves the capture's status with a null + // outcome, and that status can already be one the capture ends in. + if response.outcome == nil, !CaptureAPI.statusesWithoutOutcome.contains(status) { try notReady(reply) } + state.result = reply.body + state.end = .finished(status: status) + mark("result") + persist() + } + + /// The run and status as the server wrote them; `CaptureResult.Status` keeps no raw text. + private struct ResultHead: Decodable { + var runId: String? + var status: String? + } + + /// Always throws: a retry after the usual wait, or the end of the upload once + /// `maxNotReadyResults` reads have found no answer. + private func notReady(_ reply: HTTPReply) throws -> Never { + let notReadyReads = (state.notReadyReads ?? 0) + 1 + state.notReadyReads = notReadyReads + persist() + guard notReadyReads <= policy.maxNotReadyResults else { + throw Refused(step: "result", status: reply.status, codes: ["result_not_ready"]) + } + throw Transient(step: "result", detail: "result not ready (\(notReadyReads))", retryAfter: reply.retryAfter) + } + + // MARK: Plumbing + + private var captureID: String { state.captureID ?? "" } + + /// The server lost the capture (404: a redeploy with in-memory state). The same create body + /// opens it again, and every file goes up again under the new id. + private func captureGone(_ reply: HTTPReply) throws -> Bool { + guard reply.status == 404, CaptureAPI.errorCodes(reply.body).contains("capture_not_found") else { return false } + log?("capture-upload the server no longer has the capture; creating it again") + state.captureID = nil + state.finalized = nil + state.eventCursor = 0 + // A new server capture is a new run: the old one's limits don't carry over. + state.notReadyReads = nil + for path in state.files.keys { + state.files[path]?.phase = .queued + state.files[path]?.unacknowledged = nil + } + persist() + return true + } + + private func requeueExpiring() { + let limit = now().addingTimeInterval(policy.expiryMargin) + for (path, file) in state.files { + if case .registered(_, _, let expires?) = file.phase, expires <= limit { state.files[path]?.phase = .queued } + } + } + + private func setPhase(_ path: String, _ phase: CaptureUploadState.FilePhase) { + state.files[path]?.phase = phase + } + + private func call(_ step: String, _ request: URLRequest) async throws -> (HTTPReply, String) { + // Every API request is admitted only while the homeowner's yes stands. One already sent + // finishes, but its reply can only save the upload as abandoned. + if !canSend || Task.isCancelled { throw Withdrawn() } + let attempt = state.attemptID + let reply: HTTPReply + do { + reply = try await http.send(request) + } catch { + throw Transient(step: step, detail: Self.describe(error), retryAfter: nil) + } + if Self.isTransient(reply.status), attempt == state.attemptID { + throw Transient(step: step, detail: "status \(reply.status)", retryAfter: reply.retryAfter) + } + return (reply, attempt) + } + + private func jsonRequest(_ path: String, method: String, body: Data) -> URLRequest { + var request = URLRequest(url: base.appending(path: path)) + request.httpMethod = method + request.httpBody = body + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.setValue("application/json", forHTTPHeaderField: "Accept") + return request + } + + private func mark(_ name: String) { + guard state.marks[name] == nil else { return } + let at = now() + state.marks[name] = at + let since = state.marks["started"].map { at.timeIntervalSince($0) } ?? 0 + log?("capture-upload stage=\(name) t=\(String(format: "%.2f", since))s committed=\(state.committedCount)/\(state.files.count)") + } + + private nonisolated var isWithdrawn: Bool { Self.folders.withLock { $0[folderKey]?.withdrawn == true } } + + private nonisolated var canSend: Bool { + Self.folders.withLock { entries in + guard let entry = entries[folderKey] else { return false } + return entry.owner === self && !entry.withdrawn && !entry.stopped + } + } + + private func persist() { + do { + try Self.folders.withLock { entries in + guard entries[folderKey]?.owner === self else { throw OwnershipError.alreadyOwned } + let withdrawn = entries[folderKey]?.withdrawn == true + // A withdrawal outranks any end a late reply set (finished, failed). + if withdrawn, state.end != .abandoned(Self.withdrawnReason) { + state.end = .abandoned(Self.withdrawnReason) + // Replies still in flight belong to the old attempt and are dropped. + state.attemptID = UUID().uuidString + } + try state.save(to: Self.stateURL(in: folder)) + } + } catch { + saveFailed = true + log?("capture-upload stopped: the upload state could not be saved (\(Self.describe(error)))") + } + publish() + } + + private func publish() { + observer?(status) + } + + /// 501 is the server saying it doesn't do what was asked (multipart), which waiting won't change. + static func isTransient(_ status: Int) -> Bool { status == 0 || status == 408 || status == 429 || ((500..<600).contains(status) && status != 501) } + + /// The error's domain and code only: a URLError's description can carry the signed URL. + static func describe(_ error: any Error) -> String { + let ns = error as NSError + return "\(ns.domain) \(ns.code)" + } + + static func parseDate(_ text: String) -> Date? { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + if let date = formatter.date(from: text) { return date } + formatter.formatOptions = [.withInternetDateTime] + return formatter.date(from: text) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureFolderOwnershipTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureFolderOwnershipTests.swift new file mode 100644 index 000000000..ea16aeffa --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureFolderOwnershipTests.swift @@ -0,0 +1,224 @@ +import Foundation +import HouseScanKit +import Synchronization +import Testing + +@Suite(.serialized) struct CaptureFolderOwnershipTests { + private func create(_ folder: URL, _ base: URL) throws -> CaptureUploader { + try CaptureUploader.start( + folder: folder, base: base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), + create: .init(packetId: "ownership-test", tier: .arkit, device: .init(model: "test", systemVersion: "test", appVersion: "test")), + consentedAt: Date()) + } + + @Test func concurrentStartsSaveOnlyTheOwnersAttempt() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "owners-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let owners = await withTaskGroup(of: CaptureUploader?.self, returning: [CaptureUploader].self) { group in + for _ in 0..<12 { + group.addTask { + do { return try create(root, server.base) } + catch { #expect(error as? CaptureUploader.OwnershipError == .alreadyOwned); return nil } + } + } + var owners: [CaptureUploader] = [] + for await owner in group { if let owner { owners.append(owner) } } + return owners + } + try #require(owners.count == 1) + let before = await owners[0].snapshot + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: root)) == before) + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned) { + try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + } + await owners[0].kick() + try #require(await CaptureUploaderTests.settles(owners[0])) + #expect(server.requests("POST captures").count == 1) + } + + @Test func concurrentResumesSaveOnlyTheOwnersAttempt() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "resume-owners-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + var first: CaptureUploader? = try create(root, server.base) + weak let released = first + first = nil + try #require(released == nil) + let owners = await withTaskGroup(of: CaptureUploader?.self, returning: [CaptureUploader].self) { group in + for _ in 0..<12 { + group.addTask { + do { return try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) } + catch { #expect(error as? CaptureUploader.OwnershipError == .alreadyOwned); return nil } + } + } + var owners: [CaptureUploader] = [] + for await owner in group { if let owner { owners.append(owner) } } + return owners + } + try #require(owners.count == 1) + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: root)) == (await owners[0].snapshot)) + await owners[0].kick() + try #require(await CaptureUploaderTests.settles(owners[0])) + #expect(server.requests("POST captures").count == 1) + } + + @Test func aliasesCannotAcquireTheSameFolder() throws { + let root = FileManager.default.temporaryDirectory.appending(path: "alias-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let folder = root.appending(path: "Capture") + let owner = try create(folder, server.base) + let link = root.appending(path: "link") + try FileManager.default.createSymbolicLink(at: link, withDestinationURL: folder) + var aliases = [link, root.appending(path: "Capture/../Capture")] + let lowercased = root.appending(path: "capture") + if FileManager.default.fileExists(atPath: lowercased.path) { aliases.append(lowercased) } + for alias in aliases { + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned, "alias: \(alias.path)") { try create(alias, server.base) } + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned, "alias: \(alias.path)") { + try CaptureUploader.resume(folder: alias, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + } + } + #expect(owner.folder.path == (try folder.resourceValues(forKeys: [.canonicalPathKey]).canonicalPath)) + #expect(server.state.withLock { $0.log.isEmpty }) + } + + @Test func idleOwnershipReleasesOnlyAfterTheUploaderDies() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "lifetime-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + var owner: CaptureUploader? = try create(root, server.base) + await owner?.kick() + try #require(await CaptureUploaderTests.settles(try #require(owner))) + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned) { + try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + } + weak let released = owner + owner = nil + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + #expect(throws: CaptureUploader.OwnershipError.savedUploadExists) { try create(root, server.base) } + let resumed = try #require(try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10))) + await resumed.kick() + try #require(await CaptureUploaderTests.settles(resumed)) + #expect(server.requests("POST captures").count == 1) + } + + @Test func anInFlightRequestKeepsOwnershipAfterTheCallerReleasesIt() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "inflight-owner-\(UUID().uuidString)") + let server = try LoopbackCaptureAPI() + defer { + server.state.withLock { $0.held = [] } + server.release("POST captures") + try? FileManager.default.removeItem(at: root) + } + server.state.withLock { $0.held = ["POST captures"] } + var owner: CaptureUploader? = try create(root, server.base) + await owner?.kick() + for _ in 0..<500 where server.state.withLock({ $0.parked["POST captures"]?.isEmpty ?? true }) { + try await Task.sleep(for: .milliseconds(10)) + } + try #require(server.state.withLock { !($0.parked["POST captures"]?.isEmpty ?? true) }) + weak let runningOwner = owner + owner = nil + #expect(runningOwner != nil) + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned) { + try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + } + server.state.withLock { $0.held = [] } + server.release("POST captures") + for _ in 0..<500 where runningOwner != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(runningOwner == nil) + let resumed = try #require(try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10))) + await resumed.kick() + try #require(await CaptureUploaderTests.settles(resumed)) + #expect(server.requests("POST captures").count == 1) + } + + /// Failure injection uses permissions only on this test's temporary folder. Restoring the + /// stale saved yes after stopping models the disk contents a failed withdrawal can leave. + @Test(arguments: [0, 1, 2]) func withdrawalBlocksReuseEvenWhenDiskStillHasConsent(failedWrites: Int) async throws { + var rig = try CaptureUploaderTests.Rig(http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10))) + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: rig.capture.folder.path) + rig.cleanUp() + } + rig.server.state.withLock { $0.held = ["POST captures/files"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + for _ in 0..<500 where rig.server.state.withLock({ $0.parked["POST captures/files"]?.isEmpty ?? true }) { + try await Task.sleep(for: .milliseconds(10)) + } + try #require(rig.server.state.withLock { !($0.parked["POST captures/files"]?.isEmpty ?? true) }) + let stateURL = CaptureUploader.stateURL(in: rig.capture.folder) + let staleConsent = try Data(contentsOf: stateURL) + let marker = CaptureUploadState.withdrawnURL(in: rig.capture.folder) + if failedWrites == 1 { + try FileManager.default.createDirectory(at: marker.appending(path: "blocked"), withIntermediateDirectories: true) + } else if failedWrites == 2 { + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: rig.capture.folder.path) + } + let result = rig.uploader.withdrawConsent() + switch failedWrites { + case 0: #expect(result == .marked) + case 1: #expect(result == .savedStateRemoved) + default: + guard case .notRecorded = result else { Issue.record("Both filesystem writes must fail"); return } + #expect(try Data(contentsOf: stateURL) == staleConsent) + } + rig.server.state.withLock { $0.held = [] } + rig.server.release("POST captures/files") + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(rig.server.requests("PUT upload").isEmpty) + #expect(rig.server.requests("POST captures/files:commit").isEmpty) + #expect(await rig.uploader.snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: rig.capture.folder.path) + if FileManager.default.fileExists(atPath: marker.path) { try FileManager.default.removeItem(at: marker) } + try staleConsent.write(to: stateURL, options: .atomic) + weak let released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + #expect(try CaptureUploader.resume(folder: rig.capture.folder, base: rig.server.base, http: rig.http) == nil) + #expect(throws: CaptureUploader.OwnershipError.consentWithdrawn) { try create(rig.capture.folder, rig.server.base) } + #expect(try Data(contentsOf: stateURL) == staleConsent) + } + + @Test(arguments: [false, true]) func withdrawalDuringPutsStopsTheNextBatchAndCommit(uncancellable: Bool) async throws { + let transport = URLSessionCaptureHTTP.ephemeral(timeout: 10) + let http: any CaptureHTTP = uncancellable ? UncancellableHTTP(transport) : transport + var policy = CaptureUploaderTests.Rig.fast + policy.maxConcurrentPuts = 2 + let rig = try CaptureUploaderTests.Rig(http: http, policy: policy) + defer { + rig.server.state.withLock { $0.held = [] } + rig.server.release("PUT upload") + rig.cleanUp() + } + rig.server.state.withLock { $0.held = ["PUT upload"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 3)) + for _ in 0..<500 where rig.server.state.withLock({ ($0.parked["PUT upload"]?.count ?? 0) < 2 }) { + try await Task.sleep(for: .milliseconds(10)) + } + try #require(rig.server.state.withLock { ($0.parked["PUT upload"]?.count ?? 0) == 2 }) + #expect(rig.uploader.withdrawConsent() == .marked) + await rig.uploader.kick() + rig.server.state.withLock { $0.held = [] } + rig.server.release("PUT upload") + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(rig.server.requests("PUT upload").count == 2) + #expect(rig.server.requests("POST captures/files:commit").isEmpty) + #expect(await rig.uploader.snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) + } + + @Test func withdrawalBeforeConcurrentKicksAdmitsNoRequests() async throws { + let rig = try CaptureUploaderTests.Rig() + defer { rig.cleanUp() } + #expect(rig.uploader.withdrawConsent() == .marked) + await withTaskGroup(of: Void.self) { group in + for _ in 0..<20 { group.addTask { await rig.uploader.kick() } } + } + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(rig.server.state.withLock { $0.log.isEmpty }) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultSummaryTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultSummaryTests.swift new file mode 100644 index 000000000..e8c83cb24 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultSummaryTests.swift @@ -0,0 +1,142 @@ +import Foundation +import HouseScanKit +import simd +import Testing + +/// What the integration build shows of a capture-API result, from self-authored responses, and +/// the coordinator binding the result it fetched to its own scan. +@Suite struct CaptureResultSummaryTests { + static let world = CaptureResult.Association(sessionID: "session-A", captureID: "cap_A", runID: "run_A", epoch: "e1") + + static func response(_ json: String) throws -> CaptureResult.Response { + try JSONDecoder().decode(CaptureResult.Response.self, from: Data(json.utf8)) + } + + static func summary(_ json: String, association: CaptureResult.Association = world, analysis: CaptureResult.Analysis = .unverified) throws -> CaptureResultSummary { + CaptureResultSummary(.init(response: try response(json), association: association, analysis: analysis), world: world, meterAnchor: matrix_identity_float4x4) + } + + static let prompt = #"{"id":"vn1","kind":"wall_band","criteria":["C3"],"why":"unseen","promptId":"p1","prompt":{"title":"Step back and show the wall","body":"Include the ground under the meter."}}"# + + @Test func processingHasNoOutcomeAndNoMessage() throws { + let s = try Self.summary(#"{"runId":"run_A","status":"processing","viewsNeeded":[],"memberActions":[]}"#) + #expect(s.state == .working(.processing)) + #expect(s.message == nil && s.prompts.isEmpty && s.criteria == nil) + #expect(s.arUnavailable == .noOutcome(.processing)) + } + + /// A capture that failed or expired has no outcome, and it is over: it is not shown as + /// still working. + @Test(arguments: ["failed", "expired"]) func aCaptureThatEndedWithoutAnOutcomeIsNotWorking(status: String) throws { + let s = try Self.summary(#"{"runId":"run_A","status":"\#(status)","viewsNeeded":[],"memberActions":[]}"#) + #expect(s.state == .ended(CaptureResult.Status(rawValue: status))) + #expect(s.message == nil && s.prompts.isEmpty) + } + + @Test func manualReviewKeepsTheServersWordsAndNoInventedChecks() throws { + let s = try Self.summary(#""" + {"runId":"run_A","status":"manual_review","viewsNeeded":[],"memberActions":[], + "outcome":{"kind":"manual_review","profile":"C","message":"RECEIVER TEST: a person will look at this.","viewsNeeded":[],"reasons":[]}} + """#) + #expect(s.state == .decided(.manualReview)) + #expect(s.message == "RECEIVER TEST: a person will look at this.") + #expect(s.criteria == nil) + #expect(s.arUnavailable == .notEligible(.manualReview)) + } + + @Test func aMissingViewShowsItsPromptVerbatim() throws { + let s = try Self.summary(""" + {"runId":"run_A","status":"needs_views","viewsNeeded":[\(Self.prompt)],"memberActions":[], + "outcome":{"kind":"needs_more_photos","profile":"C","message":"One more view, please.","viewsNeeded":[\(Self.prompt)],"reasons":[]}} + """) + #expect(s.state == .decided(.needsMorePhotos)) + #expect(s.prompts.map(\.title) == ["Step back and show the wall"]) + #expect(s.prompts.map(\.body) == ["Include the ground under the meter."]) + } + + /// An eligible-shaped outcome from an unverified server stays without AR, and without the + /// criteria it didn't send. + @Test func eligibleWithoutCriteriaStaysUnverifiedAndOffAR() throws { + let s = try Self.summary(#""" + {"runId":"run_A","status":"complete","viewsNeeded":[],"memberActions":[], + "outcome":{"kind":"eligible","profile":"C","message":"A spot fits.","viewsNeeded":[],"reasons":[],"arkitEpoch":"e1", + "recommendedPlacement":{"wallId":"w1","startSM":1.2,"confidence":0.8, + "boxArkitWorld":{"pose":[1,0,0,0,0,1,0,0,0,0,1,0,0.5,0,-2,1],"size":[0.79,1.0,0.56]}}}} + """#) + #expect(s.state == .decided(.eligible)) + #expect(s.criteria == nil) + #expect(!s.analysisVerified) + #expect(s.arUnavailable == .analysisUnverified) + } + + @Test func anUnknownStatusStaysReadable() throws { + let s = try Self.summary(#"{"runId":"run_A","status":"paused_for_audit","viewsNeeded":[],"memberActions":[]}"#) + #expect(s.state == .working(.unknown("paused_for_audit"))) + #expect(s.arUnavailable == .noOutcome(.unknown("paused_for_audit"))) + } + + /// Even a verified result doesn't place a box for another session, capture or run. + @Test(arguments: [ + CaptureResult.Association(sessionID: "session-B", captureID: "cap_A", runID: "run_A", epoch: "e1"), + CaptureResult.Association(sessionID: "session-A", captureID: "cap_B", runID: "run_A", epoch: "e1"), + CaptureResult.Association(sessionID: "session-A", captureID: "cap_A", runID: "run_B", epoch: "e1"), + ]) + func aResultForAnotherScanPlacesNothing(_ association: CaptureResult.Association) throws { + let s = try Self.summary(#""" + {"runId":"run_A","status":"complete","viewsNeeded":[],"memberActions":[], + "outcome":{"kind":"eligible","profile":"C","message":"A spot fits.","viewsNeeded":[],"reasons":[],"arkitEpoch":"e1", + "recommendedPlacement":{"wallId":"w1","startSM":1.2,"confidence":0.8, + "boxArkitWorld":{"pose":[1,0,0,0,0,1,0,0,0,0,1,0,0.5,0,-2,1],"size":[0.79,1.0,0.56]}}}} + """#, association: association, analysis: .verified) + guard case .mismatch? = s.arUnavailable else { Issue.record("placed a box for another scan: \(String(describing: s.arUnavailable))"); return } + } +} + +/// The coordinator fetches the result through the real uploader and binds it to its own session. +@Suite(.serialized) @MainActor struct CaptureResultBindingTests { + @Test func theFetchedResultIsBoundToThisScanAndStaysUnverified() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "result-binding-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.resultMessage = "RECEIVER TEST: distinctive message 7f3" } + let coordinator = CaptureSessionCoordinator(environment: NativeCaptureFixture.environment( + endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: root.appending(path: "Captures"))) + var results: [CaptureResult.Record] = [] + coordinator.onResult = { results.append($0) } + try await NativeCaptureFixture(folder: root.appending(path: "store")).run(coordinator) + for _ in 0..<100 where results.isEmpty { try await Task.sleep(for: .milliseconds(20)) } + + let session = try #require(coordinator.session) + let record = try #require(results.first) + #expect(results.count == 1) + let state = await session.uploader!.snapshot + #expect(record.association == .init(sessionID: session.localID, captureID: state.captureID!, runID: state.finalized!.runID, epoch: "e1")) + #expect(record.response.outcome?.message == "RECEIVER TEST: distinctive message 7f3") + #expect(record.analysis == .unverified) + #expect(session.result == record) + } + + /// A result arriving for a session that has ended (a new scan started) is never published. + @Test func aResultForAnEndedSessionIsDropped() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "result-stale-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["GET captures/result"] } + let fixture = NativeCaptureFixture(folder: root.appending(path: "store")) + let coordinator = CaptureSessionCoordinator(environment: NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: root.appending(path: "Captures"))) + var results: [CaptureResult.Record] = [] + coordinator.onResult = { results.append($0) } + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 3)) + coordinator.captureEnded(acceptedCloseUpAt: nil) + for _ in 0..<500 where server.requests("GET captures/result").isEmpty { try await Task.sleep(for: .milliseconds(10)) } + let old = try #require(coordinator.session) + coordinator.newWorld("start over", recording: fixture.recording, newScan: true) + server.release("GET captures/result") + try await Task.sleep(for: .milliseconds(500)) + #expect(results.isEmpty) + #expect(old.result == nil) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultTests.swift new file mode 100644 index 000000000..ed3e9919a --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultTests.swift @@ -0,0 +1,320 @@ +import Foundation +import HouseScanKit +import Testing +import simd + +// Every body here is self-authored to the shape of the server's device result; none is a server +// example or a real capture. +@Suite struct CaptureResultTests { + typealias Association = CaptureResult.Association + + static let scan = Association(sessionID: "session-a", captureID: "cap-a", runID: "run-a", epoch: "e1") + + static let eligibleJSON = """ + {"runId": "run-a", "status": "complete", "viewsNeeded": [], "memberActions": [], + "outcome": {"kind": "eligible", "profile": "C", "message": "A battery fits on this wall.", + "viewsNeeded": [], "reasons": [], "arkitEpoch": "e1", + "frameToArkit": [1,0,0,0, 0,1,0,0, 0,0,1,0, 0,0,0,1], + "recommendedPlacement": {"wallId": "w1", "startSM": 1.1, "confidence": 0.7, "profile": "C", + "footprintM": {"w": 0.8, "h": 1.0, "d": 0.5}, + "boxSceneM": {"centerM": [1.5, 0.5, 0.3], "sizeM": [0.8, 1.0, 0.5], "yawDeg": 0}, + "boxArkitWorld": {"pose": [1,0,0,0, 0,1,0,0, 0,0,1,0, 1.5,0.5,0.3,1], "size": [0.8, 1.0, 0.5]}}, + "cableRoutePolylineArkit": null}, + "verdictUrl": null, "previewUrl": null, "reviewUrl": "/v1/captures/cap-a/review"} + """ + + static func decode(_ json: String) throws -> CaptureResult.Response { + try JSONDecoder().decode(CaptureResult.Response.self, from: Data(json.utf8)) + } + + static func eligible() throws -> CaptureResult.Response { try decode(eligibleJSON) } + + static func verified(_ response: CaptureResult.Response) -> CaptureResult.Record { + CaptureResult.Record(response: response, association: scan, analysis: .verified) + } + + // MARK: Decoding + + @Test func processingHasNoOutcome() throws { + let response = try Self.decode(""" + {"runId": null, "status": "processing", "viewsNeeded": [], "memberActions": [], "outcome": null} + """) + #expect(response.status == .processing) + #expect(response.runId == nil && response.outcome == nil && response.criteria == nil) + #expect(response.previewUrl == nil && response.reviewUrl == nil) + #expect(Self.verified(response).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + == .unavailable(.noOutcome(.processing))) + } + + @Test func eligibleWithoutCriteriaKeepsThemAbsent() throws { + let response = try Self.eligible() + #expect(response.outcome?.kind == .eligible) + #expect(response.criteria == nil) + #expect(response.reviewUrl == "/v1/captures/cap-a/review") + let box = try #require(response.outcome?.recommendedPlacement?.boxArkitWorld) + #expect(box.size == [0.8, 1.0, 0.5]) + guard case .box(let placed) = Self.verified(response).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) else { + Issue.record("expected a box") + return + } + #expect(placed.size == SIMD3(0.8, 1.0, 0.5)) + #expect(nearlyEqual(placed.anchorFromBox, translation(1.5, 0.5, 0.3))) + } + + @Test func eligibleWithCriteriaKeepsEachAsSent() throws { + let json = Self.eligibleJSON.replacingOccurrences( + of: "\"verdictUrl\": null", + with: """ + "criteria": [{"id": "c1", "outcome": "pass", "measuredFt": 4.2, "coverage": "observed"}, + {"id": "c2", "outcome": "unsure", "unsureCause": "unobserved"}, + {"id": "c3", "outcome": "waived"}], + "verdictUrl": null + """) + let criteria = try #require(try Self.decode(json).criteria) + #expect(criteria.map(\.outcome) == [.pass, .unsure, .unknown("waived")]) + #expect(criteria[0].measuredFt == 4.2 && criteria[0].coverage == "observed" && criteria[0].thresholdFt == nil) + #expect(criteria[1].unsureCause == "unobserved" && criteria[1].measuredFt == nil) + } + + @Test func manualReviewKeepsViewsAndPromptsVerbatim() throws { + let body = " Hold the phone at chest height,\nand include 3 ft left of the ā€œmeterā€. " + let view = """ + {"id": "vn1", "kind": "past_end", "criteria": ["c2"], "why": "The left end was not seen.", + "promptId": "p-left-end", "prompt": {"title": "Show the left end", "body": \(try jsonString(body))}, + "side": "left", "wallId": "w1", "futureKey": {"nested": true}, + "referenceCrop": {"image": "k00002", "boxPx": [10, 20, 300, 400]}} + """ + let response = try Self.decode(""" + {"runId": "run-a", "status": "manual_review", "viewsNeeded": [\(view)], + "memberActions": ["retake_meter_closeups"], + "outcome": {"kind": "manual_review", "profile": "C", "message": "An installer will look at this scan.", + "viewsNeeded": [\(view)], "reasons": [], "arkitEpoch": "e1"}, + "criteria": [{"id": "c2", "outcome": "unsure"}], + "verdictUrl": null, "previewUrl": null, "reviewUrl": "/v1/captures/cap-a/review"} + """) + let outcome = try #require(response.outcome) + #expect(outcome.kind == .manualReview && outcome.reasons.isEmpty) + #expect(outcome.message == "An installer will look at this scan.") + #expect(response.viewsNeeded == outcome.viewsNeeded) + let needed = try #require(outcome.viewsNeeded.first) + #expect(needed.prompt.title == "Show the left end") + #expect(needed.prompt.body == body) + #expect(needed.kind == "past_end" && needed.wallId == "w1" && needed.band == nil) + #expect(needed.referenceCrop?.boxPx == [10, 20, 300, 400]) + #expect(response.memberActions == ["retake_meter_closeups"]) + #expect(Self.verified(response).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + == .unavailable(.notEligible(.manualReview))) + } + + @Test func unknownStatusAndKindDecodeAndWithholdAR() throws { + let unknownKind = try Self.decode(Self.eligibleJSON + .replacingOccurrences(of: "\"status\": \"complete\"", with: "\"status\": \"archived\"") + .replacingOccurrences(of: "\"kind\": \"eligible\"", with: "\"kind\": \"deferred\"")) + #expect(unknownKind.status == .unknown("archived")) + #expect(unknownKind.outcome?.kind == .unknown("deferred")) + #expect(unknownKind.outcome?.message == "A battery fits on this wall.") + #expect(Self.verified(unknownKind).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + == .unavailable(.notEligible(.unknown("deferred")))) + + var unknownStatus = try Self.eligible() + unknownStatus.status = .unknown("archived") + #expect(Self.verified(unknownStatus).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + == .unavailable(.statusNotComplete(.unknown("archived")))) + } + + @Test func missingStatusFailsToDecode() { + #expect(throws: DecodingError.self) { + try Self.decode(#"{"runId": "run-a", "viewsNeeded": [], "memberActions": [], "outcome": null}"#) + } + } + + // MARK: Gating + + @Test func eligibleResultIsUnverifiedByDefault() throws { + let record = CaptureResult.Record(response: try Self.eligible(), association: Self.scan) + #expect(record.analysis == .unverified) + #expect(record.placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) == .unavailable(.analysisUnverified)) + } + + enum Mismatch: CaseIterable, Sendable { + case worldSession, worldCapture, worldRun, worldEpoch, responseRun, responseRunMissing, outcomeEpoch + + var field: CaptureResult.IdentityField { + switch self { + case .worldSession: .session + case .worldCapture: .capture + case .worldRun, .responseRun, .responseRunMissing: .run + case .worldEpoch, .outcomeEpoch: .epoch + } + } + } + + @Test(arguments: Mismatch.allCases) + func everyIdentityMismatchWithholdsAR(_ mismatch: Mismatch) throws { + var response = try Self.eligible() + var world = Self.scan + switch mismatch { + case .worldSession: world.sessionID = "session-b" + case .worldCapture: world.captureID = "cap-b" + case .worldRun: world.runID = "run-b" + case .worldEpoch: world.epoch = "e2" + case .responseRun: response.runId = "run-b" + case .responseRunMissing: response.runId = nil + case .outcomeEpoch: response.outcome?.arkitEpoch = "e2" + } + let record = Self.verified(response) + #expect(record.placement(in: world, meterAnchor: matrix_identity_float4x4) == .unavailable(.mismatch(mismatch.field))) + #expect(record.response.outcome?.message == "A battery fits on this wall.") + } + + @Test func epochReusedByANewSessionIsNotAMatch() throws { + let newSession = Association(sessionID: "session-b", captureID: "cap-a", runID: "run-a", epoch: "e1") + #expect(newSession.epoch == Self.scan.epoch) + #expect(Self.verified(try Self.eligible()).placement(in: newSession, meterAnchor: matrix_identity_float4x4) + == .unavailable(.mismatch(.session))) + } + + enum BadBox: CaseIterable, Sendable { + case shortPose, twoSizes, nanPose, infiniteSize, zeroSize, scaledPose, shearedPose, mirroredPose, badBottomRow + case noBox, noEpoch, noPlacement + case floatOverflowSize, floatUnderflowSize, floatOverflowTranslation + + var expected: CaptureResult.Unavailable { + switch self { + case .shortPose: .malformedBox(.poseCount(15)) + case .twoSizes: .malformedBox(.sizeCount(2)) + case .nanPose, .infiniteSize, .floatOverflowSize, .floatOverflowTranslation: .malformedBox(.nonFinite) + case .zeroSize, .floatUnderflowSize: .malformedBox(.nonPositiveSize) + case .scaledPose, .shearedPose, .mirroredPose, .badBottomRow: .malformedBox(.notRigid) + case .noBox, .noEpoch: .notInARKitWorld + case .noPlacement: .noPlacement + } + } + + func apply(to outcome: inout CaptureResult.Outcome) { + switch self { + case .shortPose: outcome.recommendedPlacement?.boxArkitWorld?.pose.removeLast() + case .twoSizes: outcome.recommendedPlacement?.boxArkitWorld?.size = [0.8, 1.0] + case .nanPose: outcome.recommendedPlacement?.boxArkitWorld?.pose[13] = .nan + case .infiniteSize: outcome.recommendedPlacement?.boxArkitWorld?.size[1] = .infinity + case .zeroSize: outcome.recommendedPlacement?.boxArkitWorld?.size[2] = 0 + case .floatOverflowSize: outcome.recommendedPlacement?.boxArkitWorld?.size[0] = 1e40 + case .floatUnderflowSize: outcome.recommendedPlacement?.boxArkitWorld?.size[0] = 1e-50 + case .floatOverflowTranslation: outcome.recommendedPlacement?.boxArkitWorld?.pose[12] = 1e40 + case .scaledPose: + for i in 0..<11 { outcome.recommendedPlacement?.boxArkitWorld?.pose[i] *= 1.25 } + case .shearedPose: outcome.recommendedPlacement?.boxArkitWorld?.pose[4] = 0.3 + case .mirroredPose: outcome.recommendedPlacement?.boxArkitWorld?.pose[0] = -1 + case .badBottomRow: outcome.recommendedPlacement?.boxArkitWorld?.pose[3] = 0.5 + case .noBox: outcome.recommendedPlacement?.boxArkitWorld = nil + case .noEpoch: outcome.arkitEpoch = nil + case .noPlacement: outcome.recommendedPlacement = nil + } + } + } + + @Test(arguments: BadBox.allCases) + func malformedGeometryWithholdsARAndKeepsTheResult(_ bad: BadBox) throws { + var response = try Self.eligible() + var outcome = try #require(response.outcome) + bad.apply(to: &outcome) + response.outcome = outcome + let record = Self.verified(response) + #expect(record.placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) == .unavailable(bad.expected)) + #expect(record.response.outcome?.kind == .eligible) + #expect(record.response.outcome?.message == "A battery fits on this wall.") + } + + @Test func nonFiniteMeterAnchorWithholdsAR() throws { + var anchor = matrix_identity_float4x4 + anchor.columns.3.x = .nan + #expect(Self.verified(try Self.eligible()).placement(in: Self.scan, meterAnchor: anchor) == .unavailable(.invalidMeterAnchor)) + } + + // MARK: Transform + + @Test func boxIsExpressedInATranslatedRotatedMeterAnchor() throws { + let worldFromAnchor = pose(rotationY(degrees: 90), SIMD3(2, 0, -1)) + let anchorFromBox = pose(rotationY(degrees: 30), SIMD3(0.5, 0.4, 0.3)) + var response = try Self.eligible() + response.outcome?.recommendedPlacement?.boxArkitWorld?.pose = columnMajor(worldFromAnchor * anchorFromBox) + + let placement = Self.verified(response).placement(in: Self.scan, meterAnchor: float(worldFromAnchor)) + guard case .box(let placed) = placement else { + Issue.record("expected a box, got \(placement)") + return + } + #expect(nearlyEqual(placed.anchorFromBox, float(anchorFromBox))) + #expect(nearlyEqual(SIMD3(placed.anchorFromBox.columns.3.x, placed.anchorFromBox.columns.3.y, placed.anchorFromBox.columns.3.z), + SIMD3(0.5, 0.4, 0.3), 1e-5)) + #expect(placed.size == SIMD3(0.8, 1.0, 0.5)) + } + + /// The server builds the box's world pose from the scene box and the scene-to-ARKit similarity: + /// the similarity's scale is divided out of the rotation and multiplied into the size. The app + /// must take that size as it is, not scale it again by `frameToArkit`. + @Test func sizeCarriesTheServersScaleOnce() throws { + let scale = 1.25 + let frameToArkit = pose(rotationY(degrees: 40) * scale, SIMD3(1, 0, 2)) + let centerM = SIMD3(1.5, 0.5, 0.3), sizeM = SIMD3(0.8, 1.0, 0.56), yawDeg = 10.0 + + let rotation = rotation3(frameToArkit) * (1 / scale) * rotationY(degrees: yawDeg) + let center = frameToArkit * SIMD4(centerM, 1) + let serverPose = pose(rotation, SIMD3(center.x, center.y, center.z)) + let serverSize = sizeM * scale + + var response = try Self.eligible() + response.outcome?.recommendedPlacement?.boxArkitWorld?.pose = columnMajor(serverPose) + response.outcome?.recommendedPlacement?.boxArkitWorld?.size = [serverSize.x, serverSize.y, serverSize.z] + + let placement = Self.verified(response).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + guard case .box(let placed) = placement else { + Issue.record("expected a box, got \(placement)") + return + } + #expect(nearlyEqual(placed.size, SIMD3(1.0, 1.25, 0.7), 1e-6)) + #expect(nearlyEqual(placed.anchorFromBox, float(serverPose))) + for column in [placed.anchorFromBox.columns.0, placed.anchorFromBox.columns.1, placed.anchorFromBox.columns.2] { + #expect(abs(simd_length(SIMD3(column.x, column.y, column.z)) - 1) < 1e-5) + } + } +} + +private func jsonString(_ text: String) throws -> String { + String(decoding: try JSONEncoder().encode(text), as: UTF8.self) +} + +private func rotationY(degrees: Double) -> simd_double3x3 { + let a = degrees * .pi / 180 + return simd_double3x3(SIMD3(cos(a), 0, -sin(a)), SIMD3(0, 1, 0), SIMD3(sin(a), 0, cos(a))) +} + +private func rotation3(_ m: simd_double4x4) -> simd_double3x3 { + simd_double3x3( + SIMD3(m.columns.0.x, m.columns.0.y, m.columns.0.z), + SIMD3(m.columns.1.x, m.columns.1.y, m.columns.1.z), + SIMD3(m.columns.2.x, m.columns.2.y, m.columns.2.z)) +} + +private func pose(_ r: simd_double3x3, _ t: SIMD3) -> simd_double4x4 { + simd_double4x4(SIMD4(r.columns.0, 0), SIMD4(r.columns.1, 0), SIMD4(r.columns.2, 0), SIMD4(t, 1)) +} + +private func translation(_ x: Float, _ y: Float, _ z: Float) -> simd_float4x4 { + var m = matrix_identity_float4x4 + m.columns.3 = SIMD4(x, y, z, 1) + return m +} + +private func columnMajor(_ m: simd_double4x4) -> [Double] { + [m.columns.0, m.columns.1, m.columns.2, m.columns.3].flatMap { [$0.x, $0.y, $0.z, $0.w] } +} + +private func float(_ m: simd_double4x4) -> simd_float4x4 { + simd_float4x4(SIMD4(m.columns.0), SIMD4(m.columns.1), SIMD4(m.columns.2), SIMD4(m.columns.3)) +} + +private func nearlyEqual(_ a: simd_float4x4, _ b: simd_float4x4, _ tolerance: Float = 1e-5) -> Bool { + [(a.columns.0, b.columns.0), (a.columns.1, b.columns.1), (a.columns.2, b.columns.2), (a.columns.3, b.columns.3)] + .allSatisfy { simd_reduce_max(simd_abs($0 - $1)) <= tolerance } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift new file mode 100644 index 000000000..1e1bfb85a --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift @@ -0,0 +1,711 @@ +import Foundation +import HouseScanKit +import simd +import Synchronization +import Testing + +/// A scan shaped like the app's live capture, for the coordinator: 1920 x 1440 sensor JPEGs, the +/// camera 2 m from a wall at 1.5 m height walking +x, the recorder's raw rows (a 60 Hz trajectory +/// with each frame's intrinsics beside it, and Core Motion's accelerometer and gyro at about +/// 100 Hz on their own jittered clocks), the meter tap's frame and pixel, and the meter close-up +/// from its own frame. Pattern images, no house; the packet says `source.kind: synthetic`. +struct NativeCaptureFixture: Sendable { + static let width = 1920, height = 1440 + static let k = SIMD4(1445, 1445, 960, 720) + static let meter = SIMD3(0.4, 1.2, -2) + let start = 5000.0 + let end = 5012.0 + let folder: URL + + func pose(_ t: Double) -> simd_float4x4 { + var m = matrix_identity_float4x4 + m.columns.3 = SIMD4(Float(t - start) * 0.1, 1.5, 0, 1) + return m + } + + /// Where `world` lands in the sensor image of the frame at `t`. + func pixel(of world: SIMD3, at t: Double) -> SIMD2 { + let p = world - SIMD3(pose(t).columns.3.x, pose(t).columns.3.y, pose(t).columns.3.z) + let depth = -p.z + return SIMD2(Double(Self.k.z + Self.k.x * p.x / depth), Double(Self.k.w - Self.k.y * p.y / depth)) + } + + var recording: RecordingSource { + let fixture = self + return RecordingSource(start: { (fixture.start, Date(timeIntervalSince1970: 1_790_000_000)) }, rows: { fixture.rows }) + } + + var rows: RecorderRows { + var trajectory: [[Double]] = [], intrinsics: [[Double]] = [] + for i in 0...Int((end - start) * 60) { + let t = start + Double(i) / 60 + let m = pose(t) + let columns: [SIMD4] = [m.columns.0, m.columns.1, m.columns.2, m.columns.3] + let values: [Double] = columns.flatMap { (c: SIMD4) -> [Double] in [Double(c.x), Double(c.y), Double(c.z), Double(c.w)] } + trajectory.append([t, 0, 0] + values) + intrinsics.append([t, 1445, 1445, 960, 720, 1920, 1440]) + } + // Two sensors on their own clocks: neither shares a timestamp with the other. + var accel: [[Double]] = [], gyro: [[Double]] = [] + for i in 0.. URL { + let url = folder.appending(path: "\(name).jpg") + if !FileManager.default.fileExists(atPath: url.path) { + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + try makeJPEG(width: Self.width, height: Self.height, at: url) + } + return url + } + + func photo(at t: Double, purpose: String? = nil) throws -> KeptPhoto { + KeptPhoto( + t: t, cameraToWorld: pose(t), cameraIntrinsics: Self.k, cameraImageSize: SIMD2(1920, 1440), width: Self.width, height: Self.height, + tracking: .normal, exposure: .init(duration: 0.004, offset: 0, iso: 64, fNumber: 1.6), jpeg: try jpeg(named: "frame-\(t)"), purpose: purpose) + } + + /// The raycast ran on the frame at `t`; the frame the app records is two frames newer, as + /// ARKit's current frame can be, and the tap is re-aimed at the hit in that frame. + func tap(at t: Double) throws -> (TapObservation, Packet04.TapHit) { + let data = try Data(contentsOf: try jpeg(named: "tap-\(t)")) + let recorded = t + 2.0 / 60 + let camera = pose(recorded).columns.3 + let stale = TapObservation( + t: recorded, cameraToWorld: pose(recorded), intrinsics: Self.k, width: Self.width, height: Self.height, tracking: .normal, + pixel: pixel(of: Self.meter, at: t), jpeg: { data }) + let tap = try #require(stale.pointing(at: Self.meter)) + let hit = Packet04.TapHit( + position: [Self.meter.x, Self.meter.y, Self.meter.z].map(Double.init), target: "estimatedPlane", alignment: "vertical", + distance: Double(simd_distance(Self.meter, SIMD3(camera.x, camera.y, camera.z)))) + return (tap, hit) + } + + static func environment( + endpoint: URL, http: any CaptureHTTP, captures: URL, sends: Bool = true, eventsWait: Int = 0, log: @escaping @Sendable (String) -> Void = { _ in } + ) -> CaptureSessionCoordinator.Environment { + var policy = CaptureUploader.Policy() + policy.eventsWait = eventsWait + let device = CaptureAPI.Device(model: "iPhone15,4", systemVersion: "26.0", appVersion: "0.1.0 (1)") + return .init( + endpoint: endpoint, sends: sends, http: http, capturesFolder: captures, + sessionInfo: { packetID, video in + Packet04SessionInfo( + packetID: packetID, sessionID: packetID, source: .synthetic, appVersion: device.appVersion, deviceModel: device.model, + systemVersion: device.systemVersion, lidarAvailable: false, sceneDepthEnabled: false, meshReconstructionSupported: nil, + sceneReconstruction: nil, planeDetection: ["horizontal", "vertical"], videoWidth: Int(video.x), videoHeight: Int(video.y), + framesPerSecond: nil, timeZone: "America/Chicago") + }, + device: device, tier: .arkit, policy: policy, log: log) + } + + /// The whole scan through `coordinator`: the tap, the close-up, keyframes (one of them the + /// close-up's own frame again), then the capture ends. + @MainActor + func run(_ coordinator: CaptureSessionCoordinator, consent: Bool? = true, beforeEnd: @MainActor () async -> Void = {}) async throws { + coordinator.begin(recording: recording) + // The integration build asks before the scan starts. + if let consent { coordinator.answerConsent(consent) } + let (tap, hit) = try self.tap(at: start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.kept(try photo(at: start + 2.5, purpose: "meter_close")) + for i in 0..<5 { coordinator.kept(try photo(at: start + 3 + Double(i) * 1.5)) } + // The app keeps the close-up's frame twice when it is also a walk frame. + coordinator.kept(try photo(at: start + 2.5)) + await coordinator.settle() + await beforeEnd() + coordinator.captureEnded(acceptedCloseUpAt: start + 2.5) + coordinator.captureEnded(acceptedCloseUpAt: start + 2.5) + await coordinator.settle() + } +} + +@Suite(.serialized) @MainActor struct CaptureSessionCoordinatorTests { + let root = FileManager.default.temporaryDirectory.appending(path: "coordinator-\(UUID().uuidString)") + var fixture: NativeCaptureFixture { NativeCaptureFixture(folder: root.appending(path: "store")) } + var captures: URL { root.appending(path: "Captures") } + + @Test func nativeScanExportsAValidPacketAndUploadsItDuringTheScan() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures)) + let committedBeforeEnd = Mutex(0) + try await fixture.run(coordinator) { + let uploader = coordinator.session!.uploader! + committedBeforeEnd.withLock { $0 = 0 } + let count = await uploader.snapshot.committedCount + committedBeforeEnd.withLock { $0 = count } + } + let session = try #require(coordinator.session) + let state = await session.uploader!.snapshot + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: try #require(state.packet)) + + #expect(Packet04Check.problems(packet, folder: session.folder).isEmpty) + // Tap frame, close-up frame and five walk frames; the repeated close-up frame is not a new keyframe. + #expect(packet.keyframes.count == 7) + #expect(packet.keyframes.map(\.reason) == ["tap", "still", "auto", "auto", "auto", "auto", "auto"]) + #expect(packet.stills?.map(\.purpose) == ["meter_close"]) + #expect(packet.scaleReference.meterCloseUp.still == "meter_close") + let tap = try #require(packet.taps?.first) + #expect(tap.label == "meter" && tap.keyframe == "k00001" && tap.epoch == "e1") + // The ray through the tap pixel reaches the meter the raycast hit. + let toMeter = simd_normalize(SIMD3(tap.hit!.position[0] - tap.rayOrigin[0], tap.hit!.position[1] - tap.rayOrigin[1], tap.hit!.position[2] - tap.rayOrigin[2])) + #expect(simd_distance(toMeter, SIMD3(tap.rayDirection[0], tap.rayDirection[1], tap.rayDirection[2])) < 1e-5) + // Raw world poses, straight from the frames. + #expect(packet.keyframes[2].pose[12...14].map { $0 } == [0.3, 1.5, 0]) + // Both sensors' own times travel beside the proposed pairing. + #expect(packet.ext?["imuRawPairingStatus"]?.hasPrefix("proposal") == true) + #expect(Set(packet.files.filter { $0.role == .stream }.map(\.path)) + == ["streams/arkit_poses.csv.gz", "streams/imu_raw.csv.gz", "streams/accelerometer_raw.csv.gz", "streams/gyroscope_raw.csv.gz"]) + + #expect(committedBeforeEnd.withLock { $0 } == packet.files.count - 4, "every image was received before the scan ended") + #expect(state.end == .finished(status: "manual_review")) + #expect(server.requests("POST captures").count == 1) + #expect(server.requests("POST captures/finalize").count == 1) + + if let dir = ProcessInfo.processInfo.environment["HOUSESCAN_EXPORT_EVIDENCE_DIR"] { + let target = URL(fileURLWithPath: dir).appending(path: "native-export-local") + try? FileManager.default.removeItem(at: target) + try FileManager.default.createDirectory(at: target.deletingLastPathComponent(), withIntermediateDirectories: true) + try FileManager.default.copyItem(at: session.folder, to: target) + } + } + + @Test func onlyTheIntegrationBuildRecordsAndOnlyAnAllowedBuildSends() { + let url = "https://capture.example.com/v1" + #expect(CaptureIntegrationMode.resolve(integrationBuild: "NO", endpoint: url, sendDeviceData: "YES") == .off("not the integration build")) + #expect(CaptureIntegrationMode.resolve(integrationBuild: nil, endpoint: url, sendDeviceData: "YES") == .off("not the integration build")) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "$(HOUSESCAN_INTEGRATION_BUILD)", endpoint: url, sendDeviceData: "YES") == .off("not the integration build")) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: "", sendDeviceData: "YES") == .off("no capture endpoint in this build")) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: url, sendDeviceData: "NO") == .recordOnly(URL(string: url)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: url, sendDeviceData: nil) == .recordOnly(URL(string: url)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: url, sendDeviceData: "YES") == .send(URL(string: url)!)) + // A replay goes only to a receiver on this machine, only when asked, whatever the device switch says. + let local = "http://127.0.0.1:8765/v1" + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: local, sendDeviceData: "NO", source: .replay(sendToLocalReceiver: true)) == .send(URL(string: local)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: local, sendDeviceData: "YES", source: .replay(sendToLocalReceiver: false)) == .recordOnly(URL(string: local)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: url, sendDeviceData: "YES", source: .replay(sendToLocalReceiver: true)) == .recordOnly(URL(string: url)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "NO", endpoint: local, sendDeviceData: "YES", source: .replay(sendToLocalReceiver: true)) == .off("not the integration build")) + } + + /// A build that may not send device data records the capture on the phone and never sends it, + /// even with a yes. + @Test func aRecordOnlyBuildKeepsTheCaptureOnThePhone() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures, sends: false)) + try await fixture.run(coordinator) + #expect(coordinator.needsConsent == false) + coordinator.answerConsent(true) + await coordinator.settle() + let session = try #require(coordinator.session) + #expect(session.uploader == nil) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: Data(contentsOf: session.folder.appending(path: "packet.json"))) + #expect(Packet04Check.problems(packet, folder: session.folder).isEmpty) + #expect(server.state.withLock { $0.log.isEmpty }) + } + + /// No answer or a no sends nothing; the next scan asks again. A yes before the scan sends + /// during it, and a world reset inside the scan keeps that yes for the new world's packet. + @Test func consentIsPerScanAndOnlyAYesSends() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let environment = NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures) + + let unanswered = CaptureSessionCoordinator(environment: environment) + try await fixture.run(unanswered, consent: nil) + #expect(unanswered.needsConsent) + #expect(unanswered.session?.uploader == nil) + + let declining = CaptureSessionCoordinator(environment: environment) + try await fixture.run(declining, consent: false) + #expect(declining.session?.uploader == nil) + #expect(server.state.withLock { $0.log.isEmpty }) + declining.newWorld("start over", recording: fixture.recording, newScan: true) + #expect(declining.consent == nil && declining.needsConsent) + + let agreeing = CaptureSessionCoordinator(environment: environment) + agreeing.begin(recording: fixture.recording) + agreeing.answerConsent(true) + let firstWorld = try #require(agreeing.session) + agreeing.newWorld("world reset", recording: fixture.recording, newScan: false) + #expect(agreeing.consent == true) + let secondWorld = try #require(agreeing.session) + #expect(secondWorld.uploader != nil && secondWorld.packetID != firstWorld.packetID) + agreeing.newWorld("start over", recording: fixture.recording, newScan: true) + #expect(agreeing.consent == nil) + #expect(agreeing.session?.uploader == nil) + } + + /// After a relaunch a frozen capture resumes only with the yes it recorded and only toward the + /// endpoint it was created on. + @Test func resumeNeedsTheCapturesYesAndTheSameEndpoint() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let original = try LoopbackCaptureAPI() + let other = try LoopbackCaptureAPI() + // A capture frozen on the phone for `original` whose upload never started. + let folder = captures.appending(path: "frozen") + let capture = try SyntheticCapture04(folder: folder) + let images = try await capture.sealImages() + let (streams, packet) = try await capture.finish() + var saved = CaptureUploadState( + attemptID: "old-process", packetID: capture.producer.info.packetID, + createBody: try JSONEncoder().encode(CaptureAPI.CreateRequest(packetId: capture.producer.info.packetID, tier: .arkit, device: .init(model: "m", systemVersion: "s", appVersion: "a")))) + saved.destination = original.base.absoluteString + // First without a recorded yes: never resumed. + for (i, file) in (images + streams).enumerated() { saved.files[file.path] = .init(sealed: file, sequence: i) } + saved.packet = packet + try saved.save(to: CaptureUploader.stateURL(in: folder)) + let http = URLSessionCaptureHTTP.ephemeral(timeout: 10) + + #expect(CaptureSessionCoordinator(environment: NativeCaptureFixture.environment(endpoint: original.base, http: http, captures: captures)) + .resumeSealedCaptures().isEmpty) + saved.consent = .init(grantedAt: Date(), destination: original.base.absoluteString) + try saved.save(to: CaptureUploader.stateURL(in: folder)) + #expect(CaptureSessionCoordinator(environment: NativeCaptureFixture.environment(endpoint: other.base, http: http, captures: captures)) + .resumeSealedCaptures().isEmpty) + #expect(CaptureSessionCoordinator(environment: NativeCaptureFixture.environment(endpoint: original.base, http: http, captures: captures, sends: false)) + .resumeSealedCaptures().isEmpty) + try await Task.sleep(for: .milliseconds(200)) + #expect(original.state.withLock { $0.log.isEmpty } && other.state.withLock { $0.log.isEmpty }) + + let resumed = CaptureSessionCoordinator(environment: NativeCaptureFixture.environment(endpoint: original.base, http: http, captures: captures)) + .resumeSealedCaptures() + #expect(resumed.count == 1) + for uploader in resumed { await uploader.kick(); await uploader.settled() } + #expect(await resumed.first?.snapshot.end == .finished(status: "manual_review")) + #expect(other.state.withLock { $0.log.isEmpty }) + } + + /// A refused close-up and its accepted retake, written over the same file as the app does: + /// both stay sealed with their own pixels, and the scale reference names the accepted one, + /// with its pixels and pose. A skipped close-up names none. + @Test func onlyTheAcceptedCloseUpIsTheScaleReference() async throws { + defer { try? FileManager.default.removeItem(at: root) } + for accepted in [true, false] { + let coordinator = CaptureSessionCoordinator(environment: NativeCaptureFixture.environment( + endpoint: URL(string: "https://capture.invalid/v1")!, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures, sends: false)) + coordinator.begin(recording: fixture.recording) + let source = fixture.folder.appending(path: "meter_close.jpg") + try FileManager.default.createDirectory(at: fixture.folder, withIntermediateDirectories: true) + try? FileManager.default.removeItem(at: source) + try makeJPEG(width: NativeCaptureFixture.width, height: NativeCaptureFixture.height, at: source, shift: 7) + let refusedBytes = try Data(contentsOf: source) + // Its depth too lives in files the retake overwrites: 4 x 3 meters and confidence. + let depthFile = fixture.folder.appending(path: "meter_close.f32"), confidenceFile = fixture.folder.appending(path: "meter_close.conf.u8") + func writeDepth(_ meters: Float, _ confidence: UInt8) throws { + try PacketFiles.depthData(meters: [Float](repeating: meters, count: 12)).write(to: depthFile) + try Data(repeating: confidence, count: 12).write(to: confidenceFile) + } + try writeDepth(1.5, 2) + func closeUp(at t: Double) -> KeptPhoto { + KeptPhoto( + t: t, cameraToWorld: fixture.pose(t), cameraIntrinsics: NativeCaptureFixture.k, cameraImageSize: SIMD2(1920, 1440), + width: NativeCaptureFixture.width, height: NativeCaptureFixture.height, tracking: .normal, exposure: nil, jpeg: source, purpose: "meter_close", + depth: { + guard let map = try? Data(contentsOf: depthFile), let confidence = try? Data(contentsOf: confidenceFile) else { return nil } + return Packet04Depth(meters: PacketFiles.floats(littleEndian: map), confidence: [UInt8](confidence), width: 4, height: 3) + }) + } + coordinator.kept(closeUp(at: fixture.start + 2)) + // The retake overwrites the same files before the first shot's sealing has run. + try FileManager.default.removeItem(at: source) + try makeJPEG(width: NativeCaptureFixture.width, height: NativeCaptureFixture.height, at: source, shift: 101) + try writeDepth(0.9, 1) + let retakeBytes = try Data(contentsOf: source) + coordinator.kept(closeUp(at: fixture.start + 4)) + coordinator.captureEnded(acceptedCloseUpAt: accepted ? fixture.start + 4 : nil) + await coordinator.settle() + + let session = try #require(coordinator.session) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: Data(contentsOf: session.folder.appending(path: "packet.json"))) + #expect(packet.stills?.map(\.id) == ["meter_close", "meter_close-2"]) + #expect(try Data(contentsOf: session.folder.appending(path: "stills/meter_close.jpg")) == refusedBytes) + #expect(try Data(contentsOf: session.folder.appending(path: "stills/meter_close-2.jpg")) == retakeBytes) + // Each shot keeps its own depth and confidence. + #expect(try Data(contentsOf: session.folder.appending(path: "keyframes/k00001.depth.f32")) == PacketFiles.depthData(meters: [Float](repeating: 1.5, count: 12))) + #expect(try Data(contentsOf: session.folder.appending(path: "keyframes/k00001.confidence.u8")) == Data(repeating: 2, count: 12)) + #expect(try Data(contentsOf: session.folder.appending(path: "keyframes/k00002.depth.f32")) == PacketFiles.depthData(meters: [Float](repeating: 0.9, count: 12))) + #expect(Packet04Check.problems(packet, folder: session.folder).isEmpty) + if accepted { + #expect(packet.scaleReference.meterCloseUp.still == "meter_close-2") + let frame = try #require(packet.keyframes.first { $0.id == packet.stills?.last?.keyframe }) + #expect(frame.timestamp == fixture.start + 4 && frame.pose[12] == 0.4) + } else { + #expect(packet.scaleReference.meterCloseUp == .notCaptured) + } + } + } + + /// A world reset while the first world's photos are still going up: that packet is abandoned, + /// the new world gets its own packet id, and nothing kept after the reset joins the old one. + @Test func aWorldResetStartsANewPacketAndDropsTheOld() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/files"] } + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures)) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + let first = try #require(coordinator.session) + for _ in 0..<500 where server.requests("POST captures/files").isEmpty { try await Task.sleep(for: .milliseconds(10)) } + + coordinator.newWorld("world reset", recording: fixture.recording, newScan: false) + coordinator.kept(try fixture.photo(at: fixture.start + 4)) + server.release("POST captures/files") + server.state.withLock { $0.held = [] } + await coordinator.settle() + await first.uploader!.settled() + + let second = try #require(coordinator.session) + #expect(second.packetID != first.packetID) + let old = await first.uploader!.snapshot + #expect(old.end == .abandoned("world reset")) + #expect(old.files.values.allSatisfy { $0.phase == .queued }) + #expect(await second.uploader!.snapshot.files.keys.sorted() == ["keyframes/k00001.jpg"]) + #expect(await second.uploader!.snapshot.committedCount == 1) + let creates = server.requests("POST captures").compactMap { try? JSONDecoder().decode(CaptureAPI.CreateRequest.self, from: $0.body).packetId } + #expect(Set(creates) == [first.packetID, second.packetID]) + } + + /// A reset that ends the session before its kept photos are sealed still deletes their staged + /// copies, so no copy of a home photo outlives the packet it was kept for. + @Test func aResetDeletesPhotosStagedForTheEndedSession() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures, sends: false)) + let staging = captures.appending(path: "staging") + func staged() -> [String] { (try? FileManager.default.contentsOfDirectory(atPath: staging.path)) ?? [] } + + coordinator.begin(recording: fixture.recording) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + coordinator.kept(try fixture.photo(at: fixture.start + 3)) + #expect(staged().count == 2) + // No suspension between keeping and the reset, so the queued work finds the session ended. + coordinator.newWorld("start over", recording: fixture.recording, newScan: true) + coordinator.kept(try fixture.photo(at: fixture.start + 4)) + await coordinator.settle() + for _ in 0..<500 where !staged().isEmpty { try await Task.sleep(for: .milliseconds(10)) } + #expect(staged().isEmpty) + } + + /// A no after a yes stops sending: the upload is abandoned, nothing kept later goes up, the + /// packet is never finalized, and a second yes in the same scan does not start it again. + @Test func aNoAfterAYesStopsSending() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures)) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + await coordinator.settle() + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + await uploader.settled() + let putsBefore = server.requests("PUT upload").count + #expect(putsBefore == 1) + + coordinator.answerConsent(false) + coordinator.kept(try fixture.photo(at: fixture.start + 4)) + coordinator.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.captureEnded(acceptedCloseUpAt: nil) + await coordinator.settle() + await uploader.settled() + + #expect(await uploader.snapshot.end == .abandoned("consent withdrawn")) + #expect(coordinator.session === session && session.uploader == nil) + #expect(server.requests("PUT upload").count == putsBefore) + #expect(server.requests("POST captures").count == 1) + #expect(server.requests("POST captures/finalize").isEmpty) + } + + /// A withdrawal is saved before `answerConsent(false)` returns: a relaunch straight after it + /// does not resume the sealed capture, even though the uploader has not saved its end yet. + @Test func aWithdrawnCaptureIsNotResumedAfterARelaunch() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/finalize"] } + let environment = NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + for i in 0..<3 { coordinator.kept(try fixture.photo(at: fixture.start + 3 + Double(i) * 1.5)) } + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + let stateURL = CaptureUploader.stateURL(in: session.folder) + for _ in 0..<500 where (try? CaptureUploadState.load(from: stateURL))?.packet == nil { try await Task.sleep(for: .milliseconds(10)) } + #expect(try CaptureUploadState.load(from: stateURL).packet != nil) + + coordinator.answerConsent(false) + // The same main-actor turn: the abandon task has not run yet. + #expect(CaptureSessionCoordinator(environment: environment).resumeSealedCaptures().isEmpty) + #expect(try CaptureUploader.resume(folder: session.folder, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) == nil) + + server.release("POST captures/finalize") + server.state.withLock { $0.held = [] } + // The withdrawal cancelled the loop; the abandon task ends the upload in its own time. + for _ in 0..<500 where await uploader.snapshot.end == nil { try await Task.sleep(for: .milliseconds(10)) } + #expect(await uploader.snapshot.end == .abandoned("consent withdrawn")) + } + + /// A relaunch scan in the live process must not create a second sender for the sealed + /// folder. On the old code B survives A's withdrawal and sends the remaining files. + @Test func withdrawalCoversTheFolderDuringResumeDiscovery() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/finalize"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + for i in 0..<3 { coordinator.kept(try fixture.photo(at: fixture.start + 3 + Double(i) * 1.5)) } + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + let uploader = try #require(coordinator.session?.uploader) + try await waitUntilParked(server, "POST captures/finalize") + + let duplicates = CaptureSessionCoordinator(environment: environment).resumeSealedCaptures() + for duplicate in duplicates { await duplicate.kick() } + if !duplicates.isEmpty { + for _ in 0..<500 where server.requests("POST captures/finalize").count < 2 { + try await Task.sleep(for: .milliseconds(10)) + } + try #require(server.requests("POST captures/finalize").count == 2) + } + #expect(duplicates.isEmpty) + coordinator.answerConsent(false) + let puts = server.requests("PUT upload").count + let commits = server.requests("POST captures/files:commit").count + server.state.withLock { $0.held = [] } + server.release("POST captures/finalize") + try #require(await CaptureUploaderTests.settles(uploader)) + for duplicate in duplicates { try #require(await CaptureUploaderTests.settles(duplicate)) } + #expect(server.requests("PUT upload").count == puts) + #expect(server.requests("POST captures/files:commit").count == commits) + #expect(server.requests("GET captures/result").isEmpty) + #expect(CaptureSessionCoordinator(environment: environment).resumeSealedCaptures().isEmpty) + } + + @Test func droppingAnIdleCoordinatorReleasesFolderOwnership() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let environment = NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures) + var coordinator: CaptureSessionCoordinator? = CaptureSessionCoordinator(environment: environment) + coordinator?.begin(recording: fixture.recording) + coordinator?.answerConsent(true) + coordinator?.kept(try fixture.photo(at: fixture.start + 2)) + await coordinator?.settle() + let folder = try #require(coordinator?.session?.folder) + weak let oldSession = coordinator?.session + weak let oldUploader = coordinator?.session?.uploader + coordinator = nil + for _ in 0..<500 where oldSession != nil || oldUploader != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(oldSession == nil && oldUploader == nil) + let resumed = try #require(try CaptureUploader.resume(folder: folder, base: server.base, http: environment.http)) + await resumed.kick() + try #require(await CaptureUploaderTests.settles(resumed)) + #expect(server.requests("POST captures").count == 1) + } + + @Test func anUnsavedWithdrawalIsReturnedToTheCaller() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/files"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + try await waitUntilParked(server, "POST captures/files") + let stateURL = CaptureUploader.stateURL(in: session.folder) + let before = try Data(contentsOf: stateURL) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: session.folder.path) + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: session.folder.path) } + + let result = coordinator.answerConsent(false) + guard case .failure(.notRecorded(let detail)) = result else { + Issue.record("The caller must receive the persistence failure") + server.state.withLock { $0.held = [] } + server.release("POST captures/files") + return + } + #expect(detail.contains("marker:") && detail.contains("state removal:")) + #expect(try Data(contentsOf: stateURL) == before) + // Repeated answers must not turn the failed withdrawal into a reported success. + if case .success = coordinator.answerConsent(false) { Issue.record("Repeated no lost the failure") } + if case .success = coordinator.answerConsent(true) { Issue.record("Later yes lost the failure") } + #expect(session.uploader == nil) + #expect(try CaptureUploader.resume(folder: session.folder, base: server.base, http: environment.http) == nil) + #expect(CaptureSessionCoordinator(environment: environment).resumeSealedCaptures().isEmpty) + server.state.withLock { $0.held = [] } + server.release("POST captures/files") + try #require(await CaptureUploaderTests.settles(uploader)) + #expect(server.requests("PUT upload").isEmpty) + #expect(server.requests("POST captures/files:commit").isEmpty) + // All writes still fail, so the app cannot promise the persisted yes was removed. + #expect(try Data(contentsOf: stateURL) == before) + } + + /// Waits, suspending, until the loopback has parked an answer for `route`. + func waitUntilParked(_ server: LoopbackCaptureAPI, _ route: String) async throws { + for _ in 0..<1000 where server.state.withLock({ $0.parked[route]?.isEmpty ?? true }) { try await Task.sleep(for: .milliseconds(10)) } + #expect(server.state.withLock { !($0.parked[route]?.isEmpty ?? true) }) + } + + /// Blocks the main actor until `stateURL` holds something other than `before`, so nothing + /// queued on the main actor (the abandon task a withdrawal schedules) can run meanwhile. + func blockUntilSaved(_ stateURL: URL, differentFrom before: Data?) { + let deadline = Date().addingTimeInterval(5) + while Date() < deadline, (try? Data(contentsOf: stateURL)) == before { usleep(10_000) } + } + + /// A reply that lands after the homeowner withdrew, but before the uploader is abandoned, + /// can't save a capture a relaunch would resume, whether or not the withdrawal marker could + /// be written. The transport lets the reply arrive even though the request's task was + /// cancelled, as a background session does. The main actor is held from the withdrawal to the + /// checks, so the abandon task the withdrawal schedules can't run first. + @Test(arguments: [false, true]) func aReplyAfterWithdrawalCannotSaveAResumableCapture(markerBlocked: Bool) async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/files"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + let stateURL = CaptureUploader.stateURL(in: session.folder) + try await waitUntilParked(server, "POST captures/files") + if markerBlocked { + // A directory where the marker goes makes writing it fail. + try FileManager.default.createDirectory( + at: CaptureUploadState.withdrawnURL(in: session.folder).appending(path: "blocked"), withIntermediateDirectories: true) + } + + coordinator.answerConsent(false) + // No suspension from here to the checks. + let before = try? Data(contentsOf: stateURL) + #expect((before == nil) == markerBlocked) + server.release("POST captures/files") + // The uploader saves the register reply on its own executor. + blockUntilSaved(stateURL, differentFrom: before) + let saved = try CaptureUploadState.load(from: stateURL) + #expect(saved.end == .abandoned("consent withdrawn")) + if markerBlocked { + // The blocking directory sits at the marker's path and would stop a resume by itself; + // without it only the saved end can. + try FileManager.default.removeItem(at: CaptureUploadState.withdrawnURL(in: session.folder)) + } else { + #expect(try CaptureUploader.resume(folder: session.folder, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) == nil) + } + #expect(CaptureSessionCoordinator(environment: environment).resumeSealedCaptures().isEmpty) + #expect(server.requests("PUT upload").isEmpty) + + server.state.withLock { $0.held = [] } + await uploader.settled() + #expect(server.requests("PUT upload").isEmpty) + } + + /// A result that arrives after the withdrawal can't save the upload as finished: the + /// withdrawal outranks the end a late reply sets. + @Test func aResultAfterWithdrawalSavesTheUploadAsAbandoned() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["GET captures/result"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + for i in 0..<3 { coordinator.kept(try fixture.photo(at: fixture.start + 3 + Double(i) * 1.5)) } + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + let stateURL = CaptureUploader.stateURL(in: session.folder) + try await waitUntilParked(server, "GET captures/result") + + coordinator.answerConsent(false) + // No suspension from here to the check. + let before = try? Data(contentsOf: stateURL) + server.release("GET captures/result") + blockUntilSaved(stateURL, differentFrom: before) + #expect(try CaptureUploadState.load(from: stateURL).end == .abandoned("consent withdrawn")) + + server.state.withLock { $0.held = [] } + await uploader.settled() + #expect(await uploader.snapshot.end == .abandoned("consent withdrawn")) + } + + /// Withdrawing cancels the request the uploader is waiting on before the abandon task runs: + /// a held register request is given up at once instead of waiting for its answer, and no + /// photo is uploaded. + @Test func withdrawalCancelsTheRequestInFlight() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let logs = Mutex<[String]>([]) + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/files"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures, + log: { line in logs.withLock { $0.append(line) } }) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + let uploader = try #require(coordinator.session?.uploader) + try await waitUntilParked(server, "POST captures/files") + + coordinator.answerConsent(false) + // The main actor stays busy, so only the withdrawal itself can end the wait. + let deadline = Date().addingTimeInterval(5) + func gaveUp() -> Bool { logs.withLock { $0.contains { $0.contains("capture-upload retry step=register") } } } + while Date() < deadline, !gaveUp() { usleep(10_000) } + #expect(gaveUp()) + + server.release("POST captures/files") + server.state.withLock { $0.held = [] } + await uploader.settled() + #expect(server.requests("PUT upload").isEmpty) + } + + /// Photos kept after the scan was sent don't change the frozen packet or start more uploads. + @Test func photosAfterTheScanWasSentAreNotAdded() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures)) + try await fixture.run(coordinator) + let before = await coordinator.session!.uploader!.snapshot + coordinator.kept(try fixture.photo(at: fixture.end - 0.5)) + await coordinator.settle() + let after = await coordinator.session!.uploader!.snapshot + #expect(after.packet == before.packet) + #expect(after.files.count == before.files.count) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift new file mode 100644 index 000000000..664d5564b --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift @@ -0,0 +1,602 @@ +import Foundation +import HouseScanKit +import Synchronization +import Testing + +/// The uploader against `LoopbackCaptureAPI` over real HTTP, through the same `URLSession` +/// transport the app uses. Retries wait 10 ms instead of seconds. +@Suite(.serialized) struct CaptureUploaderTests { + struct Rig { + let server: LoopbackCaptureAPI + let root: URL + let capture: SyntheticCapture04 + private var ownedUploader: CaptureUploader? + var uploader: CaptureUploader { + guard let ownedUploader else { preconditionFailure("The test released this uploader to simulate process exit") } + return ownedUploader + } + + mutating func releaseUploader() { ownedUploader = nil } + let http: any CaptureHTTP + + init( + packetID: String = UUID().uuidString, appVersion: String = "test (1)", server: LoopbackCaptureAPI? = nil, + http: any CaptureHTTP = URLSessionCaptureHTTP.ephemeral(timeout: 10), policy: CaptureUploader.Policy = Rig.fast, + sleep: @escaping @Sendable (Double) async throws -> Void = { _ in try await Task.sleep(for: .milliseconds(10)) }, + now: @escaping @Sendable () -> Date = { Date() } + ) throws { + self.http = http + self.server = try server ?? LoopbackCaptureAPI() + root = FileManager.default.temporaryDirectory.appending(path: "uploader-\(UUID().uuidString)") + capture = try SyntheticCapture04(folder: root.appending(path: "packet"), packetID: packetID) + ownedUploader = try CaptureUploader.start( + folder: capture.folder, base: self.server.base, http: http, + create: .init(packetId: packetID, tier: .arkit, device: .init(model: "iPhone15,4", systemVersion: "26.0", appVersion: appVersion)), + consentedAt: Date(), policy: policy, now: now, sleep: sleep) + } + + static var fast: CaptureUploader.Policy { + var policy = CaptureUploader.Policy() + policy.eventsWait = 0 + return policy + } + + func finishAndSeal() async throws { + let (streams, packet) = try await capture.finish() + await uploader.seal(packet: packet, files: streams) + } + + func cleanUp() { try? FileManager.default.removeItem(at: root) } + } + + /// Photos go up during the capture; the frozen packet finalizes early, the streams follow, and + /// the capture ends with a result for its own run. + @Test func joinedPathCommitsPhotosBeforeTheCaptureEnds() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + let images = try await rig.capture.sealImages() + await rig.uploader.add(images) + await rig.uploader.settled() + + let during = await rig.uploader.snapshot + #expect(during.committedCount == images.count) + #expect(during.packet == nil) + #expect(rig.server.requests("POST captures/finalize").isEmpty) + + try await rig.finishAndSeal() + await rig.uploader.settled() + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "manual_review")) + #expect(done.finalized?.status == "awaiting_files") + #expect(done.finalized?.missing.sorted() == [ + "streams/accelerometer_raw.csv.gz", "streams/arkit_poses.csv.gz", "streams/gyroscope_raw.csv.gz", "streams/imu_raw.csv.gz", + ]) + #expect(done.committedCount == images.count + 4) + #expect(done.result != nil) + let marks = done.marks + #expect(try #require(marks["firstCommit"]) < #require(marks["sealed"])) + #expect(try #require(marks["sealed"]) <= #require(marks["lastCommit"])) + + let puts = rig.server.requests("PUT upload") + #expect(puts.count == images.count + 4) + #expect(puts.allSatisfy { $0.headers["authorization"] == nil && $0.headers["content-md5"] != nil }) + #expect(rig.server.requests("POST captures").count == 1) + } + + /// A create and a finalize whose answers are lost are sent again as the same bytes, and the + /// server keeps one capture. + @Test func lostAnswersAreReplayedWithTheFrozenBodies() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.dropNext = ["POST captures", "POST captures/finalize"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 2)) + await rig.uploader.settled() + try await rig.finishAndSeal() + await rig.uploader.settled() + + let creates = rig.server.requests("POST captures") + let finals = rig.server.requests("POST captures/finalize") + #expect(creates.count == 2) + #expect(Set(creates.map(\.body)).count == 1) + #expect(finals.count == 2) + #expect(Set(finals.map(\.body)).count == 1) + #expect(rig.server.state.withLock { $0.captures.count } == 1) + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + } + + /// A relaunch resumes the saved capture: same packet and capture ids, no second create, and the + /// old process's attempt id is replaced. + @Test func relaunchResumesTheSameCapture() async throws { + var rig = try Rig() + defer { rig.cleanUp() } + await rig.uploader.add(try await rig.capture.sealImages(count: 2)) + await rig.uploader.settled() + let before = await rig.uploader.snapshot + + // The old process releases every uploader before the new one resumes its saved state. + weak let released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + let resumed = try #require(try CaptureUploader.resume( + folder: rig.capture.folder, base: rig.server.base, http: rig.http, policy: Rig.fast, sleep: { _ in })) + let after = await resumed.snapshot + #expect(after.packetID == before.packetID) + #expect(after.captureID == before.captureID) + #expect(after.createBody == before.createBody) + #expect(after.attemptID != before.attemptID) + + let (streams, packet) = try await rig.capture.finish() + await resumed.seal(packet: packet, files: streams) + await resumed.settled() + #expect(await resumed.snapshot.end == .finished(status: "manual_review")) + #expect(rig.server.requests("POST captures").count == 1) + } + + /// The same packet id with a different create body is the server's idempotency conflict; the + /// uploader stops instead of retrying it. + @Test func aChangedCreateBodyIsRefusedNotRetried() async throws { + let packetID = UUID().uuidString + let first = try Rig(packetID: packetID) + defer { first.cleanUp() } + await first.uploader.kick() + await first.uploader.settled() + #expect(await first.uploader.snapshot.captureID != nil) + + let second = try Rig(packetID: packetID, appVersion: "test (2)", server: first.server) + defer { second.cleanUp() } + await second.uploader.kick() + await second.uploader.settled() + #expect(await second.uploader.snapshot.end == .failed(step: "create", codes: ["idempotency_conflict"], status: 409)) + #expect(first.server.requests("POST captures").count == 2) + } + + /// A storage URL answered 403 (expired) is registered again and the file sent to the new URL. + @Test func anExpiredURLIsFetchedAgain() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.expireNextPuts = 1 } + let images = try await rig.capture.sealImages(count: 1) + await rig.uploader.add(images) + await rig.uploader.settled() + let snapshot = await rig.uploader.snapshot + #expect(snapshot.committedCount == images.count) + #expect(rig.server.requests("PUT upload").count == images.count + 1) + let registered = rig.server.requests("POST captures/files").flatMap { request -> [String] in + let body = try? JSONDecoder().decode(CaptureAPI.RegisterRequest.self, from: request.body) + return body?.files.map(\.path) ?? [] + } + #expect(registered.count == images.count + 1) + } + + /// Only paths the commit names as committed advance; notFound and mismatch go up again. + @Test func mixedCommitAnswersAdvanceOnlyCommittedFiles() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + let images = try await rig.capture.sealImages(count: 2) + // Priority order: the still, then k00001, then k00002. + let paths = ["stills/meter_close.jpg", "keyframes/k00001.jpg", "keyframes/k00002.jpg"] + #expect(Set(images.map(\.path)) == Set(paths)) + rig.server.state.withLock { $0.commitOverride = (committed: [paths[0]], notFound: [paths[1]], mismatch: [paths[2]]) } + let seen = Mutex<[Int]>([]) + await rig.uploader.observe { status in seen.withLock { $0.append(status.committed) } } + await rig.uploader.add(images) + await rig.uploader.settled() + + #expect(await rig.uploader.snapshot.committedCount == 3) + // After the first commit exactly one file counted as received. + #expect(seen.withLock { $0.first { $0 > 0 } } == 1) + let puts = rig.server.requests("PUT upload") + #expect(puts.count == 5) + } + + /// A world reset while a register answer is in flight: the late answer changes nothing, and the + /// next capture is a new packet that never reuses the old one. + @Test func resetDuringAPendingAnswerDropsTheLateReply() async throws { + // A background session's transfer outlives the task that asked for it, so the reply + // still arrives after the reset; only the attempt check keeps it out. + let rig = try Rig(http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10))) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.held = ["POST captures/files"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 2)) + for _ in 0..<500 where rig.server.requests("POST captures/files").isEmpty { try await Task.sleep(for: .milliseconds(10)) } + #expect(rig.server.requests("POST captures/files").count == 1) + + await rig.uploader.abandon("world reset") + rig.server.release("POST captures/files") + try await Task.sleep(for: .milliseconds(300)) + await rig.uploader.settled() + + let after = await rig.uploader.snapshot + #expect(after.end == .abandoned("world reset")) + #expect(after.files.values.allSatisfy { $0.phase == .queued }) + #expect(rig.server.requests("PUT upload").isEmpty) + // Nothing more is sent for the abandoned capture. + let late = try await rig.capture.producer.sealKeyframe( + jpeg: try SyntheticCapture04.jpeg(in: rig.root), observation: rig.capture.observation(SyntheticCapture04.start + 1.9), reason: "motion") + await rig.uploader.add(late) + await rig.uploader.settled() + #expect(rig.server.requests("POST captures/files").count == 1) + + let next = try Rig(server: rig.server) + defer { next.cleanUp() } + await next.uploader.add(try await next.capture.sealImages(count: 1)) + await next.uploader.settled() + #expect(await next.uploader.snapshot.captureID != after.captureID) + #expect(await next.uploader.snapshot.committedCount == 2) + } + + /// A finalize the server accepted and then lost (`failed`, next `retry_finalize`) is sent + /// again as the same bytes, and the run then completes. + @Test func aLostFinalizeIsSentAgainOnRetryFinalize() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.loseNextFinalize = true } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + let finals = rig.server.requests("POST captures/finalize") + #expect(finals.count == 2) + #expect(Set(finals.map(\.body)).count == 1) + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + #expect(await rig.uploader.snapshot.finalizeRetries == 1) + } + + /// The run is done but its answer is not readable yet (404 `result_not_ready`): the uploader + /// reads again after a wait instead of failing the capture. + @Test func aResultNotReadableYetIsReadAgain() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.notReadyResults = 2 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "manual_review")) + #expect(try Self.outcomeKind(done.result) == .manualReview) + #expect(rig.server.requests("GET captures/result").count == 3) + } + + /// The API answers a result read with the capture's status and a null outcome until the run + /// has written its answer. A terminal status with that body is not the answer. + @Test func aResultWithoutItsOutcomeIsReadAgain() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.outcomelessResults = 2 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "manual_review")) + #expect(try Self.outcomeKind(done.result) == .manualReview) + #expect(rig.server.requests("GET captures/result").count == 3) + } + + /// An answer that never becomes readable ends the upload visibly after the policy's reads, + /// rather than waiting forever. + @Test func aResultThatNeverArrivesEndsTheUpload() async throws { + var policy = Rig.fast + policy.maxNotReadyResults = 3 + let rig = try Rig(policy: policy) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.outcomelessResults = 100 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .failed(step: "result", codes: ["result_not_ready"], status: 200)) + #expect(done.result == nil) + #expect(rig.server.requests("GET captures/result").count == 4) + } + + /// A 200 whose body is not a result is refused by name instead of being kept as the answer. + @Test func anUnreadableResultStopsTheUpload() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.unreadableResults = 1 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .failed(step: "result", codes: ["result_unreadable"], status: 200)) + #expect(done.result == nil) + #expect(rig.server.requests("GET captures/result").count == 1) + } + + /// An outcome missing fields the app needs is refused by name instead of finishing the upload + /// with an answer nothing can show. + @Test func aMalformedOutcomeStopsTheUpload() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.malformedOutcomeResults = 1 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .failed(step: "result", codes: ["result_unreadable"], status: 200)) + #expect(done.result == nil) + #expect(rig.server.requests("GET captures/result").count == 1) + } + + /// Whether `uploader` goes idle within `seconds`. A loop that never goes idle fails the test + /// here instead of hanging the run; the process ending stops it. + static func settles(_ uploader: CaptureUploader, within seconds: Int = 20) async -> Bool { + let done = Mutex(false) + Task.detached { await uploader.settled(); done.withLock { $0 = true } } + let deadline = ContinuousClock.now + .seconds(seconds) + while ContinuousClock.now < deadline { + if done.withLock({ $0 }) { return true } + try? await Task.sleep(for: .milliseconds(20)) + } + return false + } + + /// A storage URL that is not https (or http to this machine) is refused when it is + /// registered, before any bytes go to it; one that doesn't parse is refused the same way + /// instead of leaving the file registered and the loop spinning. + @Test(arguments: [("http://storage.example.com/upload/x", "upload_url_insecure"), ("", "upload_url_invalid")]) + func aStorageURLThatIsNotSafeIsRefusedBeforeAnyPut(url: String, code: String) async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.uploadURLOverride = url } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "register", codes: [code], status: 200)) + #expect(rig.server.requests("POST captures/files").count == 1) + #expect(rig.server.requests("PUT upload").isEmpty) + } + + /// Storage that answers a PUT with a redirect doesn't get the photo sent on: the real + /// `URLSession` transport refuses every redirect, so the photo reaches only the URL that passed + /// the check, and the 307 comes back as a storage refusal. The redirect goes to a second local + /// server, which must see nothing. + @Test func aStorageRedirectIsNotFollowed() async throws { + let elsewhere = try LoopbackCaptureAPI() + var policy = Rig.fast + policy.maxRefusedPuts = 2 + let rig = try Rig(policy: policy) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.redirects["PUT upload"] = "http://127.0.0.1:\(elsewhere.port)/upload/elsewhere" } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "put", codes: ["storage_refused"], status: 307)) + #expect(!rig.server.requests("PUT upload").isEmpty) + #expect(elsewhere.state.withLock { $0.log.isEmpty }) + } + + /// The API's own requests aren't redirected either: a 307 on create comes back as the reply, + /// which the uploader refuses by its status, and the second server sees nothing. + @Test func anAPIRedirectIsNotFollowed() async throws { + let elsewhere = try LoopbackCaptureAPI() + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.redirects["POST captures"] = "\(elsewhere.base.absoluteString)/captures" } + await rig.uploader.kick() + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "create", codes: [], status: 307)) + #expect(rig.server.requests("POST captures").count == 1) + #expect(elsewhere.state.withLock { $0.log.isEmpty }) + } + + /// Commits that never acknowledge a file back off between tries and stop after the policy's + /// limit, instead of sending the same photo again and again. + @Test func aCommitThatNeverAcknowledgesBacksOffAndStops() async throws { + var policy = Rig.fast + policy.maxUnacknowledged = 3 + let sleeps = Mutex<[Double]>([]) + let rig = try Rig(policy: policy, sleep: { seconds in sleeps.withLock { $0.append(seconds) } }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.commitNeverAcknowledges = true } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "commit", codes: ["not_acknowledged"], status: 200)) + // The images are the close-up still and its keyframe. + #expect(rig.server.requests("POST captures/files:commit").count == 4) + #expect(sleeps.withLock { $0.count } == 3) + } + + /// Register answers that leave a file out back off and stop the same way. + @Test func aRegisterThatOmitsFilesBacksOffAndStops() async throws { + var policy = Rig.fast + policy.maxUnacknowledged = 3 + let sleeps = Mutex<[Double]>([]) + let rig = try Rig(policy: policy, sleep: { seconds in sleeps.withLock { $0.append(seconds) } }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.registerOmitsFiles = true } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "register", codes: ["not_acknowledged"], status: 200)) + #expect(rig.server.requests("POST captures/files").count == 4) + #expect(rig.server.requests("PUT upload").isEmpty) + #expect(sleeps.withLock { $0.count } == 3) + } + + /// The limit on reads that find no answer is saved with the upload, so relaunching doesn't + /// start it again: across two processes the reads still stop at `maxNotReadyResults + 1`. + @Test func theNotReadyLimitSurvivesARelaunch() async throws { + var policy = Rig.fast + policy.maxNotReadyResults = 3 + let waits = Mutex(0) + // The first process quits during its second wait. + var rig = try Rig(policy: policy, sleep: { _ in + let n = waits.withLock { $0 += 1; return $0 } + if n >= 2 { throw CancellationError() } + }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.outcomelessResults = 100 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + try #require(await Self.settles(rig.uploader)) + #expect(rig.server.requests("GET captures/result").count == 2) + #expect(await rig.uploader.snapshot.end == nil) + + // The old process releases every uploader before the new one resumes its saved state. + weak let released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + let resumed = try #require(try CaptureUploader.resume( + folder: rig.capture.folder, base: rig.server.base, http: rig.http, policy: policy, sleep: { _ in })) + await resumed.kick() + try #require(await Self.settles(resumed)) + #expect(await resumed.snapshot.end == .failed(step: "result", codes: ["result_not_ready"], status: 200)) + #expect(rig.server.requests("GET captures/result").count == 4) + } + + /// A server that answers an events poll at once with nothing new is not polled again at once: + /// the uploader waits out `minimumPollInterval` first. The clock moves only when the uploader + /// sleeps, so every empty answer arrives at once and each wait is the whole interval. The + /// terminal answer is fetched without a wait. + @Test func emptyEventsAnsweredAtOnceArePaced() async throws { + let clock = Mutex(Date(timeIntervalSince1970: 1_000_000)) + let sleeps = Mutex<[Double]>([]) + let rig = try Rig( + sleep: { seconds in sleeps.withLock { $0.append(seconds) }; clock.withLock { $0 += seconds } }, + now: { clock.withLock { $0 } }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.processingPolls = 3 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + let interval = CaptureUploader.Policy().minimumPollInterval + #expect(sleeps.withLock { $0 } == [interval, interval, interval]) + #expect(rig.server.requests("GET captures/events").count == 4) + } + + /// An empty answer the server held for the whole interval needs no further wait. + @Test func emptyEventsHeldPastTheIntervalAreNotPaced() async throws { + // Every reading of the clock is 3 s after the last, longer than the 2 s interval. + let clock = Mutex(Date(timeIntervalSince1970: 1_000_000)) + let sleeps = Mutex<[Double]>([]) + let rig = try Rig( + sleep: { seconds in sleeps.withLock { $0.append(seconds) } }, + now: { clock.withLock { $0 += 3; return $0 } }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.processingPolls = 3 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + #expect(sleeps.withLock { $0 }.isEmpty) + #expect(rig.server.requests("GET captures/events").count == 4) + } + + /// A capture the server lost and the uploader opened again is a new run, so it gets a fresh + /// not-ready limit: the old run's reads don't end it. + @Test func aRecreatedCaptureStartsANewNotReadyLimit() async throws { + var policy = Rig.fast + policy.maxNotReadyResults = 3 + let waits = Mutex(0) + // The first process quits during its third wait, after three not-ready reads. + var rig = try Rig(policy: policy, sleep: { _ in + let n = waits.withLock { $0 += 1; return $0 } + if n >= 3 { throw CancellationError() } + }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.outcomelessResults = 100 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + try #require(await Self.settles(rig.uploader)) + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: rig.capture.folder)).notReadyReads == 3) + + // The server forgets every capture (a redeploy with in-memory state). The next run has + // one not-ready read before its answer. + rig.server.state.withLock { $0.captures = [:]; $0.byPacket = [:]; $0.outcomelessResults = 1 } + // The old process releases every uploader before the new one resumes its saved state. + weak let released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + let resumed = try #require(try CaptureUploader.resume( + folder: rig.capture.folder, base: rig.server.base, http: rig.http, policy: policy, sleep: { _ in })) + await resumed.kick() + try #require(await Self.settles(resumed)) + #expect(await resumed.snapshot.end == .finished(status: "manual_review")) + #expect(rig.server.requests("POST captures").count == 2) + } + + /// `failed` ends a capture without an answer, so its outcomeless result is final at once. + @Test func aFailedCaptureEndsWithoutWaitingForAnOutcome() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.endStatus = "failed" } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "failed")) + #expect(try Self.outcomeKind(done.result) == nil) + #expect(rig.server.requests("GET captures/result").count == 1) + } + + static func outcomeKind(_ body: Data?) throws -> CaptureResult.OutcomeKind? { + try JSONDecoder().decode(CaptureResult.Response.self, from: #require(body)).outcome?.kind + } + + /// Storage that keeps refusing the sealed bytes' digest stops the upload after one retry + /// instead of cycling through register and PUT. + @Test func aDigestRefusedTwiceStopsTheUpload() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.badDigestPuts = 100 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + await rig.uploader.settled() + #expect(await rig.uploader.snapshot.end == .failed(step: "put", codes: ["bad_digest"], status: 400)) + #expect(rig.server.requests("PUT upload").count <= 4) + } + + /// A file listed in the frozen packet that reached `seal` before its own `add` is still sent. + @Test func sealQueuesListedFilesItHasNotSeen() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + let images = try await rig.capture.sealImages(count: 2) + await rig.uploader.add(Array(images.prefix(1))) + await rig.uploader.settled() + let (streams, packet) = try await rig.capture.finish() + await rig.uploader.seal(packet: packet, files: images + streams) + await rig.uploader.add(Array(images.suffix(1))) + await rig.uploader.settled() + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "manual_review")) + #expect(done.committedCount == images.count + streams.count) + } + + @Test func noEndpointOrNoConsentMeansNoUpload() { + #expect(CaptureUploadGate.decide(endpoint: nil, consented: true) == .off("no capture endpoint in this build")) + #expect(CaptureUploadGate.decide(endpoint: "", consented: true) == .off("no capture endpoint in this build")) + #expect(CaptureUploadGate.decide(endpoint: "$(HOUSESCAN_CAPTURE_API_URL)", consented: true) == .off("no capture endpoint in this build")) + #expect(CaptureUploadGate.decide(endpoint: "http://example.com/v1", consented: true) == .off("capture endpoint must be https")) + #expect(CaptureUploadGate.decide(endpoint: "https://user:secret@example.com/v1", consented: true) == .off("capture endpoint is not a URL")) + #expect(CaptureUploadGate.decide(endpoint: "https://example.com/v1", consented: false) == .off("the homeowner has not agreed to send the capture")) + #expect(CaptureUploadGate.decide(endpoint: "https://example.com/v1", consented: true) == .on(URL(string: "https://example.com/v1")!)) + } +} + +/// Runs each request in a detached task, so cancelling the caller doesn't cancel the request: +/// the way a background URLSession behaves. +struct UncancellableHTTP: CaptureHTTP { + let inner: any CaptureHTTP + init(_ inner: any CaptureHTTP) { self.inner = inner } + + func send(_ request: URLRequest) async throws -> HTTPReply { + try await Task.detached { try await inner.send(request) }.value + } + + func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + try await Task.detached { try await inner.upload(request, file: file) }.value + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift new file mode 100644 index 000000000..c10744ee1 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift @@ -0,0 +1,391 @@ +import CryptoKit +import Foundation +import Network +import Synchronization +import HouseScanKit + +/// A local stand-in for the capture API over real HTTP on 127.0.0.1, so the tests drive the +/// actual `URLSession` transport. It implements the documented behaviour the uploader relies on: +/// idempotent create and finalize bound to the exact body, register with signed PUT URLs, +/// Content-MD5 checked on the PUT, commit as the only acknowledgement, early finalize with +/// `awaiting_files`, and events leading to a result. Faults are injected per test. +/// +/// Written for these tests from the public description of the API; it is not the server's code. +final class LoopbackCaptureAPI: Sendable { + struct Request: Sendable { + var method: String + var path: String + var query: [String: String] + var headers: [String: String] + var body: Data + } + + struct Registration { + var sha256: String + var bytes: Int + var md5: String + var contentType: String + } + + struct Capture { + var id: String + var packetID: String + var createBody: Data + var registered: [String: Registration] = [:] + var stored: [String: Data] = [:] + var committed: Set = [] + var packetSHA: String? + var listed: [String] = [] + var runID: String? + var events: [(type: String, data: [String: String])] = [] + var status = "uploading" + /// Accepted a finalize, then lost it to a storage failure: no run until it is sent again. + var finalizeLost = false + } + + struct State { + var captures: [String: Capture] = [:] + var byPacket: [String: String] = [:] + var tokens: [String: (capture: String, path: String)] = [:] + var log: [Request] = [] + var nextID = 1 + /// Close the connection without answering the next request whose route matches. + var dropNext: Set = [] + /// Answer the next N storage PUTs 403, as an expired signature. + var expireNextPuts = 0 + /// Answer the next N storage PUTs 400 BadDigest. + var badDigestPuts = 0 + /// Accept the next finalize, then report it lost with `failed` / `retry_finalize`. + var loseNextFinalize = false + /// The homeowner message the result carries. + var resultMessage = "The loopback receiver's test result." + /// Answer the next N result reads 404 `result_not_ready`: the run is done but its answer + /// is not readable yet. + var notReadyResults = 0 + /// Answer the next N result reads 200 with the capture's status and run but a null + /// outcome, the body the API serves before the run has written its answer. + var outcomelessResults = 0 + /// Answer the next N result reads 200 with a body that is not JSON. + var unreadableResults = 0 + /// Answer the next N result reads 200 with an outcome that has a kind but none of the + /// other fields an outcome must carry. + var malformedOutcomeResults = 0 + /// The status a capture ends in once every listed file is committed. Only `manual_review` + /// ends with an outcome; `failed` ends with a `failed` event and none. + var endStatus = "manual_review" + /// Every register answer gives this URL for its PUTs instead of the loopback's own. + var uploadURLOverride: String? + /// Register answers leave out every file, as if the server never saw the request's list. + var registerOmitsFiles = false + /// Commits never acknowledge a file: every path comes back in `notFound`. + var commitNeverAcknowledges = false + /// Once every listed file is committed, the next N event polls answer at once with status + /// `processing` and no events, as a server that doesn't hold the poll would; then the run ends. + var processingPolls = 0 + /// Routes ("PUT upload", "POST captures", ...) answered `307` with the given `Location`, as + /// a server sending the request somewhere else would. + var redirects: [String: String] = [:] + /// Answer the next commit with this split instead of the truth. + var commitOverride: ((committed: [String], notFound: [String], mismatch: [String]))? + /// Routes whose answers wait until `release` is called. + var held: Set = [] + var parked: [String: [@Sendable () -> Void]] = [:] + } + + let state = Mutex(State()) + private let listener: NWListener + private let queue = DispatchQueue(label: "loopback-capture-api") + var port: UInt16 { listener.port?.rawValue ?? 0 } + + init() throws { + let parameters = NWParameters.tcp + parameters.acceptLocalOnly = true + parameters.requiredLocalEndpoint = .hostPort(host: "127.0.0.1", port: .any) + listener = try NWListener(using: parameters) + let ready = DispatchSemaphore(value: 0) + listener.stateUpdateHandler = { if case .ready = $0 { ready.signal() } } + listener.newConnectionHandler = { [queue] connection in + connection.start(queue: queue) + Self.receive(connection, buffer: Data()) { request in self.handle(request, connection) } + } + listener.start(queue: queue) + guard ready.wait(timeout: .now() + 5) == .success, listener.port != nil else { throw URLError(.cannotConnectToHost) } + } + + deinit { listener.cancel() } + + var base: URL { URL(string: "http://127.0.0.1:\(port)/v1")! } + + func requests(_ route: String) -> [Request] { state.withLock { $0.log.filter { Self.route($0) == route } } } + + func release(_ route: String) { + let waiting = state.withLock { s -> [@Sendable () -> Void] in + s.held.remove(route) + return s.parked.removeValue(forKey: route) ?? [] + } + waiting.forEach { $0() } + } + + /// "POST /v1/captures/{id}/files" style names, ids replaced. + static func route(_ r: Request) -> String { + let parts = r.path.split(separator: "/").map(String.init) + if parts.first == "upload" { return "\(r.method) upload" } + guard parts.count >= 2 else { return "\(r.method) \(r.path)" } + if parts.count == 2 { return "\(r.method) captures" } + return "\(r.method) captures/\(parts[2...].joined(separator: "/"))" + } + + // MARK: HTTP over the connection + + private static func receive(_ connection: NWConnection, buffer: Data, handler: @escaping @Sendable (Request) -> Void) { + connection.receive(minimumIncompleteLength: 1, maximumLength: 1 << 20) { data, _, done, error in + var buffer = buffer + if let data { buffer.append(data) } + if let request = parse(buffer) { return handler(request) } + if done || error != nil { return connection.cancel() } + receive(connection, buffer: buffer, handler: handler) + } + } + + private static func parse(_ data: Data) -> Request? { + guard let end = data.range(of: Data("\r\n\r\n".utf8)) else { return nil } + let head = String(decoding: data[..= 2 else { return nil } + var headers: [String: String] = [:] + for field in head.dropFirst() { + guard let colon = field.firstIndex(of: ":") else { continue } + headers[field[..= length else { return nil } + let target = URLComponents(string: String(line[1])) + var query: [String: String] = [:] + for item in target?.queryItems ?? [] { query[item.name] = item.value } + var path = target?.path ?? "" + if path.hasPrefix("/v1/") { path.removeFirst(4) } else if path.hasPrefix("/") { path.removeFirst() } + return Request(method: String(line[0]), path: path, query: query, headers: headers, body: Data(body.prefix(length))) + } + + private func handle(_ request: Request, _ connection: NWConnection) { + let route = Self.route(request) + let (drop, hold, redirect) = state.withLock { s -> (Bool, Bool, String?) in + s.log.append(request) + return (s.dropNext.remove(route) != nil, s.held.contains(route), s.redirects[route]) + } + let (status, body) = redirect == nil ? answer(request) : (307, Data()) + let location = redirect.map { "Location: \($0)\r\n" } ?? "" + let send: @Sendable () -> Void = { + if drop { return connection.cancel() } + let head = "HTTP/1.1 \(status) X\r\n\(location)Content-Type: application/json\r\nContent-Length: \(body.count)\r\nConnection: close\r\n\r\n" + connection.send(content: Data(head.utf8) + body, completion: .contentProcessed { _ in connection.cancel() }) + } + if hold { + state.withLock { $0.parked[route, default: []].append(send) } + } else { + send() + } + } + + // MARK: API behaviour + + private func json(_ object: Any) -> Data { try! JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]) } + private func error(_ status: Int, _ code: String) -> (Int, Data) { (status, json(["errors": [["code": code, "pointer": "/", "message": code]]])) } + + private func answer(_ r: Request) -> (Int, Data) { + state.withLock { s in + let parts = r.path.split(separator: "/").map(String.init) + if parts.first == "upload" { return put(r, token: parts.last ?? "", &s) } + if r.headers["authorization"] != nil { return error(400, "unexpected_authorization") } + if r.method == "POST", parts == ["captures"] { return create(r, &s) } + guard parts.count >= 2, var capture = s.captures[parts[1]] else { return error(404, "capture_not_found") } + defer { s.captures[capture.id] = capture } + switch (r.method, parts.dropFirst(2).joined(separator: "/")) { + case ("POST", "files"): return register(r, &capture, &s) + case ("POST", "files:commit"): return commit(r, &capture, &s) + case ("POST", "finalize"): return finalize(r, &capture, &s) + case ("GET", "events"): return events(r, &capture, &s) + case ("GET", "result"): + if s.notReadyResults > 0 { + s.notReadyResults -= 1 + return error(404, "result_not_ready") + } + if s.unreadableResults > 0 { + s.unreadableResults -= 1 + return (200, Data("not a result".utf8)) + } + if s.malformedOutcomeResults > 0 { + s.malformedOutcomeResults -= 1 + return (200, json(["runId": capture.runID ?? "", "status": capture.status, "viewsNeeded": [], "memberActions": [], + "outcome": ["kind": "manual_review"]])) + } + let withheld = s.outcomelessResults > 0 + if withheld { s.outcomelessResults -= 1 } + let outcome: Any = capture.status == "manual_review" && !withheld + ? ["kind": "manual_review", "profile": "C", "message": s.resultMessage, "viewsNeeded": [], "reasons": []] as [String: Any] + : NSNull() + return (200, json(["runId": capture.runID ?? "", "status": capture.status, "viewsNeeded": [], "memberActions": [], "outcome": outcome])) + default: return error(404, "route") + } + } + } + + private func create(_ r: Request, _ s: inout State) -> (Int, Data) { + guard let body = try? JSONSerialization.jsonObject(with: r.body) as? [String: Any], let packetID = body["packetId"] as? String else { + return error(422, "schema") + } + guard let key = r.headers["idempotency-key"] else { return error(422, "idempotency_key_missing") } + guard key == packetID else { return error(422, "idempotency_key_mismatch") } + guard body["formatVersion"] as? String == "0.4" else { return error(422, "format_version_unsupported") } + let status: Int + let capture: Capture + if let existing = s.byPacket[packetID].flatMap({ s.captures[$0] }) { + guard existing.createBody == r.body else { return error(409, "idempotency_conflict") } + (status, capture) = (200, existing) + } else { + capture = Capture(id: "cap_\(s.nextID)", packetID: packetID, createBody: r.body) + s.nextID += 1 + s.captures[capture.id] = capture + s.byPacket[packetID] = capture.id + status = 201 + } + return (status, json([ + "captureId": capture.id, "status": "uploading", "eventsUrl": "/v1/captures/\(capture.id)/events", + "finalizeBy": "2099-01-01T00:00:00Z", "upload": ["maxSinglePutBytes": 16_777_216, "maxBatch": 50, "urlTtlS": 3600, "maxFiles": 2000, "partBytes": 8_388_608], + ])) + } + + private func register(_ r: Request, _ c: inout Capture, _ s: inout State) -> (Int, Data) { + guard let body = try? JSONSerialization.jsonObject(with: r.body) as? [String: Any], let files = body["files"] as? [[String: Any]], + (1...50).contains(files.count) else { return error(422, "batch_too_large") } + var out: [[String: Any]] = [] + for file in files { + guard let path = file["path"] as? String, Packet04.isRelPath(path), let sha = file["sha256"] as? String, sha.count == 64, + let bytes = file["bytes"] as? Int, let md5 = file["md5"] as? String, Data(base64Encoded: md5)?.count == 16, + let role = file["role"] as? String, Packet04.Role(rawValue: role) != nil, let type = file["contentType"] as? String + else { return error(422, "schema") } + let meta = file["meta"] as? [String: Any] + let image = role == "keyframe" || role == "still" + if image != (meta != nil) || (meta.map { $0.count != 1 } ?? false) { return error(422, "meta_mismatch") } + if let previous = c.registered[path], previous.sha256 != sha { return error(409, "sha256_conflict") } + if c.packetSHA != nil, !c.listed.contains(path) { return error(409, "not_in_manifest") } + c.registered[path] = Registration(sha256: sha, bytes: bytes, md5: md5, contentType: type) + if c.committed.contains(path) { + out.append(["path": path, "state": "committed"]) + continue + } + if s.registerOmitsFiles { continue } + let token = UUID().uuidString + s.tokens[token] = (c.id, path) + out.append(["path": path, "state": "pending", "upload": [ + "method": "PUT", "url": s.uploadURLOverride ?? "http://127.0.0.1:\(port)/upload/\(token)", + "headers": ["Content-Type": type, "Content-MD5": md5], "expiresAt": "2099-01-01T00:00:00Z", + ]]) + } + return (200, json(["files": out])) + } + + private func put(_ r: Request, token: String, _ s: inout State) -> (Int, Data) { + guard r.method == "PUT", let (captureID, path) = s.tokens[token], var c = s.captures[captureID], let reg = c.registered[path] else { + return (403, Data("token_invalid".utf8)) + } + if r.headers["authorization"] != nil { return (400, Data("authorization sent to storage".utf8)) } + if s.badDigestPuts > 0 { + s.badDigestPuts -= 1 + return (400, Data("BadDigest".utf8)) + } + if s.expireNextPuts > 0 { + s.expireNextPuts -= 1 + s.tokens[token] = nil + return (403, Data("AccessDeniedRequest has expired".utf8)) + } + guard r.headers["content-md5"] == reg.md5, Data(Insecure.MD5.hash(data: r.body)).base64EncodedString() == reg.md5, + r.headers["content-type"] == reg.contentType + else { return (400, Data("BadDigest".utf8)) } + c.stored[path] = r.body + s.captures[captureID] = c + return (200, Data()) + } + + private func commit(_ r: Request, _ c: inout Capture, _ s: inout State) -> (Int, Data) { + guard let body = try? JSONSerialization.jsonObject(with: r.body) as? [String: Any], let files = body["files"] as? [[String: Any]] else { + return error(422, "schema") + } + var committed: [String] = [], notFound: [String] = [], mismatch: [String] = [] + for file in files { + guard let path = file["path"] as? String, let sha = file["sha256"] as? String else { return error(422, "schema") } + guard let data = c.stored[path] else { notFound.append(path); continue } + if PacketFiles.sha256(data) != sha || c.registered[path]?.sha256 != sha { mismatch.append(path); continue } + committed.append(path) + } + if let forced = s.commitOverride { + s.commitOverride = nil + (committed, notFound, mismatch) = forced + } + if s.commitNeverAcknowledges { + notFound += committed + mismatch + committed = [] + mismatch = [] + } + c.committed.formUnion(committed) + if !committed.isEmpty { c.events.append(("files_committed", ["committed": String(c.committed.count)])) } + startIfComplete(&c, endStatus: s.endStatus) + return (200, json(["committed": committed, "notFound": notFound, "mismatch": mismatch])) + } + + private func finalize(_ r: Request, _ c: inout Capture, _ s: inout State) -> (Int, Data) { + guard r.headers["idempotency-key"] == c.packetID else { return error(422, "idempotency_key_mismatch") } + let sha = PacketFiles.sha256(r.body) + if let declared = r.headers["x-packet-sha256"], declared != sha { return error(422, "packet_sha_mismatch") } + if let previous = c.packetSHA, previous != sha { return error(409, "packet_sha_conflict") } + guard let packet = try? JSONDecoder().decode(Packet04.Packet.self, from: r.body) else { return error(422, "schema") } + guard packet.packetId == c.packetID else { return error(422, "packet_id_mismatch") } + let problems = Packet04Check.problems(packet, folder: nil) + if !problems.isEmpty { return error(422, "schema") } + for entry in packet.files { + if let reg = c.registered[entry.path], reg.sha256 != entry.sha256 || reg.bytes != entry.bytes { return error(422, "sha256_mismatch") } + } + if c.packetSHA == nil { + c.packetSHA = sha + c.listed = packet.files.map(\.path) + c.runID = "run_" + sha.prefix(16) + c.status = "awaiting_files" + if s.loseNextFinalize { + s.loseNextFinalize = false + c.finalizeLost = true + c.events.append(("failed", ["code": "storage_unavailable", "next": "retry_finalize"])) + } + startIfComplete(&c, endStatus: s.endStatus) + } else if c.finalizeLost { + c.finalizeLost = false + startIfComplete(&c, endStatus: s.endStatus) + } + let missing = c.listed.filter { !c.committed.contains($0) } + return (202, json(["status": missing.isEmpty ? "processing" : "awaiting_files", "missing": missing, "runId": c.runID!, "etaS": 1])) + } + + private func startIfComplete(_ c: inout Capture, endStatus: String) { + guard c.packetSHA != nil, !c.finalizeLost, c.status == "awaiting_files", c.listed.allSatisfy(c.committed.contains) else { return } + c.status = endStatus + c.events.append(("stage", ["stage": "ingest", "status": "ok", "runId": c.runID!])) + if endStatus == "manual_review" { + c.events.append(("verdict_ready", ["kind": "manual_review", "runId": c.runID!])) + } else { + c.events.append(("failed", ["code": "loopback_run_failed", "runId": c.runID!])) + } + } + + private func events(_ r: Request, _ c: inout Capture, _ s: inout State) -> (Int, Data) { + if c.status == "manual_review" || c.status == "failed", s.processingPolls > 0, c.runID != nil { + s.processingPolls -= 1 + let after = Int(r.query["after"] ?? "0") ?? 0 + return (200, json(["status": "processing", "next": after, "events": [] as [Any]])) + } + let after = Int(r.query["after"] ?? "0") ?? 0 + let list = c.events.enumerated().filter { $0.offset + 1 > after }.map { index, event in + ["seq": index + 1, "type": event.type, "at": "2026-09-27T00:00:00Z", "data": event.data] as [String: Any] + } + return (200, json(["status": c.status, "next": c.events.count, "events": list])) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/Packet04Tests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/Packet04Tests.swift new file mode 100644 index 000000000..909bb67b2 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/Packet04Tests.swift @@ -0,0 +1,277 @@ +import Foundation +import HouseScanKit +import simd +import Testing + +/// A synthetic ARKit capture for the 0.4 tests: a camera walking along +x at 1.5 m height, +/// 96 x 72 JPEGs with intrinsics that fit them, 60 Hz poses and 100 Hz IMU with the gyro 1 ms +/// behind the accelerometer. `source` is synthetic, so no packet from it is evidence. +struct SyntheticCapture04 { + static let start = 1000.0 + let folder: URL + let producer: Packet04Producer + + init(folder: URL, packetID: String = UUID().uuidString) throws { + self.folder = folder + producer = try Packet04Producer(folder: folder, info: Self.info(packetID: packetID), startedAtUptime: Self.start, startedAt: Date(timeIntervalSince1970: 1_790_000_000)) + } + + static func info(packetID: String) -> Packet04SessionInfo { + Packet04SessionInfo( + packetID: packetID, sessionID: "S-1", source: .synthetic, appVersion: "test (1)", deviceModel: "iPhone15,4", systemVersion: "26.0", + lidarAvailable: false, sceneDepthEnabled: false, meshReconstructionSupported: false, sceneReconstruction: "none", + planeDetection: ["horizontal", "vertical"], videoWidth: 1920, videoHeight: 1440, framesPerSecond: 60, timeZone: "America/Chicago") + } + + static func pose(_ t: Double) -> simd_float4x4 { + var m = matrix_identity_float4x4 + m.columns.3 = SIMD4(Float(t - start) * 0.5, 1.5, -2, 1) + return m + } + + static let intrinsics = SIMD4(80, 80, 48, 36) + + func observation(_ t: Double) -> Packet04Observation { + Packet04Observation(t: t, cameraToWorld: Self.pose(t), intrinsics: Self.intrinsics, width: 96, height: 72, tracking: .normal) + } + + static func jpeg(in folder: URL) throws -> Data { + let url = folder.appending(path: "source-\(UUID().uuidString).jpg") + try makeJPEG(width: 96, height: 72, at: url) + defer { try? FileManager.default.removeItem(at: url) } + return try Data(contentsOf: url) + } + + /// Seals `count` keyframes and the head-on meter still on the first one; returns every file. + func sealImages(count: Int = 3) async throws -> [SealedFile] { + let scratch = folder.deletingLastPathComponent() + var files: [SealedFile] = [] + for i in 0.. [Packet04Streams.PoseRow] { + stride(from: start, through: end, by: 1.0 / 60).map { .init(t: $0, tracking: .normal, cameraToWorld: pose($0), intrinsics: SIMD4(1440, 1440, 960, 720)) } + } + + static func imu(until end: Double = start + 2) -> (accel: [Packet04Streams.MotionRow], gyro: [Packet04Streams.MotionRow]) { + let times = Array(stride(from: start, through: end, by: 0.01)) + return (times.map { .init(t: $0, value: SIMD3(0, -1, 0)) }, times.map { .init(t: $0 + 0.001, value: SIMD3(0.01, 0, 0)) }) + } + + func finish() async throws -> (streams: [SealedFile], packet: Data) { + let imu = Self.imu() + return try await producer.finish(poses: Self.poses(), accelerometer: imu.accel, gyroscope: imu.gyro, endedAtUptime: Self.start + 2, acceptedCloseUpAt: Self.start + 0.2) + } +} + +func temporaryPacketFolder(_ name: String) -> URL { + FileManager.default.temporaryDirectory.appending(path: "\(name)-\(UUID().uuidString)/packet") +} + +@Suite struct Packet04Tests { + @Test func finishedPacketPassesItsChecksAndListsEverySealedFile() async throws { + let folder = temporaryPacketFolder("p04-normal") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + let images = try await capture.sealImages() + let (streams, data) = try await capture.finish() + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: data) + + #expect(Packet04Check.problems(packet, folder: folder).isEmpty) + #expect(Set(packet.files.map(\.path)) == Set((images + streams).map(\.path))) + #expect(packet.formatVersion == "0.4") + #expect(packet.session.tier == .arkit) + #expect(packet.epochs == [.init(id: "e1", startTime: SyntheticCapture04.start, reason: "sessionStart")]) + #expect(packet.keyframes.allSatisfy { $0.epoch == "e1" }) + #expect(packet.scaleReference.meterCloseUp == .init(captured: true, still: "meter_close", arkitDistanceM: nil, side: nil)) + #expect(packet.scaleReference.ext.obliqueCloseUp == .notCaptured) + #expect(packet.tracking == [.init(time: SyntheticCapture04.start, state: "normal", epoch: "e1")]) + // The exact bytes finish returned are what is on disk. + #expect(try Data(contentsOf: folder.appending(path: "packet.json")) == data) + } + + @Test func keyframeKeepsTheRawWorldPoseAndItsOwnIntrinsics() async throws { + let folder = temporaryPacketFolder("p04-pose") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + let t = SyntheticCapture04.start + 0.7 + let files = try await capture.producer.sealKeyframe(jpeg: try SyntheticCapture04.jpeg(in: folder.deletingLastPathComponent()), observation: capture.observation(t), reason: "motion") + guard case .keyframe(let record)? = files.first?.meta else { Issue.record("no keyframe record"); return } + #expect(record.pose == [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0.35, 1.5, -2, 1]) + #expect(record.intrinsics == [80, 80, 48, 36]) + #expect(record.timestamp == t) + #expect(files.first?.path == "keyframes/k00001.jpg") + #expect(files.first?.role == .keyframe) + } + + @Test func sealedDigestsMatchTheBytesOnDisk() async throws { + let folder = temporaryPacketFolder("p04-digest") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + for file in try await capture.sealImages() { + let data = try Data(contentsOf: folder.appending(path: file.path)) + #expect(file.bytes == data.count) + #expect(file.sha256 == PacketFiles.sha256(data)) + #expect(Data(base64Encoded: file.md5)?.count == 16) + } + } + + @Test func refusesARotatedImageAndIntrinsicsOfAnotherSize() async throws { + let folder = temporaryPacketFolder("p04-refuse") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + var portrait = capture.observation(SyntheticCapture04.start + 1) + (portrait.width, portrait.height) = (72, 96) + await #expect(throws: Packet04Error.self) { + _ = try await capture.producer.sealKeyframe(jpeg: try SyntheticCapture04.jpeg(in: folder.deletingLastPathComponent()), observation: portrait, reason: "motion") + } + var wrong = capture.observation(SyntheticCapture04.start + 1) + wrong.intrinsics = SIMD4(1440, 1440, 960, 720) + await #expect(throws: Packet04Error.self) { + _ = try await capture.producer.sealKeyframe(jpeg: try SyntheticCapture04.jpeg(in: folder.deletingLastPathComponent()), observation: wrong, reason: "motion") + } + } + + @Test func nothingIsSealedAfterFinishAndFinishReturnsTheSameBytes() async throws { + let folder = temporaryPacketFolder("p04-frozen") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + _ = try await capture.sealImages() + let first = try await capture.finish() + let second = try await capture.finish() + #expect(first.packet == second.packet) + await #expect(throws: Packet04Error.sealedAfterFinish("keyframe")) { + _ = try await capture.producer.sealKeyframe(jpeg: try SyntheticCapture04.jpeg(in: folder.deletingLastPathComponent()), observation: capture.observation(SyntheticCapture04.start + 1.9), reason: "motion") + } + } + + @Test func tapRayReplaysFromItsKeyframe() async throws { + let folder = temporaryPacketFolder("p04-tap") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + _ = try await capture.sealImages(count: 1) + try await capture.producer.addTap(id: "tap1", label: "meter", keyframe: "k00001", pixel: SIMD2(48, 36), time: SyntheticCapture04.start + 0.2, hit: nil) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: try await capture.finish().packet) + let tap = try #require(packet.taps?.first) + // The principal point looks straight down -z from the camera. + #expect(tap.rayOrigin == [0.1, 1.5, -2]) + #expect(tap.rayDirection == [0, 0, -1]) + + var broken = packet + broken.taps?[0].rayDirection = [0, 0.01, -1] + #expect(Packet04Check.problems(broken, folder: nil).contains { $0.hasPrefix("tap tap1 ray replays with error") }) + } + + /// A tap ray must be two 3-vectors of finite numbers. Joined and compared pairwise, a missing + /// or misshaped vector, or a NaN, could otherwise pass the replay check unseen. + @Test(arguments: [ + ([0.1, 1.5, -2], []), + ([0.1, 1.5], [-2, 0, 0, -1]), + ([0.1, 1.5, -2], [0, 0, -1, 0]), + ([0.1, 1.5, -2], [0, Double.nan, -1]), + ([0.1, .infinity, -2], [0, 0, -1]), + ] as [([Double], [Double])]) + func aTapRayThatIsNotTwoFinite3VectorsIsRefused(origin: [Double], direction: [Double]) async throws { + let folder = temporaryPacketFolder("p04-tap-shape") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + _ = try await capture.sealImages(count: 1) + try await capture.producer.addTap(id: "tap1", label: "meter", keyframe: "k00001", pixel: SIMD2(48, 36), time: SyntheticCapture04.start + 0.2, hit: nil) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: try await capture.finish().packet) + #expect(!Packet04Check.problems(packet, folder: nil).contains { $0.hasPrefix("tap tap1") }) + + var broken = packet + broken.taps?[0].rayOrigin = origin + broken.taps?[0].rayDirection = direction + #expect(Packet04Check.problems(broken, folder: nil).contains { $0.hasPrefix("tap tap1") }) + } + + @Test func checksCatchAMalformedPacket() async throws { + let folder = temporaryPacketFolder("p04-malformed") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + _ = try await capture.sealImages() + let good = try JSONDecoder().decode(Packet04.Packet.self, from: try await capture.finish().packet) + + var unlisted = good + unlisted.files.removeAll { $0.path == "keyframes/k00002.jpg" } + #expect(Packet04Check.problems(unlisted, folder: folder) == ["keyframe k00002 image keyframes/k00002.jpg is not in files"]) + + var badPath = good + badPath.files[0].path = "../escape.jpg" + #expect(Packet04Check.problems(badPath, folder: nil).contains("file path ../escape.jpg is not a RelPath")) + + var badStill = good + badStill.stills?[0].keyframe = "k00009" + #expect(Packet04Check.problems(badStill, folder: nil).contains("still meter_close names unknown keyframe k00009")) + + try Data("changed".utf8).write(to: folder.appending(path: "keyframes/k00003.jpg")) + #expect(Packet04Check.problems(good, folder: folder).contains { $0.hasPrefix("file keyframes/k00003.jpg is 7 bytes") }) + } + + @Test func imuRowsPairOnlyCloseGyroSamplesAndReportTheMeasuredRate() throws { + let accel = (0..<100).map { Packet04Streams.MotionRow(t: 10 + Double($0) * 0.01, value: SIMD3(0, -1, 0)) } + // A gyro sample 1 ms after each accelerometer sample, except the 50th's, so the nearest to + // that accelerometer sample is 9 ms away: it is left out, not given a far reading. + var gyro = accel.map { Packet04Streams.MotionRow(t: $0.t + 0.001, value: SIMD3(0.5, 0, 0)) } + gyro.remove(at: 50) + let out = try Packet04Streams.imuRaw(accelerometer: accel, gyroscope: gyro) + let lines = String(decoding: out.csv, as: UTF8.self).split(separator: "\n") + #expect(lines.first == "t,ax,ay,az,gx,gy,gz") + #expect(lines.count == 1 + 99) + #expect(!lines.contains { $0.hasPrefix("10.5,") }) + #expect(abs(out.rateHz - 100) < 0.01) + #expect(out.pairingNote["imuRawRowsKept"] == "99 of 100") + } + + @Test func poseRowsJoinEachFrameToItsOwnIntrinsics() { + func trajectory(_ t: Double, x: Double) -> [Double] { [t, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, x, 1.5, -2, 1] } + let rows = Packet04Streams.poseRows( + trajectory: [trajectory(1, x: 0.1), trajectory(2, x: 0.2), trajectory(3, x: 0.3)], + // Frame 2 wrote no intrinsics row: it is left out, not given frame 1's or frame 3's. + intrinsics: [[1, 1440, 1441, 960, 720, 1920, 1440], [3, 1500, 1501, 961, 721, 1920, 1440]], + tracking: { state, _ in state == 0 ? .normal : .notAvailable }) + #expect(rows.map(\.t) == [1, 3]) + #expect(rows[1].intrinsics == SIMD4(1500, 1501, 961, 721)) + #expect(rows[1].cameraToWorld.columns.3 == SIMD4(0.3, 1.5, -2, 1)) + #expect(rows[0].tracking == .normal) + } + + @Test func aSlowIMUIsRefusedNotRelabelled() { + let accel = (0..<20).map { Packet04Streams.MotionRow(t: Double($0) * 0.05, value: SIMD3(0, -1, 0)) } + #expect(throws: Packet04Error.streamTooSlow(stream: "imuRaw", measuredHz: 1 / 0.05, minimumHz: 50)) { + _ = try Packet04Streams.imuRaw(accelerometer: accel, gyroscope: accel) + } + } + + @Test func gzipStreamsDecompressWithTheSystemGzip() throws { + let text = Data((0..<2000).map { "\($0),0.5,-1\n" }.joined().utf8) + let url = FileManager.default.temporaryDirectory.appending(path: "p04-\(UUID().uuidString).csv.gz") + defer { try? FileManager.default.removeItem(at: url) } + try Gzip.compress(text).write(to: url) + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/gzip") + process.arguments = ["-dc", url.path] + let pipe = Pipe() + process.standardOutput = pipe + try process.run() + let out = pipe.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + #expect(process.terminationStatus == 0) + #expect(out == text) + } + + @Test func idAndPathPatternsMatchTheServer() { + #expect(Packet04.isStorageID(UUID().uuidString)) + #expect(!Packet04.isStorageID("short")) + #expect(!Packet04.isStorageID("has.dot-in-it")) + #expect(Packet04.isSegmentID("k00001") && Packet04.isSegmentID("meter_close")) + #expect(!Packet04.isSegmentID("-lead") && !Packet04.isSegmentID("a.b")) + #expect(Packet04.isRelPath("keyframes/k00001.depth.f32") && Packet04.isRelPath("streams/imu_raw.csv.gz")) + #expect(!Packet04.isRelPath("a/../b") && !Packet04.isRelPath("a//b") && !Packet04.isRelPath("foo..bar") && !Packet04.isRelPath(".hidden")) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift index 4768300df..e171be1ab 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift @@ -21,7 +21,7 @@ enum PacketSchema { /// A JPEG of `width` x `height` with a colour pattern, written by ImageIO. `orientation` sets the /// EXIF orientation tag. -func makeJPEG(width: Int, height: Int, at url: URL, orientation: Int? = nil) throws { +func makeJPEG(width: Int, height: Int, at url: URL, orientation: Int? = nil, shift: Int = 0) throws { let space = CGColorSpaceCreateDeviceRGB() let context = try #require(CGContext( data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 4 * width, space: space, @@ -30,7 +30,7 @@ func makeJPEG(width: Int, height: Int, at url: URL, orientation: Int? = nil) thr for y in 0.. Date: Sun, 4 Oct 2026 01:01:51 -0500 Subject: [PATCH 25/71] Add a scan context and photo-processing controller to HouseScanKit A scan now has an identity that isn't the engine's generation counter: ScanContext holds the scan id, the backend profile captured at scan start (backend, who answers, origin, credential issuer) and the current spatial session (recording session id and world epoch). A world reset keeps the scan and its profile and starts a new spatial session. PhotoProcessingController drives #192's CaptureSessionCoordinator for one scan at a time and publishes a typed PhotoProcessingStatus: consent, progress (capturing, queued, uploading, processing) and either an answer (candidate, needs more photos, installer review, no candidate, unrecognized) or an end (not set up, consent not given, withdrawn with whether the phone saved it, not prepared, failed, expired, refused, unreadable, not ready, mismatch). Answers bind to the recording session, world epoch, origin, coordinator session, packet, capture and run; an answer for a world that is gone is dropped. AR stays withheld through CaptureResult.Record.placement. A needs-more-photos answer carries the parent capture as plain values for a later amendment client. Nothing resumes a saved capture. FixtureCaptureHTTP answers the capture API inside the process with a chosen outcome, for the package tests and the app's DEBUG fixture. No #192 file changed. Validated: `swift test -j 2` in ios/HouseScanKit through heavy-admit passed 835 tests in 108 suites, 17 of them new (binding refusals, each fixture answer end to end with #192's synthetic recording, consent declined and unanswered, withdrawal saved and not saved, a late answer after start over, a world reset, folder ownership, missing setup). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Processing/FixtureCaptureHTTP.swift | 265 ++++++++++++ .../Processing/PhotoCaptureBinding.swift | 72 ++++ .../PhotoProcessingController.swift | 264 ++++++++++++ .../Processing/PhotoProcessingStatus.swift | 206 ++++++++++ .../Processing/ProcessingBackend.swift | 61 +++ .../HouseScanKit/Processing/ScanContext.swift | 41 ++ .../PhotoProcessingTests.swift | 380 ++++++++++++++++++ 7 files changed, 1289 insertions(+) create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Processing/FixtureCaptureHTTP.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoCaptureBinding.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Processing/ProcessingBackend.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Processing/ScanContext.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/FixtureCaptureHTTP.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/FixtureCaptureHTTP.swift new file mode 100644 index 000000000..a9d1bae12 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/FixtureCaptureHTTP.swift @@ -0,0 +1,265 @@ +import Foundation +import Synchronization + +/// The capture API, answered inside this process, for synthetic runs: the DEBUG app's +/// `-photoProcessingFixture` and the package tests. No request leaves the process, so nothing can +/// be uploaded anywhere. It takes what the uploader sends as the documented API does (create, +/// register with PUT targets, PUT, commit as the only acknowledgement, finalize, events, result) +/// and ends every capture with the answer it was made with. Each answer's words say they come +/// from a fixture. +/// +/// Written from the public API types in `CaptureAPI` and `CaptureResult`; it is not the service. +public final class FixtureCaptureHTTP: CaptureHTTP, Sendable { + public enum Answer: String, Sendable, CaseIterable { + case candidate, needsViews, manualReview, notEligible, failed, expired + /// The run finishes, but its answer never becomes readable. + case notReady + /// The result read gets a body that isn't a result. + case unreadable + /// The service refuses to open the capture. + case refuseCreate + /// The run never finishes, so the capture stays in processing. + case hold + + /// The homeowner message the answer carries, when it has an outcome. + public var message: String? { + switch self { + case .candidate: "Fixture answer: a possible spot to the left of the meter." + case .needsViews: "Fixture answer: one more view of the wall, please." + case .manualReview: "Fixture answer: an installer needs to look at this wall." + case .notEligible: "Fixture answer: no spot on this wall." + case .failed, .expired, .notReady, .unreadable, .refuseCreate, .hold: nil + } + } + + /// The capture's status once every listed file is committed. + var endStatus: String { + switch self { + case .candidate, .notEligible, .notReady, .unreadable: "complete" + case .needsViews: "needs_views" + case .manualReview: "manual_review" + case .failed: "failed" + case .expired: "expired" + case .refuseCreate, .hold: "processing" + } + } + } + + /// The API base the fixture answers for. A `.invalid` host can't resolve, so a request that + /// somehow reached a real transport would fail rather than land somewhere. + public static let base = URL(string: "https://capture-fixture.invalid/v1")! + + /// The ARKit epoch a candidate's box is in: the name the coordinator gives every packet's + /// world (`CaptureSessionCoordinator.epoch`), which is main-actor isolated. + public static let arkitEpoch = "e1" + + private struct Capture { + var id: String + var packetID: String + var uploaded: Set = [] + var committed: Set = [] + var listed: [String]? + var runID: String? + + var allCommitted: Bool { listed.map { $0.allSatisfy(committed.contains) } ?? false } + } + + private struct State { + var captures: [String: Capture] = [:] + var byPacket: [String: String] = [:] + var tokens: [String: (capture: String, path: String)] = [:] + var routes: [String] = [] + var next = 1 + } + + public let answer: Answer + private let state = Mutex(State()) + private let onRequest: @Sendable ([String]) -> Void + + /// `onRequest` hears every route so far each time a request arrives, with the fixture's lock + /// held, so calls never overlap. It must not call back into the fixture. + public init(answer: Answer, onRequest: @escaping @Sendable ([String]) -> Void = { _ in }) { + self.answer = answer + self.onRequest = onRequest + } + + /// Every request so far as "METHOD route", ids left out: "POST captures", "PUT upload", + /// "POST captures/files:commit", "GET captures/result". + public var routes: [String] { state.withLock { $0.routes } } + + public func send(_ request: URLRequest) async throws -> HTTPReply { + reply(to: request) + } + + public func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + // The bytes stay where they are; a file that isn't there fails as a real upload would. + guard FileManager.default.fileExists(atPath: file.path) else { throw URLError(.fileDoesNotExist) } + return reply(to: request) + } + + // MARK: Routing + + private func reply(to request: URLRequest) -> HTTPReply { + let method = request.httpMethod ?? "GET" + let url = request.url ?? Self.base + var parts = url.path.split(separator: "/").map(String.init) + if parts.first == "v1" { parts.removeFirst() } + let route = Self.route(method: method, parts: parts) + state.withLock { s in + s.routes.append(route) + onRequest(s.routes) + } + let body = request.httpBody ?? Data() + return state.withLock { s in + if parts.first == "upload" { return put(token: parts.last ?? "", &s) } + if method == "POST", parts == ["captures"] { return create(body, &s) } + guard parts.count >= 3, parts[0] == "captures", var capture = s.captures[parts[1]] else { return error(404, "capture_not_found") } + defer { s.captures[capture.id] = capture } + switch (method, parts[2...].joined(separator: "/")) { + case ("POST", "files"): return register(body, &capture, &s) + case ("POST", "files:commit"): return commit(body, &capture) + case ("POST", "finalize"): return finalize(body, &capture, &s) + case ("GET", "events"): return events(url, capture) + case ("GET", "result"): return result(capture) + default: return error(404, "route") + } + } + } + + static func route(method: String, parts: [String]) -> String { + if parts.first == "upload" { return "\(method) upload" } + if parts.count <= 2 { return "\(method) captures" } + return "\(method) captures/\(parts[2...].joined(separator: "/"))" + } + + // MARK: API + + private func create(_ body: Data, _ s: inout State) -> HTTPReply { + guard let request = try? JSONDecoder().decode(CaptureAPI.CreateRequest.self, from: body) else { return error(422, "schema") } + if answer == .refuseCreate { return error(422, "fixture_refused") } + let id: String + let status: Int + if let existing = s.byPacket[request.packetId] { + (id, status) = (existing, 200) + } else { + id = "cap_fixture_\(s.next)" + s.next += 1 + s.captures[id] = Capture(id: id, packetID: request.packetId) + s.byPacket[request.packetId] = id + status = 201 + } + return json(status, ["captureId": id, "status": "uploading", "upload": ["maxBatch": 50]]) + } + + private func register(_ body: Data, _ c: inout Capture, _ s: inout State) -> HTTPReply { + guard let request = try? JSONDecoder().decode(CaptureAPI.RegisterRequest.self, from: body) else { return error(422, "schema") } + var files: [[String: Any]] = [] + for file in request.files { + if c.committed.contains(file.path) { + files.append(["path": file.path, "state": "committed"]) + continue + } + let token = UUID().uuidString + s.tokens[token] = (c.id, file.path) + files.append([ + "path": file.path, "state": "pending", + "upload": [ + "method": "PUT", "url": "https://capture-fixture.invalid/upload/\(token)", + "headers": ["Content-Type": file.contentType], "expiresAt": "2099-01-01T00:00:00Z", + ], + ]) + } + return json(200, ["files": files]) + } + + private func put(token: String, _ s: inout State) -> HTTPReply { + guard let target = s.tokens[token], var capture = s.captures[target.capture] else { return HTTPReply(status: 403, body: Data()) } + capture.uploaded.insert(target.path) + s.captures[capture.id] = capture + return HTTPReply(status: 200, body: Data()) + } + + private func commit(_ body: Data, _ c: inout Capture) -> HTTPReply { + guard let request = try? JSONDecoder().decode(CaptureAPI.CommitRequest.self, from: body) else { return error(422, "schema") } + let paths = request.files.map(\.path) + let committed = paths.filter(c.uploaded.contains) + c.committed.formUnion(committed) + return json(200, ["committed": committed, "notFound": paths.filter { !c.uploaded.contains($0) }, "mismatch": [String]()]) + } + + private func finalize(_ body: Data, _ c: inout Capture, _ s: inout State) -> HTTPReply { + guard let packet = try? JSONDecoder().decode(Packet04.Packet.self, from: body) else { return error(422, "schema") } + guard packet.packetId == c.packetID else { return error(422, "packet_id_mismatch") } + if c.runID == nil { + c.listed = packet.files.map(\.path) + c.runID = "run_fixture_\(s.next)" + s.next += 1 + } + let missing = (c.listed ?? []).filter { !c.committed.contains($0) } + return json(202, ["status": missing.isEmpty ? "processing" : "awaiting_files", "missing": missing, "runId": c.runID ?? ""]) + } + + private func events(_ url: URL, _ c: Capture) -> HTTPReply { + let after = URLComponents(url: url, resolvingAgainstBaseURL: false)?.queryItems?.first(where: { $0.name == "after" })?.value.flatMap(Int.init) ?? 0 + guard let runID = c.runID else { return json(200, ["status": "uploading", "next": after, "events": [Any]()]) } + guard c.allCommitted, answer != .hold else { + return json(200, ["status": c.allCommitted ? "processing" : "awaiting_files", "next": after, "events": [Any]()]) + } + let ended = answer.endStatus == "failed" || answer.endStatus == "expired" + let event: [String: Any] = [ + "seq": 1, "type": ended ? "failed" : "verdict_ready", "at": "2026-01-01T00:00:00Z", "data": ["runId": runID], + ] + let list: [[String: Any]] = after < 1 ? [event] : [] + return json(200, ["status": answer.endStatus, "next": 1, "events": list]) + } + + private func result(_ c: Capture) -> HTTPReply { + guard let runID = c.runID, c.allCommitted, answer != .hold else { + return json(200, ["runId": c.runID ?? "", "status": "processing", "viewsNeeded": [Any](), "memberActions": [Any](), "outcome": NSNull()]) + } + switch answer { + case .notReady: return error(404, "result_not_ready") + case .unreadable: return HTTPReply(status: 200, body: Data("not a result".utf8)) + default: break + } + let views: [[String: Any]] = answer == .needsViews ? [[ + "id": "view_fixture_1", "kind": "wall_band", "criteria": [String](), "why": "unseen", "promptId": "prompt_fixture_1", + "prompt": ["title": "Fixture: show the ground below the meter", "body": "Fixture: step back until the ground below the meter is in view."], + ]] : [] + var outcome: Any = NSNull() + if let message = answer.message { + let kind = switch answer { + case .candidate: "eligible" + case .needsViews: "needs_more_photos" + case .manualReview: "manual_review" + default: "not_eligible" + } + var decided: [String: Any] = ["kind": kind, "profile": "fixture", "message": message, "viewsNeeded": [Any](), "reasons": [Any]()] + if answer == .candidate { + // A box in the packet's ARKit world, so AR is withheld for the right reasons + // (analysis nobody verified) rather than for a missing placement. + let identity: [Double] = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, -0.9, 0.6, 0.2, 1] + decided["arkitEpoch"] = Self.arkitEpoch + decided["recommendedPlacement"] = [ + "wallId": "wall_fixture", "startSM": -1.2, "confidence": 0.5, + "boxArkitWorld": ["pose": identity, "size": [0.7, 1.2, 0.3]], + ] + } + outcome = decided + } + return json(200, [ + "runId": runID, "status": answer.endStatus, "viewsNeeded": views, "memberActions": [Any](), "outcome": outcome, + ]) + } + + // MARK: Plumbing + + private func json(_ status: Int, _ object: [String: Any]) -> HTTPReply { + let body = (try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys])) ?? Data() + return HTTPReply(status: status, body: body) + } + + private func error(_ status: Int, _ code: String) -> HTTPReply { + json(status, ["errors": [["code": code]]]) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoCaptureBinding.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoCaptureBinding.swift new file mode 100644 index 000000000..c389e6129 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoCaptureBinding.swift @@ -0,0 +1,72 @@ +import Foundation + +/// What a photo-processing answer has to match before the app shows it, taken when the scan's +/// capture ended. +public struct PhotoCaptureBinding: Sendable, Equatable { + public var spatial: SpatialSession + public var origin: URL + /// The id of the coordinator session whose packet was frozen (`CaptureSessionCoordinator.Session.localID`). + public var captureSessionID: String + public var packetID: String + /// The ARKit epoch name the packet used (`CaptureSessionCoordinator.epoch`). + public var epoch: String + + public init(spatial: SpatialSession, origin: URL, captureSessionID: String, packetID: String, epoch: String) { + self.spatial = spatial + self.origin = origin + self.captureSessionID = captureSessionID + self.packetID = packetID + self.epoch = epoch + } +} + +/// The scan on screen when an answer arrives. Nil fields mean there is nothing of that kind now. +public struct PhotoCaptureObservation: Sendable, Equatable { + public var spatial: SpatialSession? + public var origin: URL? + public var captureSessionID: String? + public var packetID: String? + /// The server's capture and run for the upload now, from its latest status. + public var captureID: String? + public var runID: String? + + public init( + spatial: SpatialSession?, origin: URL?, captureSessionID: String?, packetID: String?, captureID: String?, runID: String? + ) { + self.spatial = spatial + self.origin = origin + self.captureSessionID = captureSessionID + self.packetID = packetID + self.captureID = captureID + self.runID = runID + } +} + +/// The first identity an answer and the scan on screen disagree on. +public enum PhotoBindingMismatch: Sendable, Equatable, CaseIterable { + /// Another ARKit world, or none: an answer for a world that is gone. + case world + /// Another destination than the one the scan's profile sends to. + case origin + /// Another coordinator session than the one whose packet was frozen. + case session + case packet + case capture + case run + case epoch +} + +extension PhotoCaptureBinding { + /// The first thing `record` and the scan on screen disagree on, or nil when the answer is this + /// capture's. + public func mismatch(_ record: CaptureResult.Record, observed: PhotoCaptureObservation) -> PhotoBindingMismatch? { + guard observed.spatial == spatial else { return .world } + guard observed.origin == origin else { return .origin } + guard observed.captureSessionID == captureSessionID, record.association.sessionID == captureSessionID else { return .session } + guard observed.packetID == packetID else { return .packet } + guard let captureID = observed.captureID, record.association.captureID == captureID else { return .capture } + guard let runID = observed.runID, record.association.runID == runID, record.response.runId == runID else { return .run } + guard record.association.epoch == epoch else { return .epoch } + return nil + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift new file mode 100644 index 000000000..3e807ac8e --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift @@ -0,0 +1,264 @@ +import Foundation +import simd + +/// Photo processing for the scan under way. The app's capture events go to the process's one +/// `CaptureSessionCoordinator`, and its status and result come back as one +/// `PhotoProcessingStatus`, checked against the scan they belong to. +/// +/// Each photo-processing scan starts a new coordinator session (`beginScan`), which asks for +/// consent again. Start over, or a Legacy scan starting, ends it (`endScan`); a world reset starts +/// the scan's next packet with the same answer (`worldReset`). The coordinator drops status and +/// answers from a session that ended, and the binding check (`PhotoCaptureBinding`) refuses any +/// answer that isn't for the recording's session, world, packet, capture and run. +/// +/// A capture saved on disk is never resumed from here. Folder ownership lasts only this process, +/// so a relaunch asks for consent again rather than act on a yes it can't prove. +@MainActor +public final class PhotoProcessingController { + public enum Setup: Sendable { + /// Nothing can be sent, for the reason given. + case notSetUp(String) + /// Captures go to `environment`, which must allow sending. `standIn`: a stand-in on the + /// phone answers, such as the DEBUG capture fixture. + case ready(CaptureSessionCoordinator.Environment, standIn: Bool) + } + + private let setup: Setup + private let coordinator: CaptureSessionCoordinator? + private var begun = false + /// The photo-processing scan under way; nil when there is none. + public private(set) var context: ScanContext? + public private(set) var status: PhotoProcessingStatus? { + didSet { if status != oldValue { onChange?(status) } } + } + public var onChange: (@MainActor (PhotoProcessingStatus?) -> Void)? + /// The meter anchor's pose now, for the AR check on an answer. Nil while there is none. + public var meterAnchor: @MainActor () -> simd_float4x4? = { nil } + private var binding: PhotoCaptureBinding? + private var upload: CaptureUploadStatus? + private var captureEnded = false + + public init(setup: Setup) { + if case .ready(let environment, _) = setup, !environment.sends { + self.setup = .notSetUp("the capture endpoint may not receive this scan") + } else { + self.setup = setup + } + if case .ready(let environment, _) = self.setup { + coordinator = CaptureSessionCoordinator(environment: environment) + } else { + coordinator = nil + } + coordinator?.onStatus = { [weak self] in self?.uploadChanged($0) } + coordinator?.onResult = { [weak self] in self?.answered($0) } + } + + /// The profile a photo-processing scan starting now gets. + public var profile: ProcessingProfile { + switch setup { + case .notSetUp(let reason): + ProcessingProfile(backend: .photoProcessing, answers: .notSetUp(reason), origin: nil) + case .ready(let environment, let standIn): + ProcessingProfile(backend: .photoProcessing, answers: standIn ? .standIn : .service, origin: environment.endpoint) + } + } + + /// The current capture's folder on the phone, while a session is open. + public var captureFolder: URL? { coordinator?.session?.folder } + + // MARK: Scan events + + /// A scan that uses photo processing starts, recording into `recording`'s world. + public func beginScan(_ context: ScanContext, recording: RecordingSource) { + precondition(context.profile.backend == .photoProcessing, "a Legacy scan has no photo processing") + // Whatever the last scan left open ends here, whether or not this one can send. + startSession("new scan", recording: recording, newScan: true) + self.context = context + resetCapture() + guard coordinator != nil, context.profile == profile, case .ready(_, let standIn) = setup else { + let reason: String = switch context.profile.answers { + case .notSetUp(let why): why + case .service, .standIn: "the scan's processing setup is not this build's" + } + status = PhotoProcessingStatus(stage: .ended(.notSetUp(reason)), consent: .notNeeded, standIn: false) + return + } + status = PhotoProcessingStatus(stage: .capturing(sent: 0), consent: .asking, standIn: standIn) + } + + /// Start over, or a Legacy scan starting: the photo-processing scan's session ends, and + /// nothing it sent or receives later reaches the screen. `recording` is the next world's. + public func endScan(recording: RecordingSource) { + if begun { coordinator?.newWorld("start over", recording: recording, newScan: true) } + context = nil + resetCapture() + status = nil + } + + /// The scan's ARKit world was thrown away: its packet can't be finished, and the next one is + /// recorded in `next`'s world under the same consent. A scan whose processing already ended + /// (not set up, consent withdrawn) stays ended. + public func worldReset(_ next: ScanContext, recording: RecordingSource) { + guard let current = context, next.scanID == current.scanID, next.profile == current.profile else { return } + context = next + resetCapture() + guard var state = status, !state.isFinal else { return } + startSession("world reset", recording: recording, newScan: false) + state.stage = .capturing(sent: 0) + state.retrying = false + status = state + } + + public func answerConsent(_ yes: Bool) { + guard context != nil, let coordinator, var state = status, state.consent == .asking else { return } + // Before any yes there is no upload to withdraw, so a no can't fail here. + coordinator.answerConsent(yes) + state.consent = yes ? .granted : .declined + status = state + } + + /// A no after a yes: nothing more from this scan is sent, and the scan can't be processed + /// again. The stage says whether the phone saved the no. + public func stopSending() { + guard let coordinator, var state = status, state.consent == .granted, !state.isFinal else { return } + let recorded = switch coordinator.answerConsent(false) { + case .success: true + case .failure: false + } + state.consent = .withdrawn(recorded: recorded) + state.stage = .ended(.withdrawn(recorded: recorded)) + state.retrying = false + status = state + } + + public func kept(_ photo: KeptPhoto) { + guard acceptsCapture else { return } + coordinator?.kept(photo) + } + + public func meterTapped(_ tap: TapObservation, hit: Packet04.TapHit?) { + guard acceptsCapture else { return } + coordinator?.meterTapped(tap, hit: hit) + } + + /// The scan was sent: the packet is frozen and processing starts, if the homeowner said yes. + /// `acceptedCloseUpAt` is the frame time of the close-up the scan accepted, nil after a skip. + public func captureEnded(acceptedCloseUpAt: Double?) { + guard let context, var state = status, !captureEnded, case .capturing = state.stage else { return } + captureEnded = true + switch state.consent { + case .granted: + guard let coordinator, let session = coordinator.session, session.uploader != nil, let origin = profile.origin else { + state.stage = .ended(.notPrepared) + status = state + return + } + binding = PhotoCaptureBinding( + spatial: context.spatial, origin: origin, captureSessionID: session.localID, packetID: session.packetID, + epoch: CaptureSessionCoordinator.epoch) + coordinator.captureEnded(acceptedCloseUpAt: acceptedCloseUpAt) + if let upload { + state.stage = .progress(upload, captureEnded: true) + } else { + state.stage = .queued + } + case .asking, .declined: + state.stage = .ended(.consentNotGiven) + case .withdrawn(let recorded): + state.stage = .ended(.withdrawn(recorded: recorded)) + case .notNeeded: + return + } + status = state + } + + /// Waits until the current session's queued work and uploads are idle. For tests. + public func settle() async { + await coordinator?.settle() + } + + // MARK: Coordinator events + + private var acceptsCapture: Bool { + guard context != nil, !captureEnded, let status, case .capturing = status.stage else { return false } + return status.consent == .asking || status.consent == .granted + } + + private func startSession(_ reason: String, recording: RecordingSource, newScan: Bool) { + guard let coordinator else { return } + if begun { + coordinator.newWorld(reason, recording: recording, newScan: newScan) + } else { + coordinator.begin(recording: recording) + begun = true + } + } + + private func resetCapture() { + binding = nil + upload = nil + captureEnded = false + } + + /// The coordinator reports only its current session; nil means that session ended, and + /// whoever ended it has already set the stage. + private func uploadChanged(_ next: CaptureUploadStatus?) { + guard let next, context != nil else { return } + upload = next + guard var state = status, !state.isFinal else { return } + state.retrying = next.retryingAt != nil + switch next.phase { + case .creating, .uploading, .processing: + state.stage = .progress(next, captureEnded: captureEnded) + case .finished: + // The answer follows through `answered`, once the coordinator has decoded it. + state.stage = .processing + case .abandoned: + // Start over and world resets end the session before its uploader stops, so an + // abandoned upload still current is the packet the phone couldn't finish. + guard captureEnded else { return } + state.stage = .ended(.notPrepared) + state.retrying = false + case .failed: + readRefusal() + return + } + status = state + } + + private func readRefusal() { + guard let session = coordinator?.session, let uploader = session.uploader else { return } + Task { [weak self] in + let end = await uploader.snapshot.end + guard let self, self.coordinator?.session === session, var state = self.status, !state.isFinal, + case .failed(let step, let codes, _)? = end else { return } + state.stage = .refusal(step: step, codes: codes) + state.retrying = false + self.status = state + } + } + + private func answered(_ record: CaptureResult.Record) { + guard let binding, var state = status, !state.isFinal, let session = coordinator?.session else { return } + let observed = PhotoCaptureObservation( + spatial: context?.spatial, origin: profile.origin, captureSessionID: session.localID, packetID: session.packetID, + captureID: upload?.captureID, runID: upload?.runID) + if let mismatch = binding.mismatch(record, observed: observed) { + // An answer for a world that is gone was never this scan's to show or to end. + guard mismatch != .world else { return } + state.stage = .ended(.answerMismatch(mismatch)) + } else { + let world = CaptureResult.Association( + sessionID: session.localID, captureID: upload?.captureID ?? "", runID: upload?.runID ?? "", epoch: binding.epoch) + state.stage = .answer(record, world: world, meterAnchor: meterAnchor(), parent: amendmentParent(record, session: session)) + } + state.retrying = false + status = state + } + + private func amendmentParent(_ record: CaptureResult.Record, session: CaptureSessionCoordinator.Session) -> PhotoAmendmentParent? { + guard let context, let captureID = upload?.captureID, let runID = upload?.runID else { return nil } + let views = record.response.viewsNeeded.isEmpty ? record.response.outcome?.viewsNeeded ?? [] : record.response.viewsNeeded + return PhotoAmendmentParent(captureID: captureID, profile: context.profile, runID: runID, viewIDs: views.map(\.id), folder: session.folder) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift new file mode 100644 index 000000000..0571ffc82 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift @@ -0,0 +1,206 @@ +import Foundation +import simd + +/// Photo processing's side of one scan, as the screens show it: separate facts, never a +/// percentage or a time estimate, and nothing the service didn't say. +public struct PhotoProcessingStatus: Sendable, Equatable { + public enum Consent: Sendable, Equatable { + /// Not answered yet. Nothing is sent before a yes. + case asking + case granted + case declined + /// A no after a yes. `recorded` is false when sending stopped in this process but the phone + /// couldn't save the no (`CaptureSessionCoordinator.ConsentWithdrawalError.notRecorded`). + case withdrawn(recorded: Bool) + /// This scan can't send anything, so nobody is asked. + case notNeeded + } + + public enum Stage: Sendable, Equatable { + /// The scan is still being taken. `sent` counts files the service has acknowledged. + case capturing(sent: Int) + /// The capture ended and the service hasn't opened it yet. + case queued + /// Files still going up: `received` of `kept` acknowledged by the service. + case uploading(received: Int, kept: Int) + /// Every file is in and the service is working on it. + case processing + case answered(PhotoProcessingAnswer) + case ended(PhotoProcessingEnd) + } + + public var stage: Stage + public var consent: Consent + /// A stand-in on the phone answers rather than a service (`ProcessingProfile.Answers.standIn`). + public var standIn: Bool + /// The upload is waiting to try again after a failure a later try may fix. + public var retrying: Bool + + public init(stage: Stage, consent: Consent, standIn: Bool, retrying: Bool = false) { + self.stage = stage + self.consent = consent + self.standIn = standIn + self.retrying = retrying + } + + /// Nothing more will change for this scan. + public var isFinal: Bool { + switch stage { + case .answered, .ended: true + case .capturing, .queued, .uploading, .processing: false + } + } +} + +/// What the service answered. Every answer is provisional: an installer confirms any spot on site. +public enum PhotoProcessingAnswer: Sendable, Equatable { + /// The service proposed a spot ("eligible"), which is still only a candidate. + case candidate(Candidate) + /// The service wants more views. This build can't add photos to a capture. + case needsMorePhotos(NeedsMorePhotos) + /// The service sends the scan to an installer. + case installerReview(message: String) + /// The service found no spot. + case noCandidate(message: String) + /// An outcome this build doesn't know. Its message is shown as written. + case unrecognized(kind: String, message: String) + + public struct Candidate: Sendable, Equatable { + /// The service's words for the homeowner, verbatim. + public var message: String + /// Whether the service made a preview image. This build shows none either way. + public var previewMade: Bool + public var ar: ARAvailability + + public init(message: String, previewMade: Bool, ar: ARAvailability) { + self.message = message + self.previewMade = previewMade + self.ar = ar + } + } + + public struct NeedsMorePhotos: Sendable, Equatable { + /// Each requested view's prompt, verbatim. + public var prompts: [CaptureResult.Prompt] + /// The service's words for the homeowner, when it sent an outcome. + public var message: String? + /// What a follow-up capture would add to; nil when the upload had no capture or run yet. + public var parent: PhotoAmendmentParent? + + public init(prompts: [CaptureResult.Prompt], message: String?, parent: PhotoAmendmentParent?) { + self.prompts = prompts + self.message = message + self.parent = parent + } + } + + /// Whether the spot can be shown on the wall in AR. + public enum ARAvailability: Sendable, Equatable { + /// The result failed a check AR needs (`CaptureResult.Record.placement`), such as analysis + /// nobody verified, or another session, run or epoch than the recording's. + case withheld(CaptureResult.Unavailable) + /// Every check passed, but this build has no AR view for photo-processing results. + case notShownInThisBuild + } +} + +/// How photo processing ended without an answer to show. +public enum PhotoProcessingEnd: Sendable, Equatable { + /// This build can't send the scan for photo processing, for the reason given. Nothing was sent. + case notSetUp(String) + /// The homeowner didn't agree to send the photos, so none went. + case consentNotGiven + /// The homeowner stopped sending part way. `recorded`: whether the phone saved that. + case withdrawn(recorded: Bool) + /// The phone couldn't build a capture the service can take: the packet failed its checks on + /// the phone, or no photo was kept. + case notPrepared + /// The service's run failed. + case processingFailed + /// The capture expired before it finished. + case expired + /// The service refused a request at `step`, which sending the same thing again won't change. + case refused(step: String) + /// The service answered with something this build can't read as a result. + case answerUnreadable + /// The run finished, but its answer wasn't readable before the upload stopped asking. + case answerNotReady + /// The answer belongs to another capture than this scan's, so it isn't shown. + case answerMismatch(PhotoBindingMismatch) +} + +/// What a follow-up capture would carry from the capture it adds views to: plain values only. +/// Nothing in this build sends a follow-up, so a request for more photos ends saying so; an +/// amendment client starts from this. The follow-up keeps its own session and epoch: nothing here +/// joins its poses to the parent's. +public struct PhotoAmendmentParent: Sendable, Equatable { + /// The server's id for the parent capture. + public var captureID: String + /// Where the parent went: the scan's profile, which a follow-up must keep. + public var profile: ProcessingProfile + /// The finished run that asked for more. + public var runID: String + /// The ids of the views the answer asked for (`CaptureResult.ViewNeeded.id`). + public var viewIDs: [String] + /// The parent's capture folder. One uploader at a time owns a folder (`CaptureUploader`'s + /// registry), so a follow-up has to take it through that rather than around it. + public var folder: URL + + public init(captureID: String, profile: ProcessingProfile, runID: String, viewIDs: [String], folder: URL) { + self.captureID = captureID + self.profile = profile + self.runID = runID + self.viewIDs = viewIDs + self.folder = folder + } +} + +extension PhotoProcessingStatus.Stage { + /// The stage for a decoded answer already bound to the scan. `world` and `meterAnchor` are the + /// scan on screen now, for the AR check; `parent` is what a follow-up would add to. + public static func answer( + _ record: CaptureResult.Record, world: CaptureResult.Association, meterAnchor: simd_float4x4?, + parent: PhotoAmendmentParent? + ) -> Self { + let summary = CaptureResultSummary(record, world: world, meterAnchor: meterAnchor) + let message = summary.message ?? "" + switch summary.state { + case .decided(.eligible): + let ar: PhotoProcessingAnswer.ARAvailability = summary.arUnavailable.map { .withheld($0) } ?? .notShownInThisBuild + return .answered(.candidate(.init(message: message, previewMade: record.response.previewUrl != nil, ar: ar))) + case .decided(.needsMorePhotos), .working(.needsViews): + return .answered(.needsMorePhotos(.init(prompts: summary.prompts, message: summary.message, parent: parent))) + case .decided(.manualReview), .working(.manualReview): + return .answered(.installerReview(message: message)) + case .decided(.notEligible): + return .answered(.noCandidate(message: message)) + case .decided(.unknown(let kind)): + return .answered(.unrecognized(kind: kind, message: message)) + case .ended(.expired): + return .ended(.expired) + case .ended: + return .ended(.processingFailed) + case .working: + // The uploader only keeps an answer once it has an outcome or the capture failed or + // expired, so this is an answer without one: unreadable as a result. + return .ended(.answerUnreadable) + } + } + + /// The stage for an upload that stopped on a refusal (`CaptureUploadState.End.failed`). + public static func refusal(step: String, codes: [String]) -> Self { + guard step == "result" else { return .ended(.refused(step: step)) } + if codes.contains("result_not_ready") { return .ended(.answerNotReady) } + if codes.contains("result_for_another_run") { return .ended(.answerMismatch(.run)) } + if codes.contains("result_unreadable") { return .ended(.answerUnreadable) } + return .ended(.refused(step: step)) + } + + /// Where an upload in progress stands. Before the capture ends, only how much was sent. + public static func progress(_ upload: CaptureUploadStatus, captureEnded: Bool) -> Self { + guard captureEnded else { return .capturing(sent: upload.committed) } + if upload.phase == .creating { return .queued } + if upload.committed < upload.retained { return .uploading(received: upload.committed, kept: upload.retained) } + return upload.phase == .processing ? .processing : .uploading(received: upload.committed, kept: upload.retained) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/ProcessingBackend.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/ProcessingBackend.swift new file mode 100644 index 000000000..2a26417ed --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/ProcessingBackend.swift @@ -0,0 +1,61 @@ +import Foundation + +/// The remote service that decides where a scan's battery could go. Both run off the phone: +/// Legacy sends scene.json to the placement checker, and photo processing sends the scan's photos +/// and camera path to the capture API. Neither decides on the phone. +public enum ProcessingBackend: String, Sendable, Equatable, Hashable, CaseIterable, Codable { + case legacy + case photoProcessing + + /// What a scan uses when nobody chose. Legacy stays the default until photo processing passes + /// a real end-to-end acceptance; until then the beta offers it as a choice. + public static let operationalDefault: ProcessingBackend = .legacy +} + +/// The developer setting that picks the next scan's backend, in UserDefaults. A scan reads it once, +/// when it starts (`ScanContext`), so a change applies to the next scan. +public enum ProcessingBackendPreference { + public static let key = "processingBackend" + + /// The stored choice. Nothing stored, or a value this build doesn't know, reads as the + /// operational default rather than guessing. + public static func read(_ defaults: UserDefaults) -> ProcessingBackend { + defaults.string(forKey: key).flatMap(ProcessingBackend.init(rawValue:)) ?? .operationalDefault + } + + public static func write(_ backend: ProcessingBackend, to defaults: UserDefaults) { + defaults.set(backend.rawValue, forKey: key) + } +} + +/// The backend a scan uses, as it stood when the scan started: which one, who answers, where the +/// scan's data may go, and whose credential it may present. The scan keeps it to its end, so a +/// later change to the setting or the endpoint can't send an existing capture somewhere else. +public struct ProcessingProfile: Sendable, Equatable, Hashable { + public enum Answers: Sendable, Equatable, Hashable { + /// A service off the phone. + case service + /// A stand-in on this phone: the bundled sample result, or the DEBUG capture fixture. + /// Nothing leaves the phone. + case standIn + /// Nobody: this build has no usable setup for the backend, for the reason given. Nothing + /// is sent. + case notSetUp(String) + } + + public var backend: ProcessingBackend + public var answers: Answers + /// The API base this scan's data may go to. Nil when it goes nowhere. + public var origin: URL? + /// Who issues the scoped credential this scan may present, never the credential itself. Nil + /// when the scan presents none. A token refreshed from the same issuer leaves the profile as + /// it is. + public var credentialIssuer: String? + + public init(backend: ProcessingBackend, answers: Answers, origin: URL?, credentialIssuer: String? = nil) { + self.backend = backend + self.answers = answers + self.origin = origin + self.credentialIssuer = credentialIssuer + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/ScanContext.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/ScanContext.swift new file mode 100644 index 000000000..fbea1dee7 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/ScanContext.swift @@ -0,0 +1,41 @@ +import Foundation + +/// One ARKit world within a scan. Results and AR bind to this, not to the scan's id: a world reset +/// keeps the scan but makes every pose and answer from the old world meaningless. +public struct SpatialSession: Sendable, Equatable, Hashable { + /// The packet recorder's session id, which it makes new for every world. + public var recordingSessionID: String + /// Worlds this scan threw away before this one: 0 for its first. + public var worldEpoch: Int + + public init(recordingSessionID: String, worldEpoch: Int) { + self.recordingSessionID = recordingSessionID + self.worldEpoch = worldEpoch + } +} + +/// What a scan is, fixed when it starts: its identity, its backend profile and its current world. +/// +/// The engine's work counter isn't an identity: it moves on every reset, inside a scan as well as +/// between scans, so it can't say which scan a capture or an answer belongs to. This can. +public struct ScanContext: Sendable, Equatable { + /// The scan's id, which the app takes from its scan folder's name. Start over makes a new scan + /// and so a new id; a world reset keeps it. + public let scanID: String + public let profile: ProcessingProfile + public private(set) var spatial: SpatialSession + + public init(scanID: String, profile: ProcessingProfile, recordingSessionID: String) { + self.scanID = scanID + self.profile = profile + spatial = SpatialSession(recordingSessionID: recordingSessionID, worldEpoch: 0) + } + + /// The same scan in a new ARKit world: the id and the backend stay, the spatial session is + /// new. + public func inNewWorld(recordingSessionID: String) -> ScanContext { + var next = self + next.spatial = SpatialSession(recordingSessionID: recordingSessionID, worldEpoch: spatial.worldEpoch + 1) + return next + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift new file mode 100644 index 000000000..861ddadeb --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift @@ -0,0 +1,380 @@ +import Foundation +import HouseScanKit +import Synchronization +import Testing + +@Suite struct ScanContextTests { + let profile = ProcessingProfile(backend: .photoProcessing, answers: .standIn, origin: FixtureCaptureHTTP.base) + + @Test func aWorldResetKeepsTheScanAndItsProfileButStartsANewSpatialSession() { + let first = ScanContext(scanID: "scan-a", profile: profile, recordingSessionID: "rec-1") + let second = first.inNewWorld(recordingSessionID: "rec-2") + #expect(second.scanID == first.scanID) + #expect(second.profile == first.profile) + #expect(first.spatial == SpatialSession(recordingSessionID: "rec-1", worldEpoch: 0)) + #expect(second.spatial == SpatialSession(recordingSessionID: "rec-2", worldEpoch: 1)) + #expect(second.inNewWorld(recordingSessionID: "rec-3").spatial.worldEpoch == 2) + } + + @Test func theStoredChoiceReadsBackAndAnUnknownOneReadsAsTheDefault() throws { + let suite = "processing-backend-\(UUID().uuidString)" + let defaults = try #require(UserDefaults(suiteName: suite)) + defer { defaults.removePersistentDomain(forName: suite) } + #expect(ProcessingBackendPreference.read(defaults) == .legacy) + ProcessingBackendPreference.write(.photoProcessing, to: defaults) + #expect(ProcessingBackendPreference.read(defaults) == .photoProcessing) + defaults.set("onDevice", forKey: ProcessingBackendPreference.key) + #expect(ProcessingBackendPreference.read(defaults) == .legacy) + } +} + +@Suite struct PhotoCaptureBindingTests { + static let world = SpatialSession(recordingSessionID: "rec-1", worldEpoch: 0) + let binding = PhotoCaptureBinding( + spatial: world, origin: FixtureCaptureHTTP.base, captureSessionID: "session-1", packetID: "packet-1", epoch: "e1") + let matching = PhotoCaptureObservation( + spatial: world, origin: FixtureCaptureHTTP.base, captureSessionID: "session-1", packetID: "packet-1", captureID: "cap_1", + runID: "run_1") + + func record( + sessionID: String = "session-1", captureID: String = "cap_1", runID: String = "run_1", responseRun: String = "run_1", epoch: String = "e1" + ) throws -> CaptureResult.Record { + let body = """ + {"runId": "\(responseRun)", "status": "manual_review", "viewsNeeded": [], "memberActions": [], + "outcome": {"kind": "manual_review", "profile": "p", "message": "m", "viewsNeeded": [], "reasons": []}} + """ + let response = try JSONDecoder().decode(CaptureResult.Response.self, from: Data(body.utf8)) + return CaptureResult.Record(response: response, association: .init(sessionID: sessionID, captureID: captureID, runID: runID, epoch: epoch)) + } + + @Test func theRecordingsOwnAnswerIsAccepted() throws { + #expect(binding.mismatch(try record(), observed: matching) == nil) + } + + @Test func anAnswerFromAnotherWorldIsRefusedWhateverElseMatches() throws { + var observed = matching + observed.spatial = SpatialSession(recordingSessionID: "rec-2", worldEpoch: 1) + #expect(binding.mismatch(try record(), observed: observed) == .world) + observed.spatial = SpatialSession(recordingSessionID: "rec-1", worldEpoch: 1) + #expect(binding.mismatch(try record(), observed: observed) == .world) + observed.spatial = nil + #expect(binding.mismatch(try record(), observed: observed) == .world) + } + + @Test func anAnswerForAnotherDestinationSessionPacketCaptureRunOrEpochIsRefused() throws { + var observed = matching + observed.origin = URL(string: "https://elsewhere.invalid/v1") + #expect(binding.mismatch(try record(), observed: observed) == .origin) + observed = matching + observed.captureSessionID = "session-2" + #expect(binding.mismatch(try record(), observed: observed) == .session) + #expect(binding.mismatch(try record(sessionID: "session-2"), observed: matching) == .session) + observed = matching + observed.packetID = "packet-2" + #expect(binding.mismatch(try record(), observed: observed) == .packet) + #expect(binding.mismatch(try record(captureID: "cap_2"), observed: matching) == .capture) + observed = matching + observed.captureID = nil + #expect(binding.mismatch(try record(), observed: observed) == .capture) + #expect(binding.mismatch(try record(runID: "run_2"), observed: matching) == .run) + #expect(binding.mismatch(try record(responseRun: "run_2"), observed: matching) == .run) + #expect(binding.mismatch(try record(epoch: "e2"), observed: matching) == .epoch) + } + + @Test func aRefusalReadsAsTheStageItEndsIn() { + typealias Stage = PhotoProcessingStatus.Stage + #expect(Stage.refusal(step: "create", codes: ["schema"]) == .ended(.refused(step: "create"))) + #expect(Stage.refusal(step: "put", codes: ["storage_refused"]) == .ended(.refused(step: "put"))) + #expect(Stage.refusal(step: "result", codes: ["result_not_ready"]) == .ended(.answerNotReady)) + #expect(Stage.refusal(step: "result", codes: ["result_unreadable"]) == .ended(.answerUnreadable)) + #expect(Stage.refusal(step: "result", codes: ["result_for_another_run"]) == .ended(.answerMismatch(.run))) + #expect(Stage.refusal(step: "result", codes: []) == .ended(.refused(step: "result"))) + } +} + +/// `FixtureCaptureHTTP` with every request to one route held until `release`, to deliver an +/// answer after the scan it was for has gone. +final class HeldResultHTTP: CaptureHTTP, Sendable { + private struct Held { + var open = false + var waiting: [CheckedContinuation] = [] + } + + let inner: FixtureCaptureHTTP + private let held = Mutex(Held()) + + init(_ inner: FixtureCaptureHTTP) { + self.inner = inner + } + + var waiting: Int { held.withLock { $0.waiting.count } } + + func send(_ request: URLRequest) async throws -> HTTPReply { + if request.httpMethod == "GET", request.url?.lastPathComponent == "result" { + await withCheckedContinuation { (continuation: CheckedContinuation) in + let open = held.withLock { state -> Bool in + if state.open { return true } + state.waiting.append(continuation) + return false + } + if open { continuation.resume() } + } + } + return try await inner.send(request) + } + + func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + try await inner.upload(request, file: file) + } + + func release() { + let waiting = held.withLock { state -> [CheckedContinuation] in + state.open = true + defer { state.waiting = [] } + return state.waiting + } + waiting.forEach { $0.resume() } + } +} + +@Suite(.serialized) @MainActor struct PhotoProcessingControllerTests { + let root = FileManager.default.temporaryDirectory.appending(path: "photo-processing-\(UUID().uuidString)") + var fixture: NativeCaptureFixture { NativeCaptureFixture(folder: root.appending(path: "store")) } + + func controller(_ http: any CaptureHTTP) -> PhotoProcessingController { + var environment = NativeCaptureFixture.environment(endpoint: FixtureCaptureHTTP.base, http: http, captures: root.appending(path: "Captures")) + // Retries and unreadable answers come round in milliseconds, not the app's seconds. + environment.policy.firstDelay = 0.01 + environment.policy.maxDelay = 0.02 + return PhotoProcessingController(setup: .ready(environment, standIn: true)) + } + + func context(_ controller: PhotoProcessingController, scan: String = "scan-1", recording: String = "rec-1") -> ScanContext { + ScanContext(scanID: scan, profile: controller.profile, recordingSessionID: recording) + } + + /// The scan from the meter tap to the send: the tap, the close-up and five walk photos. + func capture(_ controller: PhotoProcessingController, consent: Bool?) async throws { + if let consent { controller.answerConsent(consent) } + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + controller.meterTapped(tap, hit: hit) + controller.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + for i in 0..<5 { controller.kept(try fixture.photo(at: fixture.start + 3 + Double(i) * 1.5)) } + await controller.settle() + controller.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + } + + func until(_ seconds: Double = 30, _ condition: () -> Bool) async throws -> Bool { + let deadline = ContinuousClock.now + .seconds(seconds) + while !condition(), ContinuousClock.now < deadline { try await Task.sleep(for: .milliseconds(20)) } + return condition() + } + + @Test func theFixtureWritesTheCoordinatorsEpoch() { + #expect(FixtureCaptureHTTP.arkitEpoch == CaptureSessionCoordinator.epoch) + } + + @Test(arguments: [ + FixtureCaptureHTTP.Answer.candidate, .needsViews, .manualReview, .notEligible, .failed, .expired, .notReady, .unreadable, + ]) + func eachAnswerReachesTheScanAsItsTypedStage(answer: FixtureCaptureHTTP.Answer) async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: answer) + let controller = controller(http) + controller.beginScan(context(controller), recording: fixture.recording) + try await capture(controller, consent: true) + try #require(try await until { controller.status?.isFinal == true }, "no final stage: \(String(describing: controller.status))") + let stage = try #require(controller.status?.stage) + let message = answer.message ?? "" + switch answer { + case .candidate: + // Proposed, never confirmed: AR stays withheld because nobody verified the analysis. + #expect(stage == .answered(.candidate(.init(message: message, previewMade: false, ar: .withheld(.analysisUnverified))))) + case .needsViews: + guard case .answered(.needsMorePhotos(let more)) = stage else { + Issue.record("expected more photos, got \(stage)") + return + } + #expect(more.prompts.map(\.title) == ["Fixture: show the ground below the meter"]) + #expect(more.message == message) + let parent = try #require(more.parent) + #expect(parent.viewIDs == ["view_fixture_1"]) + #expect(parent.profile == controller.profile) + #expect(parent.captureID.hasPrefix("cap_fixture_")) + #expect(parent.runID.hasPrefix("run_fixture_")) + #expect(parent.folder == controller.captureFolder) + case .manualReview: + #expect(stage == .answered(.installerReview(message: message))) + case .notEligible: + #expect(stage == .answered(.noCandidate(message: message))) + case .failed: + #expect(stage == .ended(.processingFailed)) + case .expired: + #expect(stage == .ended(.expired)) + case .notReady: + #expect(stage == .ended(.answerNotReady)) + case .unreadable: + #expect(stage == .ended(.answerUnreadable)) + case .refuseCreate, .hold: + Issue.record("not an argument of this test") + } + #expect(controller.status?.consent == .granted) + #expect(controller.status?.standIn == true) + #expect(http.routes.contains("POST captures/finalize")) + controller.endScan(recording: fixture.recording) + } + + @Test func aRefusedCaptureEndsWithTheStepTheServiceRefused() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .refuseCreate) + let controller = controller(http) + controller.beginScan(context(controller), recording: fixture.recording) + try await capture(controller, consent: true) + try #require(try await until { controller.status?.isFinal == true }) + #expect(controller.status?.stage == .ended(.refused(step: "create"))) + #expect(!http.routes.contains("POST captures/files")) + controller.endScan(recording: fixture.recording) + } + + @Test(arguments: [false, nil] as [Bool?]) + func withoutAYesNothingIsSent(consent: Bool?) async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .candidate) + let controller = controller(http) + controller.beginScan(context(controller), recording: fixture.recording) + #expect(controller.status?.consent == .asking) + try await capture(controller, consent: consent) + #expect(controller.status?.stage == .ended(.consentNotGiven)) + let answered: PhotoProcessingStatus.Consent = consent == false ? .declined : .asking + #expect(controller.status?.consent == answered) + try await Task.sleep(for: .milliseconds(300)) + #expect(http.routes.isEmpty) + controller.endScan(recording: fixture.recording) + } + + @Test(arguments: [true, false]) + func stoppingPartWayStopsSendingAndSaysWhetherThePhoneSavedIt(saved: Bool) async throws { + let http = FixtureCaptureHTTP(answer: .hold) + let controller = controller(http) + controller.beginScan(context(controller), recording: fixture.recording) + try await capture(controller, consent: true) + try #require(try await until { controller.status?.stage == .processing }) + let folder = try #require(controller.captureFolder) + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) + try? FileManager.default.removeItem(at: root) + } + // Neither the withdrawal marker nor the saved state's removal can be written. + if !saved { try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) } + controller.stopSending() + #expect(controller.status?.stage == .ended(.withdrawn(recorded: saved))) + #expect(controller.status?.consent == .withdrawn(recorded: saved)) + let sent = http.routes.count + // Longer than the uploader waits between polls of a service that answers at once. + try await Task.sleep(for: .seconds(2.5)) + #expect(http.routes.count == sent) + // A second yes doesn't revive the capture. + controller.answerConsent(true) + #expect(controller.status?.stage == .ended(.withdrawn(recorded: saved))) + controller.endScan(recording: fixture.recording) + } + + @Test func aLateAnswerAfterStartOverNeverReachesTheNextScan() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = HeldResultHTTP(FixtureCaptureHTTP(answer: .candidate)) + let controller = controller(http) + controller.beginScan(context(controller, scan: "scan-1"), recording: fixture.recording) + try await capture(controller, consent: true) + try #require(try await until { http.waiting > 0 }, "the result was never asked for") + controller.endScan(recording: fixture.recording) + #expect(controller.status == nil) + let next = context(controller, scan: "scan-2", recording: "rec-2") + controller.beginScan(next, recording: fixture.recording) + http.release() + try await Task.sleep(for: .milliseconds(500)) + #expect(controller.context == next) + #expect(controller.status == PhotoProcessingStatus(stage: .capturing(sent: 0), consent: .asking, standIn: true)) + #expect(http.inner.routes.filter { $0 == "POST captures" }.count == 1) + controller.endScan(recording: fixture.recording) + } + + @Test func aWorldResetSendsTheNextPacketUnderTheSameYesAndBindsItsAnswerToTheNewWorld() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .manualReview) + let controller = controller(http) + let first = context(controller) + controller.beginScan(first, recording: fixture.recording) + controller.answerConsent(true) + controller.kept(try fixture.photo(at: fixture.start + 3)) + await controller.settle() + let firstFolder = controller.captureFolder + let next = first.inNewWorld(recordingSessionID: "rec-2") + controller.worldReset(next, recording: fixture.recording) + #expect(controller.status?.consent == .granted) + #expect(controller.status?.stage == .capturing(sent: 0)) + #expect(controller.captureFolder != firstFolder) + try await capture(controller, consent: nil) + try #require(try await until { controller.status?.isFinal == true }) + #expect(controller.status?.stage == .answered(.installerReview(message: FixtureCaptureHTTP.Answer.manualReview.message ?? ""))) + #expect(controller.context?.spatial == next.spatial) + #expect(controller.context?.scanID == first.scanID) + #expect(http.routes.filter { $0 == "POST captures" }.count == 2) + controller.endScan(recording: fixture.recording) + } + + @Test func theScansUploaderOwnsItsFolderAgainstAnyOtherStartOrResume() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .hold) + let controller = controller(http) + controller.beginScan(context(controller), recording: fixture.recording) + controller.answerConsent(true) + controller.kept(try fixture.photo(at: fixture.start + 3)) + await controller.settle() + let folder = try #require(controller.captureFolder) + let create = CaptureAPI.CreateRequest(packetId: "other", tier: .arkit, device: .init(model: "m", systemVersion: "s", appVersion: "a")) + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned) { + _ = try CaptureUploader.start(folder: folder, base: FixtureCaptureHTTP.base, http: http, create: create, consentedAt: Date()) + } + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned) { + _ = try CaptureUploader.resume(folder: folder, base: FixtureCaptureHTTP.base, http: http) + } + controller.endScan(recording: fixture.recording) + } + + @Test func aBuildWithoutSetupSaysSoAndSendsNothing() throws { + let controller = PhotoProcessingController(setup: .notSetUp("live sending is off in this build")) + #expect(controller.profile.answers == .notSetUp("live sending is off in this build")) + #expect(controller.profile.origin == nil) + controller.beginScan(context(controller), recording: fixture.recording) + let ended = PhotoProcessingStatus(stage: .ended(.notSetUp("live sending is off in this build")), consent: .notNeeded, standIn: false) + #expect(controller.status == ended) + controller.answerConsent(true) + controller.captureEnded(acceptedCloseUpAt: nil) + #expect(controller.status == ended) + #expect(controller.captureFolder == nil) + } + + @Test func anEndpointThatMayNotReceiveTheScanIsNotSetUp() { + var environment = NativeCaptureFixture.environment( + endpoint: FixtureCaptureHTTP.base, http: FixtureCaptureHTTP(answer: .candidate), captures: root.appending(path: "Captures")) + environment.sends = false + let controller = PhotoProcessingController(setup: .ready(environment, standIn: true)) + guard case .notSetUp = controller.profile.answers else { + Issue.record("a record-only endpoint was offered as set up") + return + } + } + + @Test func aScanWhoseProfileIsNotThisBuildsSendsNothing() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .candidate) + let controller = controller(http) + let elsewhere = ProcessingProfile(backend: .photoProcessing, answers: .service, origin: URL(string: "https://elsewhere.invalid/v1")) + controller.beginScan(ScanContext(scanID: "scan-1", profile: elsewhere, recordingSessionID: "rec-1"), recording: fixture.recording) + #expect(controller.status?.stage == .ended(.notSetUp("the scan's processing setup is not this build's"))) + try await capture(controller, consent: true) + try await Task.sleep(for: .milliseconds(300)) + #expect(http.routes.isEmpty) + controller.endScan(recording: fixture.recording) + } +} From 7f52e096601f6f1ac9d7677859e9b6b5124857e9 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 01:21:18 -0500 Subject: [PATCH 26/71] Route photo-processing scans through their own engine path and screen A scan now starts with a ScanContext: the scan folder's id, the backend profile read once from Developer options (an App Store install always gets Legacy), and the recorder's session id with a world epoch. A world reset keeps the scan and its backend and starts a new spatial session; Start over ends the scan. startUpload sends only to the scan's own backend: Legacy keeps its scene upload, gap requests, spot check and result unchanged, and a photo-processing scan goes to STATE=processing, never to the Legacy checker, including on a ground change. Photo processing runs through HouseScanKit's PhotoProcessingController: kept photos and stills reach it from KeyframeStore.onKept, consent is asked over the camera before anything is sent (PhotoConsentCard), the capture is frozen after pending keyframe writes finish, and PhotoProcessingScreen shows the typed progress, answer or end, quoting the service's words. A possible spot stays provisional, AR is withheld, needs-more-photos says adding photos isn't supported yet, and "Stop sending photos" asks first and says whether the phone saved the no. Sending a capture off the phone is off in every build; the only way to run it is DEBUG's -photoProcessingFixture on a replay, and Developer options say when it isn't set up. Nothing resumes a saved capture. Developer options gain a Processing choice (Legacy (remote checker), Photo processing (beta)), also reachable from the processing screen, with a footer naming the scan's own backend. Demo mode renders every photo state for screenshots. Validated: nothing here was compiled; app builds and Simulator runs are left to hosted CI. Two read-only reviews checked declarations, phase switches and isolation by source; their findings (pending writes before the freeze, the demo's Legacy route, test-side isolation) are fixed. Package tests are unaffected by this commit. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/Contract/ScanContract.swift | 14 ++ ios/HouseScan/Runtime/Autopilot.swift | 22 +- ios/HouseScan/Runtime/CaptureRecorder.swift | 5 + ios/HouseScan/Runtime/KeyframeStore.swift | 5 + ios/HouseScan/Runtime/LaunchOptions.swift | 22 ++ .../Practice/ProcessingBackendSetting.swift | 37 +++ .../Runtime/ScanEngine+Actions.swift | 1 + .../Runtime/ScanEngine+GuidanceLog.swift | 2 +- .../Runtime/ScanEngine+Processing.swift | 176 +++++++++++++++ ios/HouseScan/Runtime/ScanEngine.swift | 41 +++- .../UI/Components/PhotoConsentCard.swift | 59 +++++ ios/HouseScan/UI/Copy/ProcessingCopy.swift | 204 +++++++++++++++++ .../UI/Practice/DeveloperOptions.swift | 101 ++++++++- .../Preview/DemoEngine+PhotoProcessing.swift | 74 ++++++ ios/HouseScan/UI/Preview/DemoEngine.swift | 12 + ios/HouseScan/UI/Preview/UIDemo.swift | 4 + ios/HouseScan/UI/ScanRootView.swift | 16 +- .../UI/Screens/PhotoProcessingScreen.swift | 210 ++++++++++++++++++ 18 files changed, 985 insertions(+), 20 deletions(-) create mode 100644 ios/HouseScan/Runtime/Practice/ProcessingBackendSetting.swift create mode 100644 ios/HouseScan/Runtime/ScanEngine+Processing.swift create mode 100644 ios/HouseScan/UI/Components/PhotoConsentCard.swift create mode 100644 ios/HouseScan/UI/Copy/ProcessingCopy.swift create mode 100644 ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift create mode 100644 ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift diff --git a/ios/HouseScan/Contract/ScanContract.swift b/ios/HouseScan/Contract/ScanContract.swift index cd3ffdfdb..951e04a92 100644 --- a/ios/HouseScan/Contract/ScanContract.swift +++ b/ios/HouseScan/Contract/ScanContract.swift @@ -1,5 +1,6 @@ import CoreGraphics import Foundation +import HouseScanKit import Observation import simd import SwiftUI @@ -33,6 +34,9 @@ enum ScanPhase: String, Sendable, CaseIterable { case spotConfirm case result case resultAR + /// A photo-processing scan after it was sent: its progress, then its answer + /// (`ScanViewState.photoProcessing`). Legacy scans never come here. + case processing case unsupported } @@ -965,6 +969,11 @@ final class ScanViewState { var shareableScan: URL? /// A camera permission or session failure the homeowner can act on. var failure: ScanFailure? + /// The backend this scan uses, fixed when it started; nil before a scan starts. + var scanBackend: ProcessingBackend? + /// Photo processing's side of this scan: its consent, progress and answer. Nil on a Legacy + /// scan, and before a scan starts. + var photoProcessing: PhotoProcessingStatus? init() {} } @@ -1050,6 +1059,11 @@ protocol ScanActions: AnyObject { func showAR() func closeAR() func startOver() + /// The homeowner's answer to sending this scan's photos for processing + /// (`PhotoProcessingStatus.Consent.asking`). + func answerPhotoConsent(_ yes: Bool) + /// A no after a yes: this scan's photos stop going to photo processing. + func stopSendingPhotos() /// The app came back to the foreground on the camera-access failure: if access is now on, /// the scan goes on without Start over. Does nothing otherwise. func recheckCameraAccess() diff --git a/ios/HouseScan/Runtime/Autopilot.swift b/ios/HouseScan/Runtime/Autopilot.swift index eb6809c0a..ca718cbf6 100644 --- a/ios/HouseScan/Runtime/Autopilot.swift +++ b/ios/HouseScan/Runtime/Autopilot.swift @@ -47,6 +47,12 @@ final class Autopilot { await pause(hold) await engine.waitForGate(.findMeter) + // Sending photos is the homeowner's answer, never the autopilot's: a UI test taps it. + if engine.state.photoProcessing?.consent == .asking { + guard await waitUntil(timeout: 120, { self.engine.state.photoProcessing?.consent != .asking }) else { + return fail("nobody answered whether to send the scan's photos") + } + } engine.markMeter(at: nil, viewSize: viewSize) guard await waitFor(.meterCloseUp, timeout: 10) else { return fail("meter was not marked") } await takeCloseUp() @@ -90,7 +96,10 @@ final class Autopilot { await playGapFrames(replay) } } - guard await waitFor(.uploading, timeout: 150) else { return fail("upload did not start") } + guard await waitUntil(timeout: 150, { [.uploading, .processing].contains(self.engine.state.phase) }) else { + return fail("upload did not start") + } + if engine.state.phase == .processing { return await finishPhotoProcessing() } guard await driveToResult(replay) else { return } writeSceneForTest() await pause(hold) @@ -373,6 +382,17 @@ final class Autopilot { log("answered the overhead request: open sky; \(overheadSummary)") } + /// A photo-processing scan has no Legacy steps after the send: wait for its answer or its end, + /// then hold the screen for the UI test. + private func finishPhotoProcessing() async { + guard await waitUntil(timeout: 240, { self.engine.state.photoProcessing?.isFinal == true }) else { + return fail("photo processing never ended") + } + await pause(hold) + await engine.waitForGate(.processing) + log("done") + } + /// After an upload the engine raises the answer's capturable requests one at a time, uploading /// after each, before it shows the result, and checks the answer's spot with the homeowner. /// Drives each request and check, retries a failed upload once, and returns true once the diff --git a/ios/HouseScan/Runtime/CaptureRecorder.swift b/ios/HouseScan/Runtime/CaptureRecorder.swift index e5c7d4ec5..734676e9d 100644 --- a/ios/HouseScan/Runtime/CaptureRecorder.swift +++ b/ios/HouseScan/Runtime/CaptureRecorder.swift @@ -210,6 +210,11 @@ final class CaptureRecorder: Sendable { ) } + /// The id of the session recording now, which `restart` makes new for every world. + var sessionID: String { + state.withLock { $0.sessionID } + } + /// Writes every buffered row and returns the session as it stands. func flush() -> Snapshot { state.withLock { state in diff --git a/ios/HouseScan/Runtime/KeyframeStore.swift b/ios/HouseScan/Runtime/KeyframeStore.swift index 957bbc289..584b84804 100644 --- a/ios/HouseScan/Runtime/KeyframeStore.swift +++ b/ios/HouseScan/Runtime/KeyframeStore.swift @@ -50,6 +50,9 @@ final class KeyframeStore { private var nextIndex = 1 /// Bumped by `discardKeyframes`, so a write that started before it doesn't land in the list. private var epoch = 0 + /// Hears each photo kept, once its JPEG is on disk: the photo, the still's purpose (nil for a + /// keyframe) and the JPEG. Photo processing seals it into its capture from here. + var onKept: (@MainActor (_ photo: StoredKeyframe, _ purpose: String?, _ jpeg: URL) -> Void)? /// Makes a new, empty scan folder and deletes the old ones `ScanFolderCleanup` lists: it keeps /// the two most recent completed scans (Saved scans offers them) and any folder whose bundle @@ -118,6 +121,7 @@ final class KeyframeStore { } keyframes.append(stored) keyframes.sort { $0.id < $1.id } + onKept?(stored, nil, directory.appending(path: stored.fileName)) return (true, thumbnail) } @@ -216,6 +220,7 @@ final class KeyframeStore { return .worldDiscarded } stillCatalog.save(stored, purpose: id, fileName: name) + onKept?(stored, id, directory.appending(path: name)) return .saved } diff --git a/ios/HouseScan/Runtime/LaunchOptions.swift b/ios/HouseScan/Runtime/LaunchOptions.swift index b19340cdb..cd078ff09 100644 --- a/ios/HouseScan/Runtime/LaunchOptions.swift +++ b/ios/HouseScan/Runtime/LaunchOptions.swift @@ -1,4 +1,5 @@ import Foundation +import HouseScanKit import OSLog /// Verification hooks from the launch arguments (contract C4). @@ -50,6 +51,13 @@ import OSLog /// - `-simulateAppStore`: run as an App Store install would, so the developer options and practice /// meter are unavailable whatever the stored switch says (`DeveloperSettings`). It can only take /// the switch away, never offer it. +/// - `-processingBackend ` (debug builds only): the backend choice in +/// Developer options starts at this value and stays in memory for the run, so a UI test's choice +/// never reaches the stored setting the other tests run with (`ProcessingBackendSetting`). +/// - `-photoProcessingFixture ` (debug builds only, with `-replay`): photo processing sends +/// to a capture API answered inside the app (`FixtureCaptureHTTP`), which ends every capture +/// with this answer (`FixtureCaptureHTTP.Answer`). Nothing leaves the phone. Without it, photo +/// processing isn't set up in any build. struct LaunchOptions: Equatable { var replayFolder: URL? var autopilot = false @@ -67,6 +75,8 @@ struct LaunchOptions: Equatable { var injectGroundRise: Float? var answersFromGate = false var failCloseUpSave = false + var processingBackend: ProcessingBackend? + var photoProcessingFixture: FixtureCaptureHTTP.Answer? init( arguments: [String] = ProcessInfo.processInfo.arguments, @@ -89,6 +99,18 @@ struct LaunchOptions: Equatable { precondition(!(autopilotSomethingThere && autopilotCannotCheck), "-autopilotSomethingThere and -autopilotCannotCheck each choose the first spot answer; pass one") #if DEBUG sampleResultAfterSpotAnswer = value(after: "-sampleResultAfterSpotAnswer").map { URL(fileURLWithPath: $0) } + if let text = value(after: "-processingBackend") { + guard let backend = ProcessingBackend(rawValue: text) else { + preconditionFailure("-processingBackend takes legacy or photoProcessing, got \(text)") + } + processingBackend = backend + } + if let text = value(after: "-photoProcessingFixture") { + guard let answer = FixtureCaptureHTTP.Answer(rawValue: text) else { + preconditionFailure("-photoProcessingFixture takes one of \(FixtureCaptureHTTP.Answer.allCases.map(\.rawValue)), got \(text)") + } + photoProcessingFixture = answer + } #endif simulateAppStore = arguments.contains("-simulateAppStore") failCloseUpSave = arguments.contains("-failCloseUpSave") diff --git a/ios/HouseScan/Runtime/Practice/ProcessingBackendSetting.swift b/ios/HouseScan/Runtime/Practice/ProcessingBackendSetting.swift new file mode 100644 index 000000000..dc13adf3b --- /dev/null +++ b/ios/HouseScan/Runtime/Practice/ProcessingBackendSetting.swift @@ -0,0 +1,37 @@ +import Foundation +import HouseScanKit +import Observation + +/// The backend choice in Developer options: which remote processing the next scan uses. A scan +/// reads it once, when it starts (`ScanEngine.beginScan`), so a change applies to the next scan. +/// +/// Kept in UserDefaults (`ProcessingBackendPreference`). In debug builds, `-processingBackend` +/// keeps it in memory instead, starting at that value, so a UI test's choice ends with its run. +@MainActor +@Observable +final class ProcessingBackendSetting { + static let shared = ProcessingBackendSetting(options: LaunchOptions()) + + private(set) var selection: ProcessingBackend + /// Who answers a photo-processing scan in this build, for the setting's description. The + /// engine sets it from its own setup when it starts. + var photoProcessingAnswers: ProcessingProfile.Answers = .notSetUp("not known yet") + private let defaults: UserDefaults? + + init(options: LaunchOptions) { + #if DEBUG + if let backend = options.processingBackend { + defaults = nil + selection = backend + return + } + #endif + defaults = .standard + selection = ProcessingBackendPreference.read(.standard) + } + + func choose(_ backend: ProcessingBackend) { + selection = backend + if let defaults { ProcessingBackendPreference.write(backend, to: defaults) } + } +} diff --git a/ios/HouseScan/Runtime/ScanEngine+Actions.swift b/ios/HouseScan/Runtime/ScanEngine+Actions.swift index c6cf6cbcf..574158cf0 100644 --- a/ios/HouseScan/Runtime/ScanEngine+Actions.swift +++ b/ios/HouseScan/Runtime/ScanEngine+Actions.swift @@ -9,6 +9,7 @@ extension ScanEngine: ScanActions { func finishOnboarding() { guard state.phase == .onboarding else { return } startPracticeIfOn() + beginScan() leaveOnboarding() } diff --git a/ios/HouseScan/Runtime/ScanEngine+GuidanceLog.swift b/ios/HouseScan/Runtime/ScanEngine+GuidanceLog.swift index b294b4f63..c25302ab6 100644 --- a/ios/HouseScan/Runtime/ScanEngine+GuidanceLog.swift +++ b/ios/HouseScan/Runtime/ScanEngine+GuidanceLog.swift @@ -44,7 +44,7 @@ extension ScanEngine { ) case .spotConfirm: return spotCheckGuidance - case .onboarding, .findMeter, .markFeatures, .uploading, .result, .resultAR, .unsupported: + case .onboarding, .findMeter, .markFeatures, .uploading, .result, .resultAR, .processing, .unsupported: return nil } } diff --git a/ios/HouseScan/Runtime/ScanEngine+Processing.swift b/ios/HouseScan/Runtime/ScanEngine+Processing.swift new file mode 100644 index 000000000..6129ab1ae --- /dev/null +++ b/ios/HouseScan/Runtime/ScanEngine+Processing.swift @@ -0,0 +1,176 @@ +import Foundation +import HouseScanKit +import OSLog + +/// The scan's backend, and photo processing (HouseScanKit's `PhotoProcessingController`). +/// +/// A scan reads the backend setting once, when it starts (`beginScan`), into its `ScanContext`, +/// and keeps that profile to the end: a change in Developer options applies to the next scan. +/// Start over ends the scan; a world reset keeps the scan and its backend and starts a new spatial +/// session. Sending (`startUpload`) goes to the scan's own backend: a photo-processing scan never +/// reaches the Legacy checker, not even from a resend. +extension ScanEngine { + /// The homeowner started a scan. Runs once per scan, before anything of it is kept. + func beginScan() { + guard scanContext == nil else { return } + // App Store installs have no developer options, so they ignore a stored choice. + let backend = DeveloperSettings.shared.isAvailable ? ProcessingBackendSetting.shared.selection : .operationalDefault + let profile = switch backend { + case .legacy: legacyProfile + case .photoProcessing: photoProcessing.profile + } + let context = ScanContext(scanID: store.directory.lastPathComponent, profile: profile, recordingSessionID: recorder.sessionID) + scanContext = context + state.scanBackend = backend + RuntimeLog.engine.info("scan \(context.scanID, privacy: .public) uses \(backend.rawValue, privacy: .public) processing") + guard backend == .photoProcessing else { return } + attachKeptPhotos() + photoProcessing.beginScan(context, recording: recordingSource) + if case .ended(.notSetUp(let reason))? = state.photoProcessing?.stage { + RuntimeLog.engine.info("photo processing not set up: \(reason, privacy: .public)") + } + } + + /// Start over: the scan ends, and whatever photo processing it had ends with it. `recorder` + /// is already the next scan's. + func endScan() { + scanContext = nil + state.scanBackend = nil + photoProcessing.endScan(recording: recordingSource) + } + + /// A world reset: the same scan, backend and consent in a new spatial session. `recorder` has + /// already restarted for the new world. + func scanWorldReset() { + guard let context = scanContext else { return } + let next = context.inNewWorld(recordingSessionID: recorder.sessionID) + scanContext = next + if next.profile.backend == .photoProcessing { + photoProcessing.worldReset(next, recording: recordingSource) + } + } + + /// The photo-processing scan was sent: once the photos still being written are kept, its + /// packet is frozen, and the screen follows the upload to the answer. + func startPhotoProcessing() { + go(.processing) + let context = scanContext + Task { + guard await drainPendingSaves(), scanContext == context, state.phase == .processing else { return } + let closeUp = state.closeUp == .skipped ? nil : store.stillFrames["meter_close"]?.t + photoProcessing.captureEnded(acceptedCloseUpAt: closeUp) + } + } + + func answerPhotoConsent(_ yes: Bool) { + guard state.photoProcessing?.consent == .asking else { return } + RuntimeLog.engine.info("photo processing consent: \(yes ? "yes" : "no", privacy: .public)") + photoProcessing.answerConsent(yes) + } + + func stopSendingPhotos() { + guard state.phase == .processing else { return } + photoProcessing.stopSending() + if case .withdrawn(let recorded)? = state.photoProcessing?.consent { + RuntimeLog.engine.info("photo processing: sending stopped; the phone \(recorded ? "saved" : "could not save", privacy: .public) the withdrawal") + } + } + + // MARK: Translation + + private var legacyProfile: ProcessingProfile { + if let client = resultClient as? HTTPResultClient { + return ProcessingProfile(backend: .legacy, answers: .service, origin: client.serverURL) + } + return ProcessingProfile(backend: .legacy, answers: .standIn, origin: nil) + } + + /// The current world's recording as the capture packet reads it. A replay records no poses or + /// motion, and this recorder writes no per-frame intrinsics, so the packet can't pass its + /// checks and the phone refuses it; nothing is made up to fill the gap. + var recordingSource: RecordingSource { + let recorder = recorder + return RecordingSource( + start: { + let snapshot = recorder.flush() + guard let uptime = snapshot.firstUptime, let date = snapshot.startedAt else { return nil } + return (uptime, date) + }, + rows: { + _ = recorder.flush() + return RecorderRows( + trajectory: recorder.rows(.trajectory), intrinsics: [], + accelerometer: recorder.rows(.accelerometer), gyroscope: recorder.rows(.gyroscope)) + }) + } + + private func attachKeptPhotos() { + let current = store + current.onKept = { [weak self, weak current] photo, purpose, jpeg in + guard let self, let current, current === self.store, self.scanContext?.profile.backend == .photoProcessing else { return } + self.photoProcessing.kept(ScanEngine.keptPhoto(photo, purpose: purpose, jpeg: jpeg, storeDirectory: current.directory)) + } + } + + private static func keptPhoto(_ photo: StoredKeyframe, purpose: String?, jpeg: URL, storeDirectory: URL) -> KeptPhoto { + let stored = photo.depth + return KeptPhoto( + // The pose as ARKit reported it; anchor corrections belong to scene.json and the 1.1 packet. + t: photo.t, cameraToWorld: photo.camera.cameraToWorld, cameraIntrinsics: photo.camera.intrinsics, + cameraImageSize: photo.camera.imageSize, width: photo.width, height: photo.height, + tracking: TrackingCode(photo.tracking).packetTracking, + exposure: photo.exposure.map { Packet04.Exposure(duration: $0.durationS, offset: $0.offsetEV, iso: $0.iso, fNumber: $0.fNumber) }, + jpeg: jpeg, purpose: purpose, + depth: { + // Only ARKit's own depth with its confidence; a replay's depth isn't ARKit's. + guard let stored, let depth = KeyframeStore.loadDepth(stored, in: storeDirectory), let confidence = depth.confidence, + depth.source == .arkitSceneDepth || depth.source == .arkitSmoothedSceneDepth else { return nil } + return Packet04Depth(meters: depth.meters, confidence: confidence, width: depth.width, height: depth.height) + }) + } + + // MARK: Setup + + /// What photo processing can do in this build. Sending a capture off the phone is off in this + /// slice: the 0.4 clock and privacy contract, the credential and the deployment aren't + /// settled. The one way to run it is DEBUG's fixture on a replay, which answers on the phone. + static func photoProcessingSetup(_ options: LaunchOptions) -> PhotoProcessingController.Setup { + #if DEBUG + if let answer = options.photoProcessingFixture { + guard options.replayFolder != nil else { return .notSetUp("the capture fixture runs only on a replay") } + return .ready(fixtureEnvironment(answer, gate: options.autopilotGate), standIn: true) + } + #endif + return .notSetUp("sending captures is off in this build") + } + + #if DEBUG + /// The capture fixture: every request answered in this process (`FixtureCaptureHTTP`), its + /// routes listed one per line in `/photo-transport.log` for the UI tests. + private static func fixtureEnvironment(_ answer: FixtureCaptureHTTP.Answer, gate: URL?) -> CaptureSessionCoordinator.Environment { + let log = gate?.appending(path: "photo-transport.log") + let http = FixtureCaptureHTTP(answer: answer) { routes in + guard let log else { return } + try? Data(routes.joined(separator: "\n").utf8).write(to: log, options: .atomic) + } + var policy = CaptureUploader.Policy() + // A retry against a fixture costs nothing, and the UI tests shouldn't wait out the + // service's back-off. + policy.firstDelay = 0.1 + policy.maxDelay = 0.5 + let device = CaptureAPI.Device(model: "fixture", systemVersion: "fixture", appVersion: "fixture") + let folder = FileManager.default.temporaryDirectory.appending(path: "PhotoProcessingFixture", directoryHint: .isDirectory) + return .init( + endpoint: FixtureCaptureHTTP.base, sends: true, http: http, capturesFolder: folder, + sessionInfo: { packetID, video in + Packet04SessionInfo( + packetID: packetID, sessionID: packetID, source: .replay, appVersion: device.appVersion, deviceModel: device.model, + systemVersion: device.systemVersion, lidarAvailable: false, sceneDepthEnabled: false, meshReconstructionSupported: nil, + sceneReconstruction: nil, planeDetection: nil, videoWidth: Int(video.x), videoHeight: Int(video.y), framesPerSecond: nil, + timeZone: nil) + }, + device: device, tier: .arkit, policy: policy, + log: { line in RuntimeLog.engine.info("\(line, privacy: .public)") }) + } + #endif +} diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index 1fd2527ba..5036a5d17 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -190,6 +190,11 @@ final class ScanEngine { private var resultBuiltFor: WallGeometry? // Upload + /// The scan under way: its id, backend profile and world, fixed when it starts + /// (`beginScan`); nil before a scan starts. + var scanContext: ScanContext? + /// Photo processing's capture and answer, for scans that use it (`ScanEngine+Processing`). + let photoProcessing: PhotoProcessingController let resultClient: any ResultClient private var uploadTask: Task? /// The current upload's scene has fixed its ground; keep this true through answer pacing. @@ -268,6 +273,10 @@ final class ScanEngine { } else { resultClient = SampleResultClient(pace: options.autopilot ? options.autopilotHold : 1.2) } + photoProcessing = PhotoProcessingController(setup: Self.photoProcessingSetup(options)) + ProcessingBackendSetting.shared.photoProcessingAnswers = photoProcessing.profile.answers + photoProcessing.onChange = { [weak self] status in self?.state.photoProcessing = status } + photoProcessing.meterAnchor = { [weak self] in self?.meterTracking?.pose } state.isAutopilot = options.autopilot state.isReplay = options.replayFolder != nil state.usesSampleResult = resultClient.isSample @@ -368,7 +377,7 @@ final class ScanEngine { replay.play(range: range, speed: replaySpeed) } } - case .markFeatures, .uploading, .spotConfirm, .result: + case .markFeatures, .uploading, .spotConfirm, .result, .processing: live?.setMode(.idle) replay?.stop() case .resultAR: @@ -538,11 +547,11 @@ final class ScanEngine { case .failed, .rejected, .unusableAnswer: false case .idle, .packaging, .uploading, .analyzing, .done: true } - case .onboarding, .spotConfirm, .result, .resultAR, .unsupported: false + case .onboarding, .spotConfirm, .result, .resultAR, .processing, .unsupported: false } let depthFrames = switch state.phase { case .meterCloseUp, .wallWalk, .gapRequest: true - case .onboarding, .findMeter, .markFeatures, .uploading, .spotConfirm, .result, .resultAR, .unsupported: false + case .onboarding, .findMeter, .markFeatures, .uploading, .spotConfirm, .result, .resultAR, .processing, .unsupported: false } recorder.setRecording(capturing, depthFrames: depthFrames) guard live != nil else { return } @@ -1557,7 +1566,7 @@ final class ScanEngine { // "Add something", which taps into the world frame, waits for tracking to return // (`beginMarking`). Nothing is thrown away. break - case .uploading, .spotConfirm, .result, .resultAR, .onboarding, .unsupported: + case .uploading, .spotConfirm, .result, .resultAR, .processing, .onboarding, .unsupported: // The bundle is already packed and the server's answer does not depend on the live // world frame, so the scan and the result stay. The AR result hides its overlay while // tracking is not normal and shows it again if ARKit does relocalize. @@ -1579,7 +1588,7 @@ final class ScanEngine { case .cameraDenied: // As for a failed session: once the scan is sent, the answer stays on screen. switch state.phase { - case .uploading, .spotConfirm, .result, .resultAR: + case .uploading, .spotConfirm, .result, .resultAR, .processing: RuntimeLog.engine.error("camera access lost after capture") _ = sourceState.sourceFailed(.recoverable, afterCapture: true) loseSpatialResult() @@ -1591,7 +1600,7 @@ final class ScanEngine { // longer need the camera: keep them on screen. Only the AR view needs it, and it // already hides the battery while the camera isn't tracking. switch state.phase { - case .uploading, .spotConfirm, .result, .resultAR: + case .uploading, .spotConfirm, .result, .resultAR, .processing: RuntimeLog.engine.error("camera session failed after capture: \(message, privacy: .public)") _ = sourceState.sourceFailed(.recoverable, afterCapture: true) loseSpatialResult() @@ -1612,6 +1621,7 @@ final class ScanEngine { generation += 1 // A new world frame is a new packet session: what was recorded is in the old frame. recorder.restart() + scanWorldReset() resetPacketLog() relocalizingSince = nil planes = PlaneSnapshot() @@ -1717,6 +1727,9 @@ final class ScanEngine { /// Takes the answer down when a ground change leaves it describing a ground the phone no /// longer has (`GroundFreshness`), before `publishWall` can draw it again on the new one. private func answerAfter(_ change: GroundFreshness.Change) { + // A photo-processing packet doesn't follow the phone's ground after it is sent, and + // nothing here may send that scan to the Legacy checker. + guard scanContext?.profile.backend != .photoProcessing else { return } let screen: GroundFreshness.Screen = switch state.phase { case .uploading: switch state.upload { @@ -2071,7 +2084,22 @@ final class ScanEngine { // MARK: Upload + /// Waits, for at most 10 s, for the keyframe writes this world started, so the capture holds + /// every photo kept before the send. False when the world or the scan changed meanwhile. + func drainPendingSaves() async -> Bool { + let scan = generation + for _ in 0..<200 where (pendingSaves[scan] ?? 0) > 0 { + try? await Task.sleep(for: .milliseconds(50)) + } + return scan == generation + } + func startUpload() { + // The scan's own backend only: a photo-processing scan never reaches the Legacy checker. + if scanContext?.profile.backend == .photoProcessing { + startPhotoProcessing() + return + } scenePackaged = false // "Try again" sends from the upload screen; anything else is a new send of this scan. if state.phase != .uploading { unusableAnswers = 0 } @@ -2333,6 +2361,7 @@ final class ScanEngine { store = KeyframeStore(deletingAfter: bundleTask) recorder = Self.makeRecorder(store) live?.setRecorder(recorder) + endScan() motion.stop() resetPacketLog() keptSourceIDs = [] diff --git a/ios/HouseScan/UI/Components/PhotoConsentCard.swift b/ios/HouseScan/UI/Components/PhotoConsentCard.swift new file mode 100644 index 000000000..f205c8c3e --- /dev/null +++ b/ios/HouseScan/UI/Components/PhotoConsentCard.swift @@ -0,0 +1,59 @@ +import SwiftUI + +/// Asks, once per scan and before anything is sent, whether this scan's photos may go to photo +/// processing. It covers the camera and blocks it until answered, because photos kept from the +/// meter close-up on would go up under the answer. Both choices are large, and "Don't send" +/// is a real answer: the scan goes on and ends saying it wasn't processed. +struct PhotoConsentCard: View { + var standIn: Bool + let actions: any ScanActions + + var body: some View { + ZStack(alignment: .bottom) { + Palette.ink.opacity(0.6) + .ignoresSafeArea() + .accessibilityHidden(true) + ScrollView { + VStack(alignment: .leading, spacing: 14) { + Text(ProcessingCopy.screenLabel) + .font(Typeface.caption) + .textCase(.uppercase) + .tracking(0.6) + .foregroundStyle(Palette.chalk.opacity(0.8)) + Text(ProcessingCopy.consentTitle) + .font(Typeface.instruction) + .foregroundStyle(Palette.chalk) + .fixedSize(horizontal: false, vertical: true) + .accessibilityAddTraits(.isHeader) + .accessibilityIdentifier("photoConsent.title") + Text(ProcessingCopy.consentBody(standIn: standIn)) + .font(Typeface.hint) + .foregroundStyle(Palette.chalk) + .fixedSize(horizontal: false, vertical: true) + Text(ProcessingCopy.consentDeclineNote) + .font(Typeface.hint) + .foregroundStyle(Palette.chalk.opacity(0.85)) + .fixedSize(horizontal: false, vertical: true) + VStack(spacing: 12) { + Button(ProcessingCopy.consentSend) { actions.answerPhotoConsent(true) } + .buttonStyle(.primary) + .accessibilityIdentifier("photoConsent.send") + Button(ProcessingCopy.consentDecline) { actions.answerPhotoConsent(false) } + .buttonStyle(.secondaryProminent) + .frame(maxWidth: .infinity) + .accessibilityIdentifier("photoConsent.decline") + } + .padding(.top, 6) + } + .padding(22) + .background(Palette.ink, in: .rect(cornerRadius: Metrics.cardRadius)) + .padding(Metrics.edge) + } + .scrollBounceBehavior(.basedOnSize) + .defaultScrollAnchor(.bottom) + } + .accessibilityElement(children: .contain) + .accessibilityAddTraits(.isModal) + .accessibilityIdentifier("photoConsent") + } +} diff --git a/ios/HouseScan/UI/Copy/ProcessingCopy.swift b/ios/HouseScan/UI/Copy/ProcessingCopy.swift new file mode 100644 index 000000000..91eb33a5d --- /dev/null +++ b/ios/HouseScan/UI/Copy/ProcessingCopy.swift @@ -0,0 +1,204 @@ +import HouseScanKit + +/// Words for the backend choice and photo processing. The service's own messages and prompts are +/// shown as written, never rewritten here; every answer stays provisional, as on the Legacy +/// result (`ScanCopy.installerConfirms`). +enum ProcessingCopy { + static func name(_ backend: ProcessingBackend) -> String { + switch backend { + case .legacy: "Legacy (remote checker)" + case .photoProcessing: "Photo processing (beta)" + } + } + + static func summary(_ backend: ProcessingBackend) -> String { + switch backend { + case .legacy: + "Sends the scan's measurements to the House Scan server, which checks the placement rules. Photos stay on this phone." + case .photoProcessing: + "Sends the scan's photos and how the phone moved to House Scan's photo processing service, which suggests a spot. Asks before sending." + } + } + + /// What this build can do with a photo-processing scan, under its option; nil when a service + /// answers. + static func setupNote(_ answers: ProcessingProfile.Answers) -> String? { + switch answers { + case .service: nil + case .standIn: "Test build: answers come from a stand-in on this phone, and nothing leaves it." + case .notSetUp: "Not set up in this build. A scan that uses it can't be sent, and says so at the end." + } + } + + static func choiceFooter(scanBackend: ProcessingBackend?) -> String { + guard let scanBackend else { return "Applies to the next scan you start." } + return "This scan uses \(name(scanBackend)). A change applies to the next scan." + } + + // MARK: Consent + + static let consentTitle = "Send this scan's photos?" + + static func consentBody(standIn: Bool) -> String { + let destination = standIn + ? "In this test build, a stand-in on this phone receives them, so nothing leaves it." + : "They go to House Scan's processing service while you scan." + return "Photo processing uses the photos you take of the wall around your meter, and how the phone moved, to suggest a battery spot. \(destination)" + } + + static let consentDeclineNote = "If you don't send them, nothing leaves this phone and this scan won't get an answer." + static let consentSend = "Send photos" + static let consentDecline = "Don't send" + + // MARK: Processing screen + + static let screenLabel = "Photo processing (beta)" + static let standInBadge = "Test answer from this phone, not the processing service" + static let serviceSaid = "The service says" + static let stopSending = "Stop sending photos" + static let stopQuestion = "Stop sending this scan's photos?" + static let stopDetail = "This scan won't be processed, and House Scan can't start it again. Photos already sent stay with the processing service." + static let confirmStop = "Stop sending" + static let keepSending = "Keep sending" + static let closedAppLimit = "House Scan can't come back to this scan after you close the app." + static let retrying = "Connection trouble. Trying again." + + struct Screen: Equatable { + /// A stable name for the state, for the UI tests (`photo.state.`). + var id: String + var title: String + var detail: String + /// `detail` is the service's message, shown quoted, not House Scan's words. + var fromService = false + var working = false + var tone: Tone = .neutral + var symbol: String + /// Further facts, each its own line. + var notes: [String] = [] + } + + enum Tone { + case neutral, proposed, attention + } + + static func screen(_ status: PhotoProcessingStatus?, isReplay: Bool) -> Screen { + guard let status else { + return Screen( + id: "none", title: "Nothing to show", detail: "This scan has no photo processing to show. Start over to scan again.", + tone: .attention, symbol: "questionmark.circle") + } + switch status.stage { + case .capturing, .queued: + return Screen( + id: "queued", title: "Sending your scan", detail: "Waiting for the processing service to open your scan.", working: true, + symbol: "arrow.up.circle") + case .uploading(let received, let kept): + return Screen( + id: "uploading", title: "Sending your scan", detail: "\(received) of \(kept) files received by the processing service.", + working: true, symbol: "arrow.up.circle") + case .processing: + return Screen( + id: "processing", title: "Processing your scan", detail: "The service has every file and is working on it.", working: true, + symbol: "cube.transparent") + case .answered(let answer): + return screen(answer) + case .ended(let end): + return screen(end, sentSome: status.consent != .declined && status.consent != .asking, isReplay: isReplay) + } + } + + private static func screen(_ answer: PhotoProcessingAnswer) -> Screen { + switch answer { + case .candidate(let candidate): + let ar = switch candidate.ar { + case .withheld: "Showing it on your wall isn't available for this answer." + case .notShownInThisBuild: "Showing it on your wall isn't available for photo processing yet." + } + let preview = candidate.previewMade ? "The service made a preview, which this beta doesn't show yet." : "The service made no preview for this scan." + return Screen( + id: "candidate", title: "A possible battery spot", detail: candidate.message, fromService: true, tone: .proposed, + symbol: "mappin.and.ellipse", + notes: [ScanCopy.installerConfirms, ar, preview]) + case .needsMorePhotos(let more): + return Screen( + id: "needsMorePhotos", title: "One more look needed", + detail: more.message ?? "The processing service needs more views of the wall.", fromService: more.message != nil, + tone: .attention, + symbol: "camera.viewfinder", + notes: ["Adding photos to a scan isn't supported in this beta yet. To try again, start over and include these views."]) + case .installerReview(let message): + return Screen( + id: "installerReview", title: "Needs an installer's review", detail: message, fromService: true, + symbol: "person.crop.circle.badge.checkmark") + case .noCandidate(let message): + return Screen( + id: "noCandidate", title: "No possible spot found", detail: message, fromService: true, symbol: "mappin.slash", + notes: [ScanCopy.installerConfirms]) + case .unrecognized(_, let message): + return Screen( + id: "unrecognized", title: "An answer this version can't read", + detail: message.isEmpty ? "The processing service sent a kind of answer House Scan doesn't know yet." : message, + fromService: !message.isEmpty, tone: .attention, symbol: "questionmark.bubble", + notes: ["House Scan doesn't know this kind of answer yet, so it shows the service's words as they are."]) + } + } + + private static func screen(_ end: PhotoProcessingEnd, sentSome: Bool, isReplay: Bool) -> Screen { + let kept = "Photos already sent stay with the processing service." + switch end { + case .notSetUp: + return Screen( + id: "notSetUp", title: "Photo processing isn't set up in this build", + detail: "Nothing was sent. To check a scan, choose \(name(.legacy)) in Developer options, then start over.", + tone: .attention, symbol: "wrench.and.screwdriver") + case .consentNotGiven: + return Screen( + id: "consentNotGiven", title: "Photos weren't sent", + detail: "You didn't agree to send this scan's photos, so nothing left this phone and the scan wasn't processed.", + symbol: "hand.raised") + case .withdrawn(let recorded): + guard !recorded else { + return Screen( + id: "withdrawn", title: "You stopped sending photos", + detail: "Nothing more from this scan will be sent, and it won't be processed.", symbol: "hand.raised", notes: [kept]) + } + return Screen( + id: "withdrawalNotRecorded", title: "You stopped sending photos", + detail: "Sending stopped, but House Scan couldn't save your choice on this phone. Don't use this scan again: start over for a new one.", + tone: .attention, symbol: "exclamationmark.triangle", notes: [kept]) + case .notPrepared: + var notes = sentSome ? [kept] : [] + if isReplay { notes.insert("Replays carry no motion data, which photo processing needs.", at: 0) } + return Screen( + id: "notPrepared", title: "This scan couldn't be prepared", + detail: "House Scan couldn't put this scan together for photo processing on this phone, so it wasn't processed.", + tone: .attention, symbol: "exclamationmark.triangle", notes: notes) + case .processingFailed: + return Screen( + id: "processingFailed", title: "Processing didn't finish", detail: "The processing service couldn't finish this scan.", + tone: .attention, symbol: "exclamationmark.triangle") + case .expired: + return Screen( + id: "expired", title: "This scan expired", detail: "The processing service stopped waiting before this scan was finished.", + tone: .attention, symbol: "clock.badge.exclamationmark") + case .refused: + return Screen( + id: "refused", title: "The processing service refused this scan", detail: "Sending it again wouldn't change that.", + tone: .attention, symbol: "exclamationmark.triangle") + case .answerUnreadable: + return Screen( + id: "answerUnreadable", title: "House Scan couldn't read the answer", + detail: "The processing service answered, but not in a form this version can read.", tone: .attention, + symbol: "exclamationmark.bubble") + case .answerNotReady: + return Screen( + id: "answerNotReady", title: "The answer wasn't ready", + detail: "The service finished, but its answer didn't become available in time.", tone: .attention, symbol: "clock") + case .answerMismatch: + return Screen( + id: "answerMismatch", title: "The answer didn't match this scan", + detail: "House Scan received an answer for a different capture, so it isn't shown.", tone: .attention, + symbol: "exclamationmark.triangle") + } + } +} diff --git a/ios/HouseScan/UI/Practice/DeveloperOptions.swift b/ios/HouseScan/UI/Practice/DeveloperOptions.swift index 6eb0dacb0..21a7dace3 100644 --- a/ios/HouseScan/UI/Practice/DeveloperOptions.swift +++ b/ios/HouseScan/UI/Practice/DeveloperOptions.swift @@ -1,13 +1,41 @@ import HouseScanKit import SwiftUI -/// The way into the developer options, on the first screen. Only development and TestFlight -/// installs show it (`DeveloperSettings.isAvailable`); an App Store install shows nothing here. -/// While practice meter is on it says so, so nobody starts a scan without knowing. +/// The way into the developer options, on the first screen and the photo-processing screen. Only +/// development and TestFlight installs show it (`DeveloperSettings.isAvailable`); an App Store +/// install shows nothing here. While practice meter or photo processing is on it says so, so +/// nobody starts a scan without knowing. struct DeveloperOptionsButton: View { + /// The backend of the scan under way, for the sheet to say which one it keeps; nil before a + /// scan starts. + let scanBackend: ProcessingBackend? @AppStorage(DeveloperSettings.practiceMeterKey) private var practiceMeter = false @State private var showing = false private let settings = DeveloperSettings.shared + private let processing = ProcessingBackendSetting.shared + + init(scanBackend: ProcessingBackend? = nil) { + self.scanBackend = scanBackend + } + + /// The practice meter's words stay as they were; photo processing adds its own sentence. + private var spokenLabel: String { + var parts: [String] = [] + if practiceMeter { parts.append("Practice meter is on.") } + if processing.selection == .photoProcessing { parts.append("Photo processing is chosen.") } + return parts.isEmpty ? "Developer options" : (["Developer options."] + parts).joined(separator: " ") + } + + /// What the chip says next to the wrench, when anything differs from an ordinary scan. + private var notice: String? { + let photo = processing.selection == .photoProcessing + return switch (practiceMeter, photo) { + case (true, true): "Practice meter on Ā· Photo processing" + case (true, false): "Practice meter on" + case (false, true): "Photo processing" + case (false, false): nil + } + } var body: some View { if settings.isAvailable { @@ -16,35 +44,36 @@ struct DeveloperOptionsButton: View { } label: { HStack(spacing: 6) { Image(systemName: "wrench.and.screwdriver.fill") - if practiceMeter { + if let notice { // Wraps at the largest text sizes rather than dropping out. - Text("Practice meter on") + Text(notice) .fixedSize(horizontal: false, vertical: true) } } .font(Typeface.caption) - .foregroundStyle(practiceMeter ? Palette.ink : Palette.muted) - .padding(.horizontal, practiceMeter ? 10 : 0) + .foregroundStyle(notice != nil ? Palette.ink : Palette.muted) + .padding(.horizontal, notice != nil ? 10 : 0) .padding(.vertical, 5) .background { - if practiceMeter { Capsule().fill(Palette.caution) } + if notice != nil { Capsule().fill(Palette.caution) } } .frame(minWidth: Metrics.minTarget, minHeight: Metrics.minTarget) .contentShape(.rect) } .buttonStyle(PressableStyle()) - .accessibilityLabel(practiceMeter ? "Developer options. Practice meter is on." : "Developer options") + .accessibilityLabel(spokenLabel) .accessibilityIdentifier("action.developerOptions") .sheet(isPresented: $showing) { - DeveloperOptionsSheet(environment: settings.environment) + DeveloperOptionsSheet(environment: settings.environment, scanBackend: scanBackend) } } } } -/// One switch for now: practice meter. +/// Practice meter, and which remote processing the next scan uses. private struct DeveloperOptionsSheet: View { var environment: PracticeMeter.InstallEnvironment + var scanBackend: ProcessingBackend? @AppStorage(DeveloperSettings.practiceMeterKey) private var practiceMeter = false @Environment(\.dismiss) private var dismiss @@ -57,6 +86,7 @@ private struct DeveloperOptionsSheet: View { } footer: { Text("For trying the scan where there's no electric meter. Tap any spot on a wall and a sample meter is drawn there. A photo of it stands in for the meter close-up, and the rest of the scan runs for real. Each screen and the scan's stamp say it's practice. Starts with the next scan.") } + ProcessingChoice(scanBackend: scanBackend) Section { LabeledContent("Build", value: Self.name(environment)) } @@ -82,3 +112,52 @@ private struct DeveloperOptionsSheet: View { } } } + +/// The two remote backends as a radio list. Both are services off the phone; neither decides on +/// it. A scan keeps the backend it started with, so the footer says which one a scan under way +/// uses and that a change waits for the next scan. +private struct ProcessingChoice: View { + var scanBackend: ProcessingBackend? + private let setting = ProcessingBackendSetting.shared + + var body: some View { + Section { + ForEach(ProcessingBackend.allCases, id: \.self) { backend in + let selected = setting.selection == backend + Button { + setting.choose(backend) + } label: { + HStack(alignment: .firstTextBaseline, spacing: 12) { + VStack(alignment: .leading, spacing: 4) { + Text(ProcessingCopy.name(backend)) + .foregroundStyle(.primary) + Text(ProcessingCopy.summary(backend)) + .font(.footnote) + .foregroundStyle(.secondary) + if backend == .photoProcessing, let note = ProcessingCopy.setupNote(setting.photoProcessingAnswers) { + Text(note) + .font(.footnote.weight(.semibold)) + .foregroundStyle(.primary) + } + } + .fixedSize(horizontal: false, vertical: true) + Spacer(minLength: 0) + Image(systemName: "checkmark") + .font(.body.weight(.semibold)) + .foregroundStyle(.tint) + .opacity(selected ? 1 : 0) + .accessibilityHidden(true) + } + .contentShape(.rect) + } + .accessibilityAddTraits(selected ? .isSelected : []) + .accessibilityIdentifier("developer.processing.\(backend.rawValue)") + } + } header: { + Text("Processing") + } footer: { + Text(ProcessingCopy.choiceFooter(scanBackend: scanBackend)) + .accessibilityIdentifier("developer.processing.footer") + } + } +} diff --git a/ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift b/ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift new file mode 100644 index 000000000..14642815e --- /dev/null +++ b/ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift @@ -0,0 +1,74 @@ +import Foundation +import HouseScanKit + +/// Photo processing in the UI demo. `-uiDemoPhase processing -uiDemoPhotoState ` shows one of +/// the screen's states (ids as in `ProcessingCopy.Screen.id`), and `-uiDemoPhotoConsent` on a +/// camera phase shows the consent question. The states are the types the engine publishes, with +/// the capture fixture's words, so every screen says its answer is a test one. +extension DemoEngine { + func showPhotoProcessing(photoState id: String?, consent: Bool) { + if consent { + state.scanBackend = .photoProcessing + state.photoProcessing = PhotoProcessingStatus(stage: .capturing(sent: 0), consent: .asking, standIn: true) + } + guard state.phase == .processing else { return } + state.scanBackend = .photoProcessing + state.photoProcessing = Self.photoExamples[id ?? "processing"] ?? Self.photoExamples["processing"] + } + + /// The scan was sent: a yes ends on the fixture's possible spot, anything else on photos not + /// sent, as the engine would. + func finishPhotoProcessing() { + let sent = state.photoProcessing?.consent == .granted + state.gap = nil + state.phase = .processing + state.photoProcessing = Self.photoExamples[sent ? "candidate" : "consentNotGiven"] + } + + func answerPhotoConsent(_ yes: Bool) { + guard var status = state.photoProcessing, status.consent == .asking else { return } + status.consent = yes ? .granted : .declined + state.photoProcessing = status + } + + func stopSendingPhotos() { + guard var status = state.photoProcessing, status.consent == .granted, !status.isFinal else { return } + status.consent = .withdrawn(recorded: true) + status.stage = .ended(.withdrawn(recorded: true)) + state.photoProcessing = status + } + + static let photoExamples: [String: PhotoProcessingStatus] = { + func sent(_ stage: PhotoProcessingStatus.Stage) -> PhotoProcessingStatus { + PhotoProcessingStatus(stage: stage, consent: .granted, standIn: true) + } + let prompt = try? JSONDecoder().decode( + CaptureResult.Prompt.self, + from: Data(#"{"title": "Fixture: show the ground below the meter", "body": "Fixture: step back until the ground below the meter is in view."}"#.utf8)) + let answers = FixtureCaptureHTTP.Answer.self + return [ + "queued": sent(.queued), + "uploading": sent(.uploading(received: 9, kept: 14)), + "processing": sent(.processing), + "candidate": sent(.answered(.candidate(.init( + message: answers.candidate.message ?? "", previewMade: false, ar: .withheld(.analysisUnverified))))), + "needsMorePhotos": sent(.answered(.needsMorePhotos(.init( + prompts: prompt.map { [$0] } ?? [], message: answers.needsViews.message, parent: nil)))), + "installerReview": sent(.answered(.installerReview(message: answers.manualReview.message ?? ""))), + "noCandidate": sent(.answered(.noCandidate(message: answers.notEligible.message ?? ""))), + "unrecognized": sent(.answered(.unrecognized(kind: "fixture_kind", message: "Fixture answer: a kind of answer this app doesn't know."))), + "notSetUp": PhotoProcessingStatus(stage: .ended(.notSetUp("demo")), consent: .notNeeded, standIn: false), + "consentNotGiven": PhotoProcessingStatus(stage: .ended(.consentNotGiven), consent: .declined, standIn: true), + "withdrawn": PhotoProcessingStatus(stage: .ended(.withdrawn(recorded: true)), consent: .withdrawn(recorded: true), standIn: true), + "withdrawalNotRecorded": PhotoProcessingStatus( + stage: .ended(.withdrawn(recorded: false)), consent: .withdrawn(recorded: false), standIn: true), + "notPrepared": sent(.ended(.notPrepared)), + "processingFailed": sent(.ended(.processingFailed)), + "expired": sent(.ended(.expired)), + "refused": sent(.ended(.refused(step: "create"))), + "answerUnreadable": sent(.ended(.answerUnreadable)), + "answerNotReady": sent(.ended(.answerNotReady)), + "answerMismatch": sent(.ended(.answerMismatch(.run))), + ] + }() +} diff --git a/ios/HouseScan/UI/Preview/DemoEngine.swift b/ios/HouseScan/UI/Preview/DemoEngine.swift index 739d6d6d1..e7d4e9afb 100644 --- a/ios/HouseScan/UI/Preview/DemoEngine.swift +++ b/ios/HouseScan/UI/Preview/DemoEngine.swift @@ -108,6 +108,7 @@ final class DemoEngine: ScanActions { let opensReview = arguments.contains("-uiDemoGroundQuestion") || groundAnswer != nil let phase: ScanPhase = opensReview ? .markFeatures : value("-uiDemoPhase").flatMap(ScanPhase.init(rawValue:)) ?? .onboarding jump(to: phase) + showPhotoProcessing(photoState: value("-uiDemoPhotoState"), consent: arguments.contains("-uiDemoPhotoConsent")) state.groundAnswer = groundAnswer if let raw = value("-uiDemoMarking"), let kind = FeatureKind(rawValue: raw) { state.marking = MarkingState(kind: kind, step: 0, refusal: arguments.contains("-uiDemoRefusal") ? .noSurface : nil) @@ -298,6 +299,10 @@ final class DemoEngine: ScanActions { finishedWalkState() showResult() state.phase = .resultAR + case .processing: + placeMeter() + finishedWalkState() + state.phase = .processing case .unsupported: state.failure = .arUnsupported state.phase = .unsupported @@ -454,6 +459,11 @@ final class DemoEngine: ScanActions { } private func enterUpload() { + // A photo-processing scan never takes the Legacy upload, in the demo as in the engine. + if state.scanBackend == .photoProcessing { + finishPhotoProcessing() + return + } state.phase = .uploading state.gap = nil state.path = [] @@ -1187,6 +1197,8 @@ final class DemoEngine: ScanActions { state.endScanQuestion = false state.endScanTooShort = false state.groundAnswer = nil + state.scanBackend = nil + state.photoProcessing = nil } func liveCameraView() -> AnyView { diff --git a/ios/HouseScan/UI/Preview/UIDemo.swift b/ios/HouseScan/UI/Preview/UIDemo.swift index fbcfed2e9..0141045ea 100644 --- a/ios/HouseScan/UI/Preview/UIDemo.swift +++ b/ios/HouseScan/UI/Preview/UIDemo.swift @@ -57,6 +57,10 @@ import SwiftUI /// answered "I can't check this area", so the result says so beside its spot's notice. /// - `-uiDemoFollowUp`: with `-uiDemoPhase uploading` or `gapRequest`, the check has answered /// and asked for one more view: the upload screen as it hands over, or the view itself. +/// - `-uiDemoPhase processing -uiDemoPhotoState `: a photo-processing scan after it was sent, +/// in one of its states (`ProcessingCopy.Screen.id`, such as `uploading`, `candidate` or +/// `withdrawalNotRecorded`; `processing` by default), with the capture fixture's words. +/// - `-uiDemoPhotoConsent`: on a camera phase, the question whether to send the scan's photos. /// /// Unfrozen, the demo goes back to the camera once after the first answer, as the engine does /// when the answer lists a view the camera can take. diff --git a/ios/HouseScan/UI/ScanRootView.swift b/ios/HouseScan/UI/ScanRootView.swift index 54c545573..33377abc7 100644 --- a/ios/HouseScan/UI/ScanRootView.swift +++ b/ios/HouseScan/UI/ScanRootView.swift @@ -40,7 +40,18 @@ struct ScanRootView: View { screen .id(state.phase) .transition(Self.showsCamera(state.phase) ? .opacity : .identity) + if asksForPhotoConsent { + PhotoConsentCard(standIn: state.photoProcessing?.standIn == true, actions: actions) + .transition(.opacity) + } } + .animation(Motion.screen, value: asksForPhotoConsent) + } + + /// The photo-processing question covers the camera until it is answered, on every capture + /// screen, so no photo is kept without an answer in view. + private var asksForPhotoConsent: Bool { + state.photoProcessing?.consent == .asking && Self.showsCamera(state.phase) && state.phase != .resultAR } @ViewBuilder @@ -76,6 +87,9 @@ struct ScanRootView: View { case .resultAR: ResultARScreen(state: state, actions: actions) .screenIdentifier(.resultAR) + case .processing: + PhotoProcessingScreen(state: state, actions: actions) + .screenIdentifier(.processing) case .unsupported: UnsupportedScreen(state: state, actions: actions) .screenIdentifier(.unsupported) @@ -85,7 +99,7 @@ struct ScanRootView: View { static func showsCamera(_ phase: ScanPhase) -> Bool { switch phase { case .findMeter, .meterCloseUp, .wallWalk, .gapRequest, .resultAR, .markFeatures: true - case .onboarding, .uploading, .spotConfirm, .result, .unsupported: false + case .onboarding, .uploading, .spotConfirm, .result, .processing, .unsupported: false } } } diff --git a/ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift b/ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift new file mode 100644 index 000000000..3956d2f5a --- /dev/null +++ b/ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift @@ -0,0 +1,210 @@ +import HouseScanKit +import SwiftUI + +/// A photo-processing scan after it was sent: how far the upload got, then the service's answer or +/// why there is none. Separate facts, never a percentage or a time estimate. The service's words +/// are quoted as written and every answer stays provisional. Nothing here offers AR, more photos or +/// sending again, because this build can do none of them for photo processing; the screen says so +/// instead of showing a button that leads nowhere. +struct PhotoProcessingScreen: View { + let state: ScanViewState + let actions: any ScanActions + @State private var confirmingStop = false + + init(state: ScanViewState, actions: any ScanActions) { + self.state = state + self.actions = actions + } + + var body: some View { + let status = state.photoProcessing + let copy = ProcessingCopy.screen(status, isReplay: state.isReplay) + CenteredScroll { + VStack(spacing: 24) { + ProcessingEmblem(symbol: copy.symbol, tone: copy.tone, working: copy.working) + VStack(spacing: 10) { + Text(ProcessingCopy.screenLabel) + .font(Typeface.caption) + .textCase(.uppercase) + .tracking(0.6) + .foregroundStyle(Palette.muted) + Text(copy.title) + .font(Typeface.screenTitle) + .multilineTextAlignment(.center) + .fixedSize(horizontal: false, vertical: true) + .accessibilityAddTraits(.isHeader) + .accessibilityIdentifier("photo.state.\(copy.id)") + if !copy.fromService { + Text(copy.detail) + .font(Typeface.hint) + .foregroundStyle(Palette.muted) + .multilineTextAlignment(.center) + .fixedSize(horizontal: false, vertical: true) + } + } + if status?.standIn == true { + StandInNote() + } + if copy.fromService { + ServiceWords(text: copy.detail) + } + if case .answered(.needsMorePhotos(let more))? = status?.stage, !more.prompts.isEmpty { + RequestedViews(prompts: more.prompts) + } + if !copy.notes.isEmpty || copy.working { + ProcessingNotes(lines: notes(copy, retrying: status?.retrying == true)) + } + VStack(spacing: 16) { + if copy.working, status?.consent == .granted { + Button(ProcessingCopy.stopSending) { confirmingStop = true } + .buttonStyle(.quiet) + .accessibilityIdentifier("action.stopSendingPhotos") + } + Group { + if copy.working { + Button("Start over") { actions.startOver() }.buttonStyle(.quiet) + } else { + Button("Start over") { actions.startOver() }.buttonStyle(.primary) + } + } + .accessibilityHint("Starts a new scan. This scan isn't processed any further.") + .accessibilityIdentifier("action.startOver") + } + } + .padding(24) + .frame(maxWidth: 520) + } + .safeAreaInset(edge: .top) { + HStack(alignment: .top) { + ModeBadge(isReplay: state.isReplay, isAutopilot: state.isAutopilot) + Spacer(minLength: 12) + DeveloperOptionsButton(scanBackend: state.scanBackend) + } + .padding(.horizontal, 24) + } + .background(Palette.canvas.ignoresSafeArea()) + .confirmationDialog(ProcessingCopy.stopQuestion, isPresented: $confirmingStop, titleVisibility: .visible) { + Button(ProcessingCopy.confirmStop, role: .destructive) { actions.stopSendingPhotos() } + .accessibilityIdentifier("photo.confirmStop") + Button(ProcessingCopy.keepSending, role: .cancel) {} + } message: { + Text(ProcessingCopy.stopDetail) + } + } + + private func notes(_ copy: ProcessingCopy.Screen, retrying: Bool) -> [String] { + var lines = copy.notes + if retrying { lines.insert(ProcessingCopy.retrying, at: 0) } + if copy.working { lines.append(ProcessingCopy.closedAppLimit) } + return lines + } +} + +/// The state's symbol in a tinted disc, as on the upload screen. It pulses only while the service +/// is working, and not at all with Reduce Motion. +private struct ProcessingEmblem: View { + var symbol: String + var tone: ProcessingCopy.Tone + var working: Bool + @Environment(\.accessibilityReduceMotion) private var reduceMotion + + var body: some View { + ZStack { + Circle() + .fill(tint.opacity(0.12)) + .frame(width: 120, height: 120) + Image(systemName: symbol) + .font(.system(size: 46, weight: .semibold)) + .foregroundStyle(tint) + .symbolEffect(.pulse, options: .repeating, isActive: working && !reduceMotion) + .contentTransition(.symbolEffect(.replace)) + } + .accessibilityHidden(true) + } + + private var tint: Color { + switch tone { + case .attention: Palette.caution + case .neutral, .proposed: Palette.signal + } + } +} + +/// Says the answer is a fixture's, wherever one is on screen, as the Legacy sample says it is one. +private struct StandInNote: View { + var body: some View { + Label(ProcessingCopy.standInBadge, systemImage: "testtube.2") + .font(Typeface.caption) + .foregroundStyle(Palette.ink) + .padding(.horizontal, 12) + .padding(.vertical, 8) + .background(Palette.caution, in: .rect(cornerRadius: 12)) + .fixedSize(horizontal: false, vertical: true) + .accessibilityElement(children: .combine) + .accessibilityIdentifier("photo.standIn") + } +} + +/// The service's message, quoted and attributed, so it never reads as House Scan's own judgment. +private struct ServiceWords: View { + var text: String + + var body: some View { + VStack(alignment: .leading, spacing: 6) { + Text(ProcessingCopy.serviceSaid) + .font(Typeface.caption) + .foregroundStyle(Palette.muted) + Text(text) + .font(Typeface.hint) + .foregroundStyle(Palette.ink) + .fixedSize(horizontal: false, vertical: true) + .accessibilityIdentifier("photo.serviceMessage") + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(16) + .background(Palette.surface, in: .rect(cornerRadius: 16)) + .accessibilityElement(children: .combine) + } +} + +/// The views the service asked for, each prompt as written. +private struct RequestedViews: View { + var prompts: [CaptureResult.Prompt] + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + ForEach(Array(prompts.enumerated()), id: \.offset) { _, prompt in + VStack(alignment: .leading, spacing: 4) { + Text(prompt.title) + .font(Typeface.sectionTitle) + .fixedSize(horizontal: false, vertical: true) + Text(prompt.body) + .font(Typeface.hint) + .foregroundStyle(Palette.muted) + .fixedSize(horizontal: false, vertical: true) + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(16) + .background(Palette.surface, in: .rect(cornerRadius: 16)) + .accessibilityElement(children: .combine) + } + } + .accessibilityIdentifier("photo.requestedViews") + } +} + +private struct ProcessingNotes: View { + var lines: [String] + + var body: some View { + VStack(alignment: .leading, spacing: 10) { + ForEach(lines, id: \.self) { line in + Text(line) + .font(Typeface.hint) + .foregroundStyle(Palette.muted) + .fixedSize(horizontal: false, vertical: true) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + } +} From 4a48e226e1d9e07980f58e0657db139b740c4abd Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 01:21:18 -0500 Subject: [PATCH 27/71] Test the photo-processing path in the app and document it PhotoProcessingUITests run the synthetic replay with the autopilot and the DEBUG capture fixture: the choice made in Developer options holds for its scan and a mid-scan change applies only after Start over; a yes sends photos during the walk and the scan ends on the processing screen without the Legacy upload; a no sends nothing; Start over asks again and sends nothing for the old capture; without setup the scan says so and never falls back. A replay has no motion and about 2 poses a second, so its 0.4 packet fails the phone's checks and these runs end on "not prepared", not on a service answer. Demo mode covers every typed state (default size, and the largest text for the selection, consent, progress and main answers), with accessibility audits on four, and the stop-sending confirmation. The README lists the two new debug launch arguments and how the beta choice behaves. Validated: not run locally (no app or Simulator runs here); they run on hosted CI, where they add roughly 15 minutes to the UI test job. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../PhotoProcessingUITests.swift | 351 ++++++++++++++++++ ios/README.md | 4 + 2 files changed, 355 insertions(+) create mode 100644 ios/HouseScanUITests/PhotoProcessingUITests.swift diff --git a/ios/HouseScanUITests/PhotoProcessingUITests.swift b/ios/HouseScanUITests/PhotoProcessingUITests.swift new file mode 100644 index 000000000..c82102aa1 --- /dev/null +++ b/ios/HouseScanUITests/PhotoProcessingUITests.swift @@ -0,0 +1,351 @@ +import XCTest + +/// Photo processing (beta) in the running app. +/// +/// The real-engine tests run the synthetic replay with the autopilot and the DEBUG capture fixture +/// (`-photoProcessingFixture`), which answers the capture API inside the app; its routes land in +/// `photo-transport.log` in the gate folder. A replay records no motion and only about 2 poses a +/// second, so its 0.4 packet fails the phone's own checks: these runs exercise the choice, consent, +/// sending during the walk and the end the phone reaches, not a service answer. Every answer the +/// service can give is driven end to end in HouseScanKit's `PhotoProcessingControllerTests`, and +/// rendered here from demo mode. +/// +/// `-processingBackend` keeps the choice in memory for the run, so nothing here changes the stored +/// setting the Legacy tests run with. +final class PhotoProcessingUITests: XCTestCase { + private static let largestText = ["-UIPreferredContentSizeCategoryName", "UICTContentSizeCategoryAccessibilityXXXL"] + + override func setUp() { + continueAfterFailure = false + } + + // MARK: The real engine + + /// Photo processing chosen in Developer options before the scan, a yes, photos sent during the + /// walk, and the scan ends on the processing screen, never on the Legacy upload. A change of + /// choice on that screen leaves this scan on photo processing; after Start over the next scan + /// is Legacy, asks nothing and sends nothing. + @MainActor + func testTheChoiceHoldsForItsScanAndTheNextScanUsesTheNewOne() throws { + let run = try Run.launch(backend: "legacy", fixture: "candidate", holding: ["onboarding"]) + defer { run.finish() } + run.waitFor("onboarding") + run.choose("photoProcessing", footer: "Applies to the next scan you start.", shot: "selection") + try run.open("onboarding") + + run.waitFor("findMeter") + XCTAssertTrue(run.send.waitForExistence(timeout: 20), "no photo question before the meter was marked") + run.app.buttons["photoConsent.send"].tap() + XCTAssertTrue(run.send.waitForNonExistence(timeout: 10)) + run.waitFor("processing", timeout: 300) + XCTAssertFalse(run.app.descendants(matching: .any)["screen.uploading"].exists) + let ended = run.state("notPrepared", timeout: 60) + XCTAssertTrue(ended.exists, "the replay's packet didn't end as not prepared") + run.attach("processing-notPrepared") + let routes = run.routes() + XCTAssertEqual(routes.filter { $0 == "POST captures" }.count, 1, "\(routes)") + XCTAssertTrue(routes.contains("PUT upload"), "no photo went up during the walk: \(routes)") + XCTAssertFalse(routes.contains("POST captures/finalize"), "a packet the phone refused was finalized: \(routes)") + XCTAssertFalse(FileManager.default.fileExists(atPath: run.gate.appending(path: "uploading.held").path), "the Legacy upload ran") + + // Changing the choice mid-scan: this scan stays on photo processing. + run.choose("legacy", footer: "This scan uses Photo processing (beta). A change applies to the next scan.", shot: "selection-midScan") + XCTAssertTrue(run.state("notPrepared", timeout: 5).exists, "the scan left photo processing when the choice changed") + try run.open("processing") + + run.tap("action.startOver") + run.waitFor("onboarding") + XCTAssertEqual(run.app.buttons["action.developerOptions"].label, "Developer options") + if run.app.buttons["action.onboardingSkip"].waitForExistence(timeout: 5) { run.app.buttons["action.onboardingSkip"].tap() } + run.tap("action.finishOnboarding") + run.waitFor("findMeter") + XCTAssertFalse(run.send.waitForExistence(timeout: 5), "a Legacy scan asked to send photos") + Thread.sleep(forTimeInterval: 2) + XCTAssertEqual(run.routes(), routes, "the next scan sent something to photo processing") + } + + /// A no sends nothing for the whole scan, and the scan ends saying so. + @MainActor + func testANoSendsNothingAndEndsSayingSo() throws { + let run = try Run.launch(backend: "photoProcessing", fixture: "candidate") + defer { run.finish() } + run.waitFor("findMeter") + XCTAssertTrue(run.send.waitForExistence(timeout: 20)) + run.app.buttons["photoConsent.decline"].tap() + run.waitFor("processing", timeout: 300) + XCTAssertTrue(run.state("consentNotGiven", timeout: 30).exists) + run.attach("processing-consentNotGiven") + XCTAssertEqual(run.routes(), []) + } + + /// Start over after a sent scan asks the next scan again, shows nothing of the old one and + /// sends nothing more for it. + @MainActor + func testStartingOverNeverRevivesTheOldCapture() throws { + let run = try Run.launch(backend: "photoProcessing", fixture: "hold") + defer { run.finish() } + run.waitFor("findMeter") + XCTAssertTrue(run.send.waitForExistence(timeout: 20)) + run.app.buttons["photoConsent.send"].tap() + run.waitFor("processing", timeout: 300) + // The replay's packet can't be finished, so the upload ends on the phone. + XCTAssertTrue(run.state("notPrepared", timeout: 60).exists) + run.attach("processing-ended") + let sent = run.routes() + Thread.sleep(forTimeInterval: 3) + XCTAssertEqual(run.routes(), sent, "requests went on after the scan ended") + try run.open("processing") + + run.tap("action.startOver") + run.waitFor("onboarding") + if run.app.buttons["action.onboardingSkip"].waitForExistence(timeout: 5) { run.app.buttons["action.onboardingSkip"].tap() } + run.tap("action.finishOnboarding") + run.waitFor("findMeter") + XCTAssertTrue(run.send.waitForExistence(timeout: 20), "the next scan didn't ask again") + run.app.buttons["photoConsent.decline"].tap() + XCTAssertTrue(run.send.waitForNonExistence(timeout: 10)) + Thread.sleep(forTimeInterval: 2) + XCTAssertEqual(run.routes(), sent, "the next scan revived the old capture or sent without a yes") + } + + /// Photo processing chosen in a build without its setup: Developer options say so, nothing is + /// asked, and the scan ends saying it wasn't sent. It never falls back to the Legacy checker. + @MainActor + func testWithoutSetupItSaysSoAndNeverFallsBack() throws { + let run = try Run.launch(backend: "photoProcessing", fixture: nil, holding: ["onboarding"]) + defer { run.finish() } + run.waitFor("onboarding") + run.openOptions() + let row = run.reveal("developer.processing.photoProcessing") + XCTAssertTrue(row.label.contains("Not set up in this build"), "the option doesn't say it isn't set up: \(row.label)") + run.attach("selection-notSetUp") + run.closeOptions() + try run.open("onboarding") + run.waitFor("findMeter") + XCTAssertFalse(run.send.waitForExistence(timeout: 5), "asked to send photos with nothing to send them to") + run.waitFor("processing", timeout: 300) + XCTAssertTrue(run.state("notSetUp", timeout: 10).exists) + XCTAssertFalse(FileManager.default.fileExists(atPath: run.gate.appending(path: "uploading.held").path), "the Legacy upload ran") + run.attach("processing-notSetUp") + } + + // MARK: Every state, from demo mode + + /// The consent question and each processing state, at the default text size and the largest, + /// in demo mode with the capture fixture's words. States a replay can't reach (every service + /// answer, the withdrawal the phone couldn't save) are covered here as they render, and in the + /// package tests as they are reached. + @MainActor + func testEveryStateAtDefaultAndLargestText() throws { + let consent = Self.demo(["-uiDemoPhase", "findMeter", "-uiDemoPhotoConsent"]) + XCTAssertTrue(consent.buttons["photoConsent.send"].waitForExistence(timeout: 15)) + XCTAssertGreaterThanOrEqual(consent.buttons["photoConsent.decline"].frame.height, 44) + attach(consent, "consent") + try audit(consent, "consent") + consent.terminate() + let consentLarge = Self.demo(["-uiDemoPhase", "findMeter", "-uiDemoPhotoConsent"] + Self.largestText) + XCTAssertTrue(consentLarge.buttons["photoConsent.send"].waitForExistence(timeout: 15)) + attach(consentLarge, "consent-AX5") + consentLarge.terminate() + + let large: Set = ["uploading", "processing", "candidate", "needsMorePhotos", "installerReview", "withdrawalNotRecorded", "notSetUp"] + for state in [ + "queued", "uploading", "processing", "candidate", "needsMorePhotos", "installerReview", "noCandidate", "unrecognized", + "notSetUp", "consentNotGiven", "withdrawn", "withdrawalNotRecorded", "notPrepared", "processingFailed", "expired", "refused", + "answerUnreadable", "answerNotReady", "answerMismatch", + ] { + for big in large.contains(state) ? [false, true] : [false] { + let app = Self.demo(["-uiDemoPhase", "processing", "-uiDemoPhotoState", state] + (big ? Self.largestText : [])) + XCTAssertTrue( + app.descendants(matching: .any)["photo.state.\(state)"].waitForExistence(timeout: 15), "photo.state.\(state) never appeared") + XCTAssertTrue(app.descendants(matching: .any)["photo.standIn"].exists || state == "notSetUp", "\(state) doesn't say its answer is a test one") + attach(app, "processing-\(state)\(big ? "-AX5" : "")") + if !big, ["uploading", "candidate", "needsMorePhotos", "withdrawalNotRecorded"].contains(state) { try audit(app, "processing-\(state)") } + app.terminate() + } + } + } + + /// "Stop sending photos" asks first, then ends the scan's processing. The demo stands in for the + /// engine, whose replay can't stay in processing long enough to stop; the engine's withdrawal, + /// saved or not, is driven in HouseScanKit's `PhotoProcessingControllerTests`. + @MainActor + func testStopSendingAsksFirstThenEnds() throws { + let app = Self.demo(["-uiDemoPhase", "processing", "-uiDemoPhotoState", "processing"]) + defer { app.terminate() } + let stop = app.buttons["action.stopSendingPhotos"] + XCTAssertTrue(stop.waitForExistence(timeout: 15), "no way to stop sending while processing") + stop.tap() + let confirm = app.buttons["Stop sending"] + XCTAssertTrue(confirm.waitForExistence(timeout: 5), "stopping didn't ask first") + attach(app, "processing-stopQuestion") + confirm.tap() + XCTAssertTrue(app.descendants(matching: .any)["photo.state.withdrawn"].waitForExistence(timeout: 5)) + XCTAssertFalse(app.buttons["action.stopSendingPhotos"].exists) + } + + /// The selection sheet at the largest text size. + @MainActor + func testTheSelectionAtLargestText() throws { + let app = Self.demo(Self.largestText) + let run = Run(app: app, gate: URL(fileURLWithPath: NSTemporaryDirectory())) + run.openOptions() + _ = run.reveal("developer.processing.photoProcessing") + attach(app, "selection-AX5") + app.terminate() + } + + // MARK: Helpers + + @MainActor + private static func demo(_ extra: [String]) -> XCUIApplication { + let app = XCUIApplication() + app.launchArguments = ["-uiDemo", "-uiDemoFreeze", "-practiceMeter", "NO", "-processingBackend", "photoProcessing"] + extra + app.launch() + return app + } + + @MainActor + private func attach(_ app: XCUIApplication, _ name: String) { + let shot = XCTAttachment(screenshot: app.screenshot()) + shot.name = name + shot.lifetime = .keepAlways + add(shot) + } + + @MainActor + private func audit(_ app: XCUIApplication, _ screen: String) throws { + let outcome = try AccessibilityAudit.run(app) { _ in Thread.sleep(forTimeInterval: 2.0) } + for (_, finding) in outcome.persistent { + let text = "\(screen): \(finding.message)" + if ProcessInfo.processInfo.environment["HOUSESCAN_AUDIT_REPORT_ONLY"] == "1" { + let note = XCTAttachment(string: text) + note.name = "audit-\(screen)" + note.lifetime = .keepAlways + add(note) + } else { + XCTFail(text) + } + } + } + + /// One app run on the synthetic replay with the autopilot. Every screen's gate is open from + /// the start except those in `holding` and the processing screen, which the test opens. + @MainActor + struct Run { + let app: XCUIApplication + let gate: URL + + @MainActor + static func launch(backend: String, fixture: String?, holding: Set = []) throws -> Run { + let gate = URL(fileURLWithPath: NSTemporaryDirectory()).appending(path: "housescan-gate-\(UUID().uuidString)", directoryHint: .isDirectory) + try FileManager.default.createDirectory(at: gate, withIntermediateDirectories: true) + for phase in ["onboarding", "findMeter", "meterCloseUp", "wallWalk", "markFeatures", "gapRequest"] where !holding.contains(phase) { + try Data().write(to: gate.appending(path: phase)) + } + var arguments = [ + "-replay", FullFlowUITests.fixture, "-autopilot", "-autopilotHold", "1.0", "-autopilotGate", gate.path, "-sampleResult", + "-practiceMeter", "NO", "-processingBackend", backend, + ] + if let fixture { arguments += ["-photoProcessingFixture", fixture] } + let app = XCUIApplication() + app.launchArguments = arguments + app.launch() + return Run(app: app, gate: gate) + } + + var send: XCUIElement { app.buttons["photoConsent.send"] } + + @MainActor + func waitFor(_ phase: String, timeout: TimeInterval = 60) { + XCTAssertTrue(app.descendants(matching: .any)["screen.\(phase)"].waitForExistence(timeout: timeout), "screen.\(phase) never appeared") + } + + /// Lets the autopilot leave `phase`. + func open(_ phase: String) throws { + try Data().write(to: gate.appending(path: phase)) + } + + @MainActor + func state(_ id: String, timeout: TimeInterval) -> XCUIElement { + let element = app.descendants(matching: .any)["photo.state.\(id)"] + _ = element.waitForExistence(timeout: timeout) + return element + } + + /// Every capture-API route the fixture answered, in order. + func routes() -> [String] { + guard let text = try? String(contentsOf: gate.appending(path: "photo-transport.log"), encoding: .utf8) else { return [] } + return text.split(separator: "\n").map(String.init) + } + + @MainActor + func tap(_ identifier: String) { + let target = app.descendants(matching: .any)[identifier].firstMatch + XCTAssertTrue(target.waitForExistence(timeout: 10), "missing \(identifier)") + for _ in 0..<6 where !target.isHittable { app.swipeUp() } + target.tap() + } + + @MainActor + func openOptions() { + let open = app.buttons["action.developerOptions"] + XCTAssertTrue(open.waitForExistence(timeout: 10), "no developer options here") + open.tap() + XCTAssertTrue(app.buttons["action.closeDeveloperOptions"].waitForExistence(timeout: 10), "the developer options didn't open") + _ = reveal("developer.processing.legacy") + } + + /// Scrolls the options until the row is on screen: at the largest text sizes the sheet + /// opens with the processing choice below its fold. + func reveal(_ identifier: String) -> XCUIElement { + let row = app.buttons[identifier] + let form = app.collectionViews.firstMatch + for _ in 0..<8 where !(row.exists && row.isHittable) { + if form.exists { form.swipeUp() } else { app.swipeUp() } + } + XCTAssertTrue(row.exists && row.isHittable, "\(identifier) never came on screen") + return row + } + + @MainActor + func closeOptions() { + app.buttons["action.closeDeveloperOptions"].tap() + XCTAssertTrue(app.buttons["developer.processing.legacy"].waitForNonExistence(timeout: 10)) + } + + /// Picks `backend` in Developer options and checks the footer, then closes them. + @MainActor + func choose(_ backend: String, footer: String, shot: String) { + openOptions() + let row = reveal("developer.processing.\(backend)") + // The sheet can still be presenting when the row is first on screen (see + // `FullFlowUITests.setPracticeMeter`), so a tap that changed nothing is repeated. + for _ in 0..<3 where !row.isSelected { row.tap() } + XCTAssertTrue(row.isSelected, "\(backend) wasn't selected") + let footnote = app.staticTexts["developer.processing.footer"] + for _ in 0..<4 where !footnote.exists { app.collectionViews.firstMatch.swipeUp() } + XCTAssertEqual(footnote.label, footer) + let attachment = XCTAttachment(screenshot: app.screenshot()) + attachment.name = shot + attachment.lifetime = .keepAlways + XCTContext.runActivity(named: shot) { $0.add(attachment) } + closeOptions() + } + + @MainActor + func attach(_ name: String) { + let shot = XCTAttachment(screenshot: app.screenshot()) + shot.name = name + shot.lifetime = .keepAlways + XCTContext.runActivity(named: name) { $0.add(shot) } + } + + @MainActor + func finish() { + try? open("processing") + app.terminate() + try? FileManager.default.removeItem(at: gate) + } + } +} diff --git a/ios/README.md b/ios/README.md index 788019214..47f11dbcc 100644 --- a/ios/README.md +++ b/ios/README.md @@ -25,10 +25,14 @@ Native iPhone app for the AR capture walk. The homeowner marks the electric mete | `-simulateAppStore` | Runs as an App Store install would: no developer options, and practice meter stays off whatever the stored switch says. `PracticeMeterUITests` uses it. | | `-practiceMeter YES` or `NO` | Overrides the stored practice meter switch for one run (UserDefaults' argument domain). The full-flow UI tests pass `NO`, since the switch outlives a run in the Simulator. | | `-savedScansRoot ` | Debug builds only. Saved scans lists the scan folders in this folder instead of the app's Caches/Scans. It only changes what is listed; the scan store still writes and cleans up Caches/Scans. `SavedScansUITests` passes it with synthetic scans. | +| `-processingBackend ` | Debug builds only. The processing choice in Developer options starts at this value and stays in memory for the run, so a UI test's choice never reaches the stored setting. | +| `-photoProcessingFixture ` | Debug builds only, with `-replay`. Photo processing sends to a capture API answered inside the app (HouseScanKit's `FixtureCaptureHTTP`), which ends each capture with this answer: `candidate`, `needsViews`, `manualReview`, `notEligible`, `failed`, `expired`, `notReady`, `unreadable`, `refuseCreate` or `hold`. Nothing leaves the phone. With `-autopilotGate`, the routes it answered are listed in `photo-transport.log` there. | | `-uiDemo` | Runs the screens on a scripted fake engine instead of the capture engine, for design work and for auditing states a replay can't reach. The arguments it takes are listed in `HouseScan/UI/Preview/UIDemo.swift`; for example `-uiDemoGroundQuestion` opens the feature review with the ground question showing, `-uiDemoPhase meterCloseUp -uiDemoMeterChoose` shows the meter number picker, and `-uiDemoPhase result -uiDemoCorner` shows the result model of a wall that turns a corner. | **Spot check.** Before the result, the app shows the kept photo that best sees the answer's spot, with the spot and its clearance area outlined, and asks whether anything stands in front of the wall or on the ground there (`STATE=spotConfirm`). When no kept photo shows the area, it shows no photo and gives the area in words: where it runs beside the meter, how far out from the wall, and how high up the wall. Photos without depth and the walked path claim wall, ground and clear space they never saw; "It's clear" backs those claims for that spot, and "Something's there" takes them back over the area, so scene.json reports it unseen, and sends the scan again. "I can't check this area", for a homeowner who can't see or reach it, does the same without saying anything stands there, and the result says the area wasn't checked and needs a person to check it. When the scan sent again names the same spot, the earlier answer settles it and the homeowner isn't asked again. When it names no spot, or another one, the result still says an area along the wall went unchecked, until Start over or a new wall clears the scan's spot checks. The rules are in `Runtime/ScanEngine+Confirm.swift` and HouseScanKit's `Confirm/SpotConfirmation.swift`. +**Processing (beta).** Developer options offer two backends, both remote: **Legacy (remote checker)** sends scene.json to the placement server, and **Photo processing (beta)** sends the scan's photos and camera path to the capture API. Legacy is the default. A scan reads the choice once, when it starts, and keeps it with its endpoint in its `ScanContext` (HouseScanKit); a change applies to the next scan, and an App Store install always uses Legacy. A photo-processing scan asks before sending (`PhotoConsentCard`), sends photos as they are kept, and after the review goes to `STATE=processing` instead of the upload: its progress, then the service's answer quoted as written (`PhotoProcessingScreen`). It never falls back to the Legacy checker. Answers bind to the recording session, world epoch, packet, capture and run, and AR is withheld for them. Sending off the phone is off in every build for now, so choosing photo processing ends the scan saying it isn't set up; only `-photoProcessingFixture` runs it. A replay's packet always fails the phone's 0.4 checks, since a replay has no motion data and too few poses. The code is in `Runtime/ScanEngine+Processing.swift` and HouseScanKit's `Processing/`. + **Practice meter.** For trying the whole scan where there is no electric meter, such as indoors. Debug and TestFlight builds show a wrench button at the top of the first screen. Tap it, turn on **Practice meter**, tap Done, then start the scan. At "find the meter", tap any spot on a wall: the usual raycast and refusals apply, and a drawn sample meter appears there, fixed to the meter's anchor, on every camera screen after that. The close-up keeps the live camera's pose and depth, but stores a photo of the sample (made-up maker SAMPLEWORKS, number 12345678), and the real reader reads it. The walk, fog, marks, requests, spot check, upload, result and AR all run for real. Every screen shows a "Practice meter" badge, and `scan-stamp.json` has `"practice": true`; the packet and scene.json are unchanged. The switch is kept until turned off, and the button reads "Practice meter on" while it is. App Store installs never show the button and ignore a stored switch: the app asks StoreKit's `AppTransaction` which server signed the install (sandbox means TestFlight) and treats no answer as the App Store. It is not in the iOS Settings app, because a Settings.bundle ships in the same binary that TestFlight later promotes to the App Store. The code is in `Runtime/Practice/`, `UI/Practice/` and HouseScanKit's `Practice/PracticeMeter.swift`. **Live fog.** During the walk and a gap request, a soft fog covers what the coverage map has not counted and lifts only where it counts a cell, the same rule the scan's decisions use (`FogValue.target` in `UI/Camera/LiveFog/LiveFogScene.swift`). On a LiDAR phone, dots settle on the surfaces the kept keyframes' depth measured (HouseScanKit's `Overlay/SurfaceDots.swift`, run by `Runtime/LiveDotsFeed.swift`); they are guidance and feed nothing. The look is the prototype's (experiments/live-dots on t3/experience). Phones without LiDAR get the same fog without dots. The shaders compile at launch, and the old frosted strip (`FogOverlay`) draws until they are ready, and for good if Metal fails. From 0861fa45715d5262b273f4d036e5c40b157bf9b9 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 09:45:10 -0500 Subject: [PATCH 28/71] Attach the capture API's credential only inside its configured scope ScopedCaptureHTTP wraps a CaptureHTTP: - send attaches "Authorization: Bearer ", from an injected provider, only when the request's URL is inside a CaptureAPIScope fixed at construction: one scheme, exact host and effective port, and paths under one base path, with no dot segments or encoded separators. - The scope is checked before the provider is asked for anything. - An empty token, or one with characters outside RFC 6750's b64token, is refused. - A provider error surfaces as its type name only, and nothing is sent unauthenticated. upload, for the signed storage PUTs, never consults the provider or adds the credential, even when storage shares the API's origin. A storage request that already carries Authorization is refused rather than forwarded or stripped: the signature is in the URL, so the header has no business there. Plain http is allowed only by name (.loopbackHTTP) and only to this machine. There's no retry; the uploader's policy handles thrown errors. Redirects stay refused by URLSessionCaptureHTTP. Tests: - A recording transport, with no socket, covers the header and request preservation, every refused destination with zero provider calls, refresh, malformed and unavailable credentials, and the storage cases. - LoopbackPlacementServer runs the real transport on 127.0.0.1. API requests arrive with the credential, same-origin PUTs arrive without it, and a 307 from either leaves a second listener untouched. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Upload/ScopedCaptureHTTP.swift | 172 +++++++++++ .../ScopedCaptureHTTPTests.swift | 282 ++++++++++++++++++ 2 files changed, 454 insertions(+) create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Upload/ScopedCaptureHTTP.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureHTTPTests.swift diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/ScopedCaptureHTTP.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/ScopedCaptureHTTP.swift new file mode 100644 index 000000000..760d3f3a7 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/ScopedCaptureHTTP.swift @@ -0,0 +1,172 @@ +import Foundation + +/// The capture API's own requests, and nothing else, carry the credential the API issued. +/// +/// `CaptureUploader` sends two kinds of request through `CaptureHTTP`: `send` for the capture +/// API (create, register, commit, finalize, events, result) and `upload` for the storage URLs +/// the API signs. This decorator adds `Authorization: Bearer ` to an API request only +/// when the request's URL is inside the scope fixed at construction, and asks the credential +/// provider for the token only then. A storage upload never gets the credential, and the +/// provider isn't consulted for one, even when storage shares the API's origin. +/// +/// It refuses rather than falls back: a request outside the scope, a credential the provider +/// can't supply or one that isn't a well-formed token throws `ScopedCaptureHTTPError` and nothing +/// is sent. It adds no retry; the uploader's existing policy handles thrown errors. Redirects +/// stay refused by the wrapped transport (`URLSessionCaptureHTTP` refuses every one), so the +/// credential never reaches a redirect target. +/// +/// This is the client's side of the boundary only. It doesn't establish what the server +/// authorizes, attest result bytes or obtain the credential. +public struct ScopedCaptureHTTP: CaptureHTTP { + /// Supplies the API credential, refreshing it when the issuer requires. Called once per API + /// request that passes the scope check, never for a storage upload. + public typealias Credential = @Sendable () async throws -> String + + public let scope: CaptureAPIScope + private let credential: Credential + private let inner: any CaptureHTTP + + public init(scope: CaptureAPIScope, credential: @escaping Credential, inner: any CaptureHTTP) { + self.scope = scope + self.credential = credential + self.inner = inner + } + + public func send(_ request: URLRequest) async throws -> HTTPReply { + // The destination is checked before the credential is asked for. + guard let url = request.url else { throw ScopedCaptureHTTPError.destinationNotAllowed("no_url") } + if let reason = scope.problem(with: url) { throw ScopedCaptureHTTPError.destinationNotAllowed(reason) } + let token: String + do { + token = try await credential() + } catch is CancellationError { + throw CancellationError() + } catch { + // Only the error's type: a provider's message may quote what it tried. + throw ScopedCaptureHTTPError.credentialUnavailable(String(describing: type(of: error))) + } + guard Self.isBearerToken(token) else { throw ScopedCaptureHTTPError.credentialMalformed } + var authorized = request + authorized.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") + return try await inner.send(authorized) + } + + /// A storage PUT goes out with exactly the headers the server signed. The uploader copies + /// those in and sets nothing else, and the storage URLs it receives carry their signature in + /// the query, so a signed set has no reason to hold `Authorization`. One that does is refused + /// instead of sent: forwarding it could hand a credential to storage, and stripping it could + /// break a signature in a way the server should hear about, not the bucket. + public func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + if request.value(forHTTPHeaderField: "Authorization") != nil { + throw ScopedCaptureHTTPError.storageRequestHasAuthorization + } + return try await inner.upload(request, file: file) + } + + /// RFC 6750's `b64token`: letters, digits and `-._~+/`, then optional `=` padding. Anything + /// else, a space, a quote or a line break included, could change the header or add another. + static func isBearerToken(_ token: String) -> Bool { + let body = token.utf8.reversed().drop { $0 == UInt8(ascii: "=") } + guard !body.isEmpty else { return false } + return body.allSatisfy { byte in + switch byte { + case UInt8(ascii: "a")...UInt8(ascii: "z"), UInt8(ascii: "A")...UInt8(ascii: "Z"), UInt8(ascii: "0")...UInt8(ascii: "9"): true + case UInt8(ascii: "-"), UInt8(ascii: "."), UInt8(ascii: "_"), UInt8(ascii: "~"), UInt8(ascii: "+"), UInt8(ascii: "/"): true + default: false + } + } + } +} + +/// Why `ScopedCaptureHTTP` refused a request. No case carries a credential, a URL or a header +/// value, so the description can be logged. +public enum ScopedCaptureHTTPError: Error, Equatable, CustomStringConvertible { + /// The URL is outside the configured API scope. The reason names the check that failed. + case destinationNotAllowed(String) + /// The credential provider threw. The value is the error's type name only. + case credentialUnavailable(String) + /// The provider returned an empty token, or one with characters a bearer token can't have. + case credentialMalformed + /// A storage upload already carries an `Authorization` header. + case storageRequestHasAuthorization + + public var description: String { + switch self { + case .destinationNotAllowed(let reason): "the request is outside the capture API's scope (\(reason))" + case .credentialUnavailable(let type): "the capture API credential couldn't be obtained (\(type))" + case .credentialMalformed: "the capture API credential isn't a well-formed bearer token" + case .storageRequestHasAuthorization: "a storage upload carries an Authorization header" + } + } +} + +/// The capture API a credential is for: one scheme, host and port, and the paths under one +/// base path. Fixed when the uploader is configured, so a later settings change can't redirect +/// an in-flight capture's credential. +public struct CaptureAPIScope: Sendable, Equatable { + /// Plain http is allowed only by name, and only to this machine. + public enum Transport: Sendable, Equatable { + case httpsOnly + /// For tests and local development: http to `127.0.0.1`, `::1` or `localhost`. Never a + /// public host. + case loopbackHTTP + } + + public let scheme: String + public let host: String + public let port: Int + /// The base path without a trailing slash; empty for the origin's root. + public let basePath: String + public let transport: Transport + + /// The scope of `base`, the URL the uploader is given. Throws when `base` itself couldn't + /// receive a credential: not https (or loopback http under `.loopbackHTTP`), credentials in + /// the URL, a query or fragment, or a path with dot segments or encoded separators. + public init(base: URL, transport: Transport = .httpsOnly) throws { + guard let parts = URLComponents(url: base, resolvingAgainstBaseURL: false), + let scheme = parts.scheme?.lowercased(), let host = parts.host?.lowercased(), !host.isEmpty + else { throw ScopedCaptureHTTPError.destinationNotAllowed("base_invalid") } + guard parts.user == nil, parts.password == nil else { throw ScopedCaptureHTTPError.destinationNotAllowed("base_userinfo") } + guard parts.query == nil, parts.fragment == nil else { throw ScopedCaptureHTTPError.destinationNotAllowed("base_query") } + guard Self.allows(scheme: scheme, host: host, transport: transport) else { + throw ScopedCaptureHTTPError.destinationNotAllowed("base_insecure") + } + let path = parts.percentEncodedPath + guard Self.isPlainPath(path) else { throw ScopedCaptureHTTPError.destinationNotAllowed("base_path") } + self.scheme = scheme + self.host = host + self.port = parts.port ?? (scheme == "https" ? 443 : 80) + basePath = path.hasSuffix("/") ? String(path.dropLast()) : path + self.transport = transport + } + + /// Nil when `url` may receive the credential, else the check it fails. + public func problem(with url: URL) -> String? { + guard let parts = URLComponents(url: url, resolvingAgainstBaseURL: false), + let scheme = parts.scheme?.lowercased(), let host = parts.host?.lowercased() + else { return "url_invalid" } + if parts.user != nil || parts.password != nil { return "userinfo" } + if scheme != self.scheme { return "scheme" } + // Exact host: a suffix, a prefix or a trailing dot is another name. + if host != self.host { return "host" } + if (parts.port ?? (scheme == "https" ? 443 : 80)) != port { return "port" } + let path = parts.percentEncodedPath + if !Self.isPlainPath(path) { return "path_segments" } + guard path == basePath || path.hasPrefix(basePath + "/") || (basePath.isEmpty && path.isEmpty) else { return "path" } + return nil + } + + static func allows(scheme: String, host: String, transport: Transport) -> Bool { + if scheme == "https" { return true } + guard scheme == "http", transport == .loopbackHTTP else { return false } + return ["127.0.0.1", "::1", "[::1]", "localhost"].contains(host) + } + + /// No `.` or `..` segment, and no percent-encoded dot, slash or backslash that a server + /// might decode into one. + static func isPlainPath(_ path: String) -> Bool { + let lowered = path.lowercased() + if lowered.contains("%2e") || lowered.contains("%2f") || lowered.contains("%5c") || path.contains("\\") { return false } + return !path.split(separator: "/", omittingEmptySubsequences: false).contains { $0 == "." || $0 == ".." } + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureHTTPTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureHTTPTests.swift new file mode 100644 index 000000000..c8e5a8791 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureHTTPTests.swift @@ -0,0 +1,282 @@ +import Foundation +import HouseScanKit +import Synchronization +import Testing + +/// The capture API credential reaches the configured API and nothing else. Two kinds of test: +/// a recording transport shows exactly which request would leave and with what headers (no +/// socket), and `LoopbackPlacementServer` runs the real `URLSessionCaptureHTTP` over HTTP on +/// 127.0.0.1 for what only a socket shows: the header on the wire and a refused redirect. +@Suite struct ScopedCaptureHTTPTests { + static let base = URL(string: "https://api.example.com/v1")! + static let token = "eyJhbGciOiJIUzI1NiJ9.e30.c2lnbmF0dXJl" + + /// Records what would be sent. Not a socket. + final class Recorder: CaptureHTTP, Sendable { + let sent = Mutex<[(kind: String, request: URLRequest)]>([]) + + func send(_ request: URLRequest) async throws -> HTTPReply { + sent.withLock { $0.append(("send", request)) } + return HTTPReply(status: 200, body: Data("{}".utf8)) + } + + func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + sent.withLock { $0.append(("upload", request)) } + return HTTPReply(status: 200, body: Data()) + } + } + + /// A provider that counts its calls and returns `tokens` in order, the last repeating. + final class Provider: Sendable { + let calls = Mutex(0) + let tokens: [String] + init(_ tokens: [String] = [ScopedCaptureHTTPTests.token]) { self.tokens = tokens } + + var credential: ScopedCaptureHTTP.Credential { + { [self] in + let index = calls.withLock { count in + defer { count += 1 } + return count + } + return tokens[min(index, tokens.count - 1)] + } + } + } + + static func scoped(_ provider: Provider, inner: any CaptureHTTP, base: URL = base, transport: CaptureAPIScope.Transport = .httpsOnly) throws + -> ScopedCaptureHTTP + { + ScopedCaptureHTTP(scope: try CaptureAPIScope(base: base, transport: transport), credential: provider.credential, inner: inner) + } + + static func request(_ url: String, method: String = "POST") -> URLRequest { + var request = URLRequest(url: URL(string: url)!) + request.httpMethod = method + request.httpBody = Data(#"{"packetId":"p1"}"#.utf8) + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.setValue("p1", forHTTPHeaderField: "Idempotency-Key") + return request + } + + // MARK: API requests (request inspection, no socket) + + @Test func anAPIRequestCarriesTheCredentialAndIsOtherwiseUnchanged() async throws { + let recorder = Recorder() + let provider = Provider() + let http = try Self.scoped(provider, inner: recorder) + _ = try await http.send(Self.request("https://api.example.com/v1/captures")) + let sent = try #require(recorder.sent.withLock { $0.first }) + #expect(sent.kind == "send") + #expect(sent.request.value(forHTTPHeaderField: "Authorization") == "Bearer \(Self.token)") + #expect(sent.request.httpMethod == "POST") + #expect(sent.request.httpBody == Data(#"{"packetId":"p1"}"#.utf8)) + #expect(sent.request.value(forHTTPHeaderField: "Content-Type") == "application/json") + #expect(sent.request.value(forHTTPHeaderField: "Idempotency-Key") == "p1") + #expect(provider.calls.withLock { $0 } == 1) + } + + /// The base path itself, its subpaths, a case-different host and an explicit default port + /// are all the same API. + @Test(arguments: [ + "https://api.example.com/v1", "https://api.example.com/v1/captures/c1/events?wait=25", + "https://API.example.com/v1/captures", "https://api.example.com:443/v1/captures", + ]) + func everyRequestInsideTheScopeIsAllowed(url: String) async throws { + let recorder = Recorder() + _ = try await Self.scoped(Provider(), inner: recorder).send(Self.request(url)) + #expect(recorder.sent.withLock { $0.count } == 1) + } + + /// Each of these is refused before the provider is asked for anything, and nothing is sent. + @Test(arguments: [ + ("https://api.example.com.evil.test/v1/captures", "host"), + ("https://evilapi.example.com/v1/captures", "host"), + ("https://example.com/v1/captures", "host"), + ("https://api.example.com./v1/captures", "host"), + ("https://api.example.com:8443/v1/captures", "port"), + ("http://api.example.com/v1/captures", "scheme"), + ("https://user:pass@api.example.com/v1/captures", "userinfo"), + ("https://api.example.com/v10/captures", "path"), + ("https://api.example.com/other", "path"), + ("https://api.example.com/", "path"), + ("https://api.example.com/v1/../admin", "path_segments"), + ("https://api.example.com/v1/%2e%2e/admin", "path_segments"), + ("https://api.example.com/v1/captures%2F..%2Fadmin", "path_segments"), + ]) + func aRequestOutsideTheScopeIsRefusedBeforeTheCredentialIsRead(url: String, reason: String) async throws { + let recorder = Recorder() + let provider = Provider() + let http = try Self.scoped(provider, inner: recorder) + await #expect(throws: ScopedCaptureHTTPError.destinationNotAllowed(reason)) { + try await http.send(Self.request(url)) + } + #expect(provider.calls.withLock { $0 } == 0) + #expect(recorder.sent.withLock { $0.isEmpty }) + } + + /// The provider is asked for every API request, so a token it refreshes reaches the next + /// request, at the same fixed scope. + @Test func aRefreshedCredentialIsUsedForTheNextRequest() async throws { + let recorder = Recorder() + let provider = Provider(["first-token", "second-token"]) + let http = try Self.scoped(provider, inner: recorder) + _ = try await http.send(Self.request("https://api.example.com/v1/captures")) + _ = try await http.send(Self.request("https://api.example.com/v1/captures/c1/files")) + let headers = recorder.sent.withLock { $0.map { $0.request.value(forHTTPHeaderField: "Authorization") } } + #expect(headers == ["Bearer first-token", "Bearer second-token"]) + await #expect(throws: ScopedCaptureHTTPError.destinationNotAllowed("host")) { + try await http.send(Self.request("https://other.example.com/v1/captures")) + } + #expect(provider.calls.withLock { $0 } == 2) + } + + /// An empty token, padding alone, or one with a space, quote or line break is refused, and + /// the error names no part of it. + @Test(arguments: ["", "==", "two words", "line\r\nX-Injected: 1", "quote\"d", "tab\tbed", "cafĆ©"]) + func aMalformedCredentialIsRefusedWithoutItsValue(token: String) async throws { + let recorder = Recorder() + let http = try Self.scoped(Provider([token]), inner: recorder) + let thrown = await #expect(throws: ScopedCaptureHTTPError.self) { + try await http.send(Self.request("https://api.example.com/v1/captures")) + } + #expect(thrown == .credentialMalformed) + #expect(recorder.sent.withLock { $0.isEmpty }) + if !token.isEmpty { #expect(!String(describing: thrown).contains(token)) } + } + + @Test(arguments: ["abc", "a.b-c_d~e+f/g", "dGVzdA==", ScopedCaptureHTTPTests.token]) + func wellFormedBearerTokensAreSent(token: String) async throws { + let recorder = Recorder() + _ = try await Self.scoped(Provider([token]), inner: recorder).send(Self.request("https://api.example.com/v1/captures")) + #expect(recorder.sent.withLock { $0.first?.request.value(forHTTPHeaderField: "Authorization") } == "Bearer \(token)") + } + + /// A provider that can't supply a credential stops the request. Nothing goes out + /// unauthenticated, and the error carries the type, not the provider's message. + @Test func anUnavailableCredentialSendsNothing() async throws { + struct Expired: Error, CustomStringConvertible { var description: String { "token sk-live-secret expired" } } + let recorder = Recorder() + let http = ScopedCaptureHTTP(scope: try CaptureAPIScope(base: Self.base), credential: { throw Expired() }, inner: recorder) + let thrown = await #expect(throws: ScopedCaptureHTTPError.self) { + try await http.send(Self.request("https://api.example.com/v1/captures")) + } + #expect(thrown == .credentialUnavailable("Expired")) + #expect(!String(describing: thrown).contains("sk-live-secret")) + #expect(recorder.sent.withLock { $0.isEmpty }) + } + + /// An Authorization the caller already set is replaced, never sent alongside or instead. + @Test func anAPIRequestCarriesOnlyTheIssuedCredential() async throws { + let recorder = Recorder() + var request = Self.request("https://api.example.com/v1/captures") + request.setValue("Bearer stale", forHTTPHeaderField: "Authorization") + _ = try await Self.scoped(Provider(), inner: recorder).send(request) + #expect(recorder.sent.withLock { $0.first?.request.value(forHTTPHeaderField: "Authorization") } == "Bearer \(Self.token)") + } + + // MARK: Storage uploads (request inspection, no socket) + + /// A signed PUT goes out exactly as built, without asking the provider, even to the API's + /// own origin. + @Test(arguments: ["https://storage.example.com/bucket/k0.jpg?X-Goog-Signature=abc", "https://api.example.com/v1/upload/token-1"]) + func aSignedPutIsSentUnchangedAndWithoutTheCredential(url: String) async throws { + let recorder = Recorder() + let provider = Provider() + var request = URLRequest(url: URL(string: url)!) + request.httpMethod = "PUT" + request.setValue("image/jpeg", forHTTPHeaderField: "Content-Type") + request.setValue("rL0Y20zC+Fzt72VPzMSk2A==", forHTTPHeaderField: "Content-MD5") + _ = try await Self.scoped(provider, inner: recorder).upload(request, file: URL(fileURLWithPath: "/dev/null")) + let sent = try #require(recorder.sent.withLock { $0.first }) + #expect(sent.kind == "upload") + #expect(sent.request == request) + #expect(sent.request.value(forHTTPHeaderField: "Authorization") == nil) + #expect(provider.calls.withLock { $0 } == 0) + } + + /// The chosen policy: storage requests carry their signature in the URL, so an Authorization + /// header on one is refused, not forwarded and not stripped. + @Test func aStorageRequestCarryingAuthorizationIsRefused() async throws { + let recorder = Recorder() + let provider = Provider() + var request = URLRequest(url: URL(string: "https://storage.example.com/bucket/k0.jpg?sig=abc")!) + request.httpMethod = "PUT" + request.setValue("Bearer leaked", forHTTPHeaderField: "Authorization") + let http = try Self.scoped(provider, inner: recorder) + await #expect(throws: ScopedCaptureHTTPError.storageRequestHasAuthorization) { + try await http.upload(request, file: URL(fileURLWithPath: "/dev/null")) + } + #expect(recorder.sent.withLock { $0.isEmpty }) + #expect(provider.calls.withLock { $0 } == 0) + } + + // MARK: The scope itself + + @Test(arguments: [ + ("http://api.example.com/v1", "base_insecure"), ("https://u:p@api.example.com/v1", "base_userinfo"), + ("https://api.example.com/v1?x=1", "base_query"), ("https://api.example.com/v1/../x", "base_path"), + ("ftp://api.example.com/v1", "base_insecure"), + ]) + func aBaseThatCouldNotReceiveACredentialIsRefused(base: String, reason: String) { + #expect(throws: ScopedCaptureHTTPError.destinationNotAllowed(reason)) { try CaptureAPIScope(base: URL(string: base)!) } + } + + /// Plain http is allowed only by name and only to this machine. + @Test func loopbackHTTPIsAnExplicitLocalOnlyChoice() throws { + #expect(throws: ScopedCaptureHTTPError.destinationNotAllowed("base_insecure")) { + try CaptureAPIScope(base: URL(string: "http://127.0.0.1:8080/v1")!) + } + #expect(throws: ScopedCaptureHTTPError.destinationNotAllowed("base_insecure")) { + try CaptureAPIScope(base: URL(string: "http://api.example.com/v1")!, transport: .loopbackHTTP) + } + let scope = try CaptureAPIScope(base: URL(string: "http://127.0.0.1:8080/v1")!, transport: .loopbackHTTP) + #expect(scope.problem(with: URL(string: "http://127.0.0.1:8080/v1/captures")!) == nil) + #expect(scope.problem(with: URL(string: "http://127.0.0.1:8081/v1/captures")!) == "port") + #expect(scope.problem(with: URL(string: "http://localhost:8080/v1/captures")!) == "host") + } + + // MARK: Over a real socket + + /// The real transport on 127.0.0.1: an API request arrives with the credential, a storage + /// PUT to the same origin arrives without it, and a redirect from either is not followed, + /// so the second listener receives nothing. + @Test func overHTTPTheCredentialReachesOnlyTheAPIAndNoRedirectTarget() async throws { + let elsewhere = try LoopbackPlacementServer { _ in .init(status: 200, body: Data("{}".utf8)) } + let target = elsewhere.base.absoluteString + let api = try LoopbackPlacementServer { request in + if request.path.hasSuffix("/redirect") || request.path.hasSuffix("/upload/redirect") { + return .init(status: 307, headers: ["Location": "\(target)\(request.path)"], body: Data()) + } + return .init(status: 200, body: Data("{}".utf8)) + } + let provider = Provider() + let http = try Self.scoped( + provider, inner: URLSessionCaptureHTTP.ephemeral(timeout: 10), + base: api.base.appending(path: "v1"), transport: .loopbackHTTP) + + let sent = try await http.send(Self.request("\(api.base.absoluteString)/v1/captures")) + #expect(sent.status == 200) + let redirected = try await http.send(Self.request("\(api.base.absoluteString)/v1/redirect")) + #expect(redirected.status == 307) + + let file = FileManager.default.temporaryDirectory.appending(path: "scoped-\(UUID().uuidString).bin") + try Data("photo".utf8).write(to: file) + defer { try? FileManager.default.removeItem(at: file) } + for path in ["upload/token-1", "upload/redirect"] { + var put = URLRequest(url: URL(string: "\(api.base.absoluteString)/v1/\(path)")!) + put.httpMethod = "PUT" + put.setValue("application/octet-stream", forHTTPHeaderField: "Content-Type") + _ = try await http.upload(put, file: file) + } + + let seen = api.requests + #expect(seen.map(\.path) == ["/v1/captures", "/v1/redirect", "/v1/upload/token-1", "/v1/upload/redirect"]) + #expect(seen[0].headers["authorization"] == "Bearer \(Self.token)") + #expect(seen[1].headers["authorization"] == "Bearer \(Self.token)") + #expect(seen[2].headers["authorization"] == nil) + #expect(seen[3].headers["authorization"] == nil) + #expect(seen[2].body == Data("photo".utf8)) + #expect(elsewhere.requests.isEmpty) + #expect(provider.calls.withLock { $0 } == 2) + } +} From 5120284414513c8271f3378336560edc656ed46a Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 09:51:48 -0500 Subject: [PATCH 29/71] End an upload refused at the credential boundary, without retrying it CaptureUploader turned every error from send, and every failed PUT, into Transient and retried it with backoff and no limit. A refusal by ScopedCaptureHTTP won't change on retry, so it kept the upload spinning: - a URL outside the API's scope; - a malformed credential, or one the provider couldn't supply; - a storage request carrying Authorization. Both paths now end the upload as .failed with the refusal's stable code and status 0, since no server answered. The codes are auth_destination_refused, auth_credential_unavailable, auth_credential_malformed and auth_storage_authorization. A withdrawal or cancellation that happened meanwhile outranks the refusal. A PUT batch saves its other files' results before the refusal ends the upload. Network and other errors still back off and retry as before. A credential refresh that should be retried belongs in the provider, by its own policy. LoopbackCaptureAPI gains two opt-ins: - expectedBearer: API requests must carry that bearer or get 401. - extraSignedHeaders: register signs extra headers. By default it still refuses any Authorization on an API request, and a signed PUT is refused with one either way. ScopedCaptureUploadTests run the whole uploader over HTTP through the scoped transport. A capture finishes with the bearer on every API request and on no PUT. Each refusal ends at its step with its code, sends the server nothing and sleeps for no retry. A withdrawal during the credential fetch outranks the refusal, and the default loopback keeps refusing API Authorization. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../HouseScanKit/Upload/CaptureUploader.swift | 24 ++- .../Upload/ScopedCaptureHTTP.swift | 11 ++ .../LoopbackCaptureAPI.swift | 16 +- .../ScopedCaptureUploadTests.swift | 178 ++++++++++++++++++ 4 files changed, 225 insertions(+), 4 deletions(-) create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureUploadTests.swift diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift index 6fbf75319..91f7bc463 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -432,6 +432,7 @@ public actor CaptureUploader { private func put(_ files: [CaptureUploadState.File]) async throws { let attempt = state.attemptID var transient: Transient? + var refused: Refused? var index = 0 while index < files.count { // Each batch of PUTs is admitted only while the homeowner's yes stands. @@ -482,10 +483,15 @@ public actor CaptureUploader { } setPhase(path, .queued) case .failure(let error): - transient = Transient(step: "put", detail: Self.describe(error), retryAfter: nil) + switch failure("put", error) { + case let error as Refused: refused = refused ?? error + case let error as Transient: transient = error + case let error: throw error + } } } persist() + if let refused { throw refused } } if let transient { throw transient } } @@ -657,7 +663,7 @@ public actor CaptureUploader { do { reply = try await http.send(request) } catch { - throw Transient(step: step, detail: Self.describe(error), retryAfter: nil) + throw failure(step, error) } if Self.isTransient(reply.status), attempt == state.attemptID { throw Transient(step: step, detail: "status \(reply.status)", retryAfter: reply.retryAfter) @@ -719,6 +725,20 @@ public actor CaptureUploader { static func isTransient(_ status: Int) -> Bool { status == 0 || status == 408 || status == 429 || ((500..<600).contains(status) && status != 501) } /// The error's domain and code only: a URLError's description can carry the signed URL. + /// What a transport error means for the upload. A refusal by the credential boundary + /// (`ScopedCaptureHTTP`) won't change on retry: a URL outside the API's scope, a malformed + /// credential, one the provider couldn't supply, or a storage request carrying + /// Authorization. It ends the upload with the boundary's code and status 0, since no server + /// answered. Any other error, a network one included, is retried as before. A withdrawal or + /// cancellation that happened meanwhile outranks the refusal. + private func failure(_ step: String, _ error: any Error) -> any Error { + guard let refusal = error as? ScopedCaptureHTTPError else { + return Transient(step: step, detail: Self.describe(error), retryAfter: nil) + } + if !canSend || Task.isCancelled { return Withdrawn() } + return Refused(step: step, status: 0, codes: [refusal.code]) + } + static func describe(_ error: any Error) -> String { let ns = error as NSError return "\(ns.domain) \(ns.code)" diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/ScopedCaptureHTTP.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/ScopedCaptureHTTP.swift index 760d3f3a7..7e8e5edcd 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/ScopedCaptureHTTP.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/ScopedCaptureHTTP.swift @@ -90,6 +90,17 @@ public enum ScopedCaptureHTTPError: Error, Equatable, CustomStringConvertible { /// A storage upload already carries an `Authorization` header. case storageRequestHasAuthorization + /// The stable code an upload that hit this refusal ends with (`CaptureUploadState.End.failed`). + /// It names the kind of setup or credential problem, never a value. + public var code: String { + switch self { + case .destinationNotAllowed: "auth_destination_refused" + case .credentialUnavailable: "auth_credential_unavailable" + case .credentialMalformed: "auth_credential_malformed" + case .storageRequestHasAuthorization: "auth_storage_authorization" + } + } + public var description: String { switch self { case .destinationNotAllowed(let reason): "the request is outside the capture API's scope (\(reason))" diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift index c10744ee1..4a85db542 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift @@ -87,6 +87,13 @@ final class LoopbackCaptureAPI: Sendable { var redirects: [String: String] = [:] /// Answer the next commit with this split instead of the truth. var commitOverride: ((committed: [String], notFound: [String], mismatch: [String]))? + /// Opt-in: every API request must carry `Authorization: Bearer `, or it gets 401 + /// `unauthorized`. Nil keeps the default, which refuses any Authorization on an API request. + /// A signed PUT is refused with any Authorization either way. + var expectedBearer: String? + /// Extra headers every register answer adds to a file's signed headers, as a server that + /// signs one more header would. + var extraSignedHeaders: [String: String] = [:] /// Routes whose answers wait until `release` is called. var held: Set = [] var parked: [String: [@Sendable () -> Void]] = [:] @@ -197,7 +204,11 @@ final class LoopbackCaptureAPI: Sendable { state.withLock { s in let parts = r.path.split(separator: "/").map(String.init) if parts.first == "upload" { return put(r, token: parts.last ?? "", &s) } - if r.headers["authorization"] != nil { return error(400, "unexpected_authorization") } + if let bearer = s.expectedBearer { + if r.headers["authorization"] != "Bearer \(bearer)" { return error(401, "unauthorized") } + } else if r.headers["authorization"] != nil { + return error(400, "unexpected_authorization") + } if r.method == "POST", parts == ["captures"] { return create(r, &s) } guard parts.count >= 2, var capture = s.captures[parts[1]] else { return error(404, "capture_not_found") } defer { s.captures[capture.id] = capture } @@ -280,7 +291,8 @@ final class LoopbackCaptureAPI: Sendable { s.tokens[token] = (c.id, path) out.append(["path": path, "state": "pending", "upload": [ "method": "PUT", "url": s.uploadURLOverride ?? "http://127.0.0.1:\(port)/upload/\(token)", - "headers": ["Content-Type": type, "Content-MD5": md5], "expiresAt": "2099-01-01T00:00:00Z", + "headers": ["Content-Type": type, "Content-MD5": md5].merging(s.extraSignedHeaders) { _, extra in extra }, + "expiresAt": "2099-01-01T00:00:00Z", ]]) } return (200, json(["files": out])) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureUploadTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureUploadTests.swift new file mode 100644 index 000000000..6558a58da --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureUploadTests.swift @@ -0,0 +1,178 @@ +import Foundation +import HouseScanKit +import Synchronization +import Testing + +/// The whole uploader through `ScopedCaptureHTTP`, over real HTTP to the loopback capture API +/// with an expected bearer (`LoopbackCaptureAPI.State.expectedBearer`). The credential reaches +/// every API request and no signed PUT. A refusal at the credential boundary ends the upload at +/// once with its code, instead of being retried as a network failure would be. +@Suite struct ScopedCaptureUploadTests { + typealias Rig = CaptureUploaderTests.Rig + typealias Provider = ScopedCaptureHTTPTests.Provider + static let token = "loopback-capture-token" + + /// Every backoff the uploader asked for. Empty means it never waited to retry. + final class Sleeps: Sendable { + private let values = Mutex<[Double]>([]) + var isEmpty: Bool { values.withLock { $0.isEmpty } } + var record: @Sendable (Double) async throws -> Void { { [self] seconds in values.withLock { $0.append(seconds) } } } + } + + /// A loopback API expecting `token`, and an uploader on it whose transport is scoped to + /// `scopeBase` (the loopback's own base unless a test points it elsewhere). + static func rig( + provider: Provider, scopeBase: ((LoopbackCaptureAPI) -> URL)? = nil, sleeps: Sleeps, + configure: (inout LoopbackCaptureAPI.State) -> Void = { _ in } + ) throws -> Rig { + let server = try LoopbackCaptureAPI() + server.state.withLock { + $0.expectedBearer = token + configure(&$0) + } + let scope = try CaptureAPIScope(base: scopeBase?(server) ?? server.base, transport: .loopbackHTTP) + let http = ScopedCaptureHTTP(scope: scope, credential: provider.credential, inner: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + return try Rig(server: server, http: http, sleep: sleeps.record) + } + + @Test func aWholeCaptureUploadsWithTheCredentialOnEveryAPIRequestAndNoPut() async throws { + let sleeps = Sleeps() + let provider = Provider([Self.token]) + let rig = try Self.rig(provider: provider, sleeps: sleeps) + defer { rig.cleanUp() } + let images = try await rig.capture.sealImages() + await rig.uploader.add(images) + try #require(await CaptureUploaderTests.settles(rig.uploader)) + try await rig.finishAndSeal() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "manual_review")) + #expect(done.result != nil) + let log = rig.server.state.withLock { $0.log } + let puts = log.filter { LoopbackCaptureAPI.route($0) == "PUT upload" } + let api = log.filter { LoopbackCaptureAPI.route($0) != "PUT upload" } + #expect(puts.count >= images.count + 4) + #expect(!api.isEmpty) + #expect(api.allSatisfy { $0.headers["authorization"] == "Bearer \(Self.token)" }) + #expect(puts.allSatisfy { $0.headers["authorization"] == nil }) + // At least one fetch per API request the uploader made. Not equality: URLSession may + // resend a request on a reused connection, which the server logs twice. + #expect(provider.calls.withLock { $0 } > 0) + } + + /// Each refusal ends the upload at its step with the boundary's code and status 0, sends the + /// server nothing, and sleeps for no retry. + @Test func aDestinationOutsideTheScopeEndsTheUploadWithoutARetry() async throws { + let sleeps = Sleeps() + let provider = Provider([Self.token]) + let rig = try Self.rig(provider: provider, scopeBase: { URL(string: "http://127.0.0.1:\(Int($0.port) + 1)/v1")! }, sleeps: sleeps) + defer { rig.cleanUp() } + await rig.uploader.kick() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "create", codes: ["auth_destination_refused"], status: 0)) + #expect(rig.server.state.withLock { $0.log.isEmpty }) + #expect(provider.calls.withLock { $0 } == 0) + #expect(sleeps.isEmpty) + } + + @Test(arguments: [("", "auth_credential_malformed"), ("has space", "auth_credential_malformed")]) + func aMalformedCredentialEndsTheUploadWithoutARetry(token: String, code: String) async throws { + let sleeps = Sleeps() + let provider = Provider([token]) + let rig = try Self.rig(provider: provider, sleeps: sleeps) + defer { rig.cleanUp() } + await rig.uploader.kick() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "create", codes: [code], status: 0)) + #expect(rig.server.state.withLock { $0.log.isEmpty }) + #expect(provider.calls.withLock { $0 } == 1) + #expect(sleeps.isEmpty) + } + + /// A provider that can't supply a credential ends the upload; nothing is sent without one. + /// Retrying a credential refresh belongs in the provider, by its own policy. + @Test func anUnavailableCredentialEndsTheUploadWithoutARetry() async throws { + struct Offline: Error {} + let sleeps = Sleeps() + let calls = Mutex(0) + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.expectedBearer = Self.token } + let http = ScopedCaptureHTTP( + scope: try CaptureAPIScope(base: server.base, transport: .loopbackHTTP), + credential: { + calls.withLock { $0 += 1 } + throw Offline() + }, + inner: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + let rig = try Rig(server: server, http: http, sleep: sleeps.record) + defer { rig.cleanUp() } + await rig.uploader.kick() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "create", codes: ["auth_credential_unavailable"], status: 0)) + #expect(server.state.withLock { $0.log.isEmpty }) + #expect(calls.withLock { $0 } == 1) + #expect(sleeps.isEmpty) + } + + /// Signed headers that include Authorization end the upload at the PUT, with no PUT sent + /// and no retry. The API requests before it carried the credential as usual. + @Test func signedHeadersCarryingAuthorizationEndTheUploadAtThePut() async throws { + let sleeps = Sleeps() + let rig = try Self.rig(provider: Provider([Self.token]), sleeps: sleeps) { + $0.extraSignedHeaders = ["Authorization": "Bearer storage-credential"] + } + defer { rig.cleanUp() } + await rig.uploader.add(try await rig.capture.sealImages(count: 2)) + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "put", codes: ["auth_storage_authorization"], status: 0)) + #expect(rig.server.requests("PUT upload").isEmpty) + #expect(rig.server.requests("POST captures").first?.headers["authorization"] == "Bearer \(Self.token)") + #expect(sleeps.isEmpty) + } + + /// A withdrawal that lands while the credential is being fetched outranks the refusal that + /// follows: the upload stops as withdrawn, not as a credential failure. + @Test func aWithdrawalOutranksACredentialRefusal() async throws { + struct Offline: Error {} + let sleeps = Sleeps() + let uploader = Mutex(nil) + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.expectedBearer = Self.token } + let http = ScopedCaptureHTTP( + scope: try CaptureAPIScope(base: server.base, transport: .loopbackHTTP), + credential: { + _ = uploader.withLock { $0 }?.withdrawConsent() + throw Offline() + }, + inner: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + let rig = try Rig(server: server, http: http, sleep: sleeps.record) + defer { rig.cleanUp() } + uploader.withLock { $0 = rig.uploader } + await rig.uploader.kick() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + let end = await rig.uploader.snapshot.end + #expect(end != .failed(step: "create", codes: ["auth_credential_unavailable"], status: 0)) + #expect(server.state.withLock { $0.log.isEmpty }) + #expect(sleeps.isEmpty) + } + + /// With the default loopback (no expected bearer), the old assertion stands: an API request + /// that carries Authorization is refused. + @Test func theDefaultLoopbackStillRefusesAnyAPIAuthorization() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + var request = URLRequest(url: rig.server.base.appending(path: "captures")) + request.httpMethod = "POST" + request.setValue("Bearer x", forHTTPHeaderField: "Authorization") + request.httpBody = Data("{}".utf8) + let reply = try await URLSessionCaptureHTTP.ephemeral(timeout: 10).send(request) + #expect(reply.status == 400) + #expect(CaptureAPI.errorCodes(reply.body) == ["unexpected_authorization"]) + } +} From 73e06d70b5fe00cc3737f96855988a73c675cd5c Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 09:54:42 -0500 Subject: [PATCH 30/71] Send nothing for a request cancelled while its credential is fetched ScopedCaptureHTTP.send now checks for cancellation before asking the provider for the credential and again after the provider returns. A provider can wait without watching for cancellation and then hand back a token; the second check keeps that token off a request the caller already gave up on. A CancellationError from the provider still passes straight through. This stops only new requests from this side; it doesn't withdraw anything the server already has. Two tests: - A provider held on a continuation is cancelled while waiting, then released with a good token. The request throws CancellationError and never reaches the inner transport. - A task cancelled before send never reaches the provider. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Upload/ScopedCaptureHTTP.swift | 6 ++ .../ScopedCaptureHTTPTests.swift | 69 +++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/ScopedCaptureHTTP.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/ScopedCaptureHTTP.swift index 7e8e5edcd..51633c673 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/ScopedCaptureHTTP.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/ScopedCaptureHTTP.swift @@ -36,6 +36,11 @@ public struct ScopedCaptureHTTP: CaptureHTTP { // The destination is checked before the credential is asked for. guard let url = request.url else { throw ScopedCaptureHTTPError.destinationNotAllowed("no_url") } if let reason = scope.problem(with: url) { throw ScopedCaptureHTTPError.destinationNotAllowed(reason) } + // A request cancelled before or during the credential fetch is never sent. A provider may + // wait without watching for cancellation and then return a token anyway; checking again + // after it returns keeps that token off a request the caller has given up on. This stops + // only new requests from this side: it doesn't withdraw anything the server already has. + try Task.checkCancellation() let token: String do { token = try await credential() @@ -45,6 +50,7 @@ public struct ScopedCaptureHTTP: CaptureHTTP { // Only the error's type: a provider's message may quote what it tried. throw ScopedCaptureHTTPError.credentialUnavailable(String(describing: type(of: error))) } + try Task.checkCancellation() guard Self.isBearerToken(token) else { throw ScopedCaptureHTTPError.credentialMalformed } var authorized = request authorized.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureHTTPTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureHTTPTests.swift index c8e5a8791..9c612a72c 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureHTTPTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureHTTPTests.swift @@ -165,6 +165,75 @@ import Testing #expect(recorder.sent.withLock { $0.isEmpty }) } + /// Holds the credential provider until the test releases it, ignoring cancellation while it + /// waits, as a provider that doesn't watch for it would. + final class Gate: Sendable { + private let waiting = Mutex?>(nil) + private let enteredStream: AsyncStream + private let enteredContinuation: AsyncStream.Continuation + + init() { + (enteredStream, enteredContinuation) = AsyncStream.makeStream() + } + + /// Waits for `release`; reports to `entered` once it is waiting. + func wait() async { + await withCheckedContinuation { continuation in + waiting.withLock { $0 = continuation } + enteredContinuation.yield() + } + } + + func entered() async { + for await _ in enteredStream { return } + } + + func release() { + waiting.withLock { + $0?.resume() + $0 = nil + } + } + } + + /// Cancelled while the provider is waiting, the request isn't sent, even though the + /// provider then returns a well-formed token. + @Test func aRequestCancelledDuringTheCredentialFetchIsNeverSent() async throws { + let recorder = Recorder() + let gate = Gate() + let http = ScopedCaptureHTTP( + scope: try CaptureAPIScope(base: Self.base), + credential: { + await gate.wait() + return ScopedCaptureHTTPTests.token + }, + inner: recorder) + let request = Self.request("https://api.example.com/v1/captures") + let task = Task { try await http.send(request) } + await gate.entered() + task.cancel() + gate.release() + let result = await task.result + #expect(throws: CancellationError.self) { try result.get() } + #expect(recorder.sent.withLock { $0.isEmpty }) + } + + /// A request already cancelled doesn't reach the provider at all. + @Test func aRequestCancelledBeforeTheCredentialFetchNeverReachesTheProvider() async throws { + let recorder = Recorder() + let provider = Provider() + let http = try Self.scoped(provider, inner: recorder) + let request = Self.request("https://api.example.com/v1/captures") + let task = Task { () async throws -> HTTPReply in + withUnsafeCurrentTask { $0?.cancel() } + return try await http.send(request) + } + let result = await task.result + #expect(throws: CancellationError.self) { try result.get() } + #expect(provider.calls.withLock { $0 } == 0) + #expect(recorder.sent.withLock { $0.isEmpty }) + } + /// An Authorization the caller already set is replaced, never sent alongside or instead. @Test func anAPIRequestCarriesOnlyTheIssuedCredential() async throws { let recorder = Recorder() From 12e4fa2c1fb0874c4f0bd6e16a03c1854fbdf39d Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 09:56:27 -0500 Subject: [PATCH 31/71] Add a labelled synthetic capture and say what an answer's match proves SyntheticCapture is #192's synthetic recording (the coordinator tests' NativeCaptureFixture) as a library type: a 60 Hz trajectory with each frame's intrinsics, accelerometer and gyro at about 100 Hz with a fixed jitter, six pattern photos at times on that trajectory, and a fixed start date, so two runs give the same rows and bytes. Its packet says source.kind synthetic, an existing 0.4 field. Packet04's rate and IMU checks are unchanged: the same capture without motion still ends "not prepared". CaptureFixtureLaunch decides whether a run may use the capture fixture: only a debug build, only on a replay, only when asked; the synthetic capture also needs the fixture. FixtureCaptureHTTP gains wrongRun, a result naming another run, which the uploader refuses. Wording: PhotoCaptureBinding is a local association. The answer names only its run; session, packet and capture are the phone's own record of what it sent, and the service returns no input digest or attestation, so photo processing isn't accepted for live use until it does. Two test names now say the same. FixtureCaptureHTTP and SyntheticCapture are in every build of HouseScanKit; only the app's activation is debug-only. No #192 file changed; CaptureUploader and LoopbackCaptureAPI untouched. Validated: `swift test -j 2 -Xswiftc -warnings-as-errors` in ios/HouseScanKit through heavy-admit passed 843 tests in 110 suites in 71 s (8 new: the resolver's release, live-camera and default cases, determinism and parity with #192's fixture, the synthetic packet passing Packet04Check and reaching an answer, the motionless packet refused, wrongRun refused). `swift build -c release` was not run. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Processing/FixtureCaptureHTTP.swift | 12 +- .../Processing/PhotoCaptureBinding.swift | 11 +- .../PhotoProcessingController.swift | 6 +- .../Processing/SyntheticCapture.swift | 151 ++++++++++++++++++ .../PhotoProcessingTests.swift | 8 +- .../SyntheticCaptureTests.swift | 124 ++++++++++++++ 6 files changed, 303 insertions(+), 9 deletions(-) create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Processing/SyntheticCapture.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/SyntheticCaptureTests.swift diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/FixtureCaptureHTTP.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/FixtureCaptureHTTP.swift index a9d1bae12..be965ee27 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Processing/FixtureCaptureHTTP.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/FixtureCaptureHTTP.swift @@ -8,7 +8,10 @@ import Synchronization /// and ends every capture with the answer it was made with. Each answer's words say they come /// from a fixture. /// -/// Written from the public API types in `CaptureAPI` and `CaptureResult`; it is not the service. +/// Written from the public API types in `CaptureAPI` and `CaptureResult`; it is not the service, +/// and an answer from it says nothing about any wall. The type is compiled into every build of +/// HouseScanKit; only the app's activation of it is limited to debug builds on a replay +/// (`CaptureFixtureLaunch`). public final class FixtureCaptureHTTP: CaptureHTTP, Sendable { public enum Answer: String, Sendable, CaseIterable { case candidate, needsViews, manualReview, notEligible, failed, expired @@ -20,6 +23,8 @@ public final class FixtureCaptureHTTP: CaptureHTTP, Sendable { case refuseCreate /// The run never finishes, so the capture stays in processing. case hold + /// The result names another run than the one finalize started, which the uploader refuses. + case wrongRun /// The homeowner message the answer carries, when it has an outcome. public var message: String? { @@ -28,6 +33,7 @@ public final class FixtureCaptureHTTP: CaptureHTTP, Sendable { case .needsViews: "Fixture answer: one more view of the wall, please." case .manualReview: "Fixture answer: an installer needs to look at this wall." case .notEligible: "Fixture answer: no spot on this wall." + case .wrongRun: "Fixture answer: this answer is for another run." case .failed, .expired, .notReady, .unreadable, .refuseCreate, .hold: nil } } @@ -35,7 +41,7 @@ public final class FixtureCaptureHTTP: CaptureHTTP, Sendable { /// The capture's status once every listed file is committed. var endStatus: String { switch self { - case .candidate, .notEligible, .notReady, .unreadable: "complete" + case .candidate, .notEligible, .notReady, .unreadable, .wrongRun: "complete" case .needsViews: "needs_views" case .manualReview: "manual_review" case .failed: "failed" @@ -248,7 +254,7 @@ public final class FixtureCaptureHTTP: CaptureHTTP, Sendable { outcome = decided } return json(200, [ - "runId": runID, "status": answer.endStatus, "viewsNeeded": views, "memberActions": [Any](), "outcome": outcome, + "runId": answer == .wrongRun ? "run_fixture_other" : runID, "status": answer.endStatus, "viewsNeeded": views, "memberActions": [Any](), "outcome": outcome, ]) } diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoCaptureBinding.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoCaptureBinding.swift index c389e6129..404b9e11a 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoCaptureBinding.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoCaptureBinding.swift @@ -2,6 +2,14 @@ import Foundation /// What a photo-processing answer has to match before the app shows it, taken when the scan's /// capture ended. +/// +/// This is a local association, not a binding the service declares. The answer +/// (`CaptureResult.Response`) names only its run; the session, packet and capture it is held +/// against are this phone's own record of what it sent, and the service returns no digest of the +/// input it processed and no attestation. So a match means "the answer the uploader fetched for +/// the capture this scan sent, for the run that capture started", not "the service says it read +/// these bytes". Until the capture API states its input, photo processing can't be called +/// accepted for live use. public struct PhotoCaptureBinding: Sendable, Equatable { public var spatial: SpatialSession public var origin: URL @@ -42,7 +50,8 @@ public struct PhotoCaptureObservation: Sendable, Equatable { } } -/// The first identity an answer and the scan on screen disagree on. +/// The first identity an answer and the scan on screen disagree on. All but `run` are compared +/// with the phone's own records (`PhotoCaptureBinding`). public enum PhotoBindingMismatch: Sendable, Equatable, CaseIterable { /// Another ARKit world, or none: an answer for a world that is gone. case world diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift index 3e807ac8e..28fb3b02a 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift @@ -8,8 +8,10 @@ import simd /// Each photo-processing scan starts a new coordinator session (`beginScan`), which asks for /// consent again. Start over, or a Legacy scan starting, ends it (`endScan`); a world reset starts /// the scan's next packet with the same answer (`worldReset`). The coordinator drops status and -/// answers from a session that ended, and the binding check (`PhotoCaptureBinding`) refuses any -/// answer that isn't for the recording's session, world, packet, capture and run. +/// answers from a session that ended, and `PhotoCaptureBinding` refuses an answer whose run, or +/// whose local association (the recording's world, the coordinator session, packet and capture +/// the phone sent), isn't this scan's. The service declares no input digest, so this is the +/// phone's association, not proof of what the service read. /// /// A capture saved on disk is never resumed from here. Folder ownership lasts only this process, /// so a relaunch asks for consent again rather than act on a yes it can't prove. diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/SyntheticCapture.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/SyntheticCapture.swift new file mode 100644 index 000000000..b88b57abf --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/SyntheticCapture.swift @@ -0,0 +1,151 @@ +import CoreGraphics +import Foundation +import ImageIO +import simd + +/// A made-up capture whose packet says so (`source.kind: synthetic`), for running photo +/// processing end to end without a phone's sensors: the DEBUG app's +/// `-photoProcessingSyntheticCapture` and the package tests. +/// +/// The numbers are #192's synthetic recording (`NativeCaptureFixture` in the package tests): a +/// camera 2 m from a wall at 1.5 m height walking +x at 0.1 m/s for 12 s, a 60 Hz trajectory with +/// each frame's intrinsics beside it, and accelerometer and gyro at about 100 Hz on their own +/// clocks with a fixed jitter. Photos are pattern images, no house, taken at times on that +/// trajectory with its poses and intrinsics. Everything is computed from fixed constants, so two +/// runs give the same rows and the same JPEG bytes, and no wall clock enters the data. +/// +/// It is never a recording of the scan on screen: a replay keeps no motion, and nothing here is +/// added to one. Like `FixtureCaptureHTTP`, the type is in every build; only the app's activation +/// is limited to debug builds on a replay. +public struct SyntheticCapture: Sendable { + public static let standard = SyntheticCapture() + + /// Sensor image size and its intrinsics [fx, fy, cx, cy]. + public static let imageSize = SIMD2(1920, 1440) + public static let intrinsics = SIMD4(1445, 1445, 960, 720) + /// The stored JPEGs are a quarter of the sensor size each way, with the intrinsics scaled to + /// match (`KeptPhoto.observation`), which keeps writing them fast. + public static let jpegSize = (width: 480, height: 360) + /// A fixed wall-clock start, so the packet holds no date from the phone running it. + public static let startedAt = Date(timeIntervalSince1970: 1_790_000_000) + + public let start = 5000.0 + public let end = 5012.0 + + public init() {} + + public func pose(_ t: Double) -> simd_float4x4 { + var m = matrix_identity_float4x4 + m.columns.3 = SIMD4(Float(t - start) * 0.1, 1.5, 0, 1) + return m + } + + /// The close-up the scan accepted, as the coordinator wants it at the capture's end. + public var acceptedCloseUpAt: Double { start + 2.5 } + + /// The photos, in the order they are kept: the meter close-up, then five walk frames. + public var photoTimes: [(t: Double, purpose: String?)] { + var times: [(t: Double, purpose: String?)] = [(start + 2.5, "meter_close")] + for i in 0..<5 { times.append((start + 3 + Double(i) * 1.5, nil)) } + return times + } + + public var recording: RecordingSource { + let capture = self + return RecordingSource(start: { (capture.start, Self.startedAt) }, rows: { capture.rows }) + } + + public var rows: RecorderRows { + var trajectory: [[Double]] = [], intrinsics: [[Double]] = [] + let k = Self.intrinsics + for i in 0...Int((end - start) * 60) { + let t = start + Double(i) / 60 + let m = pose(t) + let columns: [SIMD4] = [m.columns.0, m.columns.1, m.columns.2, m.columns.3] + let values: [Double] = columns.flatMap { (c: SIMD4) -> [Double] in [Double(c.x), Double(c.y), Double(c.z), Double(c.w)] } + trajectory.append([t, 0, 0] + values) + intrinsics.append([t, Double(k.x), Double(k.y), Double(k.z), Double(k.w), Double(Self.imageSize.x), Double(Self.imageSize.y)]) + } + // The fixed jitter keeps the two sensors from ever sharing a timestamp. + var accel: [[Double]] = [], gyro: [[Double]] = [] + for i in 0.., appVersion: String) -> Packet04SessionInfo { + Packet04SessionInfo( + packetID: packetID, sessionID: packetID, source: .synthetic, appVersion: appVersion, deviceModel: "synthetic", + systemVersion: "synthetic", lidarAvailable: false, sceneDepthEnabled: false, meshReconstructionSupported: nil, + sceneReconstruction: nil, planeDetection: nil, videoWidth: Int(video.x), videoHeight: Int(video.y), framesPerSecond: nil, + timeZone: nil) + } + + /// Writes the photos' JPEGs into `folder` (again only when missing) and returns them as kept + /// photos. + public func photos(in folder: URL) throws -> [KeptPhoto] { + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + return try photoTimes.enumerated().map { index, photo in + let url = folder.appending(path: "synthetic-\(index).jpg") + if !FileManager.default.fileExists(atPath: url.path) { try Self.writeJPEG(pattern: index, to: url) } + return KeptPhoto( + t: photo.t, cameraToWorld: pose(photo.t), cameraIntrinsics: Self.intrinsics, cameraImageSize: Self.imageSize, + width: Self.jpegSize.width, height: Self.jpegSize.height, tracking: .normal, + exposure: .init(duration: 0.004, offset: 0, iso: 64, fNumber: 1.6), jpeg: url, purpose: photo.purpose) + } + } + + public enum ImageError: Error, Equatable { + case notWritten + } + + /// A colour pattern, shifted per photo so no two are the same image. + static func writeJPEG(pattern: Int, to url: URL) throws { + let (width, height) = jpegSize + guard let context = CGContext( + data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 4 * width, + space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue), + let data = context.data + else { throw ImageError.notWritten } + let pixels = data.bindMemory(to: UInt8.self, capacity: 4 * width * height) + for y in 0.. Self { + guard debugBuild else { return .off("a release build has no capture fixture") } + guard let answer else { + return .off(syntheticCapture ? "the synthetic capture needs the capture fixture" : "sending captures is off in this build") + } + guard onReplay else { return .off("the capture fixture runs only on a replay, never the live camera") } + return .on(answer, syntheticCapture: syntheticCapture) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift index 861ddadeb..b08483754 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift @@ -51,7 +51,7 @@ import Testing #expect(binding.mismatch(try record(), observed: matching) == nil) } - @Test func anAnswerFromAnotherWorldIsRefusedWhateverElseMatches() throws { + @Test func anAnswerAssociatedWithAnotherWorldIsRefusedWhateverElseMatches() throws { var observed = matching observed.spatial = SpatialSession(recordingSessionID: "rec-2", worldEpoch: 1) #expect(binding.mismatch(try record(), observed: observed) == .world) @@ -61,7 +61,7 @@ import Testing #expect(binding.mismatch(try record(), observed: observed) == .world) } - @Test func anAnswerForAnotherDestinationSessionPacketCaptureRunOrEpochIsRefused() throws { + @Test func anAnswerWhoseRunOrLocalAssociationDiffersIsRefused() throws { var observed = matching observed.origin = URL(string: "https://elsewhere.invalid/v1") #expect(binding.mismatch(try record(), observed: observed) == .origin) @@ -175,7 +175,7 @@ final class HeldResultHTTP: CaptureHTTP, Sendable { } @Test(arguments: [ - FixtureCaptureHTTP.Answer.candidate, .needsViews, .manualReview, .notEligible, .failed, .expired, .notReady, .unreadable, + FixtureCaptureHTTP.Answer.candidate, .needsViews, .manualReview, .notEligible, .failed, .expired, .notReady, .unreadable, .wrongRun, ]) func eachAnswerReachesTheScanAsItsTypedStage(answer: FixtureCaptureHTTP.Answer) async throws { defer { try? FileManager.default.removeItem(at: root) } @@ -215,6 +215,8 @@ final class HeldResultHTTP: CaptureHTTP, Sendable { #expect(stage == .ended(.answerNotReady)) case .unreadable: #expect(stage == .ended(.answerUnreadable)) + case .wrongRun: + #expect(stage == .ended(.answerMismatch(.run))) case .refuseCreate, .hold: Issue.record("not an argument of this test") } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/SyntheticCaptureTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/SyntheticCaptureTests.swift new file mode 100644 index 000000000..8f2e4f5f8 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/SyntheticCaptureTests.swift @@ -0,0 +1,124 @@ +import Foundation +import HouseScanKit +import Testing + +@Suite struct CaptureFixtureLaunchTests { + @Test(arguments: [true, false]) + func aReleaseBuildNeverUsesTheFixture(onReplay: Bool) { + for synthetic in [true, false] { + let launch = CaptureFixtureLaunch.resolve(debugBuild: false, onReplay: onReplay, answer: .candidate, syntheticCapture: synthetic) + #expect(launch == .off("a release build has no capture fixture")) + } + } + + @Test func theLiveCameraNeverUsesTheFixture() { + for synthetic in [true, false] { + let launch = CaptureFixtureLaunch.resolve(debugBuild: true, onReplay: false, answer: .candidate, syntheticCapture: synthetic) + #expect(launch == .off("the capture fixture runs only on a replay, never the live camera")) + } + } + + @Test func theSyntheticCaptureNeedsTheFixtureAndNothingIsOnByDefault() { + #expect(CaptureFixtureLaunch.resolve(debugBuild: true, onReplay: true, answer: nil, syntheticCapture: true) + == .off("the synthetic capture needs the capture fixture")) + #expect(CaptureFixtureLaunch.resolve(debugBuild: true, onReplay: true, answer: nil, syntheticCapture: false) + == .off("sending captures is off in this build")) + } + + @Test func aDebugReplayUsesWhatItAskedFor() { + #expect(CaptureFixtureLaunch.resolve(debugBuild: true, onReplay: true, answer: .hold, syntheticCapture: false) == .on(.hold, syntheticCapture: false)) + #expect(CaptureFixtureLaunch.resolve(debugBuild: true, onReplay: true, answer: .candidate, syntheticCapture: true) + == .on(.candidate, syntheticCapture: true)) + } +} + +@Suite(.serialized) @MainActor struct SyntheticCaptureTests { + let root = FileManager.default.temporaryDirectory.appending(path: "synthetic-capture-\(UUID().uuidString)") + + @Test func itIsTheRecordingOf192sSyntheticFixture() { + let fixture = NativeCaptureFixture(folder: root) + let capture = SyntheticCapture.standard + #expect(capture.start == fixture.start && capture.end == fixture.end) + let ours = capture.rows, theirs = fixture.rows + #expect(ours.trajectory == theirs.trajectory) + #expect(ours.intrinsics == theirs.intrinsics) + #expect(ours.accelerometer == theirs.accelerometer) + #expect(ours.gyroscope == theirs.gyroscope) + #expect(capture.recording.start()?.uptime == capture.start) + #expect(capture.recording.start()?.date == SyntheticCapture.startedAt) + } + + @Test func twoRunsMakeTheSameRowsAndPhotos() throws { + defer { try? FileManager.default.removeItem(at: root) } + let first = try SyntheticCapture().photos(in: root.appending(path: "a")) + let second = try SyntheticCapture().photos(in: root.appending(path: "b")) + #expect(first.count == 6) + for (a, b) in zip(first, second) { + #expect(a.t == b.t && a.purpose == b.purpose && a.cameraToWorld == b.cameraToWorld) + #expect(try Data(contentsOf: a.jpeg) == Data(contentsOf: b.jpeg)) + } + #expect(Set(try first.map { try Data(contentsOf: $0.jpeg) }).count == 6, "two photos share an image") + #expect(SyntheticCapture().rows.accelerometer == SyntheticCapture.standard.rows.accelerometer) + } + + func controller(_ http: FixtureCaptureHTTP) -> PhotoProcessingController { + var policy = CaptureUploader.Policy() + policy.eventsWait = 0 + policy.firstDelay = 0.01 + policy.maxDelay = 0.02 + let environment = CaptureSessionCoordinator.Environment( + endpoint: FixtureCaptureHTTP.base, sends: true, http: http, capturesFolder: root.appending(path: "Captures"), + sessionInfo: { packetID, video in SyntheticCapture.sessionInfo(packetID: packetID, video: video, appVersion: "test") }, + device: .init(model: "synthetic", systemVersion: "synthetic", appVersion: "test"), tier: .arkit, policy: policy) + return PhotoProcessingController(setup: .ready(environment, standIn: true)) + } + + func until(_ condition: () -> Bool) async throws -> Bool { + let deadline = ContinuousClock.now + .seconds(30) + while !condition(), ContinuousClock.now < deadline { try await Task.sleep(for: .milliseconds(20)) } + return condition() + } + + /// The capture through the real coordinator and uploader: a packet the phone's checks pass, + /// labelled synthetic, and the fixture's answer as the scan's typed stage. + @Test func itsPacketPassesThePhonesChecksSaysSyntheticAndGetsAnAnswer() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .candidate) + let controller = controller(http) + let capture = SyntheticCapture.standard + controller.beginScan(ScanContext(scanID: "scan", profile: controller.profile, recordingSessionID: "rec"), recording: capture.recording) + controller.answerConsent(true) + for photo in try capture.photos(in: root.appending(path: "photos")) { controller.kept(photo) } + controller.captureEnded(acceptedCloseUpAt: capture.acceptedCloseUpAt) + try #require(try await until { controller.status?.isFinal == true }, "\(String(describing: controller.status))") + #expect(controller.status?.stage == .answered(.candidate(.init( + message: FixtureCaptureHTTP.Answer.candidate.message ?? "", previewMade: false, ar: .withheld(.analysisUnverified))))) + let folder = try #require(controller.captureFolder) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: Data(contentsOf: folder.appending(path: "packet.json"))) + #expect(packet.source.kind == .synthetic) + #expect(Packet04Check.problems(packet, folder: folder).isEmpty) + #expect(packet.stills?.map(\.purpose) == ["meter_close"]) + #expect(packet.scaleReference.meterCloseUp.still == "meter_close") + controller.endScan(recording: capture.recording) + } + + /// The same capture without its motion, as a replay is: the phone's checks still refuse it. + @Test func withoutMotionThePacketIsStillRefusedOnThePhone() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .candidate) + let controller = controller(http) + let capture = SyntheticCapture.standard + let still = RecordingSource(start: { (capture.start, SyntheticCapture.startedAt) }, rows: { + let rows = capture.rows + return RecorderRows(trajectory: rows.trajectory, intrinsics: rows.intrinsics, accelerometer: [], gyroscope: []) + }) + controller.beginScan(ScanContext(scanID: "scan", profile: controller.profile, recordingSessionID: "rec"), recording: still) + controller.answerConsent(true) + for photo in try capture.photos(in: root.appending(path: "photos")) { controller.kept(photo) } + controller.captureEnded(acceptedCloseUpAt: capture.acceptedCloseUpAt) + try #require(try await until { controller.status?.isFinal == true }) + #expect(controller.status?.stage == .ended(.notPrepared)) + #expect(!http.routes.contains("POST captures/finalize")) + controller.endScan(recording: capture.recording) + } +} From 24efd35961e4f4557ba055a6f1db4d336b34cc4c Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 09:56:27 -0500 Subject: [PATCH 32/71] Run photo processing in-app on the synthetic capture; stop unrunnable scans early Two debug launch options: -photoProcessingSyntheticCapture (with -photoProcessingFixture on a replay) sends SyntheticCapture in place of the replay's photos, so the real coordinator and uploader carry it to the fixture's typed answer on the processing screen; -photoProcessingFixtureReadOnlyCapture makes the capture folder read-only before "Stop sending photos", so the phone can't save the withdrawal and the screen says so. The setup goes through CaptureFixtureLaunch with debugBuild from #if DEBUG. The consent card and the screen say the capture is synthetic and the answer is a test one about no wall. A photo-processing scan whose profile isn't set up now stops at scan start, before the camera, the photo question or any request, and offers "Scan with Legacy instead", which chooses Legacy and starts a new scan. The old scan is never rerouted. Developer options say so up front. The processing screen's options chip gets its own full-width row; the audit found its words clipped beside the badge. Validated: built for the iPhone 17 Simulator and ran only PhotoProcessingUITests through heavy-admit: 10 of 10 passed, 459 s of test time, 858 s wall including the build. Real engine with the fixture: synthetic capture to a typed candidate shown as the main result, another run's answer refused, no consent sends nothing, a withdrawal the phone couldn't save surfaced, Start over revives nothing, a mid-scan choice applies to the next scan, not-set-up stops before capture with no request and the Legacy way out works. Demo mode: every state at default size, the main ones at the largest, five audits. heavy-admit exited 125 after the tests succeeded: its cleanup kept an Xcode DTServiceHub process. Other UI suites and Release were not run locally. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/Contract/ScanContract.swift | 6 ++ ios/HouseScan/Runtime/Autopilot.swift | 5 + ios/HouseScan/Runtime/LaunchOptions.swift | 14 ++- .../Runtime/ScanEngine+Actions.swift | 6 ++ .../Runtime/ScanEngine+Processing.swift | 89 ++++++++++++++---- ios/HouseScan/Runtime/ScanEngine.swift | 7 +- .../UI/Components/PhotoConsentCard.swift | 3 +- ios/HouseScan/UI/Copy/ProcessingCopy.swift | 15 ++- .../Preview/DemoEngine+PhotoProcessing.swift | 6 ++ ios/HouseScan/UI/ScanRootView.swift | 6 +- .../UI/Screens/PhotoProcessingScreen.swift | 23 ++++- .../PhotoProcessingUITests.swift | 91 +++++++++++++++++-- ios/README.md | 8 +- 13 files changed, 239 insertions(+), 40 deletions(-) diff --git a/ios/HouseScan/Contract/ScanContract.swift b/ios/HouseScan/Contract/ScanContract.swift index 951e04a92..ef617dfa8 100644 --- a/ios/HouseScan/Contract/ScanContract.swift +++ b/ios/HouseScan/Contract/ScanContract.swift @@ -974,6 +974,9 @@ final class ScanViewState { /// Photo processing's side of this scan: its consent, progress and answer. Nil on a Legacy /// scan, and before a scan starts. var photoProcessing: PhotoProcessingStatus? + /// Photo processing sends a synthetic test capture in place of this scan's photos (DEBUG's + /// `-photoProcessingSyntheticCapture`), so its answer says nothing about the wall on screen. + var photoCaptureIsSynthetic = false init() {} } @@ -1064,6 +1067,9 @@ protocol ScanActions: AnyObject { func answerPhotoConsent(_ yes: Bool) /// A no after a yes: this scan's photos stop going to photo processing. func stopSendingPhotos() + /// A photo-processing scan this build can't send, stopped before capture: choose Legacy and + /// start a new scan with it. + func scanWithLegacyInstead() /// The app came back to the foreground on the camera-access failure: if access is now on, /// the scan goes on without Start over. Does nothing otherwise. func recheckCameraAccess() diff --git a/ios/HouseScan/Runtime/Autopilot.swift b/ios/HouseScan/Runtime/Autopilot.swift index ca718cbf6..dbeec3fb9 100644 --- a/ios/HouseScan/Runtime/Autopilot.swift +++ b/ios/HouseScan/Runtime/Autopilot.swift @@ -39,6 +39,11 @@ final class Autopilot { await engine.waitForGate(.onboarding) engine.finishOnboarding() await prepared + // A photo-processing scan this build can't send stops before the camera: nothing to drive. + if engine.state.phase == .processing { + log("done: photo processing isn't set up, so the scan stopped before capture") + return + } if let window = replay.heldBack { log("holding back frames \(window.frames.lowerBound)..<\(window.frames.upperBound) for the gap loop; gap \(window.gap.band.rawValue) \(format(window.gap.span))") } else { diff --git a/ios/HouseScan/Runtime/LaunchOptions.swift b/ios/HouseScan/Runtime/LaunchOptions.swift index cd078ff09..c5926b554 100644 --- a/ios/HouseScan/Runtime/LaunchOptions.swift +++ b/ios/HouseScan/Runtime/LaunchOptions.swift @@ -57,7 +57,15 @@ import OSLog /// - `-photoProcessingFixture ` (debug builds only, with `-replay`): photo processing sends /// to a capture API answered inside the app (`FixtureCaptureHTTP`), which ends every capture /// with this answer (`FixtureCaptureHTTP.Answer`). Nothing leaves the phone. Without it, photo -/// processing isn't set up in any build. +/// processing isn't set up in any build. A replay's own packet always fails the phone's checks, +/// since a replay has no motion. +/// - `-photoProcessingSyntheticCapture` (debug builds only, with `-photoProcessingFixture`): the +/// capture sent is HouseScanKit's `SyntheticCapture`, labelled synthetic in its packet, in place +/// of the replay's photos, so the fixture's answer comes back through the real upload. It is a +/// test of the app's path against the fixture, never a result about the wall on screen. +/// - `-photoProcessingFixtureReadOnlyCapture` (debug builds only): just before "Stop sending +/// photos" takes effect, the capture's folder is made read-only, so the phone can't save the +/// withdrawal and says so (`PhotoProcessingEnd.withdrawn(recorded: false)`). struct LaunchOptions: Equatable { var replayFolder: URL? var autopilot = false @@ -77,6 +85,8 @@ struct LaunchOptions: Equatable { var failCloseUpSave = false var processingBackend: ProcessingBackend? var photoProcessingFixture: FixtureCaptureHTTP.Answer? + var photoProcessingSyntheticCapture = false + var photoProcessingFixtureReadOnlyCapture = false init( arguments: [String] = ProcessInfo.processInfo.arguments, @@ -111,6 +121,8 @@ struct LaunchOptions: Equatable { } photoProcessingFixture = answer } + photoProcessingSyntheticCapture = arguments.contains("-photoProcessingSyntheticCapture") + photoProcessingFixtureReadOnlyCapture = arguments.contains("-photoProcessingFixtureReadOnlyCapture") #endif simulateAppStore = arguments.contains("-simulateAppStore") failCloseUpSave = arguments.contains("-failCloseUpSave") diff --git a/ios/HouseScan/Runtime/ScanEngine+Actions.swift b/ios/HouseScan/Runtime/ScanEngine+Actions.swift index 574158cf0..dd0f03f11 100644 --- a/ios/HouseScan/Runtime/ScanEngine+Actions.swift +++ b/ios/HouseScan/Runtime/ScanEngine+Actions.swift @@ -10,6 +10,12 @@ extension ScanEngine: ScanActions { guard state.phase == .onboarding else { return } startPracticeIfOn() beginScan() + // A photo-processing scan this build can't send stops here, before the camera or any + // request, and says so with a way to a Legacy scan. + guard !photoScanCantRun else { + go(.processing) + return + } leaveOnboarding() } diff --git a/ios/HouseScan/Runtime/ScanEngine+Processing.swift b/ios/HouseScan/Runtime/ScanEngine+Processing.swift index 6129ab1ae..a35c2de22 100644 --- a/ios/HouseScan/Runtime/ScanEngine+Processing.swift +++ b/ios/HouseScan/Runtime/ScanEngine+Processing.swift @@ -24,19 +24,37 @@ extension ScanEngine { state.scanBackend = backend RuntimeLog.engine.info("scan \(context.scanID, privacy: .public) uses \(backend.rawValue, privacy: .public) processing") guard backend == .photoProcessing else { return } - attachKeptPhotos() - photoProcessing.beginScan(context, recording: recordingSource) + // A synthetic capture is sent in place of the scan's photos, never mixed with them. + if syntheticCapture == nil { attachKeptPhotos() } + photoProcessing.beginScan(context, recording: photoRecording) if case .ended(.notSetUp(let reason))? = state.photoProcessing?.stage { RuntimeLog.engine.info("photo processing not set up: \(reason, privacy: .public)") } } + /// The scan uses photo processing and this build can't send it. + var photoScanCantRun: Bool { + guard let profile = scanContext?.profile, profile.backend == .photoProcessing, case .notSetUp = profile.answers else { return false } + return true + } + + /// The homeowner's way out of a photo-processing scan this build can't run, before anything + /// was captured: Legacy becomes the choice in Developer options, and a new scan starts with + /// it. The old scan isn't rerouted; it ends. + func scanWithLegacyInstead() { + guard state.phase == .processing, photoScanCantRun else { return } + RuntimeLog.engine.info("photo processing isn't set up: the homeowner chose a Legacy scan instead") + ProcessingBackendSetting.shared.choose(.legacy) + startOver() + finishOnboarding() + } + /// Start over: the scan ends, and whatever photo processing it had ends with it. `recorder` /// is already the next scan's. func endScan() { scanContext = nil state.scanBackend = nil - photoProcessing.endScan(recording: recordingSource) + photoProcessing.endScan(recording: photoRecording) } /// A world reset: the same scan, backend and consent in a new spatial session. `recorder` has @@ -46,19 +64,30 @@ extension ScanEngine { let next = context.inNewWorld(recordingSessionID: recorder.sessionID) scanContext = next if next.profile.backend == .photoProcessing { - photoProcessing.worldReset(next, recording: recordingSource) + photoProcessing.worldReset(next, recording: photoRecording) } } /// The photo-processing scan was sent: once the photos still being written are kept, its - /// packet is frozen, and the screen follows the upload to the answer. + /// packet is frozen, and the screen follows the upload to the answer. With DEBUG's synthetic + /// capture, its photos are kept now instead, and its own close-up is the accepted one. func startPhotoProcessing() { go(.processing) let context = scanContext + let synthetic = syntheticCapture Task { guard await drainPendingSaves(), scanContext == context, state.phase == .processing else { return } - let closeUp = state.closeUp == .skipped ? nil : store.stillFrames["meter_close"]?.t - photoProcessing.captureEnded(acceptedCloseUpAt: closeUp) + guard let synthetic else { + let closeUp = state.closeUp == .skipped ? nil : store.stillFrames["meter_close"]?.t + photoProcessing.captureEnded(acceptedCloseUpAt: closeUp) + return + } + let folder = FileManager.default.temporaryDirectory.appending(path: "PhotoProcessingFixture/synthetic-photos", directoryHint: .isDirectory) + let photos = (try? await Task.detached(priority: .userInitiated) { try synthetic.photos(in: folder) }.value) ?? [] + guard scanContext == context, state.phase == .processing else { return } + if photos.isEmpty { RuntimeLog.engine.error("photo processing: the synthetic capture's photos couldn't be written") } + for photo in photos { photoProcessing.kept(photo) } + photoProcessing.captureEnded(acceptedCloseUpAt: synthetic.acceptedCloseUpAt) } } @@ -70,6 +99,11 @@ extension ScanEngine { func stopSendingPhotos() { guard state.phase == .processing else { return } + #if DEBUG + if options.photoProcessingFixtureReadOnlyCapture, photoProcessing.profile.answers == .standIn, let folder = photoProcessing.captureFolder { + try? FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) + } + #endif photoProcessing.stopSending() if case .withdrawn(let recorded)? = state.photoProcessing?.consent { RuntimeLog.engine.info("photo processing: sending stopped; the phone \(recorded ? "saved" : "could not save", privacy: .public) the withdrawal") @@ -85,6 +119,12 @@ extension ScanEngine { return ProcessingProfile(backend: .legacy, answers: .standIn, origin: nil) } + /// What photo processing records from: the synthetic capture's recording when there is one, + /// otherwise the current world's. + var photoRecording: RecordingSource { + syntheticCapture?.recording ?? recordingSource + } + /// The current world's recording as the capture packet reads it. A replay records no poses or /// motion, and this recorder writes no per-frame intrinsics, so the packet can't pass its /// checks and the phone refuses it; nothing is made up to fill the gap. @@ -132,22 +172,35 @@ extension ScanEngine { // MARK: Setup /// What photo processing can do in this build. Sending a capture off the phone is off in this - /// slice: the 0.4 clock and privacy contract, the credential and the deployment aren't - /// settled. The one way to run it is DEBUG's fixture on a replay, which answers on the phone. - static func photoProcessingSetup(_ options: LaunchOptions) -> PhotoProcessingController.Setup { + /// slice: the 0.4 clock and privacy contract, a scoped credential, the live intrinsics and + /// meter tap, and the deployment aren't settled. The one way to run it is DEBUG's fixture on a + /// replay, which answers on the phone (`CaptureFixtureLaunch`); a release build can't, since + /// `debugBuild` is decided at compile time and the launch options aren't even parsed there. + static func photoProcessingSetup(_ options: LaunchOptions) -> (setup: PhotoProcessingController.Setup, synthetic: SyntheticCapture?) { #if DEBUG - if let answer = options.photoProcessingFixture { - guard options.replayFolder != nil else { return .notSetUp("the capture fixture runs only on a replay") } - return .ready(fixtureEnvironment(answer, gate: options.autopilotGate), standIn: true) - } + let debugBuild = true + #else + let debugBuild = false #endif - return .notSetUp("sending captures is off in this build") + let launch = CaptureFixtureLaunch.resolve( + debugBuild: debugBuild, onReplay: options.replayFolder != nil, answer: options.photoProcessingFixture, + syntheticCapture: options.photoProcessingSyntheticCapture) + switch launch { + case .off(let reason): + return (.notSetUp(reason), nil) + case .on(let answer, let synthetic): + #if DEBUG + return (.ready(fixtureEnvironment(answer, synthetic: synthetic, gate: options.autopilotGate), standIn: true), synthetic ? SyntheticCapture.standard : nil) + #else + return (.notSetUp("a release build has no capture fixture"), nil) + #endif + } } #if DEBUG /// The capture fixture: every request answered in this process (`FixtureCaptureHTTP`), its /// routes listed one per line in `/photo-transport.log` for the UI tests. - private static func fixtureEnvironment(_ answer: FixtureCaptureHTTP.Answer, gate: URL?) -> CaptureSessionCoordinator.Environment { + private static func fixtureEnvironment(_ answer: FixtureCaptureHTTP.Answer, synthetic: Bool, gate: URL?) -> CaptureSessionCoordinator.Environment { let log = gate?.appending(path: "photo-transport.log") let http = FixtureCaptureHTTP(answer: answer) { routes in guard let log else { return } @@ -163,7 +216,9 @@ extension ScanEngine { return .init( endpoint: FixtureCaptureHTTP.base, sends: true, http: http, capturesFolder: folder, sessionInfo: { packetID, video in - Packet04SessionInfo( + // The synthetic capture says so in its packet; a replay's photos say replay. + if synthetic { return SyntheticCapture.sessionInfo(packetID: packetID, video: video, appVersion: device.appVersion) } + return Packet04SessionInfo( packetID: packetID, sessionID: packetID, source: .replay, appVersion: device.appVersion, deviceModel: device.model, systemVersion: device.systemVersion, lidarAvailable: false, sceneDepthEnabled: false, meshReconstructionSupported: nil, sceneReconstruction: nil, planeDetection: nil, videoWidth: Int(video.x), videoHeight: Int(video.y), framesPerSecond: nil, diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index 5036a5d17..7de314776 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -195,6 +195,8 @@ final class ScanEngine { var scanContext: ScanContext? /// Photo processing's capture and answer, for scans that use it (`ScanEngine+Processing`). let photoProcessing: PhotoProcessingController + /// DEBUG's synthetic capture, sent in place of the scan's photos; nil otherwise. + let syntheticCapture: SyntheticCapture? let resultClient: any ResultClient private var uploadTask: Task? /// The current upload's scene has fixed its ground; keep this true through answer pacing. @@ -273,13 +275,16 @@ final class ScanEngine { } else { resultClient = SampleResultClient(pace: options.autopilot ? options.autopilotHold : 1.2) } - photoProcessing = PhotoProcessingController(setup: Self.photoProcessingSetup(options)) + let photoSetup = Self.photoProcessingSetup(options) + photoProcessing = PhotoProcessingController(setup: photoSetup.setup) + syntheticCapture = photoSetup.synthetic ProcessingBackendSetting.shared.photoProcessingAnswers = photoProcessing.profile.answers photoProcessing.onChange = { [weak self] status in self?.state.photoProcessing = status } photoProcessing.meterAnchor = { [weak self] in self?.meterTracking?.pose } state.isAutopilot = options.autopilot state.isReplay = options.replayFolder != nil state.usesSampleResult = resultClient.isSample + state.photoCaptureIsSynthetic = syntheticCapture != nil } // MARK: Start diff --git a/ios/HouseScan/UI/Components/PhotoConsentCard.swift b/ios/HouseScan/UI/Components/PhotoConsentCard.swift index f205c8c3e..ffffd735e 100644 --- a/ios/HouseScan/UI/Components/PhotoConsentCard.swift +++ b/ios/HouseScan/UI/Components/PhotoConsentCard.swift @@ -6,6 +6,7 @@ import SwiftUI /// is a real answer: the scan goes on and ends saying it wasn't processed. struct PhotoConsentCard: View { var standIn: Bool + var synthetic: Bool let actions: any ScanActions var body: some View { @@ -26,7 +27,7 @@ struct PhotoConsentCard: View { .fixedSize(horizontal: false, vertical: true) .accessibilityAddTraits(.isHeader) .accessibilityIdentifier("photoConsent.title") - Text(ProcessingCopy.consentBody(standIn: standIn)) + Text(ProcessingCopy.consentBody(standIn: standIn, synthetic: synthetic)) .font(Typeface.hint) .foregroundStyle(Palette.chalk) .fixedSize(horizontal: false, vertical: true) diff --git a/ios/HouseScan/UI/Copy/ProcessingCopy.swift b/ios/HouseScan/UI/Copy/ProcessingCopy.swift index 91eb33a5d..c4237d1e8 100644 --- a/ios/HouseScan/UI/Copy/ProcessingCopy.swift +++ b/ios/HouseScan/UI/Copy/ProcessingCopy.swift @@ -26,7 +26,7 @@ enum ProcessingCopy { switch answers { case .service: nil case .standIn: "Test build: answers come from a stand-in on this phone, and nothing leaves it." - case .notSetUp: "Not set up in this build. A scan that uses it can't be sent, and says so at the end." + case .notSetUp: "Not set up in this build. A scan with it stops before the camera starts and offers a Legacy scan instead." } } @@ -39,13 +39,18 @@ enum ProcessingCopy { static let consentTitle = "Send this scan's photos?" - static func consentBody(standIn: Bool) -> String { + static func consentBody(standIn: Bool, synthetic: Bool) -> String { let destination = standIn ? "In this test build, a stand-in on this phone receives them, so nothing leaves it." : "They go to House Scan's processing service while you scan." - return "Photo processing uses the photos you take of the wall around your meter, and how the phone moved, to suggest a battery spot. \(destination)" + let body = "Photo processing uses the photos you take of the wall around your meter, and how the phone moved, to suggest a battery spot. \(destination)" + return synthetic ? body + " " + syntheticCapture : body } + /// Said wherever a synthetic capture stands in for the scan's photos. + static let syntheticCapture = "This test sends a synthetic capture made on the phone instead of this scan's photos, so its answer says nothing about your wall." + + static let consentDeclineNote = "If you don't send them, nothing leaves this phone and this scan won't get an answer." static let consentSend = "Send photos" static let consentDecline = "Don't send" @@ -60,6 +65,8 @@ enum ProcessingCopy { static let stopDetail = "This scan won't be processed, and House Scan can't start it again. Photos already sent stay with the processing service." static let confirmStop = "Stop sending" static let keepSending = "Keep sending" + static let scanWithLegacy = "Scan with Legacy instead" + static let scanWithLegacyHint = "Chooses Legacy in Developer options and starts a new scan with it." static let closedAppLimit = "House Scan can't come back to this scan after you close the app." static let retrying = "Connection trouble. Trying again." @@ -149,7 +156,7 @@ enum ProcessingCopy { case .notSetUp: return Screen( id: "notSetUp", title: "Photo processing isn't set up in this build", - detail: "Nothing was sent. To check a scan, choose \(name(.legacy)) in Developer options, then start over.", + detail: "Nothing has been captured or sent. Scan with \(name(.legacy)) instead, or choose another method in Developer options.", tone: .attention, symbol: "wrench.and.screwdriver") case .consentNotGiven: return Screen( diff --git a/ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift b/ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift index 14642815e..45c00a629 100644 --- a/ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift +++ b/ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift @@ -31,6 +31,12 @@ extension DemoEngine { state.photoProcessing = status } + func scanWithLegacyInstead() { + startOver() + finishOnboarding() + state.scanBackend = .legacy + } + func stopSendingPhotos() { guard var status = state.photoProcessing, status.consent == .granted, !status.isFinal else { return } status.consent = .withdrawn(recorded: true) diff --git a/ios/HouseScan/UI/ScanRootView.swift b/ios/HouseScan/UI/ScanRootView.swift index 33377abc7..1eb893acd 100644 --- a/ios/HouseScan/UI/ScanRootView.swift +++ b/ios/HouseScan/UI/ScanRootView.swift @@ -28,9 +28,11 @@ struct ScanRootView: View { ZStack { if Self.showsCamera(state.phase) { CameraBackdrop(feed: state.feed, actions: actions) + .accessibilityHidden(asksForPhotoConsent) .transition(.opacity) if state.isPracticeScan { PracticeMeterOverlay(state: state) + .accessibilityHidden(asksForPhotoConsent) } CameraEdgeShade() } @@ -40,8 +42,10 @@ struct ScanRootView: View { screen .id(state.phase) .transition(Self.showsCamera(state.phase) ? .opacity : .identity) + // The question is modal: what it covers isn't there for VoiceOver until it's answered. + .accessibilityHidden(asksForPhotoConsent) if asksForPhotoConsent { - PhotoConsentCard(standIn: state.photoProcessing?.standIn == true, actions: actions) + PhotoConsentCard(standIn: state.photoProcessing?.standIn == true, synthetic: state.photoCaptureIsSynthetic, actions: actions) .transition(.opacity) } } diff --git a/ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift b/ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift index 3956d2f5a..68f211870 100644 --- a/ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift +++ b/ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift @@ -51,7 +51,7 @@ struct PhotoProcessingScreen: View { if case .answered(.needsMorePhotos(let more))? = status?.stage, !more.prompts.isEmpty { RequestedViews(prompts: more.prompts) } - if !copy.notes.isEmpty || copy.working { + if !copy.notes.isEmpty || copy.working || state.photoCaptureIsSynthetic { ProcessingNotes(lines: notes(copy, retrying: status?.retrying == true)) } VStack(spacing: 16) { @@ -60,8 +60,14 @@ struct PhotoProcessingScreen: View { .buttonStyle(.quiet) .accessibilityIdentifier("action.stopSendingPhotos") } + if copy.id == "notSetUp" { + Button(ProcessingCopy.scanWithLegacy) { actions.scanWithLegacyInstead() } + .buttonStyle(.primary) + .accessibilityHint(ProcessingCopy.scanWithLegacyHint) + .accessibilityIdentifier("action.scanWithLegacy") + } Group { - if copy.working { + if copy.working || copy.id == "notSetUp" { Button("Start over") { actions.startOver() }.buttonStyle(.quiet) } else { Button("Start over") { actions.startOver() }.buttonStyle(.primary) @@ -75,10 +81,12 @@ struct PhotoProcessingScreen: View { .frame(maxWidth: 520) } .safeAreaInset(edge: .top) { - HStack(alignment: .top) { + // Stacked, each on its own full-width row, so the options chip's words can wrap rather + // than be squeezed beside the badge (the audit found them clipped there). + VStack(alignment: .leading, spacing: 4) { ModeBadge(isReplay: state.isReplay, isAutopilot: state.isAutopilot) - Spacer(minLength: 12) DeveloperOptionsButton(scanBackend: state.scanBackend) + .frame(maxWidth: .infinity, alignment: .leading) } .padding(.horizontal, 24) } @@ -95,6 +103,11 @@ struct PhotoProcessingScreen: View { private func notes(_ copy: ProcessingCopy.Screen, retrying: Bool) -> [String] { var lines = copy.notes if retrying { lines.insert(ProcessingCopy.retrying, at: 0) } + let sent = switch state.photoProcessing?.consent { + case .granted?, .withdrawn(_)?: true + default: false + } + if state.photoCaptureIsSynthetic, sent { lines.insert(ProcessingCopy.syntheticCapture, at: 0) } if copy.working { lines.append(ProcessingCopy.closedAppLimit) } return lines } @@ -158,12 +171,12 @@ private struct ServiceWords: View { .font(Typeface.hint) .foregroundStyle(Palette.ink) .fixedSize(horizontal: false, vertical: true) - .accessibilityIdentifier("photo.serviceMessage") } .frame(maxWidth: .infinity, alignment: .leading) .padding(16) .background(Palette.surface, in: .rect(cornerRadius: 16)) .accessibilityElement(children: .combine) + .accessibilityIdentifier("photo.serviceMessage") } } diff --git a/ios/HouseScanUITests/PhotoProcessingUITests.swift b/ios/HouseScanUITests/PhotoProcessingUITests.swift index c82102aa1..0719aa6f7 100644 --- a/ios/HouseScanUITests/PhotoProcessingUITests.swift +++ b/ios/HouseScanUITests/PhotoProcessingUITests.swift @@ -108,10 +108,11 @@ final class PhotoProcessingUITests: XCTestCase { XCTAssertEqual(run.routes(), sent, "the next scan revived the old capture or sent without a yes") } - /// Photo processing chosen in a build without its setup: Developer options say so, nothing is - /// asked, and the scan ends saying it wasn't sent. It never falls back to the Legacy checker. + /// Photo processing chosen in a build without its setup: Developer options say so, and the + /// scan stops before the camera starts, says why, and offers a Legacy scan, which starts as a + /// new scan. No photo question, no capture and nothing sent. @MainActor - func testWithoutSetupItSaysSoAndNeverFallsBack() throws { + func testWithoutSetupTheScanStopsBeforeCaptureAndOffersLegacy() throws { let run = try Run.launch(backend: "photoProcessing", fixture: nil, holding: ["onboarding"]) defer { run.finish() } run.waitFor("onboarding") @@ -121,12 +122,85 @@ final class PhotoProcessingUITests: XCTestCase { run.attach("selection-notSetUp") run.closeOptions() try run.open("onboarding") + run.waitFor("processing", timeout: 30) + XCTAssertTrue(run.state("notSetUp", timeout: 5).exists) + let any = run.app.descendants(matching: .any) + XCTAssertFalse(any["screen.findMeter"].exists, "the camera started for a scan that can't be sent") + XCTAssertFalse(run.send.exists, "asked to send photos with nothing to send them to") + XCTAssertEqual(run.routes(), [], "a capture request was made") + run.attach("processing-notSetUp-beforeCapture") + + run.tap("action.scanWithLegacy") + run.waitFor("findMeter", timeout: 20) + XCTAssertFalse(run.send.waitForExistence(timeout: 3), "the Legacy scan asked to send photos") + XCTAssertFalse(FileManager.default.fileExists(atPath: run.gate.appending(path: "uploading.held").path)) + } + + // MARK: The real engine with the synthetic capture + + /// The real coordinator and uploader carry `SyntheticCapture` (labelled synthetic in its + /// packet) to the fixture's answer, which the processing screen shows as the scan's main + /// result: provisional, quoted, and marked as a test answer about a synthetic capture. + @MainActor + func testTheSyntheticCaptureReachesATypedAnswerAsTheMainResult() throws { + let run = try Run.launch(backend: "photoProcessing", fixture: "candidate", extra: ["-photoProcessingSyntheticCapture"]) + defer { run.finish() } run.waitFor("findMeter") - XCTAssertFalse(run.send.waitForExistence(timeout: 5), "asked to send photos with nothing to send them to") + XCTAssertTrue(run.send.waitForExistence(timeout: 20)) + run.app.buttons["photoConsent.send"].tap() run.waitFor("processing", timeout: 300) - XCTAssertTrue(run.state("notSetUp", timeout: 10).exists) + XCTAssertTrue(run.state("candidate", timeout: 60).exists, "no answer reached the screen") + let any = run.app.descendants(matching: .any) + let message = any["photo.serviceMessage"].firstMatch + XCTAssertTrue(message.label.contains("Fixture answer: a possible spot to the left of the meter."), "the service's words aren't shown as written: \(message.label)") + XCTAssertTrue(any["photo.standIn"].firstMatch.exists, "the fixture's answer isn't marked as a test one") + XCTAssertTrue(run.app.staticTexts.containing(NSPredicate(format: "label CONTAINS 'synthetic capture'")).firstMatch.exists, "no word that the capture was synthetic") + run.attach("processing-candidate-engine") + let routes = run.routes() + XCTAssertTrue(routes.contains("POST captures/finalize"), "\(routes)") + XCTAssertTrue(routes.contains("GET captures/result"), "\(routes)") XCTAssertFalse(FileManager.default.fileExists(atPath: run.gate.appending(path: "uploading.held").path), "the Legacy upload ran") - run.attach("processing-notSetUp") + } + + /// An answer naming another run than the capture started is refused, and the scan says so + /// instead of showing it. + @MainActor + func testAnAnswerForAnotherRunIsRefused() throws { + let run = try Run.launch(backend: "photoProcessing", fixture: "wrongRun", extra: ["-photoProcessingSyntheticCapture"]) + defer { run.finish() } + run.waitFor("findMeter") + XCTAssertTrue(run.send.waitForExistence(timeout: 20)) + run.app.buttons["photoConsent.send"].tap() + run.waitFor("processing", timeout: 300) + XCTAssertTrue(run.state("answerMismatch", timeout: 60).exists) + XCTAssertFalse(run.app.descendants(matching: .any)["photo.serviceMessage"].firstMatch.exists, "the refused answer's words were shown") + run.attach("processing-answerMismatch-engine") + } + + /// "Stop sending photos" while the service works, with a capture folder the phone can't write + /// (`-photoProcessingFixtureReadOnlyCapture`): sending stops and the screen says the phone + /// couldn't save the choice. + @MainActor + func testStoppingWhenThePhoneCantSaveItSaysSo() throws { + let run = try Run.launch( + backend: "photoProcessing", fixture: "hold", extra: ["-photoProcessingSyntheticCapture", "-photoProcessingFixtureReadOnlyCapture"]) + defer { run.finish() } + run.waitFor("findMeter") + XCTAssertTrue(run.send.waitForExistence(timeout: 20)) + run.app.buttons["photoConsent.send"].tap() + run.waitFor("processing", timeout: 300) + XCTAssertTrue(run.state("processing", timeout: 60).exists, "the capture never reached processing") + run.attach("processing-engine") + run.tap("action.stopSendingPhotos") + let confirm = run.app.buttons["Stop sending"].firstMatch + XCTAssertTrue(confirm.waitForExistence(timeout: 5), "stopping didn't ask first") + confirm.tap() + XCTAssertTrue(run.state("withdrawalNotRecorded", timeout: 10).exists) + run.attach("processing-withdrawalNotRecorded-engine") + let sent = run.routes() + // Longer than the uploader waits between polls of a capture still processing. + Thread.sleep(forTimeInterval: 3) + XCTAssertEqual(run.routes(), sent, "requests went on after the homeowner stopped sending") } // MARK: Every state, from demo mode @@ -176,7 +250,7 @@ final class PhotoProcessingUITests: XCTestCase { let stop = app.buttons["action.stopSendingPhotos"] XCTAssertTrue(stop.waitForExistence(timeout: 15), "no way to stop sending while processing") stop.tap() - let confirm = app.buttons["Stop sending"] + let confirm = app.buttons["Stop sending"].firstMatch XCTAssertTrue(confirm.waitForExistence(timeout: 5), "stopping didn't ask first") attach(app, "processing-stopQuestion") confirm.tap() @@ -237,7 +311,7 @@ final class PhotoProcessingUITests: XCTestCase { let gate: URL @MainActor - static func launch(backend: String, fixture: String?, holding: Set = []) throws -> Run { + static func launch(backend: String, fixture: String?, holding: Set = [], extra: [String] = []) throws -> Run { let gate = URL(fileURLWithPath: NSTemporaryDirectory()).appending(path: "housescan-gate-\(UUID().uuidString)", directoryHint: .isDirectory) try FileManager.default.createDirectory(at: gate, withIntermediateDirectories: true) for phase in ["onboarding", "findMeter", "meterCloseUp", "wallWalk", "markFeatures", "gapRequest"] where !holding.contains(phase) { @@ -248,6 +322,7 @@ final class PhotoProcessingUITests: XCTestCase { "-practiceMeter", "NO", "-processingBackend", backend, ] if let fixture { arguments += ["-photoProcessingFixture", fixture] } + arguments += extra let app = XCUIApplication() app.launchArguments = arguments app.launch() diff --git a/ios/README.md b/ios/README.md index 47f11dbcc..6a392fbd2 100644 --- a/ios/README.md +++ b/ios/README.md @@ -26,12 +26,16 @@ Native iPhone app for the AR capture walk. The homeowner marks the electric mete | `-practiceMeter YES` or `NO` | Overrides the stored practice meter switch for one run (UserDefaults' argument domain). The full-flow UI tests pass `NO`, since the switch outlives a run in the Simulator. | | `-savedScansRoot ` | Debug builds only. Saved scans lists the scan folders in this folder instead of the app's Caches/Scans. It only changes what is listed; the scan store still writes and cleans up Caches/Scans. `SavedScansUITests` passes it with synthetic scans. | | `-processingBackend ` | Debug builds only. The processing choice in Developer options starts at this value and stays in memory for the run, so a UI test's choice never reaches the stored setting. | -| `-photoProcessingFixture ` | Debug builds only, with `-replay`. Photo processing sends to a capture API answered inside the app (HouseScanKit's `FixtureCaptureHTTP`), which ends each capture with this answer: `candidate`, `needsViews`, `manualReview`, `notEligible`, `failed`, `expired`, `notReady`, `unreadable`, `refuseCreate` or `hold`. Nothing leaves the phone. With `-autopilotGate`, the routes it answered are listed in `photo-transport.log` there. | +| `-photoProcessingFixture ` | Debug builds only, with `-replay`. Photo processing sends to a capture API answered inside the app (HouseScanKit's `FixtureCaptureHTTP`), which ends each capture with this answer: `candidate`, `needsViews`, `manualReview`, `notEligible`, `failed`, `expired`, `notReady`, `unreadable`, `refuseCreate`, `hold` or `wrongRun`. Nothing leaves the phone. With `-autopilotGate`, the routes it answered are listed in `photo-transport.log` there. | +| `-photoProcessingSyntheticCapture` | Debug builds only, with `-photoProcessingFixture`. Sends HouseScanKit's `SyntheticCapture` (packet `source.kind: synthetic`) in place of the replay's photos, so the fixture's answer comes back through the real upload. | +| `-photoProcessingFixtureReadOnlyCapture` | Debug builds only. Makes the capture's folder read-only just before "Stop sending photos" takes effect, so the phone can't save the withdrawal and says so. | | `-uiDemo` | Runs the screens on a scripted fake engine instead of the capture engine, for design work and for auditing states a replay can't reach. The arguments it takes are listed in `HouseScan/UI/Preview/UIDemo.swift`; for example `-uiDemoGroundQuestion` opens the feature review with the ground question showing, `-uiDemoPhase meterCloseUp -uiDemoMeterChoose` shows the meter number picker, and `-uiDemoPhase result -uiDemoCorner` shows the result model of a wall that turns a corner. | **Spot check.** Before the result, the app shows the kept photo that best sees the answer's spot, with the spot and its clearance area outlined, and asks whether anything stands in front of the wall or on the ground there (`STATE=spotConfirm`). When no kept photo shows the area, it shows no photo and gives the area in words: where it runs beside the meter, how far out from the wall, and how high up the wall. Photos without depth and the walked path claim wall, ground and clear space they never saw; "It's clear" backs those claims for that spot, and "Something's there" takes them back over the area, so scene.json reports it unseen, and sends the scan again. "I can't check this area", for a homeowner who can't see or reach it, does the same without saying anything stands there, and the result says the area wasn't checked and needs a person to check it. When the scan sent again names the same spot, the earlier answer settles it and the homeowner isn't asked again. When it names no spot, or another one, the result still says an area along the wall went unchecked, until Start over or a new wall clears the scan's spot checks. The rules are in `Runtime/ScanEngine+Confirm.swift` and HouseScanKit's `Confirm/SpotConfirmation.swift`. -**Processing (beta).** Developer options offer two backends, both remote: **Legacy (remote checker)** sends scene.json to the placement server, and **Photo processing (beta)** sends the scan's photos and camera path to the capture API. Legacy is the default. A scan reads the choice once, when it starts, and keeps it with its endpoint in its `ScanContext` (HouseScanKit); a change applies to the next scan, and an App Store install always uses Legacy. A photo-processing scan asks before sending (`PhotoConsentCard`), sends photos as they are kept, and after the review goes to `STATE=processing` instead of the upload: its progress, then the service's answer quoted as written (`PhotoProcessingScreen`). It never falls back to the Legacy checker. Answers bind to the recording session, world epoch, packet, capture and run, and AR is withheld for them. Sending off the phone is off in every build for now, so choosing photo processing ends the scan saying it isn't set up; only `-photoProcessingFixture` runs it. A replay's packet always fails the phone's 0.4 checks, since a replay has no motion data and too few poses. The code is in `Runtime/ScanEngine+Processing.swift` and HouseScanKit's `Processing/`. +**Processing (beta).** Developer options offer two backends, both remote: **Legacy (remote checker)** sends scene.json to the placement server, and **Photo processing (beta)** sends the scan's photos and camera path to the capture API. Legacy is the default. A scan reads the choice once, when it starts, and keeps it with its endpoint in its `ScanContext` (HouseScanKit); a change applies to the next scan, and an App Store install always uses Legacy. A photo-processing scan asks before sending (`PhotoConsentCard`), sends photos as they are kept, and after the review goes to `STATE=processing` instead of the upload: its progress, then the service's answer quoted as written (`PhotoProcessingScreen`). It never falls back to the Legacy checker. An answer is shown only when the run it names, and the phone's own record of what it sent (the recording session and world epoch, the coordinator session, packet and capture), match this scan; that is a local association, since the service returns no digest of its input and no attestation. AR is withheld for every answer. Sending off the phone is off in every build for now, so choosing photo processing ends the scan saying it isn't set up. + +Two debug-only switches run the path on a replay against `FixtureCaptureHTTP`, a stand-in for the capture API inside the app (the type is in every build; only its activation is debug-only): `-photoProcessingFixture` alone sends the replay's photos, whose packet always fails the phone's 0.4 checks because a replay has no motion data and too few poses, so the scan ends "couldn't be prepared"; adding `-photoProcessingSyntheticCapture` sends HouseScanKit's `SyntheticCapture` instead, labelled `source.kind: synthetic`, so the real coordinator and uploader carry it to the fixture's answer. Either way the answer is the fixture's, never a service's or a GPU reconstruction's, and the screens say so. Still missing before live use: a scoped server-issued credential, live per-frame intrinsics and the meter tap in the 0.4 packet, the clock and privacy contract, and an input digest or attestation from the service. The code is in `Runtime/ScanEngine+Processing.swift` and HouseScanKit's `Processing/`. **Practice meter.** For trying the whole scan where there is no electric meter, such as indoors. Debug and TestFlight builds show a wrench button at the top of the first screen. Tap it, turn on **Practice meter**, tap Done, then start the scan. At "find the meter", tap any spot on a wall: the usual raycast and refusals apply, and a drawn sample meter appears there, fixed to the meter's anchor, on every camera screen after that. The close-up keeps the live camera's pose and depth, but stores a photo of the sample (made-up maker SAMPLEWORKS, number 12345678), and the real reader reads it. The walk, fog, marks, requests, spot check, upload, result and AR all run for real. Every screen shows a "Practice meter" badge, and `scan-stamp.json` has `"practice": true`; the packet and scene.json are unchanged. The switch is kept until turned off, and the button reads "Practice meter on" while it is. App Store installs never show the button and ignore a stored switch: the app asks StoreKit's `AppTransaction` which server signed the install (sandbox means TestFlight) and treats no answer as the App Store. It is not in the iOS Settings app, because a Settings.bundle ships in the same binary that TestFlight later promotes to the App Store. The code is in `Runtime/Practice/`, `UI/Practice/` and HouseScanKit's `Practice/PracticeMeter.swift`. From 6ca1bbbfda404e828521a2dea30e2491c9bd3c4c Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 10:02:33 -0500 Subject: [PATCH 33/71] Test a withdrawal while the credential provider waits and then returns A provider that ignores cancellation is held on a continuation. The upload is withdrawn while the provider waits, then the provider returns a good token. The upload must end abandoned for withdrawal, saved that way, with no request, no retry scheduled and no backoff. At 73e06d70 the wrapper throws CancellationError there, and CaptureUploader.failure classifies it as Transient before its withdrawal check, so the end is never saved (T3 Sol review, P2). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../ScopedCaptureUploadTests.swift | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureUploadTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureUploadTests.swift index 6558a58da..ecaeac30a 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureUploadTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedCaptureUploadTests.swift @@ -162,6 +162,39 @@ import Testing #expect(sleeps.isEmpty) } + /// A withdrawal that lands while a provider is waiting, one that ignores cancellation and + /// then returns a good token, ends the upload as withdrawn: nothing is sent, no retry is + /// scheduled, and the end is saved. The wrapper throws CancellationError there, which the + /// uploader must not read as a network failure to retry. + @Test func aWithdrawalWhileTheProviderWaitsEndsTheUploadAsWithdrawn() async throws { + let sleeps = Sleeps() + let gate = ScopedCaptureHTTPTests.Gate() + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.expectedBearer = Self.token } + let http = ScopedCaptureHTTP( + scope: try CaptureAPIScope(base: server.base, transport: .loopbackHTTP), + credential: { + await gate.wait() + return Self.token + }, + inner: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + let rig = try Rig(server: server, http: http, sleep: sleeps.record) + defer { rig.cleanUp() } + await rig.uploader.kick() + await gate.entered() + #expect(rig.uploader.withdrawConsent() == .marked) + gate.release() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + let snapshot = await rig.uploader.snapshot + #expect(snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) + #expect(await rig.uploader.status.retryingAt == nil) + #expect(server.state.withLock { $0.log.isEmpty }) + #expect(sleeps.isEmpty) + let saved = try CaptureUploadState.load(from: CaptureUploader.stateURL(in: rig.capture.folder)) + #expect(saved.end == .abandoned(CaptureUploader.withdrawnReason)) + } + /// With the default loopback (no expected bearer), the old assertion stands: an API request /// that carries Authorization is refused. @Test func theDefaultLoopbackStillRefusesAnyAPIAuthorization() async throws { From 3ae118fc2de46855196c1e7b301c734da5de918e Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 10:03:02 -0500 Subject: [PATCH 34/71] Read any transport error after a withdrawal as the withdrawal CaptureUploader.failure checked for a withdrawal or cancellation only after recognizing a credential-boundary refusal. Suppose the upload was withdrawn while a credential provider waited, and the provider then returned. ScopedCaptureHTTP threw CancellationError, which failure turned into Transient. drive then scheduled a retry, its sleep was cancelled, and it exited without saving the end. No request or credential left the phone, but the upload stayed open with a retry pending (T3 Sol review, P2). The withdrawal and cancellation check now comes first, for every error. That only changes errors raised after the homeowner withdrew or the task was cancelled, which end the upload in any case. Network errors otherwise still back off and retry as before. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../HouseScanKit/Upload/CaptureUploader.swift | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift index 91f7bc463..334f117b3 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -725,17 +725,19 @@ public actor CaptureUploader { static func isTransient(_ status: Int) -> Bool { status == 0 || status == 408 || status == 429 || ((500..<600).contains(status) && status != 501) } /// The error's domain and code only: a URLError's description can carry the signed URL. - /// What a transport error means for the upload. A refusal by the credential boundary - /// (`ScopedCaptureHTTP`) won't change on retry: a URL outside the API's scope, a malformed - /// credential, one the provider couldn't supply, or a storage request carrying - /// Authorization. It ends the upload with the boundary's code and status 0, since no server - /// answered. Any other error, a network one included, is retried as before. A withdrawal or - /// cancellation that happened meanwhile outranks the refusal. + /// What a transport error means for the upload. A withdrawal or cancellation that happened + /// meanwhile outranks the error, whatever it is: the request may have been cut short by that + /// very withdrawal (`ScopedCaptureHTTP` throws CancellationError once a credential wait is + /// cancelled), and retrying it would leave the upload waiting to retry instead of saved as + /// withdrawn. A refusal by the credential boundary won't change on retry: a URL outside the + /// API's scope, a malformed credential, one the provider couldn't supply, or a storage + /// request carrying Authorization. It ends the upload with the boundary's code and status 0, + /// since no server answered. Any other error, a network one included, is retried as before. private func failure(_ step: String, _ error: any Error) -> any Error { + if !canSend || Task.isCancelled { return Withdrawn() } guard let refusal = error as? ScopedCaptureHTTPError else { return Transient(step: step, detail: Self.describe(error), retryAfter: nil) } - if !canSend || Task.isCancelled { return Withdrawn() } return Refused(step: step, status: 0, codes: [refusal.code]) } From 2e08c88f9fcd9ac40f78fe3d17ff64a5554e8cf0 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 10:05:28 -0500 Subject: [PATCH 35/71] Log a withdrawal that overtakes an error, and test the in-flight give-up by it When a withdrawal or cancellation overtakes a transport error, CaptureUploader.failure now logs "capture-upload withdrawn step= detail=". It holds no URL or credential. CaptureSessionCoordinatorTests.withdrawalCancelsTheRequestInFlight detected the given-up register request by its "retry" log line. That line existed only because the cut-short request was misread as a network failure, which 3ae118fc fixed. The test now looks for the withdrawn line instead; its intent is unchanged. The app owner approved this edit to that file. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Sources/HouseScanKit/Upload/CaptureUploader.swift | 5 ++++- .../HouseScanKitTests/CaptureSessionCoordinatorTests.swift | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift index 334f117b3..911bccc3e 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -734,7 +734,10 @@ public actor CaptureUploader { /// request carrying Authorization. It ends the upload with the boundary's code and status 0, /// since no server answered. Any other error, a network one included, is retried as before. private func failure(_ step: String, _ error: any Error) -> any Error { - if !canSend || Task.isCancelled { return Withdrawn() } + if !canSend || Task.isCancelled { + log?("capture-upload withdrawn step=\(step) detail=\(Self.describe(error))") + return Withdrawn() + } guard let refusal = error as? ScopedCaptureHTTPError else { return Transient(step: step, detail: Self.describe(error), retryAfter: nil) } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift index 1e1bfb85a..008d40758 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift @@ -684,7 +684,8 @@ struct NativeCaptureFixture: Sendable { coordinator.answerConsent(false) // The main actor stays busy, so only the withdrawal itself can end the wait. let deadline = Date().addingTimeInterval(5) - func gaveUp() -> Bool { logs.withLock { $0.contains { $0.contains("capture-upload retry step=register") } } } + // The cut-short request ends as a withdrawal, not as a network failure to retry. + func gaveUp() -> Bool { logs.withLock { $0.contains { $0.contains("capture-upload withdrawn step=register") } } } while Date() < deadline, !gaveUp() { usleep(10_000) } #expect(gaveUp()) From 0b3c79d0e6042f9976e52027dddd8232c78547c8 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 10:06:24 -0500 Subject: [PATCH 36/71] Split hosted iOS UI tests into journey and photo-processing jobs One job no longer fits the 90-minute cap: #214's UI step took 72.3 min and BRIEF-19's photo-processing UI tests add about 7.7 min measured locally, before setup and the build. The iOS workflow now runs a matrix with fail-fast off, each group with its own 90-minute cap: - journey: package tests (warnings as errors), the project drift check, the Debug device build and every HouseScanUITests class except PhotoProcessingUITests; the every-state audit still follows the full-ui policy. - photoProcessing: an unsigned Release build, which shows the Release configuration compiles, not what it does at run time; then PhotoProcessingUITests with all its default and largest-text audits. ios/Tools/ui-test-groups.py holds both groups' selectors, which the workflow reads, and `check` lists the UI tests from source and fails unless the groups run each exactly once, with only the every-state audit left out without full-ui. CI runs the check first. Artifacts are named per group. Triggers, concurrency and pinned actions are unchanged. No product test changed. Validated: `ui-test-groups.py check` passes (119 tests, journey 109, photoProcessing 10, overlap 0, missing 0), and it fails when fed an overlap, a dropped class or a stale selector. The selector loop ran under /bin/bash 3.2 for both groups with and without full-ui. The workflow parses with Ruby's YAML loader; actionlint isn't installed. The local Release build was refused by heavy-admit (memory free 34%, below 35%), so Release compiling is left to hosted CI. Nothing ran on hosted CI yet. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/ios.yml | 55 +++++++++++++--- CONTRIBUTING.md | 2 + ios/Tools/ui-test-groups.py | 122 ++++++++++++++++++++++++++++++++++++ 3 files changed, 169 insertions(+), 10 deletions(-) create mode 100755 ios/Tools/ui-test-groups.py diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 6d1c66192..df665f0cd 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -28,7 +28,17 @@ concurrency: jobs: ios-build: - name: iOS build + # Two groups of the UI tests, one job each (ios/Tools/ui-test-groups.py holds their selectors + # and checks that together they run every test once). One job no longer fits the cap below: + # #214's UI step alone took 72.3 min, and photo processing's tests add about 7.7 min measured + # locally (BRIEF-19), before setup and the build. journey runs the package tests and the Debug + # device build; photoProcessing runs the unsigned Release build. fail-fast is off so a failure + # in one group still leaves the other's results. + name: iOS build (${{ matrix.group }}) + strategy: + fail-fast: false + matrix: + group: [journey, photoProcessing] # A free account runs 5 macOS jobs at once, shared by every open pull request, so drafts and # labels other than full-ui don't take a slot. if: >- @@ -42,7 +52,8 @@ jobs: # freshness UI tests add about 161 s more (run 36744881974), so 60 would cut the # every-state audit short. 75 left room until the 2026-10-03 candidate's 86 UI cases: # run 37121918073 attempt 2 spent 10 min 34 s before the UI step and 64 min 39 s in it, - # and the cap stopped it after 76 cases with 0 failures. 90 runs all of them. + # and the cap stopped it after 76 cases with 0 failures. 90 ran all of them; since photo + # processing's tests moved to their own job, 90 is the cap for each group. timeout-minutes: 90 env: # Newest stable Xcode on macos-26 per images/macos/macos-26-arm64-Readme.md (image 20260907). @@ -55,7 +66,9 @@ jobs: XCODEGEN_VERSION: 2.46.0 XCODEGEN_SHA256: 4d9e34b62172d645eed6457cac13fc222569974098ef4ee9c3368bedf0196806 # The every-state accessibility audit relaunches the app 65 times (about 10 min). Pull - # requests skip it unless labelled full-ui; pushes and manual runs always include it. + # requests skip it unless labelled full-ui; pushes and manual runs always include it. It is + # in the journey group; photoProcessing always runs its own audits. + GROUP: ${{ matrix.group }} FULL_UI: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'full-ui') }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -71,6 +84,10 @@ jobs: xcodebuild -version swift --version + # Before anything slow: the two groups' selectors must cover every UI test exactly once. + - name: Check the UI test groups + run: python3 ios/Tools/ui-test-groups.py check + # Booting takes 2 to 3 minutes and needs nothing from the build, so it runs alongside the # steps below; the UI test step waits for it. - name: Boot the Simulator in the background @@ -94,6 +111,7 @@ jobs: # than skip, so CI can't go green without comparing. The checkout counts only when it holds # every compared file: a folder alone isn't enough (main has a server/ without schemas). - name: Fetch the server contract + if: matrix.group == 'journey' run: | fetch_unless_present() { local branch="$1"; shift @@ -111,6 +129,7 @@ jobs: fetch_unless_present t3/server server/schemas/scene.schema.json server/schemas/result.schema.json server/tests/fixtures/example-scene.json - name: Install XcodeGen + if: matrix.group == 'journey' run: | curl -fsSL -o "$RUNNER_TEMP/xcodegen.zip" \ "https://github.com/yonaskolb/XcodeGen/releases/download/${XCODEGEN_VERSION}/xcodegen.zip" @@ -119,9 +138,11 @@ jobs: echo "$RUNNER_TEMP/xcodegen/bin" >> "$GITHUB_PATH" - name: Generate project + if: matrix.group == 'journey' run: xcodegen generate --spec ios/project.yml - name: Check committed project matches project.yml + if: matrix.group == 'journey' run: | if ! git diff --exit-code -- ios; then echo "::error::HouseScan.xcodeproj differs from project.yml. Run 'make ios-project' (XcodeGen ${XCODEGEN_VERSION}) and commit the result." @@ -136,42 +157,56 @@ jobs: - name: Test HouseScanKit # The capture logic is a plain Swift package; its tests run on macOS without a Simulator. + # Once per run: photoProcessing builds the same package sources into its Release build. + if: matrix.group == 'journey' env: HOUSESCAN_REQUIRE_UPSTREAM: "1" run: swift test --package-path ios/HouseScanKit -Xswiftc -warnings-as-errors # CI builds once from clean, so the index Xcode keeps for editing is wasted work. - name: Build + if: matrix.group == 'journey' run: >- xcodebuild -project ios/HouseScan.xcodeproj -scheme HouseScan -configuration Debug -destination "generic/platform=iOS" CODE_SIGNING_ALLOWED=NO COMPILER_INDEX_STORE_ENABLE=NO build + # Proves the Release configuration compiles, the photo-processing setup gate included, whose + # fixture switches exist only in Debug. It doesn't show what Release does at run time. + - name: Build Release, unsigned + if: matrix.group == 'photoProcessing' + run: >- + xcodebuild -project ios/HouseScan.xcodeproj -scheme HouseScan -configuration Release + -destination "generic/platform=iOS" CODE_SIGNING_ALLOWED=NO COMPILER_INDEX_STORE_ENABLE=NO build + - name: Wait for the Simulator timeout-minutes: 10 run: | xcrun simctl bootstatus "$SIM_UDID" -b echo "Booted $SIM_NAME on iOS $SIM_OS ($SIM_UDID)." - - name: UI test the whole flow in the Simulator + - name: UI test the group in the Simulator # Replays the committed synthetic wall (ios/HouseScanUITests/Fixtures/synthetic-wall) with - # -autopilot and the bundled sample result. + # -autopilot and the bundled sample result; photoProcessing adds its in-app fixture. run: | - skip=() - if [ "$FULL_UI" != "true" ]; then - skip=(-skip-testing:HouseScanUITests/ScreenStatesUITests/testEveryStatePassesTheAccessibilityAudit) + if [ "$GROUP" = journey ] && [ "$FULL_UI" != "true" ]; then echo "::notice::Skipped the every-state accessibility audit (about 10 min). Add the full-ui label to run it on this pull request." fi + # Read without mapfile: the runner's /bin/bash is 3.2. + list="$(python3 ios/Tools/ui-test-groups.py args "$GROUP" --full-ui "$FULL_UI")" + selectors=() + while IFS= read -r line; do selectors+=("$line"); done <<< "$list" + echo "Selectors: ${selectors[*]}" xcodebuild -project ios/HouseScan.xcodeproj -scheme HouseScan -configuration Debug \ -destination "id=$SIM_UDID" \ -resultBundlePath "$RUNNER_TEMP/HouseScanUITests.xcresult" \ - -only-testing:HouseScanUITests "${skip[@]}" \ + "${selectors[@]}" \ COMPILER_INDEX_STORE_ENABLE=NO test - name: Keep UI test screenshots if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: ui-test-results + name: ui-test-results-${{ matrix.group }} path: ${{ runner.temp }}/HouseScanUITests.xcresult if-no-files-found: ignore retention-days: 7 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c0b53fc1a..d434dc5bc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,6 +38,8 @@ Greptile reviews new commits through [`.greptile/config.json`](.greptile/config. The iOS UI tests skip the every-state accessibility audit on pull requests, because it adds about 10 minutes and macOS runners are scarce. Add the `full-ui` label when a pull request changes screens or copy; pushes to `t3/ios-mvf` and `main` always run it. +The iOS check runs as two jobs, each under its own 90-minute cap: `journey` runs the package tests, the Debug build and every UI test class but `PhotoProcessingUITests`, and `photoProcessing` runs an unsigned Release build and that class with its own audits. `ios/Tools/ui-test-groups.py` holds both groups' test selectors; `python3 ios/Tools/ui-test-groups.py check` lists the UI tests from source and fails unless the two groups run each exactly once. Run it after adding or renaming a UI test class. + `make check` runs the server, web and iOS suites, then `make scoring`, `make measure-lab`, `make evals`, `make recon` and `make meter-closeup` for each of those folders the branch has. They need uv, Node 24 with pnpm, and Xcode 26 or newer; each directory's README has details. No check is required by branch rules yet. Don't call one required until the rules require its status. Keep workflows that run pull-request code away from production credentials and destructive external systems. A green CI run is evidence for the checks it ran, not proof that a capture works on a real house. diff --git a/ios/Tools/ui-test-groups.py b/ios/Tools/ui-test-groups.py new file mode 100755 index 000000000..66ff7d88f --- /dev/null +++ b/ios/Tools/ui-test-groups.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""The two hosted UI test groups, and a check that together they run every UI test once. + +CI splits HouseScanUITests across two jobs so neither passes the 90-minute cap: `journey`, every +class but PhotoProcessingUITests, and `photoProcessing`, that class alone. The workflow takes each +group's xcodebuild selectors from here, so the check below reads the same selectors CI passes. + + ui-test-groups.py args [--full-ui true|false] print the group's selectors, one per line + ui-test-groups.py check list the tests from source and check them + +`check` finds every `func test...()` in an XCTestCase class under ios/HouseScanUITests, works out +which tests each group's selectors select as xcodebuild does (an -only-testing prefix includes, a +-skip-testing prefix excludes), and fails unless, with the full audit on, each test is in exactly +one group; with it off, the only test left out must be the every-state audit, the existing +pull-request policy. It also fails on a selector that names no class or test. It reads source +only, so a test hidden behind `#if` would be listed although the build leaves it out; there is none +today. +""" +import pathlib +import re +import sys + +TARGET = "HouseScanUITests" +PHOTO_CLASS = "PhotoProcessingUITests" +# Pull requests skip this without the full-ui label (CONTRIBUTING.md, "Checks"). +FULL_UI_ONLY = f"{TARGET}/ScreenStatesUITests/testEveryStatePassesTheAccessibilityAudit" +GROUPS = ("journey", "photoProcessing") + + +def selectors(group, full_ui): + if group == "journey": + args = [f"-only-testing:{TARGET}", f"-skip-testing:{TARGET}/{PHOTO_CLASS}"] + if not full_ui: + args.append(f"-skip-testing:{FULL_UI_ONLY}") + return args + if group == "photoProcessing": + # Its default and largest-text audits always run: they are this group's own evidence. + return [f"-only-testing:{TARGET}/{PHOTO_CLASS}"] + sys.exit(f"unknown group {group!r}; the groups are {', '.join(GROUPS)}") + + +def tests_in_source(folder): + """Every `Class/testMethod` XCTest would run, read from the Swift sources.""" + found = [] + for path in sorted(folder.glob("*.swift")): + current = None + for line in path.read_text(encoding="utf-8").splitlines(): + declared = re.match(r"\s*(?:final\s+)?class\s+(\w+)\s*:\s*([\w ,]+)", line) + if declared: + current = declared.group(1) if "XCTestCase" in declared.group(2) else None + continue + if re.match(r"\s*(?:final\s+)?class\s+\w+", line): + current = None + continue + extended = re.match(r"\s*extension\s+(\w+)", line) + if extended: + current = extended.group(1) if extended.group(1).endswith("Tests") else None + continue + method = re.match(r"\s*(?:@MainActor\s+)?func\s+(test\w*)\s*\(\s*\)", line) + if method and current: + found.append(f"{TARGET}/{current}/{method.group(1)}") + return found + + +def selected(tests, args): + only = [a.split(":", 1)[1] for a in args if a.startswith("-only-testing:")] + skip = [a.split(":", 1)[1] for a in args if a.startswith("-skip-testing:")] + covers = lambda prefix, test: test == prefix or test.startswith(prefix + "/") + return {t for t in tests if any(covers(p, t) for p in only) and not any(covers(p, t) for p in skip)} + + +def check(): + folder = pathlib.Path(__file__).resolve().parent.parent / TARGET + tests = tests_in_source(folder) + problems = [] + if len(tests) != len(set(tests)): + problems.append("a test name appears twice in source") + names = set(tests) + for full_ui in (True, False): + for group in GROUPS: + for arg in selectors(group, full_ui): + prefix = arg.split(":", 1)[1] + if prefix != TARGET and not any(t == prefix or t.startswith(prefix + "/") for t in names): + problems.append(f"{group}: {arg} names no test") + full = {g: selected(names, selectors(g, True)) for g in GROUPS} + overlap = full["journey"] & full["photoProcessing"] + missing = names - full["journey"] - full["photoProcessing"] + pr = {g: selected(names, selectors(g, False)) for g in GROUPS} + left_out = names - pr["journey"] - pr["photoProcessing"] + print(f"tests in source: {len(names)}") + print(f"journey: {len(full['journey'])} (pull request without full-ui: {len(pr['journey'])})") + print(f"photoProcessing: {len(full['photoProcessing'])} (pull request without full-ui: {len(pr['photoProcessing'])})") + print(f"overlap: {len(overlap)}") + print(f"missing: {len(missing)}") + print(f"left out without full-ui: {sorted(left_out)}") + problems += [f"in both groups: {t}" for t in sorted(overlap)] + problems += [f"in no group: {t}" for t in sorted(missing)] + if left_out != {FULL_UI_ONLY}: + problems.append(f"without full-ui exactly {FULL_UI_ONLY} should be left out, not {sorted(left_out)}") + if not full["photoProcessing"] or any(not t.startswith(f"{TARGET}/{PHOTO_CLASS}/") for t in full["photoProcessing"]): + problems.append("photoProcessing must hold PhotoProcessingUITests and nothing else") + for problem in problems: + print(f"error: {problem}") + return 1 if problems else 0 + + +def main(argv): + if argv[1:2] == ["check"] and len(argv) == 2: + return check() + if argv[1:2] == ["args"] and len(argv) in (3, 5): + full_ui = True + if len(argv) == 5: + if argv[3] != "--full-ui" or argv[4] not in ("true", "false"): + sys.exit(__doc__) + full_ui = argv[4] == "true" + print("\n".join(selectors(argv[2], full_ui))) + return 0 + sys.exit(__doc__) + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) From db301cb4792341c1d000ecc426f71e394c33bf83 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 10:28:43 -0500 Subject: [PATCH 37/71] Show capture-scoped auth refusals as one setup state The uploader ends an upload refused at the credential boundary with .failed(step:codes:status: 0) and one of aa72's four codes. The photo-processing stage now maps those, at status 0 only, to PhotoProcessingEnd.setupRefused(step:problem:). PhotoSetupProblem takes each code from ScopedCaptureHTTPError.code rather than copying the text. The same code at a server status stays a plain refusal. The controller logs auth_storage_authorization as a service or contract setup failure and the other three as boundary refusals. The screen says the connection isn't complete and that the problem is setup, not the homeowner's photos or marks. It doesn't say whether the scan was processed, since the refusal can come at the events or result step, and it says the connection needs fixing before trying again, since an unavailable credential can be temporary. It offers Start over, Share only when the scan's archive exists, and no Legacy fallback. A withdrawal set first is never replaced by a refusal read later. Also: the demo and the every-state audit gain the state (default and largest text, with an audit and a text check at default), and the iOS workflow comment now says one job risks exceeding the cap. Validated, before the copy fix: `swift test -j 2 -Xswiftc -warnings-as-errors` through heavy-admit passed 868 tests in 113 suites in 79 s. That run covered the merged auth tests, the four codes driven through the real ScopedCaptureHTTP and uploader to the setup state, and a withdrawal during a credential fetch ending withdrawn. Only PhotoProcessingUITests then ran on the Simulator: 10 of 10 passed, 459 s of test time; the job ended cleanup_failed, as before, because an Xcode DTServiceHub process outlived it. After the copy fix nothing has run: the result-step test and the copy assertions wait for a free slot or hosted CI. `ui-test-groups.py check` passes: 119 tests, overlap 0, missing 0. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/ios.yml | 6 +- ios/HouseScan/UI/Copy/ProcessingCopy.swift | 10 ++ .../Preview/DemoEngine+PhotoProcessing.swift | 2 + .../UI/Screens/PhotoProcessingScreen.swift | 4 + .../PhotoProcessingController.swift | 13 +- .../Processing/PhotoProcessingStatus.swift | 38 ++++- .../PhotoProcessingTests.swift | 17 +- .../ScopedPhotoProcessingTests.swift | 152 ++++++++++++++++++ .../PhotoProcessingUITests.swift | 15 +- 9 files changed, 242 insertions(+), 15 deletions(-) create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/ScopedPhotoProcessingTests.swift diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index df665f0cd..07fc42123 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -29,9 +29,9 @@ concurrency: jobs: ios-build: # Two groups of the UI tests, one job each (ios/Tools/ui-test-groups.py holds their selectors - # and checks that together they run every test once). One job no longer fits the cap below: - # #214's UI step alone took 72.3 min, and photo processing's tests add about 7.7 min measured - # locally (BRIEF-19), before setup and the build. journey runs the package tests and the Debug + # and checks that together they run every test once). One job risks exceeding the cap below: + # #214's hosted UI step took 72.3 min, and photo processing's tests add about 7.7 min measured + # locally (BRIEF-19), before setup and the build. No combined run has been seen to fail. journey runs the package tests and the Debug # device build; photoProcessing runs the unsigned Release build. fail-fast is off so a failure # in one group still leaves the other's results. name: iOS build (${{ matrix.group }}) diff --git a/ios/HouseScan/UI/Copy/ProcessingCopy.swift b/ios/HouseScan/UI/Copy/ProcessingCopy.swift index c4237d1e8..68a8e874b 100644 --- a/ios/HouseScan/UI/Copy/ProcessingCopy.swift +++ b/ios/HouseScan/UI/Copy/ProcessingCopy.swift @@ -65,6 +65,7 @@ enum ProcessingCopy { static let stopDetail = "This scan won't be processed, and House Scan can't start it again. Photos already sent stay with the processing service." static let confirmStop = "Stop sending" static let keepSending = "Keep sending" + static let setupRefusedDetail = "House Scan couldn't complete its connection to the processing service. This is a setup problem, not a problem with your photos or marks." static let scanWithLegacy = "Scan with Legacy instead" static let scanWithLegacyHint = "Chooses Legacy in Developer options and starts a new scan with it." static let closedAppLimit = "House Scan can't come back to this scan after you close the app." @@ -192,6 +193,15 @@ enum ProcessingCopy { return Screen( id: "refused", title: "The processing service refused this scan", detail: "Sending it again wouldn't change that.", tone: .attention, symbol: "exclamationmark.triangle") + case .setupRefused: + // The boundary's four codes read the same to the homeowner: this build's connection + // isn't right, nothing they did. The code itself goes to the log. It can come at the + // events or result step, after the service already has the scan, so the words don't + // say whether it was processed, and an unavailable credential can be temporary. + return Screen( + id: "setupRefused", title: "Photo processing isn't connected correctly", + detail: setupRefusedDetail, tone: .attention, symbol: "wrench.and.screwdriver", + notes: ["The connection needs to be fixed before you try again. Start over to scan again, or choose \(name(.legacy)) in Developer options first."]) case .answerUnreadable: return Screen( id: "answerUnreadable", title: "House Scan couldn't read the answer", diff --git a/ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift b/ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift index 45c00a629..2e0ee2961 100644 --- a/ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift +++ b/ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift @@ -72,6 +72,8 @@ extension DemoEngine { "processingFailed": sent(.ended(.processingFailed)), "expired": sent(.ended(.expired)), "refused": sent(.ended(.refused(step: "create"))), + // At the result step: the service may already have processed the scan. + "setupRefused": sent(.ended(.setupRefused(step: "result", .credentialUnavailable))), "answerUnreadable": sent(.ended(.answerUnreadable)), "answerNotReady": sent(.ended(.answerNotReady)), "answerMismatch": sent(.ended(.answerMismatch(.run))), diff --git a/ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift b/ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift index 68f211870..7e011da25 100644 --- a/ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift +++ b/ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift @@ -60,6 +60,10 @@ struct PhotoProcessingScreen: View { .buttonStyle(.quiet) .accessibilityIdentifier("action.stopSendingPhotos") } + // The scan is kept: shared from here only when its archive exists. + if copy.id == "setupRefused", let scan = state.shareableScan { + ShareScanButton(url: scan) + } if copy.id == "notSetUp" { Button(ProcessingCopy.scanWithLegacy) { actions.scanWithLegacyInstead() } .buttonStyle(.primary) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift index 28fb3b02a..82437bcf6 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift @@ -27,6 +27,7 @@ public final class PhotoProcessingController { private let setup: Setup private let coordinator: CaptureSessionCoordinator? + private let log: @Sendable (String) -> Void private var begun = false /// The photo-processing scan under way; nil when there is none. public private(set) var context: ScanContext? @@ -48,8 +49,10 @@ public final class PhotoProcessingController { } if case .ready(let environment, _) = self.setup { coordinator = CaptureSessionCoordinator(environment: environment) + log = environment.log } else { coordinator = nil + log = { _ in } } coordinator?.onStatus = { [weak self] in self?.uploadChanged($0) } coordinator?.onResult = { [weak self] in self?.answered($0) } @@ -232,9 +235,15 @@ public final class PhotoProcessingController { guard let session = coordinator?.session, let uploader = session.uploader else { return } Task { [weak self] in let end = await uploader.snapshot.end + // A withdrawal already set its own end, which a refusal read afterwards never replaces. guard let self, self.coordinator?.session === session, var state = self.status, !state.isFinal, - case .failed(let step, let codes, _)? = end else { return } - state.stage = .refusal(step: step, codes: codes) + case .failed(let step, let codes, let status)? = end else { return } + state.stage = .refusal(step: step, codes: codes, status: status) + if case .ended(.setupRefused(_, .storageAuthorization)) = state.stage { + self.log("photo processing: a service or contract setup failure: the service signed a storage upload carrying Authorization (step \(step))") + } else if case .ended(.setupRefused(_, let problem)) = state.stage { + self.log("photo processing: refused at the credential boundary, \(problem.code) (step \(step))") + } state.retrying = false self.status = state } diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift index 0571ffc82..90de57a8a 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift @@ -121,6 +121,10 @@ public enum PhotoProcessingEnd: Sendable, Equatable { case expired /// The service refused a request at `step`, which sending the same thing again won't change. case refused(step: String) + /// The phone refused to send a request at `step` because this build's connection to the + /// service isn't set up right (`ScopedCaptureHTTPError`): nothing about the scan caused it, and + /// sending again won't change it. + case setupRefused(step: String, PhotoSetupProblem) /// The service answered with something this build can't read as a result. case answerUnreadable /// The run finished, but its answer wasn't readable before the upload stopped asking. @@ -129,6 +133,37 @@ public enum PhotoProcessingEnd: Sendable, Equatable { case answerMismatch(PhotoBindingMismatch) } +/// The credential boundary's refusals (`ScopedCaptureHTTPError`), read back from the stable code +/// the uploader ends with at status 0, since no server answered. +public enum PhotoSetupProblem: Sendable, Equatable, CaseIterable { + /// A request's URL is outside the capture API's configured scope. + case destinationRefused + /// The credential provider couldn't supply a credential. + case credentialUnavailable + /// The credential isn't a well-formed bearer token. + case credentialMalformed + /// The service signed a storage upload that carries an Authorization header: a service or + /// contract setup failure, not the phone's credential. + case storageAuthorization + + /// The code the uploader ends with, taken from `ScopedCaptureHTTPError.code` so the two can't + /// drift apart. + public var code: String { + switch self { + case .destinationRefused: ScopedCaptureHTTPError.destinationNotAllowed("").code + case .credentialUnavailable: ScopedCaptureHTTPError.credentialUnavailable("").code + case .credentialMalformed: ScopedCaptureHTTPError.credentialMalformed.code + case .storageAuthorization: ScopedCaptureHTTPError.storageRequestHasAuthorization.code + } + } + + /// The problem an upload refusal names: one of the boundary's codes at status 0. + public init?(codes: [String], status: Int) { + guard status == 0, let problem = Self.allCases.first(where: { codes.contains($0.code) }) else { return nil } + self = problem + } +} + /// What a follow-up capture would carry from the capture it adds views to: plain values only. /// Nothing in this build sends a follow-up, so a request for more photos ends saying so; an /// amendment client starts from this. The follow-up keeps its own session and epoch: nothing here @@ -188,7 +223,8 @@ extension PhotoProcessingStatus.Stage { } /// The stage for an upload that stopped on a refusal (`CaptureUploadState.End.failed`). - public static func refusal(step: String, codes: [String]) -> Self { + public static func refusal(step: String, codes: [String], status: Int) -> Self { + if let problem = PhotoSetupProblem(codes: codes, status: status) { return .ended(.setupRefused(step: step, problem)) } guard step == "result" else { return .ended(.refused(step: step)) } if codes.contains("result_not_ready") { return .ended(.answerNotReady) } if codes.contains("result_for_another_run") { return .ended(.answerMismatch(.run)) } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift index b08483754..60d5d2d50 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift @@ -83,12 +83,17 @@ import Testing @Test func aRefusalReadsAsTheStageItEndsIn() { typealias Stage = PhotoProcessingStatus.Stage - #expect(Stage.refusal(step: "create", codes: ["schema"]) == .ended(.refused(step: "create"))) - #expect(Stage.refusal(step: "put", codes: ["storage_refused"]) == .ended(.refused(step: "put"))) - #expect(Stage.refusal(step: "result", codes: ["result_not_ready"]) == .ended(.answerNotReady)) - #expect(Stage.refusal(step: "result", codes: ["result_unreadable"]) == .ended(.answerUnreadable)) - #expect(Stage.refusal(step: "result", codes: ["result_for_another_run"]) == .ended(.answerMismatch(.run))) - #expect(Stage.refusal(step: "result", codes: []) == .ended(.refused(step: "result"))) + #expect(Stage.refusal(step: "create", codes: ["schema"], status: 422) == .ended(.refused(step: "create"))) + #expect(Stage.refusal(step: "put", codes: ["storage_refused"], status: 403) == .ended(.refused(step: "put"))) + #expect(Stage.refusal(step: "result", codes: ["result_not_ready"], status: 404) == .ended(.answerNotReady)) + #expect(Stage.refusal(step: "result", codes: ["result_unreadable"], status: 200) == .ended(.answerUnreadable)) + #expect(Stage.refusal(step: "result", codes: ["result_for_another_run"], status: 200) == .ended(.answerMismatch(.run))) + #expect(Stage.refusal(step: "result", codes: [], status: 500) == .ended(.refused(step: "result"))) + // The boundary's codes at status 0, where no server answered; the same text from a server isn't the phone's setup. + for problem in PhotoSetupProblem.allCases { + #expect(Stage.refusal(step: "create", codes: [problem.code], status: 0) == .ended(.setupRefused(step: "create", problem))) + #expect(Stage.refusal(step: "create", codes: [problem.code], status: 401) == .ended(.refused(step: "create"))) + } } } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedPhotoProcessingTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedPhotoProcessingTests.swift new file mode 100644 index 000000000..f23f5403c --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedPhotoProcessingTests.swift @@ -0,0 +1,152 @@ +import Foundation +import HouseScanKit +import Synchronization +import Testing + +/// `FixtureCaptureHTTP` whose register answers sign each storage upload with an Authorization +/// header, as a misconfigured service might. +final class AuthorizingStorageHTTP: CaptureHTTP, Sendable { + let inner: FixtureCaptureHTTP + + init(_ inner: FixtureCaptureHTTP) { + self.inner = inner + } + + func send(_ request: URLRequest) async throws -> HTTPReply { + let reply = try await inner.send(request) + guard request.httpMethod == "POST", request.url?.lastPathComponent == "files", + var body = try JSONSerialization.jsonObject(with: reply.body) as? [String: Any], + let files = body["files"] as? [[String: Any]] else { return reply } + body["files"] = files.map { file -> [String: Any] in + guard var upload = file["upload"] as? [String: Any], var headers = upload["headers"] as? [String: String] else { return file } + headers["Authorization"] = "Bearer storage" + upload["headers"] = headers + var signed = file + signed["upload"] = upload + return signed + } + return HTTPReply(status: reply.status, body: try JSONSerialization.data(withJSONObject: body), retryAfter: reply.retryAfter) + } + + func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + try await inner.upload(request, file: file) + } +} + +/// The scoped credential boundary's refusals as photo processing sees them: the real +/// `ScopedCaptureHTTP` and `CaptureUploader` end the upload with their code at status 0, and the +/// scan ends in the setup state, never as the homeowner's scan being refused. +@Suite(.serialized) @MainActor struct ScopedPhotoProcessingTests { + struct NoCredential: Error {} + + let root = FileManager.default.temporaryDirectory.appending(path: "scoped-photo-\(UUID().uuidString)") + + func controller(_ http: any CaptureHTTP) -> PhotoProcessingController { + var policy = CaptureUploader.Policy() + policy.eventsWait = 0 + policy.firstDelay = 0.01 + policy.maxDelay = 0.02 + let environment = CaptureSessionCoordinator.Environment( + endpoint: FixtureCaptureHTTP.base, sends: true, http: http, capturesFolder: root.appending(path: "Captures"), + sessionInfo: { packetID, video in SyntheticCapture.sessionInfo(packetID: packetID, video: video, appVersion: "test") }, + device: .init(model: "synthetic", systemVersion: "synthetic", appVersion: "test"), tier: .arkit, policy: policy) + return PhotoProcessingController(setup: .ready(environment, standIn: true)) + } + + func scoped(base: URL = FixtureCaptureHTTP.base, credential: @escaping ScopedCaptureHTTP.Credential, inner: any CaptureHTTP) throws + -> ScopedCaptureHTTP + { + ScopedCaptureHTTP(scope: try CaptureAPIScope(base: base), credential: credential, inner: inner) + } + + func until(_ seconds: Double = 30, _ condition: () -> Bool) async throws -> Bool { + let deadline = ContinuousClock.now + .seconds(seconds) + while !condition(), ContinuousClock.now < deadline { try await Task.sleep(for: .milliseconds(20)) } + return condition() + } + + func run(_ controller: PhotoProcessingController) async throws { + let capture = SyntheticCapture.standard + controller.beginScan(ScanContext(scanID: "scan", profile: controller.profile, recordingSessionID: "rec"), recording: capture.recording) + controller.answerConsent(true) + for photo in try capture.photos(in: root.appending(path: "photos")) { controller.kept(photo) } + controller.captureEnded(acceptedCloseUpAt: capture.acceptedCloseUpAt) + } + + @Test(arguments: [PhotoSetupProblem.destinationRefused, .credentialUnavailable, .credentialMalformed, .storageAuthorization]) + func eachBoundaryRefusalEndsInTheSetupState(problem: PhotoSetupProblem) async throws { + defer { try? FileManager.default.removeItem(at: root) } + let fixture = FixtureCaptureHTTP(answer: .candidate) + let http: ScopedCaptureHTTP + switch problem { + case .destinationRefused: + let elsewhere = try #require(URL(string: "https://elsewhere.invalid/v1")) + http = try scoped(base: elsewhere, credential: { "token" }, inner: fixture) + case .credentialUnavailable: + http = try scoped(credential: { throw NoCredential() }, inner: fixture) + case .credentialMalformed: + http = try scoped(credential: { "not a token" }, inner: fixture) + case .storageAuthorization: + http = try scoped(credential: { "token" }, inner: AuthorizingStorageHTTP(fixture)) + } + let controller = controller(http) + try await run(controller) + try #require(try await until { controller.status?.isFinal == true }, "\(String(describing: controller.status))") + let step = problem == .storageAuthorization ? "put" : "create" + #expect(controller.status?.stage == .ended(.setupRefused(step: step, problem))) + #expect(controller.status?.consent == .granted) + // Refused on the phone: nothing reached the API, or past register for the storage case. + if problem == .storageAuthorization { + #expect(!fixture.routes.contains("PUT upload") && !fixture.routes.contains("POST captures/finalize")) + } else { + #expect(fixture.routes.isEmpty) + } + controller.endScan(recording: SyntheticCapture.standard.recording) + } + + /// A credential that stops being available once the service has the whole capture, so the + /// refusal comes at the result read, after the service may have processed the scan. The scan + /// ends in the same setup state, at the result step. + @Test func aRefusalAtTheResultStepIsTheSameSetupState() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let fixture = FixtureCaptureHTTP(answer: .candidate) + // The fixture lists a route once the request got past the boundary, so the events poll + // that finds the run finished gets its credential, and the result read after it doesn't. + let http = try scoped(credential: { + if fixture.routes.contains("GET captures/events") { throw NoCredential() } + return "token" + }, inner: fixture) + let controller = controller(http) + try await run(controller) + try #require(try await until { controller.status?.isFinal == true }, "\(String(describing: controller.status))") + #expect(controller.status?.stage == .ended(.setupRefused(step: "result", .credentialUnavailable))) + #expect(fixture.routes.contains("POST captures/finalize")) + #expect(!fixture.routes.contains("GET captures/result")) + controller.endScan(recording: SyntheticCapture.standard.recording) + } + + /// The homeowner stops sending while the credential is being fetched, and the provider then + /// fails: the scan ends withdrawn, never in the setup state. + @Test func aWithdrawalOutranksARefusalThatFollowsIt() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let fixture = FixtureCaptureHTTP(answer: .candidate) + let asked = Mutex(0) + let http = try scoped(credential: { + asked.withLock { $0 += 1 } + // Waits without watching for cancellation, then fails. + await Task.detached { try? await Task.sleep(for: .milliseconds(400)) }.value + throw NoCredential() + }, inner: fixture) + let controller = controller(http) + controller.beginScan( + ScanContext(scanID: "scan", profile: controller.profile, recordingSessionID: "rec"), recording: SyntheticCapture.standard.recording) + controller.answerConsent(true) + try #require(try await until(10) { asked.withLock { $0 } > 0 }, "the credential was never asked for") + controller.stopSending() + #expect(controller.status?.stage == .ended(.withdrawn(recorded: true))) + try await Task.sleep(for: .seconds(1)) + #expect(controller.status?.stage == .ended(.withdrawn(recorded: true))) + #expect(fixture.routes.isEmpty) + controller.endScan(recording: SyntheticCapture.standard.recording) + } +} diff --git a/ios/HouseScanUITests/PhotoProcessingUITests.swift b/ios/HouseScanUITests/PhotoProcessingUITests.swift index 0719aa6f7..1f836eace 100644 --- a/ios/HouseScanUITests/PhotoProcessingUITests.swift +++ b/ios/HouseScanUITests/PhotoProcessingUITests.swift @@ -222,10 +222,10 @@ final class PhotoProcessingUITests: XCTestCase { attach(consentLarge, "consent-AX5") consentLarge.terminate() - let large: Set = ["uploading", "processing", "candidate", "needsMorePhotos", "installerReview", "withdrawalNotRecorded", "notSetUp"] + let large: Set = ["uploading", "processing", "candidate", "needsMorePhotos", "installerReview", "withdrawalNotRecorded", "notSetUp", "setupRefused"] for state in [ "queued", "uploading", "processing", "candidate", "needsMorePhotos", "installerReview", "noCandidate", "unrecognized", - "notSetUp", "consentNotGiven", "withdrawn", "withdrawalNotRecorded", "notPrepared", "processingFailed", "expired", "refused", + "notSetUp", "consentNotGiven", "withdrawn", "withdrawalNotRecorded", "notPrepared", "processingFailed", "expired", "refused", "setupRefused", "answerUnreadable", "answerNotReady", "answerMismatch", ] { for big in large.contains(state) ? [false, true] : [false] { @@ -233,8 +233,17 @@ final class PhotoProcessingUITests: XCTestCase { XCTAssertTrue( app.descendants(matching: .any)["photo.state.\(state)"].waitForExistence(timeout: 15), "photo.state.\(state) never appeared") XCTAssertTrue(app.descendants(matching: .any)["photo.standIn"].exists || state == "notSetUp", "\(state) doesn't say its answer is a test one") + if state == "setupRefused", !big { + // Shown here at the result step, after the service may have processed the scan: + // the words claim neither way, and don't rule out trying again once it's fixed. + let words = app.staticTexts.allElementsBoundByIndex.map(\.label).joined(separator: " ") + XCTAssertTrue(words.contains("This is a setup problem, not a problem with your photos or marks."), words) + XCTAssertTrue(words.contains("The connection needs to be fixed before you try again."), words) + XCTAssertFalse(words.contains("wasn't processed"), words) + XCTAssertFalse(words.contains("won't help"), words) + } attach(app, "processing-\(state)\(big ? "-AX5" : "")") - if !big, ["uploading", "candidate", "needsMorePhotos", "withdrawalNotRecorded"].contains(state) { try audit(app, "processing-\(state)") } + if !big, ["uploading", "candidate", "needsMorePhotos", "withdrawalNotRecorded", "setupRefused"].contains(state) { try audit(app, "processing-\(state)") } app.terminate() } } From 3df2da74474948bb4fddf47c2e0e456057a65ae9 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 10:33:08 -0500 Subject: [PATCH 38/71] ios: the stop-sending question says only what House Scan stops doing Stop sending can come after the photos reached the processing service and its job began, and the app can't cancel that job, so "This scan won't be processed" could be false. The question now reads "House Scan will stop sending from this scan and won't resume it. Photos already sent may still be processed by the service." The stop-confirmation UI test asserts that wording and the absence of "won't be processed". Declining at consent keeps its own words: nothing was sent then. Wording only, not run locally; the hosted photoProcessing group runs the test. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/UI/Copy/ProcessingCopy.swift | 4 +++- ios/HouseScanUITests/PhotoProcessingUITests.swift | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ios/HouseScan/UI/Copy/ProcessingCopy.swift b/ios/HouseScan/UI/Copy/ProcessingCopy.swift index 68a8e874b..eb533e702 100644 --- a/ios/HouseScan/UI/Copy/ProcessingCopy.swift +++ b/ios/HouseScan/UI/Copy/ProcessingCopy.swift @@ -62,7 +62,9 @@ enum ProcessingCopy { static let serviceSaid = "The service says" static let stopSending = "Stop sending photos" static let stopQuestion = "Stop sending this scan's photos?" - static let stopDetail = "This scan won't be processed, and House Scan can't start it again. Photos already sent stay with the processing service." + /// Stopping can come after the photos reached the service and its processing began, and the app + /// can't cancel the service's job, so this says only what House Scan stops doing. + static let stopDetail = "House Scan will stop sending from this scan and won't resume it. Photos already sent may still be processed by the service." static let confirmStop = "Stop sending" static let keepSending = "Keep sending" static let setupRefusedDetail = "House Scan couldn't complete its connection to the processing service. This is a setup problem, not a problem with your photos or marks." diff --git a/ios/HouseScanUITests/PhotoProcessingUITests.swift b/ios/HouseScanUITests/PhotoProcessingUITests.swift index 1f836eace..a497ffb9a 100644 --- a/ios/HouseScanUITests/PhotoProcessingUITests.swift +++ b/ios/HouseScanUITests/PhotoProcessingUITests.swift @@ -261,6 +261,12 @@ final class PhotoProcessingUITests: XCTestCase { stop.tap() let confirm = app.buttons["Stop sending"].firstMatch XCTAssertTrue(confirm.waitForExistence(timeout: 5), "stopping didn't ask first") + // The question says only what House Scan stops doing: photos already sent may still be + // processed, since the app can't cancel the service's job. + let detail = app.staticTexts.matching(NSPredicate(format: "label CONTAINS %@", "Photos already sent may still be processed by the service.")).firstMatch + XCTAssertTrue(detail.waitForExistence(timeout: 5), "the stop question must say photos already sent may still be processed") + XCTAssertTrue(detail.label.contains("House Scan will stop sending from this scan and won't resume it."), detail.label) + XCTAssertFalse(app.staticTexts.matching(NSPredicate(format: "label CONTAINS %@", "won't be processed")).firstMatch.exists, "the stop question must not promise the scan won't be processed") attach(app, "processing-stopQuestion") confirm.tap() XCTAssertTrue(app.descendants(matching: .any)["photo.state.withdrawn"].waitForExistence(timeout: 5)) From f0468d4f1ab7073c00061bb4a87c860f3f35d7ea Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 10:35:02 -0500 Subject: [PATCH 39/71] ios: after Stop sending or Start over, say only what House Scan stops doing Both can come after photos reached the processing service, and the app can't cancel the service's job. Three claims said otherwise: - the screen after a recorded withdrawal ("Nothing more from this scan will be sent, and it won't be processed") now reads "House Scan won't send anything more from this scan. Photos already sent may still be processed by the service."; - Start over's accessibility hint ("This scan isn't processed any further") now says House Scan stops sending from this scan and won't show its answer, and that photos already sent may still be processed; - the not-prepared screen, which can follow some sending, now says the scan couldn't be sent for processing rather than that it wasn't processed. Declining at consent keeps "wasn't processed": nothing was sent then. The stop-flow UI test asserts the stopped screen's wording and the absence of "won't be processed". Wording only, not run locally; the hosted photoProcessing group runs the tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/UI/Copy/ProcessingCopy.swift | 7 +++++-- ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift | 3 ++- ios/HouseScanUITests/PhotoProcessingUITests.swift | 4 ++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ios/HouseScan/UI/Copy/ProcessingCopy.swift b/ios/HouseScan/UI/Copy/ProcessingCopy.swift index eb533e702..bf628cb09 100644 --- a/ios/HouseScan/UI/Copy/ProcessingCopy.swift +++ b/ios/HouseScan/UI/Copy/ProcessingCopy.swift @@ -170,7 +170,9 @@ enum ProcessingCopy { guard !recorded else { return Screen( id: "withdrawn", title: "You stopped sending photos", - detail: "Nothing more from this scan will be sent, and it won't be processed.", symbol: "hand.raised", notes: [kept]) + // Stopping can follow sending, and the app can't cancel the service's job. + detail: "House Scan won't send anything more from this scan. Photos already sent may still be processed by the service.", + symbol: "hand.raised", notes: [kept]) } return Screen( id: "withdrawalNotRecorded", title: "You stopped sending photos", @@ -181,7 +183,8 @@ enum ProcessingCopy { if isReplay { notes.insert("Replays carry no motion data, which photo processing needs.", at: 0) } return Screen( id: "notPrepared", title: "This scan couldn't be prepared", - detail: "House Scan couldn't put this scan together for photo processing on this phone, so it wasn't processed.", + // Some photos may already have been sent; the phone knows only that it couldn't send the scan. + detail: "House Scan couldn't put this scan together for photo processing on this phone, so it couldn't be sent for processing.", tone: .attention, symbol: "exclamationmark.triangle", notes: notes) case .processingFailed: return Screen( diff --git a/ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift b/ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift index 7e011da25..7ae68a8ab 100644 --- a/ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift +++ b/ios/HouseScan/UI/Screens/PhotoProcessingScreen.swift @@ -77,7 +77,8 @@ struct PhotoProcessingScreen: View { Button("Start over") { actions.startOver() }.buttonStyle(.primary) } } - .accessibilityHint("Starts a new scan. This scan isn't processed any further.") + // Start over can come while the service is processing, and the app can't cancel that. + .accessibilityHint("Starts a new scan. House Scan stops sending from this one and won't show its answer. Photos already sent may still be processed by the service.") .accessibilityIdentifier("action.startOver") } } diff --git a/ios/HouseScanUITests/PhotoProcessingUITests.swift b/ios/HouseScanUITests/PhotoProcessingUITests.swift index a497ffb9a..ab1402a09 100644 --- a/ios/HouseScanUITests/PhotoProcessingUITests.swift +++ b/ios/HouseScanUITests/PhotoProcessingUITests.swift @@ -270,6 +270,10 @@ final class PhotoProcessingUITests: XCTestCase { attach(app, "processing-stopQuestion") confirm.tap() XCTAssertTrue(app.descendants(matching: .any)["photo.state.withdrawn"].waitForExistence(timeout: 5)) + let ended = app.staticTexts.matching(NSPredicate(format: "label CONTAINS %@", "House Scan won't send anything more from this scan.")).firstMatch + XCTAssertTrue(ended.waitForExistence(timeout: 5), "the stopped screen must say only what House Scan stops doing") + XCTAssertTrue(ended.label.contains("Photos already sent may still be processed by the service."), ended.label) + XCTAssertFalse(app.staticTexts.matching(NSPredicate(format: "label CONTAINS %@", "won't be processed")).firstMatch.exists, "the stopped screen must not promise the scan won't be processed") XCTAssertFalse(app.buttons["action.stopSendingPhotos"].exists) } From 0c68d92172ff3761e2fe88554bbde1a78e9845d9 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 11:02:09 -0500 Subject: [PATCH 40/71] Test fresh retry limits on a recreated capture and the end of an upload whose state can't be saved Co-Authored-By: Claude Opus 5.5 (1M context) --- .../CaptureUploaderRecoveryTests.swift | 241 ++++++++++++++++++ 1 file changed, 241 insertions(+) create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderRecoveryTests.swift diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderRecoveryTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderRecoveryTests.swift new file mode 100644 index 000000000..cd9318601 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderRecoveryTests.swift @@ -0,0 +1,241 @@ +import Foundation +import HouseScanKit +import Synchronization +import Testing + +/// Two ways an upload could keep going on limits or state that no longer hold. A server that +/// lost the capture gets a new run, so the old run's retry limits must not end it early. A state +/// save that fails ends the upload in this process, so it never sends what the saved state +/// doesn't record. +/// +/// The loopback decides an answer when a request arrives and only delays sending it while its +/// route is held, so these tests change the server between one request and the next. +@Suite struct CaptureUploaderRecoveryTests { + typealias Rig = CaptureUploaderTests.Rig + + /// Waits until at least `count` requests on `route` are parked. + static func parked(_ server: LoopbackCaptureAPI, _ route: String, count: Int = 1, within seconds: Int = 20) async -> Bool { + let deadline = ContinuousClock.now + .seconds(seconds) + while ContinuousClock.now < deadline { + if server.state.withLock({ ($0.parked[route] ?? []).count >= count }) { return true } + try? await Task.sleep(for: .milliseconds(10)) + } + return false + } + + /// Sends the answers parked on `route` and keeps holding it. + static func deliver(_ server: LoopbackCaptureAPI, _ route: String) { + let waiting = server.state.withLock { $0.parked.removeValue(forKey: route) ?? [] } + waiting.forEach { $0() } + } + + static func saved(_ rig: Rig) throws -> CaptureUploadState { + try CaptureUploadState.load(from: CaptureUploader.stateURL(in: rig.capture.folder)) + } + + static func routes(_ server: LoopbackCaptureAPI) -> [String] { server.state.withLock { $0.log.map(LoopbackCaptureAPI.route) } } + + // MARK: A recreated capture is a new run + + enum PutFault: String, CaseIterable, Sendable { + /// A second 400 BadDigest for a file whose one resend was spent. + case badDigest + /// A refused PUT past `maxRefusedPuts`. + case refused + } + + /// Every file of one sealed photo spends its PUT limits on the first server capture: a 400 + /// BadDigest (its one resend), a 403 refusal and a 200, three PUTs against `maxRefusedPuts` 3. + /// The server then loses the capture. The same fault once more for every file on the new + /// capture is retried, not the end of the upload, and the state a relaunch would load already + /// holds the new run's counts. + @Test(arguments: PutFault.allCases) + func aRecreatedCaptureGetsFreshPutLimits(fault: PutFault) async throws { + var policy = Rig.fast + policy.maxRefusedPuts = 3 + let rig = try Rig(policy: policy) + defer { rig.cleanUp() } + let files = try await rig.capture.sealImages(count: 1) + // One register answer and one round of PUTs covers every file, so each fault below lands + // once on each file. + let n = files.count + try #require(n <= policy.maxConcurrentPuts) + rig.server.state.withLock { + $0.badDigestPuts = n + $0.expireNextPuts = n + $0.held = ["PUT upload"] + } + await rig.uploader.add(files) + // Rounds one and two go through. A round starts only once the one before is answered, so + // a parked PUT past the 2n-th belongs to round three. + while true { + try #require(await Self.parked(rig.server, "PUT upload")) + if rig.server.requests("PUT upload").count > 2 * n { break } + Self.deliver(rig.server, "PUT upload") + } + try #require(await Self.parked(rig.server, "PUT upload", count: n)) + #expect(rig.server.requests("PUT upload").count == 3 * n) + + // Round three's 200s are decided. The server forgets the capture before the commit. + rig.server.state.withLock { s in + s.captures = [:] + s.byPacket = [:] + switch fault { + case .badDigest: s.badDigestPuts = n + case .refused: s.expireNextPuts = n + } + s.held = ["POST captures"] + } + Self.deliver(rig.server, "PUT upload") + try #require(await Self.parked(rig.server, "POST captures")) + let gone = try Self.saved(rig) + #expect(gone.captureID == nil) + #expect(gone.files.values.map(\.attempts) == Array(repeating: 0, count: n)) + #expect(gone.files.values.allSatisfy { $0.phase == .queued }) + rig.server.release("POST captures") + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(await rig.uploader.snapshot.end == nil) + + try await rig.finishAndSeal() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + #expect(rig.server.requests("POST captures").count == 2) + } + + /// The first server capture loses its finalize and follows `retry_finalize` once, the whole + /// limit. The server then loses the capture. A lost finalize on the new capture is sent again, + /// and the saved state reads 0 follows before the new capture exists. + @Test func aRecreatedCaptureGetsAFreshFinalizeLimit() async throws { + var policy = Rig.fast + policy.maxFinalizeRetries = 1 + let rig = try Rig(policy: policy) + defer { rig.cleanUp() } + rig.server.state.withLock { + $0.loseNextFinalize = true + $0.held = ["POST captures/finalize"] + } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await CaptureUploaderTests.settles(rig.uploader)) + try await rig.finishAndSeal() + try #require(await Self.parked(rig.server, "POST captures/finalize")) + Self.deliver(rig.server, "POST captures/finalize") + try #require(await Self.parked(rig.server, "POST captures/finalize")) + #expect(try Self.saved(rig).finalizeRetries == 1) + + // The second finalize is accepted. The server forgets the capture before the next poll. + rig.server.state.withLock { s in + s.captures = [:] + s.byPacket = [:] + s.loseNextFinalize = true + s.held = ["POST captures"] + } + Self.deliver(rig.server, "POST captures/finalize") + try #require(await Self.parked(rig.server, "POST captures")) + let gone = try Self.saved(rig) + #expect(gone.captureID == nil) + #expect(gone.finalizeRetries == 0) + rig.server.release("POST captures") + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + #expect(rig.server.requests("POST captures").count == 2) + #expect(rig.server.requests("POST captures/finalize").count == 4) + } + + // MARK: A state save that fails + + /// A failed save ends the upload in this process with the local code and status 0. The + /// observer hears it, nothing more is sent or retried, the saved file keeps its last good + /// bytes, and this process doesn't resume the folder from those older bytes. + @Test func aFailedSaveEndsTheUploadAndSendsNothingMore() async throws { + let sleeps = ScopedCaptureUploadTests.Sleeps() + var rig = try Rig(sleep: sleeps.record) + defer { rig.cleanUp() } + let folder = rig.capture.folder + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) } + let statuses = Mutex<[CaptureUploadStatus]>([]) + await rig.uploader.observe { status in statuses.withLock { $0.append(status) } } + rig.server.state.withLock { $0.held = ["POST captures"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 2)) + try #require(await Self.parked(rig.server, "POST captures")) + + let url = CaptureUploader.stateURL(in: folder) + let before = try Data(contentsOf: url) + // A folder that takes no new file: the atomic save can't write its replacement. + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) + rig.server.release("POST captures") + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + let unsaved = CaptureUploadState.End.failed(step: "save", codes: ["local_state_unsaved"], status: 0) + #expect(await rig.uploader.snapshot.end == unsaved) + let last = try #require(statuses.withLock { $0.last }) + #expect(last.phase == .failed) + #expect(last.detail == "save 0 local_state_unsaved") + #expect(last.retryingAt == nil) + #expect(try Data(contentsOf: url) == before) + + await rig.uploader.kick() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(Self.routes(rig.server) == ["POST captures"]) + #expect(sleeps.isEmpty) + + // Writable again, the folder still holds the older saved yes. Once the uploader is gone, + // this process doesn't pick that up and send the capture again. + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) + weak let released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + #expect(try CaptureUploader.resume(folder: folder, base: rig.server.base, http: rig.http, policy: Rig.fast, sleep: { _ in }) == nil) + #expect(try Data(contentsOf: url) == before) + #expect(Self.routes(rig.server) == ["POST captures"]) + } + + enum Order: String, CaseIterable, Sendable { + /// The withdrawal lands while the create is in flight; the reply's save then fails. + case withdrawalFirst + /// The save fails and ends the upload; the withdrawal comes after. + case saveFailureFirst + } + + /// A withdrawal outranks the save failure in either order. The folder is unwritable, so the + /// withdrawal itself can't be recorded (`notRecorded`) and no save succeeds. The coordinator + /// then abandons the upload as withdrawn, as it does on every withdrawal: the upload ends as + /// withdrawn, not as a local save failure, and sends nothing more. + @Test(arguments: Order.allCases) + func aWithdrawalOutranksAFailedSave(order: Order) async throws { + let sleeps = ScopedCaptureUploadTests.Sleeps() + let rig = try Rig(sleep: sleeps.record) + defer { rig.cleanUp() } + let folder = rig.capture.folder + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) } + rig.server.state.withLock { $0.held = ["POST captures"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 2)) + try #require(await Self.parked(rig.server, "POST captures")) + + let url = CaptureUploader.stateURL(in: folder) + let before = try Data(contentsOf: url) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) + if order == .saveFailureFirst { + rig.server.release("POST captures") + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(await rig.uploader.snapshot.end == .failed(step: "save", codes: ["local_state_unsaved"], status: 0)) + } + let record = rig.uploader.withdrawConsent() + guard case .notRecorded = record else { + Issue.record("expected the withdrawal to go unrecorded in an unwritable folder, got \(record)") + return + } + if order == .withdrawalFirst { rig.server.release("POST captures") } + try #require(await CaptureUploaderTests.settles(rig.uploader)) + await rig.uploader.abandon(CaptureUploader.withdrawnReason) + + #expect(await rig.uploader.snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) + #expect(await rig.uploader.status.phase == .abandoned) + #expect(try Data(contentsOf: url) == before) + await rig.uploader.kick() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(Self.routes(rig.server) == ["POST captures"]) + #expect(sleeps.isEmpty) + } +} From 9ccce0f89aae6390373e848b84e42be1219dcac9 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 10:50:18 -0500 Subject: [PATCH 41/71] ios: don't seal a photo-processing capture while kept photos are still being saved MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex on #216 (f0468d4): when a keyframe write takes longer than the ten-second wait, `drainPendingSaves` still returned true because the scan hadn't changed. `startPhotoProcessing` then sealed the capture, and the late photo's `onKept` was refused, so the packet silently left out a photo taken before the send. The wait now says whether writes are still pending. If they are, the capture is never sealed: `PhotoProcessingController.capturePreparationFailed` ends the scan as not prepared ("This scan couldn't be prepared ā€¦ couldn't be sent for processing"). It also ends the capture's coordinator session the way Start over does (`newWorld` with a new scan), so the uploader already running for the scan stops sending and is abandoned: no new request, no retry, and nothing it reports later reaches the screen. A request already on its way may still be answered by the service, and photos uploaded during the scan stay there. Legacy's own wait is unchanged. Test `aCaptureMissingAPhotoIsNeverSealedAndItsUploaderStops` drives the controller directly with a storage upload held in flight. It proves what happens once the engine's wait gives up, not the engine's 10 s wait itself. After the held request finishes, no further request is made, no finalize happens, and the stage stays not prepared. Not run locally: the shared heavy slot was held by other runs. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Runtime/ScanEngine+Processing.swift | 8 +- ios/HouseScan/Runtime/ScanEngine.swift | 10 ++- .../PhotoProcessingController.swift | 26 ++++++ .../PhotoProcessingTests.swift | 84 ++++++++++++++++++- 4 files changed, 123 insertions(+), 5 deletions(-) diff --git a/ios/HouseScan/Runtime/ScanEngine+Processing.swift b/ios/HouseScan/Runtime/ScanEngine+Processing.swift index a35c2de22..001a5d9fd 100644 --- a/ios/HouseScan/Runtime/ScanEngine+Processing.swift +++ b/ios/HouseScan/Runtime/ScanEngine+Processing.swift @@ -76,7 +76,13 @@ extension ScanEngine { let context = scanContext let synthetic = syntheticCapture Task { - guard await drainPendingSaves(), scanContext == context, state.phase == .processing else { return } + let saves = await drainPendingSaves() + guard saves != .scanChanged, scanContext == context, state.phase == .processing else { return } + if saves == .stillWriting { + RuntimeLog.engine.error("photo processing: photos were still being saved after 10 s; the capture isn't sealed") + photoProcessing.capturePreparationFailed() + return + } guard let synthetic else { let closeUp = state.closeUp == .skipped ? nil : store.stillFrames["meter_close"]?.t photoProcessing.captureEnded(acceptedCloseUpAt: closeUp) diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index 7de314776..c1de48bf8 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -2089,14 +2089,18 @@ final class ScanEngine { // MARK: Upload + enum PendingSaves { case done, scanChanged, stillWriting } + /// Waits, for at most 10 s, for the keyframe writes this world started, so the capture holds - /// every photo kept before the send. False when the world or the scan changed meanwhile. - func drainPendingSaves() async -> Bool { + /// every photo kept before the send. `stillWriting` when some are still being written then: a + /// capture sealed now would leave them out, and their late `onKept` would be refused. + func drainPendingSaves() async -> PendingSaves { let scan = generation for _ in 0..<200 where (pendingSaves[scan] ?? 0) > 0 { try? await Task.sleep(for: .milliseconds(50)) } - return scan == generation + guard scan == generation else { return .scanChanged } + return (pendingSaves[scan] ?? 0) > 0 ? .stillWriting : .done } func startUpload() { diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift index 82437bcf6..f7ff099cc 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift @@ -29,6 +29,9 @@ public final class PhotoProcessingController { private let coordinator: CaptureSessionCoordinator? private let log: @Sendable (String) -> Void private var begun = false + /// The world the current session records, so the session can be ended in place + /// (`capturePreparationFailed`). + private var recording: RecordingSource? /// The photo-processing scan under way; nil when there is none. public private(set) var context: ScanContext? public private(set) var status: PhotoProcessingStatus? { @@ -177,6 +180,28 @@ public final class PhotoProcessingController { status = state } + /// The scan was sent, but the phone couldn't finish saving every photo it kept before the send, + /// so the capture would leave one out: it is never sealed, and the scan ends as not prepared. + /// + /// The capture's coordinator session ends too, the way Start over ends one (`newWorld` with a + /// new scan): its uploader stops sending and is abandoned, so it starts no new request and + /// retries nothing, and anything it reports later never reaches the screen. A request already + /// on its way may still be answered by the service; photos already uploaded stay there. The + /// empty session that replaces it has no consent, so it sends nothing. + public func capturePreparationFailed() { + guard context != nil, var state = status, !captureEnded, case .capturing = state.stage else { return } + captureEnded = true + switch state.consent { + case .granted: state.stage = .ended(.notPrepared) + case .asking, .declined: state.stage = .ended(.consentNotGiven) + case .withdrawn(let recorded): state.stage = .ended(.withdrawn(recorded: recorded)) + case .notNeeded: return + } + state.retrying = false + status = state + if begun, let recording { coordinator?.newWorld("capture not prepared", recording: recording, newScan: true) } + } + /// Waits until the current session's queued work and uploads are idle. For tests. public func settle() async { await coordinator?.settle() @@ -191,6 +216,7 @@ public final class PhotoProcessingController { private func startSession(_ reason: String, recording: RecordingSource, newScan: Bool) { guard let coordinator else { return } + self.recording = recording if begun { coordinator.newWorld(reason, recording: recording, newScan: newScan) } else { diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift index 60d5d2d50..f0202c509 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift @@ -146,11 +146,12 @@ final class HeldResultHTTP: CaptureHTTP, Sendable { let root = FileManager.default.temporaryDirectory.appending(path: "photo-processing-\(UUID().uuidString)") var fixture: NativeCaptureFixture { NativeCaptureFixture(folder: root.appending(path: "store")) } - func controller(_ http: any CaptureHTTP) -> PhotoProcessingController { + func controller(_ http: any CaptureHTTP, maxConcurrentPuts: Int? = nil) -> PhotoProcessingController { var environment = NativeCaptureFixture.environment(endpoint: FixtureCaptureHTTP.base, http: http, captures: root.appending(path: "Captures")) // Retries and unreadable answers come round in milliseconds, not the app's seconds. environment.policy.firstDelay = 0.01 environment.policy.maxDelay = 0.02 + if let maxConcurrentPuts { environment.policy.maxConcurrentPuts = maxConcurrentPuts } return PhotoProcessingController(setup: .ready(environment, standIn: true)) } @@ -243,6 +244,43 @@ final class HeldResultHTTP: CaptureHTTP, Sendable { controller.endScan(recording: fixture.recording) } + /// The phone couldn't finish saving a photo kept before the send. This drives the controller + /// directly: it proves what happens once the engine's wait gives up, not the engine's 10 s wait. + /// A storage upload is held in flight when the capture is marked unprepared. The scan ends as + /// not prepared; the held request, already admitted, may finish; after it the uploader starts + /// no new request and retries nothing, a photo kept later isn't forwarded, and the capture is + /// never sealed or finalized, nor does any later report change the stage. + @Test func aCaptureMissingAPhotoIsNeverSealedAndItsUploaderStops() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let inner = FixtureCaptureHTTP(answer: .candidate) + let http = HoldFirstPut(inner) + // Released even if an expectation below fails, so the held upload can't strand the test. + defer { http.release() } + // One storage upload at a time, so the held one is the only request admitted when the + // capture ends; with the default of three, others could land after the snapshot below. + let controller = controller(http, maxConcurrentPuts: 1) + controller.beginScan(context(controller), recording: fixture.recording) + controller.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + controller.meterTapped(tap, hit: hit) + controller.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + controller.kept(try fixture.photo(at: fixture.start + 3)) + try #require(try await until(10) { http.isHolding }, "no storage upload was held") + let before = inner.routes + controller.capturePreparationFailed() + #expect(controller.status?.stage == .ended(.notPrepared)) + http.release() + controller.kept(try fixture.photo(at: fixture.start + 4)) + controller.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + // Longer than the uploader waits between retries and polls with this test's policy. + try await Task.sleep(for: .seconds(2.5)) + let after = Array(inner.routes.dropFirst(before.count)) + #expect(after.count <= 1 && after.allSatisfy { $0 == "PUT upload" }, "requests after the capture ended: \(after)") + #expect(!inner.routes.contains("POST captures/finalize")) + #expect(controller.status?.stage == .ended(.notPrepared)) + controller.endScan(recording: fixture.recording) + } + @Test(arguments: [false, nil] as [Bool?]) func withoutAYesNothingIsSent(consent: Bool?) async throws { defer { try? FileManager.default.removeItem(at: root) } @@ -385,3 +423,47 @@ final class HeldResultHTTP: CaptureHTTP, Sendable { controller.endScan(recording: fixture.recording) } } + +/// The capture fixture with its first storage upload held until `release()`: a request already +/// admitted, still waiting for its answer. +private final class HoldFirstPut: CaptureHTTP, @unchecked Sendable { + private let inner: FixtureCaptureHTTP + private let lock = NSLock() + private var held = false + private var released = false + private var waiting: CheckedContinuation? + + init(_ inner: FixtureCaptureHTTP) { self.inner = inner } + + var isHolding: Bool { lock.withLock { waiting != nil } } + + func release() { + let next = lock.withLock { () -> CheckedContinuation? in + released = true + defer { waiting = nil } + return waiting + } + next?.resume() + } + + func send(_ request: URLRequest) async throws -> HTTPReply { try await inner.send(request) } + + func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + let hold = lock.withLock { () -> Bool in + guard !held, !released else { return false } + held = true + return true + } + if hold { + await withCheckedContinuation { continuation in + let resumeNow = lock.withLock { () -> Bool in + if released { return true } + waiting = continuation + return false + } + if resumeNow { continuation.resume() } + } + } + return try await inner.upload(request, file: file) + } +} From d1d3076fe87ba801720d2a6148b052abcb626b22 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 11:02:09 -0500 Subject: [PATCH 42/71] Give a recreated capture fresh retry limits, and end an upload whose state can't be saved A server that lost the capture opens a new run, but the old run's PUT attempts, BadDigest resends and retry_finalize follows still counted against it, so the new capture could fail on its first fault. captureGone now resets them with the limits it already reset. A failed state save stopped the loop but left the status reading Sending or Processing forever, and once the uploader was gone this process could resume the folder from its older saved yes. It now ends the upload in memory at step save, status 0, code local_state_unsaved, drops replies in flight and refuses to resume the folder in this process. A withdrawal still outranks it. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../HouseScanKit/Upload/CaptureUploader.swift | 42 +++++++++++++++---- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift index 911bccc3e..64d9726af 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -20,9 +20,10 @@ public actor CaptureUploader { public var eventsWait = 20 /// A signed URL this close to expiry is fetched again before the PUT. public var expiryMargin = 60.0 - /// PUTs of one file the storage refuses (not transient) before the upload stops. + /// PUTs of one file the storage refuses (not transient) before the upload stops, counted per + /// server capture. public var maxRefusedPuts = 6 - /// `retry_finalize` answers followed before the upload stops. + /// `retry_finalize` answers followed before the upload stops, counted per server capture. public var maxFinalizeRetries = 3 /// Result reads that find the run's answer not readable yet before the upload stops. Each /// waits the retry delay first, so the default allows about three minutes. No measurement @@ -82,6 +83,9 @@ public actor CaptureUploader { private struct FolderEntry { weak var owner: CaptureUploader? var withdrawn = false + /// A save failed here: the saved state is older than what this process sent, so this + /// process never resumes it. A relaunch may, under the consent it records. + var unsaved = false var stopped = false var running: Task? } @@ -95,8 +99,8 @@ public actor CaptureUploader { case savedUploadExists case folderIdentityUnavailable } - /// Set when a state change could not be saved: the loop stops rather than send a request the - /// saved state doesn't know about. + /// Set when a state change could not be saved. The upload ends here (`unsavedStateCode`) + /// rather than send a request the saved state doesn't know about. private var saveFailed = false private var observer: (@Sendable (CaptureUploadStatus) -> Void)? private var log: (@Sendable (String) -> Void)? @@ -138,7 +142,7 @@ public actor CaptureUploader { try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) let folder = try canonicalFolder(folder) return try folders.withLock { entries in - entries = entries.filter { $0.value.owner != nil || $0.value.withdrawn } + entries = entries.filter { $0.value.owner != nil || $0.value.withdrawn || $0.value.unsaved } guard entries[folder.path]?.withdrawn != true, !FileManager.default.fileExists(atPath: CaptureUploadState.withdrawnURL(in: folder).path) else { throw OwnershipError.consentWithdrawn } @@ -168,9 +172,9 @@ public actor CaptureUploader { guard FileManager.default.fileExists(atPath: folder.path) else { return nil } let folder = try canonicalFolder(folder) return try folders.withLock { entries in - entries = entries.filter { $0.value.owner != nil || $0.value.withdrawn } + entries = entries.filter { $0.value.owner != nil || $0.value.withdrawn || $0.value.unsaved } let url = stateURL(in: folder) - guard entries[folder.path]?.withdrawn != true, + guard entries[folder.path]?.withdrawn != true, entries[folder.path]?.unsaved != true, FileManager.default.fileExists(atPath: url.path), !FileManager.default.fileExists(atPath: CaptureUploadState.withdrawnURL(in: folder).path) else { return nil } @@ -231,6 +235,12 @@ public actor CaptureUploader { public static let withdrawnReason = "consent withdrawn" + /// The code an upload ends with, at step `save` and status 0, when this phone couldn't save + /// the upload's state. It is a local storage problem: it says nothing about the photos or + /// the server, and no server answered. + public static let unsavedStateCode = "local_state_unsaved" + static let unsavedEnd = CaptureUploadState.End.failed(step: "save", codes: [unsavedStateCode], status: 0) + /// How a withdrawal was recorded on disk. public enum WithdrawalRecord: Sendable, Equatable { /// The marker `resume` refuses was written. @@ -272,7 +282,8 @@ public actor CaptureUploader { /// dropped when they arrive. public func abandon(_ reason: String) { stopSending() - guard state.end == nil else { return } + // An end the app asks for, a withdrawal included, replaces a save failure. + guard state.end == nil || state.end == Self.unsavedEnd else { return } state.end = .abandoned(reason) state.attemptID = UUID().uuidString loop?.cancel() @@ -633,10 +644,14 @@ public actor CaptureUploader { state.captureID = nil state.finalized = nil state.eventCursor = 0 - // A new server capture is a new run: the old one's limits don't carry over. + // A new server capture is a new run: none of the old one's retry limits carry over. + // Each still bounds the new run on its own. state.notReadyReads = nil + state.finalizeRetries = 0 + digestRetried.removeAll() for path in state.files.keys { state.files[path]?.phase = .queued + state.files[path]?.attempts = 0 state.files[path]?.unacknowledged = nil } persist() @@ -712,6 +727,15 @@ public actor CaptureUploader { } } catch { saveFailed = true + Self.folders.withLock { entries in + if entries[folderKey]?.owner === self { entries[folderKey]?.unsaved = true } + } + // The saved state is now behind this one, so the upload ends in memory only: nothing + // claims it was saved, and nothing more is sent. An end already set stays: a + // withdrawal's, set before the write was tried, or a result or refusal. + // Replies in flight belong to the old attempt and are dropped. + if state.end == nil { state.end = Self.unsavedEnd } + state.attemptID = UUID().uuidString log?("capture-upload stopped: the upload state could not be saved (\(Self.describe(error)))") } publish() From d63cc6e137e05f6a15ba8d5d1d8b05723e167ec5 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 11:12:50 -0500 Subject: [PATCH 43/71] Test that a failed save stops a request waiting for its credential, and that a withdrawal outranks an end whose save failed Co-Authored-By: Claude Opus 5.5 (1M context) --- .../CaptureUploaderRecoveryTests.swift | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderRecoveryTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderRecoveryTests.swift index cd9318601..1ab403919 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderRecoveryTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderRecoveryTests.swift @@ -238,4 +238,130 @@ import Testing #expect(Self.routes(rig.server) == ["POST captures"]) #expect(sleeps.isEmpty) } + + /// Counts what reaches the network under `ScopedCaptureHTTP`. + final class CountingHTTP: CaptureHTTP { + let inner: any CaptureHTTP + let sends = Mutex(0) + init(_ inner: any CaptureHTTP) { self.inner = inner } + func send(_ request: URLRequest) async throws -> HTTPReply { + sends.withLock { $0 += 1 } + return try await inner.send(request) + } + func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + sends.withLock { $0 += 1 } + return try await inner.upload(request, file: file) + } + } + + /// A save that fails while a request waits for its credential stops that request. The create + /// waits on a provider that ignores cancellation. Meanwhile `add` saves into an unwritable + /// folder and fails. When the provider then returns a good token, nothing reaches the + /// network, nothing is retried, and the upload ends as the local save failure, not as a + /// withdrawal. + @Test func aFailedSaveStopsARequestWaitingForItsCredential() async throws { + let sleeps = ScopedCaptureUploadTests.Sleeps() + let gate = ScopedCaptureHTTPTests.Gate() + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.expectedBearer = ScopedCaptureUploadTests.token } + let network = CountingHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)) + let http = ScopedCaptureHTTP( + scope: try CaptureAPIScope(base: server.base, transport: .loopbackHTTP), + credential: { + await gate.wait() + return ScopedCaptureUploadTests.token + }, + inner: network) + let rig = try Rig(server: server, http: http, sleep: sleeps.record) + defer { rig.cleanUp() } + let folder = rig.capture.folder + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) } + let images = try await rig.capture.sealImages(count: 1) + await rig.uploader.kick() + await gate.entered() + + let url = CaptureUploader.stateURL(in: folder) + let before = try Data(contentsOf: url) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) + await rig.uploader.add(images) + gate.release() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + + #expect(network.sends.withLock { $0 } == 0) + #expect(server.state.withLock { $0.log.isEmpty }) + #expect(await rig.uploader.snapshot.end == .failed(step: "save", codes: ["local_state_unsaved"], status: 0)) + #expect(await rig.uploader.status.phase == .failed) + #expect(await rig.uploader.status.retryingAt == nil) + #expect(sleeps.isEmpty) + #expect(try Data(contentsOf: url) == before) + await rig.uploader.kick() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(network.sends.withLock { $0 } == 0) + } + + enum UnsavedEnd: String, CaseIterable, Sendable { + /// The result arrives and ends the upload, and that save fails. + case finished + /// The server's answer ends the upload as refused, and that save fails. + case refused + } + + /// A withdrawal outranks an end this process reached but couldn't save. The end stays when + /// its save fails; the folder is unwritable, so the withdrawal goes unrecorded too. The + /// coordinator's `abandon(withdrawnReason)` must still end the upload as withdrawn. + /// + /// This holds at the library and coordinator boundary. The app doesn't offer Stop sending + /// once its screen shows a final state, so no button reaches this path today. + @Test(arguments: UnsavedEnd.allCases) + func aWithdrawalOutranksAnEndWhoseSaveFailed(end: UnsavedEnd) async throws { + let sleeps = ScopedCaptureUploadTests.Sleeps() + let rig = try Rig(sleep: sleeps.record) + defer { rig.cleanUp() } + let folder = rig.capture.folder + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) } + let route: String + switch end { + case .finished: + route = "GET captures/result" + rig.server.state.withLock { $0.held = [route] } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await CaptureUploaderTests.settles(rig.uploader)) + try await rig.finishAndSeal() + case .refused: + route = "POST captures/files" + // A register answer with a plain-http storage URL to another host, which ends the upload. + rig.server.state.withLock { + $0.held = [route] + $0.uploadURLOverride = "http://example.com/upload/x" + } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + } + try #require(await Self.parked(rig.server, route)) + + let url = CaptureUploader.stateURL(in: folder) + let before = try Data(contentsOf: url) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) + rig.server.release(route) + try #require(await CaptureUploaderTests.settles(rig.uploader)) + let reached = await rig.uploader.snapshot.end + switch end { + case .finished: #expect(reached == .finished(status: "manual_review")) + case .refused: #expect(reached == .failed(step: "register", codes: ["upload_url_insecure"], status: 200)) + } + #expect(try Data(contentsOf: url) == before) + let sent = Self.routes(rig.server) + + let record = rig.uploader.withdrawConsent() + guard case .notRecorded = record else { + Issue.record("expected the withdrawal to go unrecorded in an unwritable folder, got \(record)") + return + } + await rig.uploader.abandon(CaptureUploader.withdrawnReason) + #expect(await rig.uploader.snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) + #expect(await rig.uploader.status.phase == .abandoned) + #expect(try Data(contentsOf: url) == before) + await rig.uploader.kick() + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(Self.routes(rig.server) == sent) + } } From 04aeff676311404897b3c48b5802800466b59656 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 11:13:24 -0500 Subject: [PATCH 44/71] Stop a waiting request when a save fails, and let a withdrawal outrank an end that wasn't saved A failed save ended the upload but left admission open, so a request whose credential fetch was still waiting could send once the token arrived. The failure now closes admission and cancels the running task, as stopSending does. A result or refusal whose save failed kept its end, and abandon(withdrawnReason) returned without replacing it. A withdrawal now replaces an end this process couldn't save. A saved end still stays. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../HouseScanKit/Upload/CaptureUploader.swift | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift index 64d9726af..30816cd37 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -282,9 +282,12 @@ public actor CaptureUploader { /// dropped when they arrive. public func abandon(_ reason: String) { stopSending() - // An end the app asks for, a withdrawal included, replaces a save failure. - guard state.end == nil || state.end == Self.unsavedEnd else { return } - state.end = .abandoned(reason) + // An end the app asks for replaces a save failure. A withdrawal also replaces an end this + // process reached but couldn't save, such as a result or a refusal. Otherwise an end + // stays: Start over doesn't rewrite a finished upload. + let unsavedWithdrawal = saveFailed && isWithdrawn + guard state.end == nil || state.end == Self.unsavedEnd || unsavedWithdrawal else { return } + state.end = .abandoned(unsavedWithdrawal ? Self.withdrawnReason : reason) state.attemptID = UUID().uuidString loop?.cancel() persist() @@ -727,8 +730,14 @@ public actor CaptureUploader { } } catch { saveFailed = true - Self.folders.withLock { entries in - if entries[folderKey]?.owner === self { entries[folderKey]?.unsaved = true } + // Admission closes before any suspended request resumes, as `stopSending` closes it: + // a credential fetch that returns after this sends nothing. One already handed to the + // transport may finish. + let running = Self.folders.withLock { entries -> Task? in + guard entries[folderKey]?.owner === self else { return nil } + entries[folderKey]?.unsaved = true + entries[folderKey]?.stopped = true + return entries[folderKey]?.running } // The saved state is now behind this one, so the upload ends in memory only: nothing // claims it was saved, and nothing more is sent. An end already set stays: a @@ -737,6 +746,8 @@ public actor CaptureUploader { if state.end == nil { state.end = Self.unsavedEnd } state.attemptID = UUID().uuidString log?("capture-upload stopped: the upload state could not be saved (\(Self.describe(error)))") + // Outside the lock: cancellation handlers belong to the transport. + running?.cancel() } publish() } From eea5f7a3ed5f4db0372d99c682654c5062dbe11f Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 11:18:10 -0500 Subject: [PATCH 45/71] ios: a scan whose upload state the phone couldn't save ends as a local failure With 04aeff67 merged, an upload whose state this phone can't save ends at step `save`, status 0, with `CaptureUploader.unsavedStateCode`. Only that exact end now maps to a new stage, `uploadStateUnsaved`: the same code with a server status, or at another step, stays an ordinary refusal. The screen says "This phone stopped sending this scan": House Scan couldn't save how far the upload had got and stopped sending from this scan, and photos already sent may still be processed by the service. It offers Start over. It doesn't say the service refused or cancelled anything, and it doesn't mention resuming, since this app doesn't reopen a saved capture after it closes. A withdrawal keeps its own stage. Tests: the pure mapping (save/0 maps; save/500 and put/0 don't); `anUploadStateThePhoneCantSaveEndsTheScanAsALocalFailure`, where the real uploader ends once the capture folder turns read-only and nothing more is sent; and the new state in the photo-processing every-state UI test at default and largest text, with wording assertions and the default-size audit. Not run locally (admission refused); the next hosted run covers it. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/UI/Copy/ProcessingCopy.swift | 9 +++++ .../Preview/DemoEngine+PhotoProcessing.swift | 1 + .../PhotoProcessingController.swift | 2 ++ .../Processing/PhotoProcessingStatus.swift | 6 ++++ .../PhotoProcessingTests.swift | 34 +++++++++++++++++++ .../PhotoProcessingUITests.swift | 15 ++++++-- 6 files changed, 64 insertions(+), 3 deletions(-) diff --git a/ios/HouseScan/UI/Copy/ProcessingCopy.swift b/ios/HouseScan/UI/Copy/ProcessingCopy.swift index bf628cb09..2fdc004fb 100644 --- a/ios/HouseScan/UI/Copy/ProcessingCopy.swift +++ b/ios/HouseScan/UI/Copy/ProcessingCopy.swift @@ -207,6 +207,15 @@ enum ProcessingCopy { id: "setupRefused", title: "Photo processing isn't connected correctly", detail: setupRefusedDetail, tone: .attention, symbol: "wrench.and.screwdriver", notes: ["The connection needs to be fixed before you try again. Start over to scan again, or choose \(name(.legacy)) in Developer options first."]) + case .uploadStateUnsaved: + // The phone, not the service: it couldn't save how far the upload had got, so it + // stopped sending. That stops nothing on the service, and this app doesn't reopen a + // saved capture after it closes, so the way on is a new scan. + return Screen( + id: "uploadStateUnsaved", title: "This phone stopped sending this scan", + detail: "House Scan couldn't save how far this scan's upload had got, so it stopped sending from this scan. Photos already sent may still be processed by the service.", + tone: .attention, symbol: "exclamationmark.triangle", + notes: ["Start over to scan again. If your phone's storage is nearly full, freeing some space may help."]) case .answerUnreadable: return Screen( id: "answerUnreadable", title: "House Scan couldn't read the answer", diff --git a/ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift b/ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift index 2e0ee2961..80170fdbc 100644 --- a/ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift +++ b/ios/HouseScan/UI/Preview/DemoEngine+PhotoProcessing.swift @@ -74,6 +74,7 @@ extension DemoEngine { "refused": sent(.ended(.refused(step: "create"))), // At the result step: the service may already have processed the scan. "setupRefused": sent(.ended(.setupRefused(step: "result", .credentialUnavailable))), + "uploadStateUnsaved": sent(.ended(.uploadStateUnsaved)), "answerUnreadable": sent(.ended(.answerUnreadable)), "answerNotReady": sent(.ended(.answerNotReady)), "answerMismatch": sent(.ended(.answerMismatch(.run))), diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift index f7ff099cc..13117e4c4 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift @@ -269,6 +269,8 @@ public final class PhotoProcessingController { self.log("photo processing: a service or contract setup failure: the service signed a storage upload carrying Authorization (step \(step))") } else if case .ended(.setupRefused(_, let problem)) = state.stage { self.log("photo processing: refused at the credential boundary, \(problem.code) (step \(step))") + } else if case .ended(.uploadStateUnsaved) = state.stage { + self.log("photo processing: this phone couldn't save the upload's progress; sending stopped") } state.retrying = false self.status = state diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift index 90de57a8a..659735bd2 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift @@ -125,6 +125,10 @@ public enum PhotoProcessingEnd: Sendable, Equatable { /// service isn't set up right (`ScopedCaptureHTTPError`): nothing about the scan caused it, and /// sending again won't change it. case setupRefused(step: String, PhotoSetupProblem) + /// This phone couldn't save the upload's progress (`CaptureUploader.unsavedStateCode`, step + /// `save`, status 0), so the uploader stopped sending from this scan. A local problem: no + /// server refused anything, and what the service already has may still be processed. + case uploadStateUnsaved /// The service answered with something this build can't read as a result. case answerUnreadable /// The run finished, but its answer wasn't readable before the upload stopped asking. @@ -224,6 +228,8 @@ extension PhotoProcessingStatus.Stage { /// The stage for an upload that stopped on a refusal (`CaptureUploadState.End.failed`). public static func refusal(step: String, codes: [String], status: Int) -> Self { + // Only the uploader's own end (`CaptureUploader.unsavedEnd`): step `save`, status 0, its code. + if step == "save", status == 0, codes.contains(CaptureUploader.unsavedStateCode) { return .ended(.uploadStateUnsaved) } if let problem = PhotoSetupProblem(codes: codes, status: status) { return .ended(.setupRefused(step: step, problem)) } guard step == "result" else { return .ended(.refused(step: step)) } if codes.contains("result_not_ready") { return .ended(.answerNotReady) } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift index f0202c509..902084a59 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift @@ -94,6 +94,12 @@ import Testing #expect(Stage.refusal(step: "create", codes: [problem.code], status: 0) == .ended(.setupRefused(step: "create", problem))) #expect(Stage.refusal(step: "create", codes: [problem.code], status: 401) == .ended(.refused(step: "create"))) } + // The uploader's own end when this phone couldn't save its state: status 0, no server. The + // same code with a server status isn't the phone's failure. + #expect(Stage.refusal(step: "save", codes: [CaptureUploader.unsavedStateCode], status: 0) == .ended(.uploadStateUnsaved)) + #expect(Stage.refusal(step: "save", codes: [CaptureUploader.unsavedStateCode], status: 500) == .ended(.refused(step: "save"))) + // Status 0 alone doesn't make it this failure: the code at another step isn't the uploader's own end. + #expect(Stage.refusal(step: "put", codes: [CaptureUploader.unsavedStateCode], status: 0) == .ended(.refused(step: "put"))) } } @@ -281,6 +287,34 @@ final class HeldResultHTTP: CaptureHTTP, Sendable { controller.endScan(recording: fixture.recording) } + /// While the service processes, the phone can no longer save the upload's state (its folder + /// turned read-only): the real uploader ends and the scan says sending stopped on this phone, + /// not that the service refused it, and nothing more is sent. A withdrawal that couldn't be + /// saved keeps its own stage instead (`stoppingPartWayStopsSendingAndSaysWhetherThePhoneSavedIt`). + @Test func anUploadStateThePhoneCantSaveEndsTheScanAsALocalFailure() async throws { + let http = FixtureCaptureHTTP(answer: .hold) + let controller = controller(http) + // Before anything can throw: the session ends and the folders go even if a wait times out. + var lockedFolder: URL? + defer { + if let lockedFolder { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: lockedFolder.path) } + controller.endScan(recording: fixture.recording) + try? FileManager.default.removeItem(at: root) + } + controller.beginScan(context(controller), recording: fixture.recording) + try await capture(controller, consent: true) + try #require(try await until { controller.status?.stage == .processing }) + let folder = try #require(controller.captureFolder) + lockedFolder = folder + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) + try #require(try await until { controller.status?.isFinal == true }, "no final stage: \(String(describing: controller.status))") + #expect(controller.status?.stage == .ended(.uploadStateUnsaved)) + #expect(controller.status?.consent == .granted) + let sent = http.routes.count + try await Task.sleep(for: .seconds(2.5)) + #expect(http.routes.count == sent) + } + @Test(arguments: [false, nil] as [Bool?]) func withoutAYesNothingIsSent(consent: Bool?) async throws { defer { try? FileManager.default.removeItem(at: root) } diff --git a/ios/HouseScanUITests/PhotoProcessingUITests.swift b/ios/HouseScanUITests/PhotoProcessingUITests.swift index ab1402a09..3729028a7 100644 --- a/ios/HouseScanUITests/PhotoProcessingUITests.swift +++ b/ios/HouseScanUITests/PhotoProcessingUITests.swift @@ -222,11 +222,11 @@ final class PhotoProcessingUITests: XCTestCase { attach(consentLarge, "consent-AX5") consentLarge.terminate() - let large: Set = ["uploading", "processing", "candidate", "needsMorePhotos", "installerReview", "withdrawalNotRecorded", "notSetUp", "setupRefused"] + let large: Set = ["uploading", "processing", "candidate", "needsMorePhotos", "installerReview", "withdrawalNotRecorded", "notSetUp", "setupRefused", "uploadStateUnsaved"] for state in [ "queued", "uploading", "processing", "candidate", "needsMorePhotos", "installerReview", "noCandidate", "unrecognized", "notSetUp", "consentNotGiven", "withdrawn", "withdrawalNotRecorded", "notPrepared", "processingFailed", "expired", "refused", "setupRefused", - "answerUnreadable", "answerNotReady", "answerMismatch", + "uploadStateUnsaved", "answerUnreadable", "answerNotReady", "answerMismatch", ] { for big in large.contains(state) ? [false, true] : [false] { let app = Self.demo(["-uiDemoPhase", "processing", "-uiDemoPhotoState", state] + (big ? Self.largestText : [])) @@ -242,8 +242,17 @@ final class PhotoProcessingUITests: XCTestCase { XCTAssertFalse(words.contains("wasn't processed"), words) XCTAssertFalse(words.contains("won't help"), words) } + if state == "uploadStateUnsaved", !big { + // The phone's own failure: it says sending stopped here, and promises nothing + // about what the service does with photos it already has. + let words = app.staticTexts.allElementsBoundByIndex.map(\.label).joined(separator: " ") + XCTAssertTrue(words.contains("so it stopped sending from this scan."), words) + XCTAssertTrue(words.contains("Photos already sent may still be processed by the service."), words) + XCTAssertFalse(words.contains("refused"), words) + XCTAssertFalse(words.contains("cancel"), words) + } attach(app, "processing-\(state)\(big ? "-AX5" : "")") - if !big, ["uploading", "candidate", "needsMorePhotos", "withdrawalNotRecorded", "setupRefused"].contains(state) { try audit(app, "processing-\(state)") } + if !big, ["uploading", "candidate", "needsMorePhotos", "withdrawalNotRecorded", "setupRefused", "uploadStateUnsaved"].contains(state) { try audit(app, "processing-\(state)") } app.terminate() } } From 8cc8cec879822917eb2a429205434bf165342596 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 12:12:55 -0500 Subject: [PATCH 46/71] ios: stop un-hiding the camera's decoration for VoiceOver when no photo question shows Run 37213636011 on #216 (f0468d4f): the journey group's every-state accessibility audit failed on every camera screen (findMeter, meterCloseUp, wallWalk, markFeatures, gapRequest, resultAR) with "Element has no description (id '' label '')", as did the audit step of five FullFlowUITests on screen.findMeter. The extra 28 minutes of failures pushed the job past its 90-minute cap. ScanRootView applied `.accessibilityHidden(asksForPhotoConsent)` to CameraBackdrop, the practice-meter overlay and the screen. With no question showing, that is `.accessibilityHidden(false)`, which overrode the hiding CameraBackdrop and other decoration set on themselves, and exposed the feed as an unlabeled element. PhotoConsentCard already carries `.isModal`, which is what keeps VoiceOver on the question while it shows, so the three modifiers are removed. Not run locally; the next hosted journey run's audit is the check. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/UI/ScanRootView.swift | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ios/HouseScan/UI/ScanRootView.swift b/ios/HouseScan/UI/ScanRootView.swift index 1eb893acd..68db7a6fe 100644 --- a/ios/HouseScan/UI/ScanRootView.swift +++ b/ios/HouseScan/UI/ScanRootView.swift @@ -28,11 +28,9 @@ struct ScanRootView: View { ZStack { if Self.showsCamera(state.phase) { CameraBackdrop(feed: state.feed, actions: actions) - .accessibilityHidden(asksForPhotoConsent) .transition(.opacity) if state.isPracticeScan { PracticeMeterOverlay(state: state) - .accessibilityHidden(asksForPhotoConsent) } CameraEdgeShade() } @@ -42,8 +40,11 @@ struct ScanRootView: View { screen .id(state.phase) .transition(Self.showsCamera(state.phase) ? .opacity : .identity) - // The question is modal: what it covers isn't there for VoiceOver until it's answered. - .accessibilityHidden(asksForPhotoConsent) + // The question is modal for VoiceOver through its own `.isModal` trait + // (`PhotoConsentCard`). Its siblings aren't hidden here: `.accessibilityHidden(false)`, + // as that was whenever no question showed, overrode the hiding the camera feed and + // other decoration set on themselves, and the audit found an element with no + // description on every camera screen (#216, run 37213636011). if asksForPhotoConsent { PhotoConsentCard(standIn: state.photoProcessing?.standIn == true, synthetic: state.photoCaptureIsSynthetic, actions: actions) .transition(.opacity) From e1a5cae4c97a9aee9c0c9f5735e0f50af7e9e06e Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 12:18:13 -0500 Subject: [PATCH 47/71] Record each ARFrame's intrinsics with its pose for the 0.4 pose stream The recorder gains a frameIntrinsics stream, rows [t, fx, fy, cx, cy, width, height]: ARKit's camera.intrinsics in the unrotated sensor image's pixels and imageResolution, as reported, built by HouseScanKit's FrameCalibrationRow. recordPose now writes it at the one LiveCapture call site, from the same ARFrame and at the same t as the trajectory row, under one lock so recording toggled between them can't keep one without the other. It uses the recorder's existing storage, restart and recording-off path like every stream. Photo processing's recording source forwards these rows instead of [] to Packet04Streams.poseRows, which joins them by exact t and omits a pose without its own row. The 1.1 packet doesn't get the stream: writeMotion now takes each stream's 1.1 destination from Stream.motionPacketStream, nil for the trajectory and for frame intrinsics, and PacketStream has no case for it. No 1.1 or 0.4 wire field or schema changed. No keyframe or depth calibration is borrowed and no clock is rebased. Gates, camera activation, sending, Share, the meter tap and #192's files are unchanged. Comments and the README no longer say live intrinsics are missing; they aren't checked on a phone yet. Tests: FrameCalibrationRowTests (HouseScanKit) check the row's values and units, that frames whose calibration and resolution change independently each join their own row through poseRows, and that a missing row or one off by one ulp omits the pose. There is no app unit test target, so the recorder's own behaviour and the 1.1 exclusion are not tested here. Validated: nothing ran. leased-admit refused the package tests: "heavy lease refused for run frame-intrinsics: refused: count limit 1 reached ; nothing started" (exit 75). Not retried. The app wasn't built. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/Runtime/CaptureRecorder.swift | 63 ++++++++++++----- ios/HouseScan/Runtime/LiveCapture.swift | 7 +- ios/HouseScan/Runtime/ScanEngine+Packet.swift | 15 ++-- .../Runtime/ScanEngine+Processing.swift | 13 ++-- .../Packet/FrameCalibrationRow.swift | 13 ++++ .../FrameCalibrationRowTests.swift | 68 +++++++++++++++++++ ios/README.md | 2 +- 7 files changed, 144 insertions(+), 37 deletions(-) create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Packet/FrameCalibrationRow.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/FrameCalibrationRowTests.swift diff --git a/ios/HouseScan/Runtime/CaptureRecorder.swift b/ios/HouseScan/Runtime/CaptureRecorder.swift index 734676e9d..f48b78cca 100644 --- a/ios/HouseScan/Runtime/CaptureRecorder.swift +++ b/ios/HouseScan/Runtime/CaptureRecorder.swift @@ -35,6 +35,11 @@ final class CaptureRecorder: Sendable { case deviceMotion /// t, pressure in kPa, relative altitude in m. case barometer + /// t, then that ARFrame's own fx, fy, cx, cy and image width and height in pixels of the + /// unrotated sensor image (`FrameCalibrationRow`), written with its trajectory row at the + /// same t. Read by photo processing's 0.4 pose stream only; the 1.1 packet has no column + /// for it (`motionPacketStream`). + case frameIntrinsics var width: Int { switch self { @@ -42,6 +47,20 @@ final class CaptureRecorder: Sendable { case .accelerometer, .gyroscope, .magnetometer: 4 case .deviceMotion: 15 case .barometer: 3 + case .frameIntrinsics: 7 + } + } + + /// The 1.1 packet's stream a motion row goes to; nil for the camera's own streams, which + /// the 1.1 packet writes elsewhere (the trajectory) or not at all (frame intrinsics). + var motionPacketStream: PacketStream? { + switch self { + case .accelerometer: .accelerometer + case .gyroscope: .gyroscope + case .magnetometer: .magnetometer + case .deviceMotion: .deviceMotion + case .barometer: .barometer + case .trajectory, .frameIntrinsics: nil } } @@ -124,34 +143,42 @@ final class CaptureRecorder: Sendable { } } - func recordPose(t: Double, tracking: TrackingCode, cameraToWorld m: simd_float4x4) { + /// One ARFrame's pose and its own calibration (`ARCamera.intrinsics`, `imageResolution`), under + /// one lock, so recording turned off or on between them can't keep one without the other. + func recordPose(t: Double, tracking: TrackingCode, cameraToWorld m: simd_float4x4, intrinsics: simd_float3x3, imageWidth: Double, imageHeight: Double) { let columns = [m.columns.0, m.columns.1, m.columns.2, m.columns.3] let pose = columns.flatMap { [Double($0.x), Double($0.y), Double($0.z), Double($0.w)] } - append(.trajectory, [t, Double(tracking.state), Double(tracking.reason)] + pose) + let calibration = FrameCalibrationRow.row(t: t, intrinsics: intrinsics, imageWidth: imageWidth, imageHeight: imageHeight) + state.withLock { state in + append(.trajectory, [t, Double(tracking.state), Double(tracking.reason)] + pose, &state) + append(.frameIntrinsics, calibration, &state) + } } /// Appends one row. A row not later than the stream's last one is dropped: every stream's `t` /// strictly increases. func append(_ stream: Stream, _ row: [Double]) { + state.withLock { append(stream, row, &$0) } + } + + private func append(_ stream: Stream, _ row: [Double], _ state: inout State) { precondition(row.count == stream.width, "\(stream) row has \(row.count) values, expected \(stream.width)") let t = row[0] - state.withLock { state in - guard state.recording, !state.failed, t >= state.since, t > state.lastT[stream.rawValue] else { return } - state.lastT[stream.rawValue] = t - if stream == .trajectory { - if state.firstUptime == nil { - state.firstUptime = t - // The same instant on the wall clock: the frame is `now - t` seconds old. - state.startedAt = Date(timeIntervalSinceNow: t - ProcessInfo.processInfo.systemUptime) - } - state.lastUptime = t - } - row.withUnsafeBufferPointer { values in - for value in values { withUnsafeBytes(of: value.bitPattern.littleEndian) { state.buffers[stream.rawValue].append(contentsOf: $0) } } - } - if state.buffers[stream.rawValue].count >= Self.flushBytes { - write(stream, &state) + guard state.recording, !state.failed, t >= state.since, t > state.lastT[stream.rawValue] else { return } + state.lastT[stream.rawValue] = t + if stream == .trajectory { + if state.firstUptime == nil { + state.firstUptime = t + // The same instant on the wall clock: the frame is `now - t` seconds old. + state.startedAt = Date(timeIntervalSinceNow: t - ProcessInfo.processInfo.systemUptime) } + state.lastUptime = t + } + var bytes = Data(capacity: row.count * 8) + for value in row { withUnsafeBytes(of: value.bitPattern.littleEndian) { bytes.append(contentsOf: $0) } } + state.buffers[stream.rawValue].append(bytes) + if state.buffers[stream.rawValue].count >= Self.flushBytes { + write(stream, &state) } } diff --git a/ios/HouseScan/Runtime/LiveCapture.swift b/ios/HouseScan/Runtime/LiveCapture.swift index c15dcd44e..011eec7bb 100644 --- a/ios/HouseScan/Runtime/LiveCapture.swift +++ b/ios/HouseScan/Runtime/LiveCapture.swift @@ -426,8 +426,11 @@ final class LiveSessionDelegate: NSObject, ARSessionDelegate, Sendable { } let shared = shared.withLock { $0 } let tracking = Self.tracking(frame.camera.trackingState) - // Every frame's pose goes to the packet's trajectory, before any sampling. - shared.recorder?.recordPose(t: frame.timestamp, tracking: TrackingCode(tracking), cameraToWorld: frame.camera.transform) + // Every frame's pose goes to the packet's trajectory, before any sampling, with that frame's + // own calibration at the same t. + shared.recorder?.recordPose( + t: frame.timestamp, tracking: TrackingCode(tracking), cameraToWorld: frame.camera.transform, intrinsics: frame.camera.intrinsics, + imageWidth: Double(frame.camera.imageResolution.width), imageHeight: Double(frame.camera.imageResolution.height)) let intrinsics = frame.camera.intrinsics let resolution = frame.camera.imageResolution let camera = CameraFrame( diff --git a/ios/HouseScan/Runtime/ScanEngine+Packet.swift b/ios/HouseScan/Runtime/ScanEngine+Packet.swift index 253daedb8..dda3f27b7 100644 --- a/ios/HouseScan/Runtime/ScanEngine+Packet.swift +++ b/ios/HouseScan/Runtime/ScanEngine+Packet.swift @@ -451,17 +451,12 @@ extension ScanEngine { /// Core Motion's streams, rows inside the capture only. Returns how many rows were refused. private nonisolated static func writeMotion(_ inputs: PacketInputs, window: ClosedRange, into writer: inout PacketWriter) -> Int { var refused = 0 - for stream in CaptureRecorder.Stream.allCases where stream != .trajectory && inputs.motionStreams.contains(stream) { + // Motion streams only: the trajectory is written elsewhere, and frame intrinsics have no + // column in the 1.1 packet (`CaptureRecorder.Stream.motionPacketStream`). + for stream in CaptureRecorder.Stream.allCases where inputs.motionStreams.contains(stream) { + guard let packetStream = stream.motionPacketStream else { continue } let rows = inputs.recorder.rows(stream).filter { window.contains($0[0]) } guard !rows.isEmpty else { continue } - let packetStream: PacketStream = switch stream { - case .trajectory: .trajectory - case .accelerometer: .accelerometer - case .gyroscope: .gyroscope - case .magnetometer: .magnetometer - case .deviceMotion: .deviceMotion - case .barometer: .barometer - } // CMAltimeter sets its own rate (about 1 Hz), so the barometer claims none. if stream != .barometer { try? writer.setNominalRate(MotionSource.rate, for: packetStream) } for r in rows { @@ -476,7 +471,7 @@ extension ScanEngine { userAcceleration: SIMD3(r[8], r[9], r[10]), rotationRate: SIMD3(r[11], r[12], r[13]), headingDegrees: r[14] )) case .barometer: try writer.appendBarometer(t: r[0], pressureKPa: r[1], relativeAltitudeM: r[2]) - case .trajectory: break + case .trajectory, .frameIntrinsics: break } } catch { refused += 1 diff --git a/ios/HouseScan/Runtime/ScanEngine+Processing.swift b/ios/HouseScan/Runtime/ScanEngine+Processing.swift index 001a5d9fd..17cccb31c 100644 --- a/ios/HouseScan/Runtime/ScanEngine+Processing.swift +++ b/ios/HouseScan/Runtime/ScanEngine+Processing.swift @@ -131,9 +131,10 @@ extension ScanEngine { syntheticCapture?.recording ?? recordingSource } - /// The current world's recording as the capture packet reads it. A replay records no poses or - /// motion, and this recorder writes no per-frame intrinsics, so the packet can't pass its - /// checks and the phone refuses it; nothing is made up to fill the gap. + /// The current world's recording as the capture packet reads it: live, each ARFrame's pose with + /// its own intrinsics at the same t, which the packet joins by exact time + /// (`Packet04Streams.poseRows`). A replay records no poses, intrinsics or motion, so its packet + /// can't pass its checks and the phone refuses it; nothing is made up to fill the gap. var recordingSource: RecordingSource { let recorder = recorder return RecordingSource( @@ -145,7 +146,7 @@ extension ScanEngine { rows: { _ = recorder.flush() return RecorderRows( - trajectory: recorder.rows(.trajectory), intrinsics: [], + trajectory: recorder.rows(.trajectory), intrinsics: recorder.rows(.frameIntrinsics), accelerometer: recorder.rows(.accelerometer), gyroscope: recorder.rows(.gyroscope)) }) } @@ -178,8 +179,8 @@ extension ScanEngine { // MARK: Setup /// What photo processing can do in this build. Sending a capture off the phone is off in this - /// slice: the 0.4 clock and privacy contract, a scoped credential, the live intrinsics and - /// meter tap, and the deployment aren't settled. The one way to run it is DEBUG's fixture on a + /// slice: the 0.4 clock and privacy contract, a scoped credential, the live meter tap, and + /// the deployment aren't settled; per-frame intrinsics are recorded but unchecked on a phone. The one way to run it is DEBUG's fixture on a /// replay, which answers on the phone (`CaptureFixtureLaunch`); a release build can't, since /// `debugBuild` is decided at compile time and the launch options aren't even parsed there. static func photoProcessingSetup(_ options: LaunchOptions) -> (setup: PhotoProcessingController.Setup, synthetic: SyntheticCapture?) { diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/FrameCalibrationRow.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/FrameCalibrationRow.swift new file mode 100644 index 000000000..f786520fe --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/FrameCalibrationRow.swift @@ -0,0 +1,13 @@ +import simd + +/// One ARFrame's own calibration as the recorder stores it beside that frame's pose: the row +/// `Packet04Streams.poseRows(trajectory:intrinsics:tracking:)` joins to the pose with the same `t`. +public enum FrameCalibrationRow { + /// `[t, fx, fy, cx, cy, width, height]`: ARKit's `camera.intrinsics` (column-major, so fx is + /// column 0's x, fy column 1's y, cx and cy column 2's x and y) in pixels of the unrotated + /// landscape sensor image, whose size `camera.imageResolution` gives. Values are kept as ARKit + /// reported them, with no rotation, scaling or rounding. + public static func row(t: Double, intrinsics k: simd_float3x3, imageWidth: Double, imageHeight: Double) -> [Double] { + [t, Double(k.columns.0.x), Double(k.columns.1.y), Double(k.columns.2.x), Double(k.columns.2.y), imageWidth, imageHeight] + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/FrameCalibrationRowTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/FrameCalibrationRowTests.swift new file mode 100644 index 000000000..a5df48783 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/FrameCalibrationRowTests.swift @@ -0,0 +1,68 @@ +import HouseScanKit +import simd +import Testing + +@Suite struct FrameCalibrationRowTests { + /// ARKit's intrinsics, column-major: fx and fy on the diagonal, cx and cy in the last column. + static func intrinsics(fx: Float, fy: Float, cx: Float, cy: Float) -> simd_float3x3 { + simd_float3x3(SIMD3(fx, 0, 0), SIMD3(0, fy, 0), SIMD3(cx, cy, 1)) + } + + /// A trajectory row as the recorder writes it: t, tracking state and reason, then the pose's + /// 16 values column by column. + static func trajectoryRow(_ t: Double, x: Float) -> [Double] { + var m = matrix_identity_float4x4 + m.columns.3 = SIMD4(x, 1.5, 0, 1) + let columns = [m.columns.0, m.columns.1, m.columns.2, m.columns.3] + return [t, 0, 0] + columns.flatMap { [Double($0.x), Double($0.y), Double($0.z), Double($0.w)] } + } + + @Test func theRowIsTheFramesOwnValuesInTheirOriginalUnits() { + // Every value distinct, so a swapped column or axis would show. + let k = Self.intrinsics(fx: 1445.25, fy: 1446.5, cx: 961.75, cy: 719.125) + #expect(FrameCalibrationRow.row(t: 12.345, intrinsics: k, imageWidth: 1920, imageHeight: 1440) + == [12.345, 1445.25, 1446.5, 961.75, 719.125, 1920, 1440]) + // Off-diagonal entries (skew, the bottom row) don't leak into the row. + var skewed = k + skewed.columns.1.x = 3 + skewed.columns.0.z = 7 + #expect(FrameCalibrationRow.row(t: 1, intrinsics: skewed, imageWidth: 1920, imageHeight: 1440) + == FrameCalibrationRow.row(t: 1, intrinsics: k, imageWidth: 1920, imageHeight: 1440)) + } + + /// Calibration and resolution change from frame to frame independently; each pose gets the + /// calibration of the frame with its own t, through #192's join. + @Test func eachPoseJoinsItsOwnFramesCalibration() throws { + let frames: [(t: Double, k: SIMD4, size: SIMD2)] = [ + (100.000, SIMD4(1445, 1445, 960, 720), SIMD2(1920, 1440)), + (100.017, SIMD4(1450, 1449, 958, 722), SIMD2(1920, 1440)), + (100.033, SIMD4(1450, 1449, 958, 722), SIMD2(1280, 960)), + (100.050, SIMD4(963, 964, 640, 480), SIMD2(1280, 960)), + ] + let trajectory = frames.enumerated().map { Self.trajectoryRow($1.t, x: Float($0)) } + let intrinsics = frames.map { + FrameCalibrationRow.row( + t: $0.t, intrinsics: Self.intrinsics(fx: $0.k.x, fy: $0.k.y, cx: $0.k.z, cy: $0.k.w), imageWidth: $0.size.x, imageHeight: $0.size.y) + } + // Rows out of order, as nothing promises they arrive sorted. + let poses = Packet04Streams.poseRows(trajectory: trajectory, intrinsics: intrinsics.reversed(), tracking: PacketTracking.init(recorderState:reason:)) + #expect(poses.map(\.t) == frames.map(\.t)) + #expect(poses.map(\.intrinsics) == frames.map(\.k)) + #expect(poses.map { $0.cameraToWorld.columns.3.x } == [0, 1, 2, 3]) + #expect(intrinsics.map { [$0[5], $0[6]] } == frames.map { [$0.size.x, $0.size.y] }) + } + + /// A pose whose frame has no calibration row is left out, and a row at another t, however + /// close, isn't borrowed for it. + @Test func aPoseWithoutItsOwnRowIsOmittedNotGuessed() { + let k = Self.intrinsics(fx: 1445, fy: 1445, cx: 960, cy: 720) + let trajectory = [100.0, 100.5, 101.0].enumerated().map { Self.trajectoryRow($1, x: Float($0)) } + let intrinsics = [ + FrameCalibrationRow.row(t: 100.0, intrinsics: k, imageWidth: 1920, imageHeight: 1440), + // 100.5 has no row; 101.0's row is a hair off its pose's t. + FrameCalibrationRow.row(t: 101.0.nextUp, intrinsics: k, imageWidth: 1920, imageHeight: 1440), + ] + let poses = Packet04Streams.poseRows(trajectory: trajectory, intrinsics: intrinsics, tracking: PacketTracking.init(recorderState:reason:)) + #expect(poses.map(\.t) == [100.0]) + } +} diff --git a/ios/README.md b/ios/README.md index 6a392fbd2..a21abbceb 100644 --- a/ios/README.md +++ b/ios/README.md @@ -35,7 +35,7 @@ Native iPhone app for the AR capture walk. The homeowner marks the electric mete **Processing (beta).** Developer options offer two backends, both remote: **Legacy (remote checker)** sends scene.json to the placement server, and **Photo processing (beta)** sends the scan's photos and camera path to the capture API. Legacy is the default. A scan reads the choice once, when it starts, and keeps it with its endpoint in its `ScanContext` (HouseScanKit); a change applies to the next scan, and an App Store install always uses Legacy. A photo-processing scan asks before sending (`PhotoConsentCard`), sends photos as they are kept, and after the review goes to `STATE=processing` instead of the upload: its progress, then the service's answer quoted as written (`PhotoProcessingScreen`). It never falls back to the Legacy checker. An answer is shown only when the run it names, and the phone's own record of what it sent (the recording session and world epoch, the coordinator session, packet and capture), match this scan; that is a local association, since the service returns no digest of its input and no attestation. AR is withheld for every answer. Sending off the phone is off in every build for now, so choosing photo processing ends the scan saying it isn't set up. -Two debug-only switches run the path on a replay against `FixtureCaptureHTTP`, a stand-in for the capture API inside the app (the type is in every build; only its activation is debug-only): `-photoProcessingFixture` alone sends the replay's photos, whose packet always fails the phone's 0.4 checks because a replay has no motion data and too few poses, so the scan ends "couldn't be prepared"; adding `-photoProcessingSyntheticCapture` sends HouseScanKit's `SyntheticCapture` instead, labelled `source.kind: synthetic`, so the real coordinator and uploader carry it to the fixture's answer. Either way the answer is the fixture's, never a service's or a GPU reconstruction's, and the screens say so. Still missing before live use: a scoped server-issued credential, live per-frame intrinsics and the meter tap in the 0.4 packet, the clock and privacy contract, and an input digest or attestation from the service. The code is in `Runtime/ScanEngine+Processing.swift` and HouseScanKit's `Processing/`. +Two debug-only switches run the path on a replay against `FixtureCaptureHTTP`, a stand-in for the capture API inside the app (the type is in every build; only its activation is debug-only): `-photoProcessingFixture` alone sends the replay's photos, whose packet always fails the phone's 0.4 checks because a replay has no motion data and too few poses, so the scan ends "couldn't be prepared"; adding `-photoProcessingSyntheticCapture` sends HouseScanKit's `SyntheticCapture` instead, labelled `source.kind: synthetic`, so the real coordinator and uploader carry it to the fixture's answer. Either way the answer is the fixture's, never a service's or a GPU reconstruction's, and the screens say so. Live captures now record each frame's intrinsics with its pose for the 0.4 pose stream, not yet checked on a phone. Still missing before live use: a scoped server-issued credential, the meter tap in the 0.4 packet, the clock and privacy contract, and an input digest or attestation from the service. The code is in `Runtime/ScanEngine+Processing.swift` and HouseScanKit's `Processing/`. **Practice meter.** For trying the whole scan where there is no electric meter, such as indoors. Debug and TestFlight builds show a wrench button at the top of the first screen. Tap it, turn on **Practice meter**, tap Done, then start the scan. At "find the meter", tap any spot on a wall: the usual raycast and refusals apply, and a drawn sample meter appears there, fixed to the meter's anchor, on every camera screen after that. The close-up keeps the live camera's pose and depth, but stores a photo of the sample (made-up maker SAMPLEWORKS, number 12345678), and the real reader reads it. The walk, fog, marks, requests, spot check, upload, result and AR all run for real. Every screen shows a "Practice meter" badge, and `scan-stamp.json` has `"practice": true`; the packet and scene.json are unchanged. The switch is kept until turned off, and the button reads "Practice meter on" while it is. App Store installs never show the button and ignore a stored switch: the app asks StoreKit's `AppTransaction` which server signed the install (sandbox means TestFlight) and treats no answer as the App Store. It is not in the iOS Settings app, because a Settings.bundle ships in the same binary that TestFlight later promotes to the App Store. The code is in `Runtime/Practice/`, `UI/Practice/` and HouseScanKit's `Practice/PracticeMeter.swift`. From c3e095d7d6b14e08645065dc542f1e7525adac91 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 12:37:10 -0500 Subject: [PATCH 48/71] Test that settles still sees an uploader settle through a stalled test process Co-Authored-By: Claude Opus 5.5 (1M context) --- .../CaptureUploaderSettlesTests.swift | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderSettlesTests.swift diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderSettlesTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderSettlesTests.swift new file mode 100644 index 000000000..a7d41c5eb --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderSettlesTests.swift @@ -0,0 +1,45 @@ +import Foundation +import HouseScanKit +import Testing + +/// `CaptureUploaderTests.settles` is a hang safeguard, not a timing requirement, so a test +/// process that can't run its tasks for a while must not read as an uploader that never settles. +/// Hosted run 37219668935 stalled every test task for about 42 s after start (trivial tests +/// reported 28ā€“43 s); four ScopedCaptureUploadTests then failed `settles` at its 20 s budget +/// although their uploads end at the first request. +/// +/// These tests block every cooperative thread (sleeping, not spinning) for longer than a 1 s +/// budget while a real uploader settles. That stalls the whole test process, so they run only +/// when asked: `HSK_STARVE_COOPERATIVE_POOL=1 swift test --filter CaptureUploaderSettlesTests`. +@Suite(.enabled(if: ProcessInfo.processInfo.environment["HSK_STARVE_COOPERATIVE_POOL"] == "1"), .serialized) +struct CaptureUploaderSettlesTests { + typealias Rig = CaptureUploaderTests.Rig + + /// Occupies every cooperative thread for `seconds`. Twice the processor count, so the threads + /// a waiter frees by suspending are taken too. + static func starve(for seconds: Double) { + for _ in 0..<(2 * ProcessInfo.processInfo.activeProcessorCount) { + Task.detached { usleep(useconds_t(seconds * 1_000_000)) } + } + } + + @Test func anIdleUploaderSettlesThroughAStall() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + Self.starve(for: 1.5) + #expect(await CaptureUploaderTests.settles(rig.uploader, within: 1)) + } + + /// The failing hosted case: a malformed credential ends the upload at once, with a stall + /// longer than the budget in between. + @Test func aRefusedCredentialSettlesThroughAStall() async throws { + let sleeps = ScopedCaptureUploadTests.Sleeps() + let rig = try ScopedCaptureUploadTests.rig(provider: ScopedCaptureHTTPTests.Provider([""]), sleeps: sleeps) + defer { rig.cleanUp() } + await rig.uploader.kick() + Self.starve(for: 1.5) + #expect(await CaptureUploaderTests.settles(rig.uploader, within: 1)) + #expect(await rig.uploader.snapshot.end == .failed(step: "create", codes: ["auth_credential_malformed"], status: 0)) + #expect(sleeps.isEmpty) + } +} From 6c1f68eade3be7cac03d123f113af4d95f8330c6 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 12:37:59 -0500 Subject: [PATCH 49/71] Count settles' budget in its own turns and read the answer after the last wait settles is a hang safeguard. Its wall-clock deadline expired while the test process was stalled, before either the uploader or the waiter could run, and the loop returned without reading the answer once more. Hosted run 37219668935 stalled every task for about 42 s and four ScopedCaptureUploadTests failed there. The budget now counts the waiter's 20 ms turns, like the suite's other waits, and a held uploader still fails it. parked in the recovery tests gets the same change. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../CaptureUploaderRecoveryTests.swift | 11 ++++++----- .../CaptureUploaderSettlesTests.swift | 16 ++++++++++++++++ .../HouseScanKitTests/CaptureUploaderTests.swift | 15 ++++++++++----- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderRecoveryTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderRecoveryTests.swift index 1ab403919..d7b586417 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderRecoveryTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderRecoveryTests.swift @@ -13,14 +13,15 @@ import Testing @Suite struct CaptureUploaderRecoveryTests { typealias Rig = CaptureUploaderTests.Rig - /// Waits until at least `count` requests on `route` are parked. + /// Waits until at least `count` requests on `route` are parked. Like `settles`, the budget is + /// this waiter's own turns, not wall time, and the answer is read after the last wait. static func parked(_ server: LoopbackCaptureAPI, _ route: String, count: Int = 1, within seconds: Int = 20) async -> Bool { - let deadline = ContinuousClock.now + .seconds(seconds) - while ContinuousClock.now < deadline { - if server.state.withLock({ ($0.parked[route] ?? []).count >= count }) { return true } + let ready = { server.state.withLock { ($0.parked[route] ?? []).count >= count } } + for _ in 0..<(seconds * 100) { + if ready() { return true } try? await Task.sleep(for: .milliseconds(10)) } - return false + return ready() } /// Sends the answers parked on `route` and keeps holding it. diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderSettlesTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderSettlesTests.swift index a7d41c5eb..85ced8153 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderSettlesTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderSettlesTests.swift @@ -43,3 +43,19 @@ struct CaptureUploaderSettlesTests { #expect(sleeps.isEmpty) } } + +/// The safeguard still ends: an uploader held at its first request never settles, and `settles` +/// says so after at least its budget, every turn being a wait of 20 ms or more. +@Suite struct CaptureUploaderSettlesBoundTests { + @Test func aHeldUploaderDoesNotSettle() async throws { + let rig = try CaptureUploaderTests.Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.held = ["POST captures"] } + await rig.uploader.kick() + let started = ContinuousClock.now + #expect(await !CaptureUploaderTests.settles(rig.uploader, within: 1)) + #expect(ContinuousClock.now - started >= .seconds(1)) + rig.server.release("POST captures") + #expect(await CaptureUploaderTests.settles(rig.uploader)) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift index 664d5564b..90c8fab2a 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift @@ -323,17 +323,22 @@ import Testing #expect(rig.server.requests("GET captures/result").count == 1) } - /// Whether `uploader` goes idle within `seconds`. A loop that never goes idle fails the test - /// here instead of hanging the run; the process ending stops it. + /// Whether `uploader` goes idle within about `seconds`. A loop that never goes idle fails the + /// test here instead of hanging the run; the process ending stops it. + /// + /// A hang safeguard, not a timing requirement, so the budget is this waiter's own turns, 20 ms + /// apart, like the suite's `for _ in 0..<500 where` waits, not wall time. A stalled test process + /// holds the uploader and this waiter up together, and a wall-clock deadline expired on time + /// neither got to run (CaptureUploaderSettlesTests). The answer is read once more after the + /// last wait, which the old loop skipped. static func settles(_ uploader: CaptureUploader, within seconds: Int = 20) async -> Bool { let done = Mutex(false) Task.detached { await uploader.settled(); done.withLock { $0 = true } } - let deadline = ContinuousClock.now + .seconds(seconds) - while ContinuousClock.now < deadline { + for _ in 0..<(seconds * 50) { if done.withLock({ $0 }) { return true } try? await Task.sleep(for: .milliseconds(20)) } - return false + return done.withLock { $0 } } /// A storage URL that is not https (or http to this machine) is refused when it is From ac5a512038703a9d011aff170ae55fee732fb523 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 12:39:31 -0500 Subject: [PATCH 50/71] ios: hide the screen under the photo question from accessibility only while it shows Run 37219668935 on #216 (8cc8cec8): photoProcessing's every-state audit failed on the consent card, "Contrast nearly passed" on the find-meter words showing dimmed through its scrim. 8cc8cec8 had removed `.accessibilityHidden(asksForPhotoConsent)` because, set to false whenever no question showed, it overrode the hiding the camera's decoration sets on itself (an element with no description on every camera screen, run 37213636011). The screen is now hidden with `.accessibilityHidden(true)` only while the question shows, and gets no accessibility modifier otherwise, so the hiding its subviews set stays. The camera feed and the practice overlay hide themselves already and need nothing. The screen is rebuilt once, when the question is answered: the find-meter screen's state there is its camera size, tap ripples and camera window, which it reads again. Not run: the next hosted run, or a local focused run once a slot frees, checks both audits. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/UI/ScanRootView.swift | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/ios/HouseScan/UI/ScanRootView.swift b/ios/HouseScan/UI/ScanRootView.swift index 68db7a6fe..2048254b4 100644 --- a/ios/HouseScan/UI/ScanRootView.swift +++ b/ios/HouseScan/UI/ScanRootView.swift @@ -40,11 +40,13 @@ struct ScanRootView: View { screen .id(state.phase) .transition(Self.showsCamera(state.phase) ? .opacity : .identity) - // The question is modal for VoiceOver through its own `.isModal` trait - // (`PhotoConsentCard`). Its siblings aren't hidden here: `.accessibilityHidden(false)`, - // as that was whenever no question showed, overrode the hiding the camera feed and - // other decoration set on themselves, and the audit found an element with no - // description on every camera screen (#216, run 37213636011). + // While the photo question shows, the screen under it is out of the accessibility + // tree: its words show dimmed through the card's scrim, and the audit failed them + // on contrast (#216, run 37219668935). Only ever set to hidden, never to + // `.accessibilityHidden(false)`, which overrode the hiding the camera's decoration + // sets on itself and left an element with no description on every camera screen + // (run 37213636011). The camera feed and practice overlay hide themselves already. + .hiddenFromAccessibility(while: asksForPhotoConsent) if asksForPhotoConsent { PhotoConsentCard(standIn: state.photoProcessing?.standIn == true, synthetic: state.photoCaptureIsSynthetic, actions: actions) .transition(.opacity) @@ -155,3 +157,16 @@ private extension View { .accessibilityIdentifier("screen.\(phase.rawValue)") } } + +private extension View { + /// `.accessibilityHidden(true)` while `hidden`, and no accessibility modifier at all otherwise, + /// so the hiding a subview sets on itself still holds. The view is rebuilt when `hidden` + /// changes; here that happens once, when the photo question is answered. + @ViewBuilder func hiddenFromAccessibility(while hidden: Bool) -> some View { + if hidden { + accessibilityHidden(true) + } else { + self + } + } +} From 1be8ededae8a6a9f26366d92bfad9232f0dde399 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 12:54:33 -0500 Subject: [PATCH 51/71] Send the accepted meter mark's own frame as the capture packet's tap Until now the app never called meterTapped, so a live photo-processing packet had no meter tap. When the homeowner's mark is accepted (raycast, MeterTap.refusal and setWall all passed), the engine reads ARKit's current frame once, in that same main-actor turn, and takes its time, raw pose, intrinsics, image size, tracking and image from it (LiveCapture.meterTapSnapshot). HouseScanKit's MeterTapFrame projects the raycast's hit through that frame's own camera (TapObservation.pointing). Tracking that isn't normal, or a hit behind the camera or outside the image, gives no tap. The image is encoded off the main actor with the walk's existing encoder, now internal, and the tap is forwarded only with its JPEG already in hand. Legacy scans, replays and the synthetic capture take no snapshot; LiveMode and the walk's encoding are unchanged. PhotoProcessingController keeps each packet's tap (none, forwarded, failed). It resets with every packet and is fenced on the ScanContext captured at the mark, so a result after a world reset or Start over is dropped and logged. A failed tap ends the packet as not prepared at once after a yes; before an answer it is kept and a later yes ends it the same way; a no or a withdrawal keeps its own end. A live send waits for the tap and seals the capture only once it was forwarded. No wire field, gate, endpoint, Share path or #192/aa72 file changed. Tests (HouseScanKit): MeterTapFrameTests, MeterTapControllerTests, and the two existing PhotoProcessingTests call sites updated to pass the scan context. Validated: see the slice report; this commit is what the wrapper jobs build. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Runtime/LiveCapture+MeterTap.swift | 39 +++ ios/HouseScan/Runtime/LiveCapture.swift | 12 +- .../Runtime/ScanEngine+Actions.swift | 2 + .../Runtime/ScanEngine+Processing.swift | 51 ++++ ios/HouseScan/Runtime/ScanEngine.swift | 3 + .../Processing/MeterTapObservation.swift | 58 +++++ .../PhotoProcessingController.swift | 48 +++- .../MeterTapObservationTests.swift | 235 ++++++++++++++++++ .../PhotoProcessingTests.swift | 4 +- 9 files changed, 443 insertions(+), 9 deletions(-) create mode 100644 ios/HouseScan/Runtime/LiveCapture+MeterTap.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Processing/MeterTapObservation.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/MeterTapObservationTests.swift diff --git a/ios/HouseScan/Runtime/LiveCapture+MeterTap.swift b/ios/HouseScan/Runtime/LiveCapture+MeterTap.swift new file mode 100644 index 000000000..c85f43f28 --- /dev/null +++ b/ios/HouseScan/Runtime/LiveCapture+MeterTap.swift @@ -0,0 +1,39 @@ +import ARKit +import HouseScanKit + +/// The meter tap's one-frame snapshot, for photo processing's capture packet: taken only when the +/// homeowner's meter mark has been accepted, never on a timer (`LiveMode` and the walk's encoding +/// are unchanged). +extension LiveCapture { + /// What one ARFrame gave for the tap: its own camera, the image to encode, or why there is none. + enum MeterTapSnapshot { + case captured(MeterTapFrame, image: LiveSessionDelegate.PixelBufferBox) + case failed(MeterTapFailure) + } + + /// ARKit's current frame, read once, in the same main-actor turn as the accepted raycast. Its + /// time, raw pose, intrinsics, image size, tracking and image all come from that one frame; + /// nothing is taken from the engine's sampled frames. The image is kept only until it is + /// encoded (`encodeMeterTapImage`), as the walk's encode keeps its own. + func meterTapSnapshot() -> MeterTapSnapshot { + guard let frame = arView.session.currentFrame else { return .failed(.noFrame) } + let k = frame.camera.intrinsics + let size = frame.camera.imageResolution + let tracking: PacketTracking = switch frame.camera.trackingState { + case .normal: .normal + case .notAvailable: .notAvailable + case .limited(_): .limited(nil) + } + let camera = MeterTapFrame( + t: frame.timestamp, cameraToWorld: frame.camera.transform, + intrinsics: SIMD4(k.columns.0.x, k.columns.1.y, k.columns.2.x, k.columns.2.y), + width: Int(size.width), height: Int(size.height), tracking: tracking) + return .captured(camera, image: LiveSessionDelegate.PixelBufferBox(buffer: frame.capturedImage)) + } + + /// Encodes the snapshot's image off the main actor with the walk's encoder. Nil when it fails. + func encodeMeterTapImage(_ image: LiveSessionDelegate.PixelBufferBox) async -> Data? { + let delegate = delegate + return await Task.detached(priority: .userInitiated) { delegate.encode(image.buffer) }.value + } +} diff --git a/ios/HouseScan/Runtime/LiveCapture.swift b/ios/HouseScan/Runtime/LiveCapture.swift index c15dcd44e..663e5e02b 100644 --- a/ios/HouseScan/Runtime/LiveCapture.swift +++ b/ios/HouseScan/Runtime/LiveCapture.swift @@ -40,7 +40,7 @@ struct VerticalPlaneHit { @MainActor final class LiveCapture { let arView: ARView - private let delegate: LiveSessionDelegate + let delegate: LiveSessionDelegate /// True when this phone gives per-frame LiDAR depth. static var supportsDepth: Bool { ARWorldTrackingConfiguration.supportsFrameSemantics(.sceneDepth) } @@ -393,8 +393,9 @@ final class LiveSessionDelegate: NSObject, ARSessionDelegate, Sendable { } /// The camera image handed to the encode queue. CVPixelBuffer is not Sendable; ARKit doesn't - /// write to a delivered frame's image, and only the encode queue reads it. - private struct PixelBufferBox: @unchecked Sendable { + /// write to a delivered frame's image, and only the encode queue reads it. The meter tap's + /// snapshot hands its one image off the main actor the same way. + struct PixelBufferBox: @unchecked Sendable { let buffer: CVPixelBuffer } @@ -533,8 +534,9 @@ final class LiveSessionDelegate: NSObject, ARSessionDelegate, Sendable { } } - /// JPEG of the sensor image as captured: landscape, unrotated, matching the intrinsics. - private func encode(_ buffer: CVPixelBuffer) -> Data? { + /// JPEG of the sensor image as captured: landscape, unrotated, matching the intrinsics. The + /// walk's keyframes and the meter tap's snapshot both use it. + func encode(_ buffer: CVPixelBuffer) -> Data? { let image = CIImage(cvPixelBuffer: buffer) let space = CGColorSpace(name: CGColorSpace.sRGB) ?? CGColorSpaceCreateDeviceRGB() let options = [CIImageRepresentationOption(rawValue: kCGImageDestinationLossyCompressionQuality as String): 0.8] diff --git a/ios/HouseScan/Runtime/ScanEngine+Actions.swift b/ios/HouseScan/Runtime/ScanEngine+Actions.swift index dd0f03f11..5061f89ed 100644 --- a/ios/HouseScan/Runtime/ScanEngine+Actions.swift +++ b/ios/HouseScan/Runtime/ScanEngine+Actions.swift @@ -72,6 +72,8 @@ extension ScanEngine: ScanActions { // takes the server's error for it. updateCoverage { $0.setWallLineSource(meterLineSource) } setMeterAnchor(live.addMeterAnchor(at: hit.transform), pose: hit.transform) + // In this same turn as the raycast: photo processing's tap, from ARKit's current frame. + takeMeterTap(live, hit: hit) markTimes[MarkKey.meter] = captureClock go(.meterCloseUp) } diff --git a/ios/HouseScan/Runtime/ScanEngine+Processing.swift b/ios/HouseScan/Runtime/ScanEngine+Processing.swift index 001a5d9fd..0bdfce973 100644 --- a/ios/HouseScan/Runtime/ScanEngine+Processing.swift +++ b/ios/HouseScan/Runtime/ScanEngine+Processing.swift @@ -53,6 +53,7 @@ extension ScanEngine { /// is already the next scan's. func endScan() { scanContext = nil + meterTapTask = nil state.scanBackend = nil photoProcessing.endScan(recording: photoRecording) } @@ -63,6 +64,8 @@ extension ScanEngine { guard let context = scanContext else { return } let next = context.inNewWorld(recordingSessionID: recorder.sessionID) scanContext = next + // The old world's tap, if still encoding, is dropped when it finishes (`takeMeterTap`). + meterTapTask = nil if next.profile.backend == .photoProcessing { photoProcessing.worldReset(next, recording: photoRecording) } @@ -84,6 +87,16 @@ extension ScanEngine { return } guard let synthetic else { + // A live capture is sealed only with its meter tap; a replay has none to give. + if replay == nil { + await meterTapTask?.value + guard scanContext == context, state.phase == .processing else { return } + guard photoProcessing.meterTap == .forwarded else { + RuntimeLog.engine.error("photo processing: the capture has no meter tap, so it isn't sealed") + photoProcessing.capturePreparationFailed() + return + } + } let closeUp = state.closeUp == .skipped ? nil : store.stillFrames["meter_close"]?.t photoProcessing.captureEnded(acceptedCloseUpAt: closeUp) return @@ -97,6 +110,44 @@ extension ScanEngine { } } + /// Photo processing's meter tap for an accepted mark: one snapshot of ARKit's current frame, + /// taken now, then its image encoded off the main actor and the tap forwarded. Legacy scans, + /// the synthetic capture and a packet that wants no tap take none. A snapshot that can't give + /// a tap ends the packet as not prepared (`PhotoProcessingController.meterTapFailed`). The + /// result is used only while the scan and its world are the ones the mark was made in. + func takeMeterTap(_ live: LiveCapture, hit: VerticalPlaneHit) { + guard let context = scanContext, context.profile.backend == .photoProcessing, syntheticCapture == nil, + photoProcessing.context == context, photoProcessing.wantsMeterTap else { return } + let position = hit.position + let estimated = hit.source == .estimatedPlane + switch live.meterTapSnapshot() { + case .failed(let reason): + photoProcessing.meterTapFailed(reason, scan: context) + case .captured(let frame, let image): + // The geometry is checked before anything is encoded. + if case .failure(let reason) = frame.observation(hit: position, jpeg: { nil }) { + photoProcessing.meterTapFailed(reason, scan: context) + return + } + meterTapTask = Task { [weak self, live] in + let jpeg = await live.encodeMeterTapImage(image) + guard let self else { return } + guard self.scanContext == context, self.photoProcessing.context == context else { + RuntimeLog.engine.info("photo processing: a meter tap from a scan or world that is gone was dropped") + return + } + guard let jpeg else { + self.photoProcessing.meterTapFailed(.encodeFailed, scan: context) + return + } + switch frame.observation(hit: position, jpeg: { jpeg }) { + case .success(let tap): self.photoProcessing.meterTapped(tap, hit: frame.tapHit(position, estimatedPlane: estimated), scan: context) + case .failure(let reason): self.photoProcessing.meterTapFailed(reason, scan: context) + } + } + } + } + func answerPhotoConsent(_ yes: Bool) { guard state.photoProcessing?.consent == .asking else { return } RuntimeLog.engine.info("photo processing consent: \(yes ? "yes" : "no", privacy: .public)") diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index c1de48bf8..98512e833 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -193,6 +193,9 @@ final class ScanEngine { /// The scan under way: its id, backend profile and world, fixed when it starts /// (`beginScan`); nil before a scan starts. var scanContext: ScanContext? + /// The meter tap's encode and forward, from an accepted mark on a live photo-processing scan + /// (`takeMeterTap`); the send waits for it. + var meterTapTask: Task? /// Photo processing's capture and answer, for scans that use it (`ScanEngine+Processing`). let photoProcessing: PhotoProcessingController /// DEBUG's synthetic capture, sent in place of the scan's photos; nil otherwise. diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/MeterTapObservation.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/MeterTapObservation.swift new file mode 100644 index 000000000..31847cab7 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/MeterTapObservation.swift @@ -0,0 +1,58 @@ +import Foundation +import simd + +/// Why the homeowner's accepted meter mark gave the capture packet no tap. Each ends the packet as +/// not prepared rather than letting it go out without its tap (`PhotoProcessingController.meterTapFailed`). +public enum MeterTapFailure: String, Sendable, Equatable, CaseIterable { + /// ARKit had no current frame to take. + case noFrame + /// The frame's tracking wasn't normal, so its pose can't place the tap. + case trackingLimited + /// The raycast's hit is behind that frame's camera or outside its image. + case hitOutsideImage + /// The frame's image couldn't be encoded as a JPEG. + case encodeFailed +} + +/// One ARFrame's own camera, read once from that frame: its time, raw pose, intrinsics and image +/// size in pixels of the unrotated landscape sensor image, and its tracking. +public struct MeterTapFrame: Sendable, Equatable { + public var t: Double + public var cameraToWorld: simd_float4x4 + /// [fx, fy, cx, cy]: ARKit's pinhole K, focal lengths and principal point in pixels from the + /// image's top-left corner. + public var intrinsics: SIMD4 + public var width: Int + public var height: Int + public var tracking: PacketTracking + + public init(t: Double, cameraToWorld: simd_float4x4, intrinsics: SIMD4, width: Int, height: Int, tracking: PacketTracking) { + self.t = t + self.cameraToWorld = cameraToWorld + self.intrinsics = intrinsics + self.width = width + self.height = height + self.tracking = tracking + } + + /// The tap on this frame: the raycast's world hit projected through this frame's own camera + /// (`TapObservation.pointing(at:)`), so pixel, pose and intrinsics all belong to one frame. + /// `jpeg` returns this frame's image, already encoded. + public func observation(hit: SIMD3, jpeg: @escaping @Sendable () -> Data?) -> Result { + guard tracking == .normal else { return .failure(.trackingLimited) } + let unaimed = TapObservation( + t: t, cameraToWorld: cameraToWorld, intrinsics: intrinsics, width: width, height: height, tracking: tracking, + pixel: SIMD2(Double(width) / 2, Double(height) / 2), jpeg: jpeg) + guard let aimed = unaimed.pointing(at: hit) else { return .failure(.hitOutsideImage) } + return .success(aimed) + } + + /// What the packet records about the raycast's hit, its distance measured from this frame's + /// camera. + public func tapHit(_ hit: SIMD3, estimatedPlane: Bool) -> Packet04.TapHit { + let camera = SIMD3(cameraToWorld.columns.3.x, cameraToWorld.columns.3.y, cameraToWorld.columns.3.z) + return Packet04.TapHit( + position: [hit.x, hit.y, hit.z].map(Double.init), target: estimatedPlane ? "estimatedPlane" : "existingPlaneGeometry", + alignment: "vertical", distance: Double(simd_distance(hit, camera))) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift index 13117e4c4..bef45c00f 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift @@ -44,6 +44,20 @@ public final class PhotoProcessingController { private var upload: CaptureUploadStatus? private var captureEnded = false + /// What became of the current packet's meter tap. Every packet needs its own: a world reset + /// starts a packet with none. + public enum MeterTap: Sendable, Equatable { + case none + case forwarded + case failed(MeterTapFailure) + } + + public private(set) var meterTap: MeterTap = .none + + /// Whether the current packet still wants its meter tap: it is being captured, the homeowner + /// hasn't said no, and no tap result came yet. + public var wantsMeterTap: Bool { acceptsCapture && meterTap == .none } + public init(setup: Setup) { if case .ready(let environment, _) = setup, !environment.sends { self.setup = .notSetUp("the capture endpoint may not receive this scan") @@ -123,6 +137,8 @@ public final class PhotoProcessingController { coordinator.answerConsent(yes) state.consent = yes ? .granted : .declined status = state + // A yes after this packet lost its tap: the packet can't be sent, so it ends now. + if yes, case .failed = meterTap { capturePreparationFailed() } } /// A no after a yes: nothing more from this scan is sent, and the scan can't be processed @@ -144,15 +160,42 @@ public final class PhotoProcessingController { coordinator?.kept(photo) } - public func meterTapped(_ tap: TapObservation, hit: Packet04.TapHit?) { - guard acceptsCapture else { return } + /// The homeowner's accepted meter mark, as the frame it was taken on saw it, for the scan in + /// `scan` as it stood at the mark. Nothing is forwarded when that scan or its world is gone: + /// a tap from an old world belongs to a packet that no longer exists. `tap`'s image is already + /// encoded, so the coordinator's own read of it can't come back empty. + public func meterTapped(_ tap: TapObservation, hit: Packet04.TapHit?, scan: ScanContext) { + guard scan == context, meterTap == .none, acceptsCapture else { + log("photo processing: a meter tap for another scan, world or packet was dropped") + return + } + meterTap = .forwarded coordinator?.meterTapped(tap, hit: hit) } + /// The accepted mark gave no tap, for the scan in `scan` as it stood at the mark. The packet + /// can't be sent without it: with a yes it ends as not prepared now, so photos of a packet that + /// can't be finished stop going up; before an answer the failure is kept and a later yes ends + /// it the same way; a no or a withdrawal keeps its own end. Dropped when that scan or world is + /// gone, so it never ends a newer packet. + public func meterTapFailed(_ reason: MeterTapFailure, scan: ScanContext) { + guard scan == context, meterTap == .none, acceptsCapture else { + log("photo processing: a meter tap failure for another scan, world or packet was dropped") + return + } + meterTap = .failed(reason) + log("photo processing: the meter mark gave no tap (\(reason.rawValue)); the capture can't be prepared") + if status?.consent == .granted { capturePreparationFailed() } + } + /// The scan was sent: the packet is frozen and processing starts, if the homeowner said yes. /// `acceptedCloseUpAt` is the frame time of the close-up the scan accepted, nil after a skip. public func captureEnded(acceptedCloseUpAt: Double?) { guard let context, var state = status, !captureEnded, case .capturing = state.stage else { return } + if state.consent == .granted, case .failed = meterTap { + capturePreparationFailed() + return + } captureEnded = true switch state.consent { case .granted: @@ -229,6 +272,7 @@ public final class PhotoProcessingController { binding = nil upload = nil captureEnded = false + meterTap = .none } /// The coordinator reports only its current session; nil means that session ended, and diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/MeterTapObservationTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/MeterTapObservationTests.swift new file mode 100644 index 000000000..fa0646595 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/MeterTapObservationTests.swift @@ -0,0 +1,235 @@ +import Foundation +import HouseScanKit +import simd +import Testing + +/// Two ARFrames that differ in pose, intrinsics and image size, and a meter hit both can see. +enum TwoFrames { + static let hit = SIMD3(0.4, 1.2, -2) + + static let a = MeterTapFrame( + t: 10, cameraToWorld: pose(x: 0, y: 1.5, z: 0, yawDegrees: 0), intrinsics: SIMD4(1445, 1445, 960, 720), width: 1920, height: 1440, + tracking: .normal) + static let b = MeterTapFrame( + t: 10.033, cameraToWorld: pose(x: 0.3, y: 1.4, z: 0.1, yawDegrees: 10), intrinsics: SIMD4(1001, 1003, 641, 479), width: 1280, + height: 960, tracking: .normal) + + static func pose(x: Float, y: Float, z: Float, yawDegrees: Float) -> simd_float4x4 { + let r = yawDegrees * .pi / 180 + return simd_float4x4( + SIMD4(cos(r), 0, -sin(r), 0), SIMD4(0, 1, 0, 0), SIMD4(sin(r), 0, cos(r), 0), SIMD4(x, y, z, 1)) + } + + static func columns(_ m: simd_float4x4) -> [Double] { + [m.columns.0, m.columns.1, m.columns.2, m.columns.3].flatMap { [Double($0.x), Double($0.y), Double($0.z), Double($0.w)] } + } + + /// How far `point` lies from the ray, meters. + static func miss(_ ray: (origin: [Double], direction: [Double]), _ point: SIMD3) -> Double { + let o = SIMD3(ray.origin[0], ray.origin[1], ray.origin[2]), d = simd_normalize(SIMD3(ray.direction[0], ray.direction[1], ray.direction[2])) + let v = SIMD3(point) - o + return simd_length(v - simd_dot(v, d) * d) + } + + static func ray(_ frame: MeterTapFrame, pixel: SIMD2) -> (origin: [Double], direction: [Double]) { + let k = frame.intrinsics + return Packet04Check.ray(pose: columns(frame.cameraToWorld), intrinsics: [k.x, k.y, k.z, k.w].map(Double.init), pixel: [pixel.x, pixel.y]) + } +} + +@Suite struct MeterTapFrameTests { + func observation(_ frame: MeterTapFrame) throws -> TapObservation { + try frame.observation(hit: TwoFrames.hit, jpeg: { Data([0xFF]) }).get() + } + + /// The tap carries one frame's time, pose, intrinsics and image size, and its pixel is the hit + /// seen through that frame's camera: the ray rebuilt the way the packet rebuilds it reaches the + /// hit. The same pixel with the other frame's camera, or this frame's pose with the other's + /// intrinsics, misses it by centimeters, so pairing frames would fail this test. + @Test func thePixelAndRayComeFromTheSnapshotsOwnCamera() throws { + let a = try observation(TwoFrames.a), b = try observation(TwoFrames.b) + #expect(a.t == TwoFrames.a.t && a.cameraToWorld == TwoFrames.a.cameraToWorld && a.intrinsics == TwoFrames.a.intrinsics) + #expect(a.width == 1920 && a.height == 1440) + #expect(TwoFrames.miss(TwoFrames.ray(TwoFrames.a, pixel: a.pixel), TwoFrames.hit) < 1e-4) + #expect(TwoFrames.miss(TwoFrames.ray(TwoFrames.b, pixel: b.pixel), TwoFrames.hit) < 1e-4) + #expect(simd_distance(a.pixel, b.pixel) > 50, "the two frames should see the hit at different pixels") + // Mixed: B's camera with A's pixel, and A's pose with B's intrinsics. + #expect(TwoFrames.miss(TwoFrames.ray(TwoFrames.b, pixel: a.pixel), TwoFrames.hit) > 0.05) + var mixed = TwoFrames.a + mixed.intrinsics = TwoFrames.b.intrinsics + #expect(TwoFrames.miss(TwoFrames.ray(mixed, pixel: a.pixel), TwoFrames.hit) > 0.05) + } + + @Test func aFrameWithoutNormalTrackingGivesNoTap() { + for tracking in [PacketTracking.limited(.relocalizing), .limited(nil), .notAvailable] { + var frame = TwoFrames.a + frame.tracking = tracking + #expect(frame.observation(hit: TwoFrames.hit, jpeg: { Data() }).failureValue == .trackingLimited) + } + } + + @Test func aHitBehindTheCameraOrOutsideTheImageGivesNoTap() { + // Behind: the camera looks along -z from z = 0. + #expect(TwoFrames.a.observation(hit: SIMD3(0.4, 1.2, 2), jpeg: { Data() }).failureValue == .hitOutsideImage) + // In front, far to the side: past the image's right edge. + #expect(TwoFrames.a.observation(hit: SIMD3(5, 1.2, -2), jpeg: { Data() }).failureValue == .hitOutsideImage) + } + + @Test func theHitRecordsItsDistanceFromTheSnapshotsCamera() { + let hit = TwoFrames.a.tapHit(TwoFrames.hit, estimatedPlane: true) + #expect(hit.target == "estimatedPlane" && hit.alignment == "vertical") + #expect(abs((hit.distance ?? 0) - Double(simd_distance(TwoFrames.hit, SIMD3(0, 1.5, 0)))) < 1e-6) + #expect(TwoFrames.a.tapHit(TwoFrames.hit, estimatedPlane: false).target == "existingPlaneGeometry") + } +} + +extension Result { + var failureValue: Failure? { + if case .failure(let failure) = self { return failure } + return nil + } +} + +/// The meter tap through the controller and the real coordinator, against the in-process fixture. +@Suite(.serialized) @MainActor struct MeterTapControllerTests { + let root = FileManager.default.temporaryDirectory.appending(path: "meter-tap-\(UUID().uuidString)") + var fixture: NativeCaptureFixture { NativeCaptureFixture(folder: root.appending(path: "store")) } + + func controller(_ http: FixtureCaptureHTTP) -> PhotoProcessingController { + var environment = NativeCaptureFixture.environment(endpoint: FixtureCaptureHTTP.base, http: http, captures: root.appending(path: "Captures")) + environment.policy.firstDelay = 0.01 + environment.policy.maxDelay = 0.02 + return PhotoProcessingController(setup: .ready(environment, standIn: true)) + } + + func begin(_ controller: PhotoProcessingController, scan: String = "scan-1", recording: String = "rec-1") -> ScanContext { + let context = ScanContext(scanID: scan, profile: controller.profile, recordingSessionID: recording) + controller.beginScan(context, recording: fixture.recording) + return context + } + + /// The fixture's frame one second in, as the snapshot would read it, with its image encoded. + func snapshotTap() throws -> (TapObservation, Packet04.TapHit) { + let t = fixture.start + 1 + let frame = MeterTapFrame( + t: t, cameraToWorld: fixture.pose(t), intrinsics: NativeCaptureFixture.k, width: NativeCaptureFixture.width, + height: NativeCaptureFixture.height, tracking: .normal) + let jpeg = try Data(contentsOf: try fixture.jpeg(named: "tap-\(t)")) + return (try frame.observation(hit: NativeCaptureFixture.meter, jpeg: { jpeg }).get(), frame.tapHit(NativeCaptureFixture.meter, estimatedPlane: true)) + } + + func until(_ seconds: Double = 30, _ condition: () -> Bool) async throws -> Bool { + let deadline = ContinuousClock.now + .seconds(seconds) + while !condition(), ContinuousClock.now < deadline { try await Task.sleep(for: .milliseconds(20)) } + return condition() + } + + /// The forwarded tap is sealed with a ray, rebuilt from its keyframe's pose and intrinsics, + /// that reaches the hit within the server's replay tolerance; the packet passes its checks, and + /// every request went to the in-process fixture. + @Test func aForwardedTapIsSealedWithARayThroughTheHit() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .candidate) + let controller = controller(http) + let scan = begin(controller) + controller.answerConsent(true) + let (tap, hit) = try snapshotTap() + controller.meterTapped(tap, hit: hit, scan: scan) + #expect(controller.meterTap == .forwarded) + controller.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + for i in 0..<5 { controller.kept(try fixture.photo(at: fixture.start + 3 + Double(i) * 1.5)) } + await controller.settle() + controller.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + try #require(try await until { controller.status?.isFinal == true }, "\(String(describing: controller.status))") + let folder = try #require(controller.captureFolder) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: Data(contentsOf: folder.appending(path: "packet.json"))) + #expect(Packet04Check.problems(packet, folder: folder).isEmpty) + let sealed = try #require(packet.taps?.first) + #expect(sealed.label == "meter" && sealed.time == tap.t) + let toHit = simd_normalize(SIMD3(NativeCaptureFixture.meter) - SIMD3(sealed.rayOrigin[0], sealed.rayOrigin[1], sealed.rayOrigin[2])) + let error = zip([toHit.x, toHit.y, toHit.z], sealed.rayDirection).map { abs($0 - $1) }.max() ?? .infinity + #expect(error <= Packet04Check.tapReplayTolerance, "ray misses the hit by \(error)") + #expect(http.routes.allSatisfy { $0.hasPrefix("POST captures") || $0.hasPrefix("GET captures") || $0 == "PUT upload" }) + controller.endScan(recording: fixture.recording) + } + + /// After a yes, a failed tap ends the packet as not prepared at once, and nothing more of it + /// is sealed or finalized. + @Test(arguments: MeterTapFailure.allCases) + func aFailedTapAfterAYesEndsThePacketNotPrepared(reason: MeterTapFailure) async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .candidate) + let controller = controller(http) + let scan = begin(controller) + controller.answerConsent(true) + controller.meterTapFailed(reason, scan: scan) + #expect(controller.meterTap == .failed(reason)) + #expect(controller.status?.stage == .ended(.notPrepared)) + controller.kept(try fixture.photo(at: fixture.start + 3)) + controller.captureEnded(acceptedCloseUpAt: nil) + try await Task.sleep(for: .milliseconds(500)) + #expect(controller.status?.stage == .ended(.notPrepared)) + #expect(!http.routes.contains("POST captures/finalize")) + controller.endScan(recording: fixture.recording) + } + + /// Before an answer, a failed tap is kept: a later yes ends the packet as not prepared, and a + /// no keeps its own end. + @Test func aFailedTapBeforeAnAnswerIsKeptForIt() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let yes = controller(FixtureCaptureHTTP(answer: .candidate)) + let scan = begin(yes) + yes.meterTapFailed(.encodeFailed, scan: scan) + #expect(yes.status?.stage == .capturing(sent: 0)) + yes.answerConsent(true) + #expect(yes.status?.stage == .ended(.notPrepared)) + yes.endScan(recording: fixture.recording) + + let no = controller(FixtureCaptureHTTP(answer: .candidate)) + let other = begin(no, scan: "scan-2") + no.meterTapFailed(.noFrame, scan: other) + no.answerConsent(false) + no.captureEnded(acceptedCloseUpAt: nil) + #expect(no.status?.stage == .ended(.consentNotGiven)) + no.endScan(recording: fixture.recording) + } + + /// A tap result that arrives after a world reset or Start over belongs to a packet that no + /// longer exists: it is neither forwarded nor allowed to end the new packet. + @Test func aLateTapAfterAWorldResetOrStartOverIsDropped() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let controller = controller(FixtureCaptureHTTP(answer: .candidate)) + let first = begin(controller) + controller.answerConsent(true) + let (tap, hit) = try snapshotTap() + + let reset = first.inNewWorld(recordingSessionID: "rec-2") + controller.worldReset(reset, recording: fixture.recording) + controller.meterTapFailed(.encodeFailed, scan: first) + controller.meterTapped(tap, hit: hit, scan: first) + #expect(controller.meterTap == .none) + #expect(controller.status?.stage == .capturing(sent: 0)) + + controller.endScan(recording: fixture.recording) + let next = begin(controller, scan: "scan-2", recording: "rec-3") + controller.answerConsent(true) + controller.meterTapFailed(.trackingLimited, scan: reset) + controller.meterTapped(tap, hit: hit, scan: first) + #expect(controller.meterTap == .none) + #expect(controller.status?.stage == .capturing(sent: 0)) + #expect(controller.context == next) + controller.endScan(recording: fixture.recording) + } + + /// One tap per packet: a second result for the same packet changes nothing. + @Test func onlyThePacketsFirstTapResultCounts() throws { + defer { try? FileManager.default.removeItem(at: root) } + let controller = controller(FixtureCaptureHTTP(answer: .candidate)) + let scan = begin(controller) + let (tap, hit) = try snapshotTap() + controller.meterTapped(tap, hit: hit, scan: scan) + controller.meterTapFailed(.encodeFailed, scan: scan) + #expect(controller.meterTap == .forwarded) + controller.endScan(recording: fixture.recording) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift index 902084a59..83067deac 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift @@ -169,7 +169,7 @@ final class HeldResultHTTP: CaptureHTTP, Sendable { func capture(_ controller: PhotoProcessingController, consent: Bool?) async throws { if let consent { controller.answerConsent(consent) } let (tap, hit) = try fixture.tap(at: fixture.start + 1) - controller.meterTapped(tap, hit: hit) + if let scan = controller.context { controller.meterTapped(tap, hit: hit, scan: scan) } controller.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) for i in 0..<5 { controller.kept(try fixture.photo(at: fixture.start + 3 + Double(i) * 1.5)) } await controller.settle() @@ -268,7 +268,7 @@ final class HeldResultHTTP: CaptureHTTP, Sendable { controller.beginScan(context(controller), recording: fixture.recording) controller.answerConsent(true) let (tap, hit) = try fixture.tap(at: fixture.start + 1) - controller.meterTapped(tap, hit: hit) + if let scan = controller.context { controller.meterTapped(tap, hit: hit, scan: scan) } controller.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) controller.kept(try fixture.photo(at: fixture.start + 3)) try #require(try await until(10) { http.isHolding }, "no storage upload was held") From d4dcd96b1942283f69a843e1c12bef8177ead689 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 12:54:50 -0500 Subject: [PATCH 52/71] Let MeterTapFailure be a Result's failure Result requires an Error; the package build refused the type without it (wrapper job meter-tap-1be8eded-swift-test, exit 1). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Sources/HouseScanKit/Processing/MeterTapObservation.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/MeterTapObservation.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/MeterTapObservation.swift index 31847cab7..0dcd59fb5 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Processing/MeterTapObservation.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/MeterTapObservation.swift @@ -3,7 +3,7 @@ import simd /// Why the homeowner's accepted meter mark gave the capture packet no tap. Each ends the packet as /// not prepared rather than letting it go out without its tap (`PhotoProcessingController.meterTapFailed`). -public enum MeterTapFailure: String, Sendable, Equatable, CaseIterable { +public enum MeterTapFailure: String, Error, Sendable, Equatable, CaseIterable { /// ARKit had no current frame to take. case noFrame /// The frame's tracking wasn't normal, so its pose can't place the tap. From a3d10ec48042194577f6a88f82bee6275c7786b7 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 12:56:59 -0500 Subject: [PATCH 53/71] Keep a yes after a failed meter tap away from the coordinator A yes on a packet that already lost its tap used to reach the coordinator first, which starts an uploader and saves a consented upload state (CaptureUploader.start) before the packet is ended as not prepared. Now the packet ends without the coordinator hearing the yes, so no upload starts and nothing a relaunch could find is saved. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Processing/PhotoProcessingController.swift | 10 ++++++++-- .../HouseScanKitTests/MeterTapObservationTests.swift | 5 +++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift index bef45c00f..92336efe1 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift @@ -133,12 +133,18 @@ public final class PhotoProcessingController { public func answerConsent(_ yes: Bool) { guard context != nil, let coordinator, var state = status, state.consent == .asking else { return } + // A yes after this packet lost its tap: the packet can't be sent, so it ends now as not + // prepared, and the coordinator never hears the yes, so no upload starts for it. + if yes, case .failed = meterTap { + state.consent = .granted + status = state + capturePreparationFailed() + return + } // Before any yes there is no upload to withdraw, so a no can't fail here. coordinator.answerConsent(yes) state.consent = yes ? .granted : .declined status = state - // A yes after this packet lost its tap: the packet can't be sent, so it ends now. - if yes, case .failed = meterTap { capturePreparationFailed() } } /// A no after a yes: nothing more from this scan is sent, and the scan can't be processed diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/MeterTapObservationTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/MeterTapObservationTests.swift index fa0646595..5c8f48ecc 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/MeterTapObservationTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/MeterTapObservationTests.swift @@ -183,6 +183,11 @@ extension Result { #expect(yes.status?.stage == .capturing(sent: 0)) yes.answerConsent(true) #expect(yes.status?.stage == .ended(.notPrepared)) + // The yes never reached the coordinator: no upload started, so no consented upload state + // was saved for a packet that can't be sent (a relaunch could otherwise find it). + let saved = FileManager.default.enumerator(at: root.appending(path: "Captures"), includingPropertiesForKeys: nil)? + .compactMap { $0 as? URL }.filter { $0.lastPathComponent == CaptureUploader.stateURL(in: root).lastPathComponent } ?? [] + #expect(saved.isEmpty, "upload state saved after a yes on a packet without its tap: \(saved)") yes.endScan(recording: fixture.recording) let no = controller(FixtureCaptureHTTP(answer: .candidate)) From 3be7f64d6814b97b2a9c50a420b640b958df684a Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 12:58:31 -0500 Subject: [PATCH 54/71] Test that a photo or tap the phone couldn't keep stops the capture's upload Co-Authored-By: Claude Opus 5.5 (1M context) --- .../CaptureLifecycleFailClosedTests.swift | 192 ++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift new file mode 100644 index 000000000..a0a6e969e --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift @@ -0,0 +1,192 @@ +import Foundation +import HouseScanKit +import simd +import Synchronization +import Testing + +/// A photo or tap the scan accepted but the phone couldn't keep makes the capture one that can't be +/// prepared: the running upload stops at once with a local code, nothing is finalized, and an +/// upload that hasn't started never starts. A withdrawal still outranks it. Frames the producer +/// leaves out on purpose (a repeated frame, photos after the scan was sent) are not losses. A +/// failure from a world that was reset never reaches the next one. +@Suite(.serialized) @MainActor struct CaptureLifecycleFailClosedTests { + let root = FileManager.default.temporaryDirectory.appending(path: "fail-closed-\(UUID().uuidString)") + var fixture: NativeCaptureFixture { NativeCaptureFixture(folder: root.appending(path: "store")) } + var captures: URL { root.appending(path: "Captures") } + + static let preparationFailed = CaptureUploadState.End.failed(step: "prepare", codes: ["capture_input_lost"], status: 0) + + func coordinator(_ server: LoopbackCaptureAPI, log: @escaping @Sendable (String) -> Void = { _ in }) -> CaptureSessionCoordinator { + CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment( + endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures, log: log)) + } + + /// A pose the producer refuses: scaled, so not rigid. + static func scaled(_ m: simd_float4x4) -> simd_float4x4 { + var out = m + out.columns.0 *= 2 + out.columns.1 *= 2 + out.columns.2 *= 2 + return out + } + + enum Loss: String, CaseIterable, Sendable { + /// The JPEG the scan kept is gone before it could be staged. + case stagingCopy + /// Staged, then refused by the producer when its turn came. + case queuedSeal + /// The tap's frame couldn't be encoded. + case tapFrame + /// The tap's seal was refused. + case tapSeal + } + + func lose(_ loss: Loss, in coordinator: CaptureSessionCoordinator) throws { + let t = fixture.start + 9 + switch loss { + case .stagingCopy: + var photo = try fixture.photo(at: t) + photo.jpeg = root.appending(path: "missing.jpg") + coordinator.kept(photo) + case .queuedSeal: + var photo = try fixture.photo(at: t) + photo.cameraToWorld = Self.scaled(photo.cameraToWorld) + coordinator.kept(photo) + case .tapFrame: + var (tap, hit) = try fixture.tap(at: t) + tap.jpeg = { nil } + coordinator.meterTapped(tap, hit: hit) + case .tapSeal: + var (tap, hit) = try fixture.tap(at: t) + tap.cameraToWorld = Self.scaled(tap.cameraToWorld) + coordinator.meterTapped(tap, hit: hit) + } + } + + /// The scan's photos so far, sent while it runs. + func startScan(_ coordinator: CaptureSessionCoordinator, consent: Bool?) async throws { + coordinator.begin(recording: fixture.recording) + if let consent { coordinator.answerConsent(consent) } + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + coordinator.kept(try fixture.photo(at: fixture.start + 4)) + await coordinator.settle() + } + + @Test(arguments: Loss.allCases) + func aLostInputStopsTheRunningUploadBeforeTheScanEnds(loss: Loss) async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + try await startScan(coordinator, consent: true) + let uploader = try #require(coordinator.session?.uploader) + #expect(await uploader.snapshot.end == nil) + + try lose(loss, in: coordinator) + await coordinator.settle() + // Ended by the loss itself, before the scan was sent. + #expect(await uploader.snapshot.end == Self.preparationFailed) + #expect(await uploader.status.retryingAt == nil) + + coordinator.kept(try fixture.photo(at: fixture.start + 10)) + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + await coordinator.settle() + #expect(await uploader.snapshot.end == Self.preparationFailed) + #expect(await uploader.snapshot.packet == nil) + #expect(server.requests("POST captures/finalize").isEmpty) + } + + /// Lost before the homeowner answered: a later yes sends nothing, so no capture is opened for + /// a packet that can't be complete. + @Test(arguments: Loss.allCases) + func aLostInputBeforeConsentOpensNoCapture(loss: Loss) async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + try await startScan(coordinator, consent: nil) + try lose(loss, in: coordinator) + await coordinator.settle() + coordinator.answerConsent(true) + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + await coordinator.settle() + #expect(coordinator.session?.uploader == nil) + #expect(server.state.withLock { $0.log.isEmpty }) + } + + /// The homeowner's no stands: a loss afterwards doesn't relabel the upload's end. + @Test func aWithdrawalOutranksALostInput() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + try await startScan(coordinator, consent: true) + let uploader = try #require(coordinator.session?.uploader) + guard case .success = coordinator.answerConsent(false) else { + Issue.record("the withdrawal wasn't recorded") + return + } + try lose(.queuedSeal, in: coordinator) + await coordinator.settle() + await uploader.settled() + #expect(await uploader.snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) + } + + /// What the producer leaves out on purpose isn't a loss: the close-up's frame kept again, a + /// photo for a purpose that isn't a still, and photos after the scan was sent. + @Test func framesLeftOutOnPurposeStillFinish() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + try await startScan(coordinator, consent: true) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5)) + coordinator.kept(try fixture.photo(at: fixture.start + 6, purpose: "door_view")) + await coordinator.settle() + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + coordinator.kept(try fixture.photo(at: fixture.start + 11)) + await coordinator.settle() + #expect(await coordinator.session?.uploader?.snapshot.end == .finished(status: "manual_review")) + #expect(server.requests("POST captures/finalize").count == 1) + } + + /// A tap whose frame is still being encoded when the world is reset, and then fails, doesn't + /// touch the next world's session, upload or status. + @Test func aFailureFromAResetWorldDoesNotReachTheNextOne() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let lines = Mutex<[String]>([]) + let coordinator = coordinator(server) { line in lines.withLock { $0.append(line) } } + let statuses = Mutex<[CaptureUploadStatus?]>([]) + coordinator.onStatus = { status in statuses.withLock { $0.append(status) } } + try await startScan(coordinator, consent: true) + + let entered = Mutex(false) + let gate = DispatchSemaphore(value: 0) + var (tap, hit) = try fixture.tap(at: fixture.start + 9) + tap.jpeg = { + entered.withLock { $0 = true } + gate.wait() + return nil + } + coordinator.meterTapped(tap, hit: hit) + for _ in 0..<500 where !entered.withLock({ $0 }) { try await Task.sleep(for: .milliseconds(10)) } + try #require(entered.withLock { $0 }) + + coordinator.newWorld("world reset", recording: fixture.recording, newScan: false) + let next = try #require(coordinator.session) + let seen = statuses.withLock { $0.count } + gate.signal() + // The old world's work has run to its end once it logs the failed frame. + let failed = { lines.withLock { $0.contains { $0.contains("meter tap") } } } + for _ in 0..<500 where !failed() { try await Task.sleep(for: .milliseconds(10)) } + try #require(failed()) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + await coordinator.settle() + let nextUploader = try #require(next.uploader) + await nextUploader.settled() + + #expect(coordinator.session === next) + #expect(await nextUploader.snapshot.end == nil) + #expect(statuses.withLock { $0.dropFirst(seen) }.allSatisfy { status in status?.phase != .failed }) + } +} From 63f43485a9653eaeee40c7d0311195f966974ed4 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 12:59:36 -0500 Subject: [PATCH 55/71] Stop a capture's upload when the phone loses a photo or tap the scan accepted A failed staging copy, a refused seal or a tap frame that couldn't be encoded was logged and dropped, and the capture went on to seal and finalize a packet missing it. The session now records capture_input_lost, closes the running upload's admission at once and ends it at step prepare with status 0. It never seals and never starts an upload. A withdrawal still outranks it, and late work from a reset world is ignored. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Upload/CaptureSessionCoordinator.swift | 34 ++++++++++++++++--- .../HouseScanKit/Upload/CaptureUploader.swift | 13 +++++++ .../CaptureLifecycleFailClosedTests.swift | 3 ++ 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift index d35fea631..48dbeeab2 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift @@ -165,6 +165,12 @@ public final class CaptureSessionCoordinator { /// Checklist purposes the packet's `stills` take; any other still stays a keyframe only. public static let stillPurposes: Set = ["meter_close", "meter_oblique"] + /// The local code a capture ends with when the phone lost a photo or tap the scan accepted: + /// the packet can't be prepared complete. The upload, if one is running, ends + /// `.failed(step: "prepare", codes: [inputLostCode], status: 0)`; the session's + /// `preparationFailure` holds it either way. + public static let inputLostCode = "capture_input_lost" + @MainActor public final class Session { /// This app's id for this session, new for every session even when two packets both call @@ -186,6 +192,9 @@ public final class CaptureSessionCoordinator { var frozen: Data? /// The server's result for this session's capture, decoded and bound to it. public fileprivate(set) var result: CaptureResult.Record? + /// Set once a photo or tap the scan accepted couldn't be kept (`inputLostCode`). This + /// session then never seals, and never starts an upload. + public fileprivate(set) var preparationFailure: String? init(packetID: String, folder: URL, recording: RecordingSource) { self.packetID = packetID @@ -302,7 +311,7 @@ public final class CaptureSessionCoordinator { try FileManager.default.createDirectory(at: staged.deletingLastPathComponent(), withIntermediateDirectories: true) try FileManager.default.copyItem(at: photo.jpeg, to: staged) } catch { - environment.log("capture packet: a kept photo could not be copied: \(error)") + inputLost(session, "a kept photo could not be copied: \(error)") return } // The copy is deleted even when a reset ends the session before this work runs. @@ -320,7 +329,7 @@ public final class CaptureSessionCoordinator { } await session.uploader?.add(files) } catch { - environment.log("capture packet: a kept photo was not sealed: \(error)") + self.inputLost(session, "a kept photo was not sealed: \(error)") } }, cleanup: { try? FileManager.default.removeItem(at: staged) }) } @@ -332,7 +341,7 @@ public final class CaptureSessionCoordinator { do { let producer = try session.producer(first: (tap.t, SIMD2(Float(tap.width), Float(tap.height))), info: environment.sessionInfo) guard let jpeg = await Task.detached(priority: .userInitiated, operation: { tap.jpeg() }).value else { - environment.log("capture packet: the meter tap's frame could not be encoded; no tap") + self.inputLost(session, "the meter tap's frame could not be encoded") return } let observation = Packet04Observation( @@ -340,7 +349,7 @@ public final class CaptureSessionCoordinator { let files = try await producer.sealTap(id: "meter", label: "meter", jpeg: jpeg, observation: observation, pixel: tap.pixel, hit: hit) await session.uploader?.add(files) } catch { - environment.log("capture packet: the meter tap was not recorded: \(error)") + self.inputLost(session, "the meter tap was not recorded: \(error)") } } } @@ -351,6 +360,8 @@ public final class CaptureSessionCoordinator { public func captureEnded(acceptedCloseUpAt: Double?) { guard let session, !session.sealing, let environment else { return } session.sealing = true + // A packet missing an accepted photo or tap is never frozen; its upload already ended. + guard session.preparationFailure == nil else { return } session.enqueue { session in guard let producer = session.producer else { await session.uploader?.abandon("no photos were kept") @@ -415,7 +426,7 @@ public final class CaptureSessionCoordinator { /// packet if the scan was already sent. Photos kept later go up as they are sealed. private func startUploading(_ session: Session) { guard let environment, environment.sends, consent == true, let consentedAt, session.uploader == nil, !session.ended, - !session.consentWithdrawn else { return } + !session.consentWithdrawn, session.preparationFailure == nil else { return } do { let uploader = try CaptureUploader.start( folder: session.folder, base: environment.endpoint, http: environment.http, @@ -470,6 +481,19 @@ public final class CaptureSessionCoordinator { } } + /// A photo or tap the scan accepted couldn't be kept, so `session`'s packet can't be complete. + /// Admission closes now, before the uploader's own turn, so its loop sends nothing more; the + /// upload then ends with `inputLostCode`. A session that already ended (a reset world's late + /// work) is left alone: its upload was abandoned, and the next world isn't this one. + private func inputLost(_ session: Session, _ detail: String) { + environment?.log("capture packet: \(detail); this capture can't be prepared") + guard !session.ended, session.preparationFailure == nil else { return } + session.preparationFailure = Self.inputLostCode + guard let uploader = session.uploader else { return } + uploader.stopSending() + Task { await uploader.failPreparation(Self.inputLostCode) } + } + private func endSession(_ reason: String) { guard let session else { return } self.session = nil diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift index 30816cd37..f3d65ecad 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -293,6 +293,19 @@ public actor CaptureUploader { persist() } + /// Ends the upload because the phone lost a photo or tap the packet needs: step `prepare`, + /// status 0, `code`. Admission closes first, so nothing more is sent; a request already + /// admitted may finish, and its reply is dropped. A withdrawal still outranks it (`persist`), + /// and an end already reached stays. + public func failPreparation(_ code: String) { + stopSending() + guard state.end == nil else { return } + state.end = .failed(step: "prepare", codes: [code], status: 0) + state.attemptID = UUID().uuidString + loop?.cancel() + persist() + } + /// Starts the loop if it is idle: after a relaunch, or when the network is back. public func kick() { guard loop == nil, state.end == nil, !saveFailed else { return } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift index a0a6e969e..507b3a444 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift @@ -89,6 +89,7 @@ import Testing // Ended by the loss itself, before the scan was sent. #expect(await uploader.snapshot.end == Self.preparationFailed) #expect(await uploader.status.retryingAt == nil) + #expect(coordinator.session?.preparationFailure == CaptureSessionCoordinator.inputLostCode) coordinator.kept(try fixture.photo(at: fixture.start + 10)) coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) @@ -112,6 +113,7 @@ import Testing coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) await coordinator.settle() #expect(coordinator.session?.uploader == nil) + #expect(coordinator.session?.preparationFailure == CaptureSessionCoordinator.inputLostCode) #expect(server.state.withLock { $0.log.isEmpty }) } @@ -186,6 +188,7 @@ import Testing await nextUploader.settled() #expect(coordinator.session === next) + #expect(next.preparationFailure == nil) #expect(await nextUploader.snapshot.end == nil) #expect(statuses.withLock { $0.dropFirst(seen) }.allSatisfy { status in status?.phase != .failed }) } From f47fda390df5c00b56cf66fe12a0ef02c7e79227 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 13:00:56 -0500 Subject: [PATCH 56/71] Test that a world reset records its capture's end before newWorld returns Co-Authored-By: Claude Opus 5.5 (1M context) --- .../CaptureSessionEndTests.swift | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionEndTests.swift diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionEndTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionEndTests.swift new file mode 100644 index 000000000..24edf45df --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionEndTests.swift @@ -0,0 +1,60 @@ +import Foundation +import HouseScanKit +import Testing + +/// A world reset ends its capture for good before `newWorld` returns. The upload's own abandonment +/// still runs on the uploader's turn, so the ended record written synchronously here is what keeps +/// a relaunch from finding the old yes and resuming the capture the homeowner left. It is a +/// reset's record, not a withdrawal's. +@Suite(.serialized) @MainActor struct CaptureSessionEndTests { + let root = FileManager.default.temporaryDirectory.appending(path: "session-end-\(UUID().uuidString)") + var fixture: NativeCaptureFixture { NativeCaptureFixture(folder: root.appending(path: "store")) } + var captures: URL { root.appending(path: "Captures") } + static let endedFile = "capture-ended" + + func coordinator(_ server: LoopbackCaptureAPI) -> CaptureSessionCoordinator { + CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures)) + } + + func scan(_ coordinator: CaptureSessionCoordinator) async throws -> URL { + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + coordinator.kept(try fixture.photo(at: fixture.start + 4)) + await coordinator.settle() + let folder = try #require(coordinator.session?.folder) + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: folder)).end == nil) + return folder + } + + @Test func aResetRecordsTheEndBeforeNewWorldReturns() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + let folder = try await scan(coordinator) + + coordinator.newWorld("world reset", recording: fixture.recording, newScan: false) + // No suspension point since newWorld: only what it did synchronously is on disk. + #expect(FileManager.default.fileExists(atPath: folder.appending(path: Self.endedFile).path)) + #expect(!FileManager.default.fileExists(atPath: CaptureUploadState.withdrawnURL(in: folder).path)) + await coordinator.settle() + } + + /// The record is what a relaunch reads: a folder carrying it isn't resumed, even with its + /// saved state still unended and nothing in this process remembering it. + @Test func aFolderWithTheEndedRecordIsNotResumed() async throws { + var rig = try CaptureUploaderTests.Rig() + defer { rig.cleanUp() } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await CaptureUploaderTests.settles(rig.uploader)) + weak let released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + try Data("world reset".utf8).write(to: rig.capture.folder.appending(path: Self.endedFile)) + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: rig.capture.folder)).end == nil) + + #expect(try CaptureUploader.resume(folder: rig.capture.folder, base: rig.server.base, http: rig.http, policy: CaptureUploaderTests.Rig.fast) == nil) + } +} From 756b86f74251bc9de7632557dc513522ddd30bca Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 13:01:52 -0500 Subject: [PATCH 57/71] Record a world's end before newWorld returns, and never resume that capture The reset's abandonment ran in an unqueued Task, so the app could quit with the capture's saved yes still unended, and a relaunch could resume what the homeowner left. retire now closes admission, bars this process from resuming the folder and writes a capture-ended marker under the folder registry's lock before returning. resume refuses a folder with that marker. newWorld returns notRecorded when the marker can't be written. A reset is kept distinct from a withdrawal. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Upload/CaptureSessionCoordinator.swift | 30 +++++++++---- .../Upload/CaptureUploadState.swift | 7 ++++ .../HouseScanKit/Upload/CaptureUploader.swift | 42 +++++++++++++++++-- .../CaptureSessionEndTests.swift | 33 ++++++++++++++- 4 files changed, 99 insertions(+), 13 deletions(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift index 48dbeeab2..d788c812d 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift @@ -247,6 +247,12 @@ public final class CaptureSessionCoordinator { environment?.sends == true && consent == nil && session != nil } + public enum SessionEndError: Error, Sendable, Equatable { + /// Sending stopped and this process won't resume the capture, but its ended marker + /// couldn't be written, so a relaunch could still find the saved yes. + case notRecorded(String) + } + public enum ConsentWithdrawalError: Error, Sendable, Equatable { /// Sending stopped in this process, but a relaunch may still find the old saved yes. case notRecorded(String) @@ -291,14 +297,18 @@ public final class CaptureSessionCoordinator { /// The ARKit world was thrown away, or (`newScan`) the scan started over: this packet can't /// be finished, and the next one belongs to `recording`'s new world. A new scan asks again. - public func newWorld(_ reason: String, recording: RecordingSource, newScan: Bool) { - endSession(reason) + /// Before this returns the old capture is ended in this process and its ended marker is + /// written; failure means the marker couldn't be, so only this process is sure not to resume it. + @discardableResult + public func newWorld(_ reason: String, recording: RecordingSource, newScan: Bool) -> Result { + let ended = endSession(reason) if newScan { consent = nil consentedAt = nil } self.recording = recording if environment != nil { startSession() } + return ended } public func kept(_ photo: KeptPhoto) { @@ -494,15 +504,19 @@ public final class CaptureSessionCoordinator { Task { await uploader.failPreparation(Self.inputLostCode) } } - private func endSession(_ reason: String) { - guard let session else { return } + private func endSession(_ reason: String) -> Result { + guard let session else { return .success(()) } self.session = nil session.ended = true onStatus?(nil) - if let uploader = session.uploader { - uploader.stopSending() - Task { await uploader.abandon(reason) } - } + guard let uploader = session.uploader else { return .success(()) } + // Synchronous: closes admission, bars this process from resuming it, writes the marker a + // relaunch reads. The abandonment saved in the upload's own state follows on its turn. + let record = uploader.retire(reason) + Task { await uploader.abandon(reason) } + guard case .notRecorded(let error) = record else { return .success(()) } + environment?.log("capture upload: the end of this world's capture could not be recorded (\(error)); it won't resume in this process") + return .failure(.notRecorded(error)) } } diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift index f27754c2c..f24ecce7a 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift @@ -11,6 +11,13 @@ public struct CaptureUploadState: Codable, Sendable, Equatable { public static func withdrawnURL(in folder: URL) -> URL { folder.appending(path: withdrawnFileName) } + /// Written into a capture's folder when its world ends before the capture was finished (a + /// reset, or starting over). It holds the reason. `CaptureUploader.resume` never resumes a folder + /// that has it. Not a withdrawal: the homeowner's yes wasn't taken back. + public static let endedFileName = "capture-ended" + + public static func endedURL(in folder: URL) -> URL { folder.appending(path: endedFileName) } + /// Called with the uploader's folder registry locked so no initial save or reply can restore /// a yes between these writes. Neither failing proves anything durable after a process exit. static func recordWithdrawal(in folder: URL) -> CaptureUploader.WithdrawalRecord { diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift index f3d65ecad..10dbeec7c 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -86,6 +86,8 @@ public actor CaptureUploader { /// A save failed here: the saved state is older than what this process sent, so this /// process never resumes it. A relaunch may, under the consent it records. var unsaved = false + /// Its world ended (`retire`): this process never resumes it, whatever reached the disk. + var retired = false var stopped = false var running: Task? } @@ -142,7 +144,7 @@ public actor CaptureUploader { try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) let folder = try canonicalFolder(folder) return try folders.withLock { entries in - entries = entries.filter { $0.value.owner != nil || $0.value.withdrawn || $0.value.unsaved } + entries = entries.filter { $0.value.owner != nil || $0.value.withdrawn || $0.value.unsaved || $0.value.retired } guard entries[folder.path]?.withdrawn != true, !FileManager.default.fileExists(atPath: CaptureUploadState.withdrawnURL(in: folder).path) else { throw OwnershipError.consentWithdrawn } @@ -172,11 +174,12 @@ public actor CaptureUploader { guard FileManager.default.fileExists(atPath: folder.path) else { return nil } let folder = try canonicalFolder(folder) return try folders.withLock { entries in - entries = entries.filter { $0.value.owner != nil || $0.value.withdrawn || $0.value.unsaved } + entries = entries.filter { $0.value.owner != nil || $0.value.withdrawn || $0.value.unsaved || $0.value.retired } let url = stateURL(in: folder) - guard entries[folder.path]?.withdrawn != true, entries[folder.path]?.unsaved != true, + guard entries[folder.path]?.withdrawn != true, entries[folder.path]?.unsaved != true, entries[folder.path]?.retired != true, FileManager.default.fileExists(atPath: url.path), - !FileManager.default.fileExists(atPath: CaptureUploadState.withdrawnURL(in: folder).path) + !FileManager.default.fileExists(atPath: CaptureUploadState.withdrawnURL(in: folder).path), + !FileManager.default.fileExists(atPath: CaptureUploadState.endedURL(in: folder).path) else { return nil } guard entries[folder.path]?.owner == nil else { throw OwnershipError.alreadyOwned } var state = try CaptureUploadState.load(from: url) @@ -268,6 +271,37 @@ public actor CaptureUploader { return record } + /// How a world's end was recorded on disk (`retire`). + public enum RetirementRecord: Sendable, Equatable { + /// The ended marker `resume` refuses was written. + case marked + /// It couldn't be written: this process won't resume the capture, but a relaunch could + /// still find its saved yes until a later save records the upload as abandoned. + case notRecorded(String) + } + + /// Ends this capture's world before returning: admission closes, this process never resumes + /// the folder, and the ended marker (`CaptureUploadState.endedURL`) is written, synchronously + /// and under the folder registry's lock, as `withdrawConsent` writes its own. The running task + /// is cancelled. The upload's saved end still comes from `abandon`, on the uploader's turn. + @discardableResult + public nonisolated func retire(_ reason: String) -> RetirementRecord { + let (record, task) = Self.folders.withLock { entries -> (RetirementRecord, Task?) in + entries[folderKey, default: FolderEntry()].stopped = true + entries[folderKey]?.retired = true + let record: RetirementRecord + do { + try Data(reason.utf8).write(to: CaptureUploadState.endedURL(in: folder), options: .atomic) + record = .marked + } catch { + record = .notRecorded(Self.describe(error)) + } + return (record, entries[folderKey]?.running) + } + task?.cancel() + return record + } + /// Closes admission synchronously when a coordinator ends a world, before its queued actor /// work can kick the upload. Ownership stays until the old uploader and its requests die. nonisolated func stopSending() { diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionEndTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionEndTests.swift index 24edf45df..c73602bc8 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionEndTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionEndTests.swift @@ -34,13 +34,44 @@ import Testing let coordinator = coordinator(server) let folder = try await scan(coordinator) - coordinator.newWorld("world reset", recording: fixture.recording, newScan: false) + let ended = coordinator.newWorld("world reset", recording: fixture.recording, newScan: false) // No suspension point since newWorld: only what it did synchronously is on disk. + guard case .success = ended else { + Issue.record("the reset's end wasn't recorded: \(ended)") + return + } #expect(FileManager.default.fileExists(atPath: folder.appending(path: Self.endedFile).path)) #expect(!FileManager.default.fileExists(atPath: CaptureUploadState.withdrawnURL(in: folder).path)) await coordinator.settle() } + /// When the marker can't be written, newWorld says so, and this process still never resumes + /// the capture, even after its uploader is gone and with its saved state unended. + @Test func aResetThatCantBeRecordedSaysSoAndStillNeverResumesHere() async throws { + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + let folder = try await scan(coordinator) + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) + try? FileManager.default.removeItem(at: root) + } + weak let old = coordinator.session?.uploader + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) + + let ended = coordinator.newWorld("world reset", recording: fixture.recording, newScan: false) + guard case .failure(.notRecorded) = ended else { + Issue.record("expected the unwritable folder to leave the end unrecorded, got \(ended)") + return + } + #expect(!FileManager.default.fileExists(atPath: folder.appending(path: Self.endedFile).path)) + await coordinator.settle() + for _ in 0..<500 where old != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(old == nil) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: folder)).end == nil) + #expect(try CaptureUploader.resume(folder: folder, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) == nil) + } + /// The record is what a relaunch reads: a folder carrying it isn't resumed, even with its /// saved state still unended and nothing in this process remembering it. @Test func aFolderWithTheEndedRecordIsNotResumed() async throws { From 4fe929dcfdb61f49bcb5b276e2b78da282a5da7e Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 13:02:45 -0500 Subject: [PATCH 58/71] Test that a storage URL to ::1 counts as this machine and near matches don't Co-Authored-By: Claude Opus 5.5 (1M context) --- .../CaptureUploaderTests.swift | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift index 90c8fab2a..224cecef5 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift @@ -344,7 +344,12 @@ import Testing /// A storage URL that is not https (or http to this machine) is refused when it is /// registered, before any bytes go to it; one that doesn't parse is refused the same way /// instead of leaving the file registered and the loop spinning. - @Test(arguments: [("http://storage.example.com/upload/x", "upload_url_insecure"), ("", "upload_url_invalid")]) + @Test(arguments: [ + ("http://storage.example.com/upload/x", "upload_url_insecure"), ("", "upload_url_invalid"), + // Near matches of a loopback name are public hosts. + ("http://[::2]:9/upload/x", "upload_url_insecure"), ("http://localhost.example.com/upload/x", "upload_url_insecure"), + ("http://127.0.0.1.example.com/upload/x", "upload_url_insecure"), + ]) func aStorageURLThatIsNotSafeIsRefusedBeforeAnyPut(url: String, code: String) async throws { let rig = try Rig() defer { rig.cleanUp() } @@ -357,6 +362,28 @@ import Testing #expect(rig.server.requests("PUT upload").isEmpty) } + /// Plain http to this machine over IPv6 is allowed, as ScopedCaptureHTTP's `.loopbackHTTP` + /// allows it: the file is registered for its PUT instead of refused. Nothing listens on `::1` + /// here, so the PUT itself fails as a network error and the upload waits to retry. + @Test func aLoopbackStorageURLOverIPv6IsNotRefused() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + let url = "http://[::1]:\(rig.server.port)/v1/uploads/x" + rig.server.state.withLock { $0.uploadURLOverride = url } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + let registered = { () async -> Bool in + await rig.uploader.snapshot.files.values.contains { if case .registered(url, _, _) = $0.phase { true } else { false } } + } + for _ in 0..<500 { + if await rig.uploader.snapshot.end != nil { break } + if await registered() { break } + try await Task.sleep(for: .milliseconds(10)) + } + #expect(await rig.uploader.snapshot.end == nil) + #expect(await registered()) + await rig.uploader.abandon("test done") + } + /// Storage that answers a PUT with a redirect doesn't get the photo sent on: the real /// `URLSession` transport refuses every redirect, so the photo reaches only the URL that passed /// the check, and the 307 comes back as a storage refusal. The redirect goes to a second local From 22b932fb5de5b09d9516ca3872a96d7a04c39862 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 13:03:17 -0500 Subject: [PATCH 59/71] Treat a storage URL to ::1 as this machine, by the API scope's own rule uploadURLProblem allowed plain http only to 127.0.0.1 and localhost, while ScopedCaptureHTTP's loopback transport also allows ::1. It now asks CaptureAPIScope.allows, so both use one list. Public hosts, near matches included, stay refused. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Sources/HouseScanKit/Upload/CaptureUploader.swift | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift index 10dbeec7c..115fa99ed 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -486,8 +486,9 @@ public actor CaptureUploader { guard let url = URL(string: text), let scheme = url.scheme?.lowercased(), let host = url.host(), !host.isEmpty else { return "upload_url_invalid" } - let local = host == "127.0.0.1" || host == "localhost" - return scheme == "https" || (scheme == "http" && local) ? nil : "upload_url_insecure" + // The same rule as the API's own scope under `.loopbackHTTP`: https anywhere, plain http only + // to 127.0.0.1, ::1 or localhost, so the two lists can't drift apart. + return CaptureAPIScope.allows(scheme: scheme, host: host.lowercased(), transport: .loopbackHTTP) ? nil : "upload_url_insecure" } private func put(_ files: [CaptureUploadState.File]) async throws { From 7f80a19fa4e4b402d8e22e42de1dbb8c40a47066 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 13:10:38 -0500 Subject: [PATCH 60/71] Make inputLostCode readable outside the main actor The app maps an upload's end to a stage in a nonisolated function, which can't read a main-actor static. The code is a Sendable constant. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Upload/CaptureSessionCoordinator.swift | 3 ++- .../CaptureLifecycleFailClosedTests.swift | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift index d788c812d..3ec3a9d1d 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift @@ -169,7 +169,8 @@ public final class CaptureSessionCoordinator { /// the packet can't be prepared complete. The upload, if one is running, ends /// `.failed(step: "prepare", codes: [inputLostCode], status: 0)`; the session's /// `preparationFailure` holds it either way. - public static let inputLostCode = "capture_input_lost" + /// Nonisolated, so status mapping outside the main actor can match it. + nonisolated public static let inputLostCode = "capture_input_lost" @MainActor public final class Session { diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift index 507b3a444..44d3f0f84 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift @@ -193,3 +193,13 @@ import Testing #expect(statuses.withLock { $0.dropFirst(seen) }.allSatisfy { status in status?.phase != .failed }) } } + +/// The code is readable where the app maps an upload's end to a stage, outside the main actor. +struct InputLostCodeIsNonisolated { + static func matches(_ codes: [String]) -> Bool { codes.contains(CaptureSessionCoordinator.inputLostCode) } + + @Test func aNonisolatedMappingCanMatchTheCode() { + #expect(Self.matches(["capture_input_lost"])) + #expect(!Self.matches(["local_state_unsaved"])) + } +} From 91b673a7fa4c0b28dee7e4e52e4c2802b4083784 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 13:12:46 -0500 Subject: [PATCH 61/71] Test a withdrawal after a loss, a reset after an unrecorded withdrawal, and a loss queued before the end Co-Authored-By: Claude Opus 5.5 (1M context) --- .../CaptureLifecycleFailClosedTests.swift | 55 +++++++++++++++++++ .../CaptureSessionEndTests.swift | 26 +++++++++ 2 files changed, 81 insertions(+) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift index 44d3f0f84..d10af08df 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureLifecycleFailClosedTests.swift @@ -134,6 +134,61 @@ import Testing #expect(await uploader.snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) } + /// The other order: the loss ends the upload first, then the homeowner says no. The recorded + /// withdrawal replaces the local end. A reset after a loss doesn't: it's not a withdrawal. + @Test func aWithdrawalAfterALossStillEndsTheUploadAsWithdrawn() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + try await startScan(coordinator, consent: true) + let uploader = try #require(coordinator.session?.uploader) + try lose(.queuedSeal, in: coordinator) + await coordinator.settle() + try #require(await uploader.snapshot.end == Self.preparationFailed) + guard case .success = coordinator.answerConsent(false) else { + Issue.record("the withdrawal wasn't recorded") + return + } + await uploader.settled() + for _ in 0..<500 { + if await uploader.snapshot.end != Self.preparationFailed { break } + try await Task.sleep(for: .milliseconds(10)) + } + #expect(await uploader.snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: uploader.folder)).end == .abandoned(CaptureUploader.withdrawnReason)) + } + + @Test func aResetAfterALossKeepsTheLossAsTheEnd() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + try await startScan(coordinator, consent: true) + let uploader = try #require(coordinator.session?.uploader) + try lose(.queuedSeal, in: coordinator) + await coordinator.settle() + coordinator.newWorld("world reset", recording: fixture.recording, newScan: false) + await uploader.settled() + try await Task.sleep(for: .milliseconds(50)) + #expect(await uploader.snapshot.end == Self.preparationFailed) + } + + /// A loss still queued when the scan is sent: the packet is never frozen, even with no upload + /// to refuse it. Nothing is settled between the lost input and the end of the capture. + @Test(arguments: [Loss.queuedSeal, Loss.tapSeal, Loss.tapFrame]) + func aLossQueuedBeforeTheEndKeepsThePacketUnfrozen(loss: Loss) async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + try await startScan(coordinator, consent: nil) + let folder = try #require(coordinator.session?.folder) + try lose(loss, in: coordinator) + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + await coordinator.settle() + #expect(coordinator.session?.preparationFailure == CaptureSessionCoordinator.inputLostCode) + #expect(!FileManager.default.fileExists(atPath: folder.appending(path: "packet.json").path)) + #expect(server.state.withLock { $0.log.isEmpty }) + } + /// What the producer leaves out on purpose isn't a loss: the close-up's frame kept again, a /// photo for a purpose that isn't a still, and photos after the scan was sent. @Test func framesLeftOutOnPurposeStillFinish() async throws { diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionEndTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionEndTests.swift index c73602bc8..da6465217 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionEndTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionEndTests.swift @@ -72,6 +72,32 @@ import Testing #expect(try CaptureUploader.resume(folder: folder, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) == nil) } + /// A no that couldn't be recorded (unwritable folder) leaves the old yes on disk. The reset + /// that follows has no uploader left to retire, and must report that failure, not success. + @Test func aResetAfterAnUnrecordedWithdrawalReportsTheFailure() async throws { + let server = try LoopbackCaptureAPI() + let coordinator = coordinator(server) + let folder = try await scan(coordinator) + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: folder.path) + try? FileManager.default.removeItem(at: root) + } + let saved = try Data(contentsOf: CaptureUploader.stateURL(in: folder)) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: folder.path) + guard case .failure(.notRecorded) = coordinator.answerConsent(false) else { + Issue.record("expected the withdrawal to go unrecorded in an unwritable folder") + return + } + let ended = coordinator.newWorld("world reset", recording: fixture.recording, newScan: false) + guard case .failure(.notRecorded) = ended else { + Issue.record("the reset reported \(ended) with the old yes still on disk") + return + } + await coordinator.settle() + #expect(try Data(contentsOf: CaptureUploader.stateURL(in: folder)) == saved) + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: folder)).end == nil) + } + /// The record is what a relaunch reads: a folder carrying it isn't resumed, even with its /// saved state still unended and nothing in this process remembering it. @Test func aFolderWithTheEndedRecordIsNotResumed() async throws { From 0d6e29248345dd4f3b4b2b41dcb7208e1d79c087 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 13:13:44 -0500 Subject: [PATCH 62/71] Let a withdrawal outrank a lost input, report an unrecorded no at reset, and recheck before freezing A no after a loss left the upload's end at the local prepare failure, because abandon kept any end but a save failure. A recorded withdrawal now replaces that end; a plain reset still keeps it. newWorld after a no that couldn't be recorded returned success with the old yes still on disk; it now returns that failure. captureEnded checked for a lost input only when queued, so a loss queued ahead of it could still let the packet freeze. It checks again when its turn comes. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Upload/CaptureSessionCoordinator.swift | 9 ++++++++- .../HouseScanKit/Upload/CaptureUploader.swift | 19 ++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift index 3ec3a9d1d..364a96d01 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift @@ -374,6 +374,8 @@ public final class CaptureSessionCoordinator { // A packet missing an accepted photo or tap is never frozen; its upload already ended. guard session.preparationFailure == nil else { return } session.enqueue { session in + // A photo or tap queued before this one may have been lost since: never freeze then. + guard session.preparationFailure == nil else { return } guard let producer = session.producer else { await session.uploader?.abandon("no photos were kept") return @@ -510,7 +512,12 @@ public final class CaptureSessionCoordinator { self.session = nil session.ended = true onStatus?(nil) - guard let uploader = session.uploader else { return .success(()) } + guard let uploader = session.uploader else { + // A no that couldn't be recorded took the uploader away and left the old yes on disk; + // this end can't record anything either, so it says so. + if case .notRecorded(let error)? = session.withdrawalFailure { return .failure(.notRecorded(error)) } + return .success(()) + } // Synchronous: closes admission, bars this process from resuming it, writes the marker a // relaunch reads. The abandonment saved in the upload's own state follows on its turn. let record = uploader.retire(reason) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift index 115fa99ed..a8ce5732b 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -242,6 +242,12 @@ public actor CaptureUploader { /// the upload's state. It is a local storage problem: it says nothing about the photos or /// the server, and no server answered. public static let unsavedStateCode = "local_state_unsaved" + + /// The local end `failPreparation` writes: step `prepare`, status 0, no server answer. + static func isPreparationEnd(_ end: CaptureUploadState.End?) -> Bool { + guard case .failed(step: "prepare", _, status: 0)? = end else { return false } + return true + } static let unsavedEnd = CaptureUploadState.End.failed(step: "save", codes: [unsavedStateCode], status: 0) /// How a withdrawal was recorded on disk. @@ -317,11 +323,14 @@ public actor CaptureUploader { public func abandon(_ reason: String) { stopSending() // An end the app asks for replaces a save failure. A withdrawal also replaces an end this - // process reached but couldn't save, such as a result or a refusal. Otherwise an end - // stays: Start over doesn't rewrite a finished upload. - let unsavedWithdrawal = saveFailed && isWithdrawn - guard state.end == nil || state.end == Self.unsavedEnd || unsavedWithdrawal else { return } - state.end = .abandoned(unsavedWithdrawal ? Self.withdrawnReason : reason) + // process reached but couldn't save, such as a result or a refusal, and the local end of + // a capture that lost an input (`failPreparation`): the homeowner's no is the reason it + // stopped. Otherwise an end stays: Start over doesn't rewrite a finished upload, and a + // reset after a loss keeps the loss. + let withdrawn = isWithdrawn + let replacedByWithdrawal = withdrawn && (saveFailed || Self.isPreparationEnd(state.end)) + guard state.end == nil || state.end == Self.unsavedEnd || replacedByWithdrawal else { return } + state.end = .abandoned(replacedByWithdrawal ? Self.withdrawnReason : reason) state.attemptID = UUID().uuidString loop?.cancel() persist() From afd16da813e86d83fa45db962a3a4f30dedd6071 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 13:07:51 -0500 Subject: [PATCH 63/71] End a scan whose phone lost an accepted photo or tap as not prepared The capture library now stops an upload whose photo or tap the phone couldn't keep, ending it failed at step "prepare" with CaptureSessionCoordinator.inputLostCode at status 0 (22b932fb). The screen showed that as the service refusing the scan; it is the phone's own failure, and the capture is never sealed, so it reads as not prepared. The same code with a server status, or at another step, keeps its existing reading. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../PhotoProcessingController.swift | 2 ++ .../Processing/PhotoProcessingStatus.swift | 4 +++ .../PhotoProcessingTests.swift | 33 +++++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift index 13117e4c4..7a57f072b 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingController.swift @@ -271,6 +271,8 @@ public final class PhotoProcessingController { self.log("photo processing: refused at the credential boundary, \(problem.code) (step \(step))") } else if case .ended(.uploadStateUnsaved) = state.stage { self.log("photo processing: this phone couldn't save the upload's progress; sending stopped") + } else if case .ended(.notPrepared) = state.stage { + self.log("photo processing: the phone lost a photo or tap the scan accepted (step \(step)); the capture can't be prepared") } state.retrying = false self.status = state diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift index 659735bd2..3ae95d264 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Processing/PhotoProcessingStatus.swift @@ -230,6 +230,10 @@ extension PhotoProcessingStatus.Stage { public static func refusal(step: String, codes: [String], status: Int) -> Self { // Only the uploader's own end (`CaptureUploader.unsavedEnd`): step `save`, status 0, its code. if step == "save", status == 0, codes.contains(CaptureUploader.unsavedStateCode) { return .ended(.uploadStateUnsaved) } + // The coordinator's own end when the phone lost a photo or tap the scan accepted + // (`CaptureSessionCoordinator.inputLostCode`): the capture can't be prepared, so it is + // never sealed and the scan ends the way a capture that couldn't be prepared does. + if step == "prepare", status == 0, codes.contains(CaptureSessionCoordinator.inputLostCode) { return .ended(.notPrepared) } if let problem = PhotoSetupProblem(codes: codes, status: status) { return .ended(.setupRefused(step: step, problem)) } guard step == "result" else { return .ended(.refused(step: step)) } if codes.contains("result_not_ready") { return .ended(.answerNotReady) } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift index 902084a59..ef3cc6dd0 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift @@ -100,6 +100,11 @@ import Testing #expect(Stage.refusal(step: "save", codes: [CaptureUploader.unsavedStateCode], status: 500) == .ended(.refused(step: "save"))) // Status 0 alone doesn't make it this failure: the code at another step isn't the uploader's own end. #expect(Stage.refusal(step: "put", codes: [CaptureUploader.unsavedStateCode], status: 0) == .ended(.refused(step: "put"))) + // The coordinator's end for a photo or tap the phone lost: not prepared, never the service's + // refusal. With a server status, or at another step, the code isn't that end. + #expect(Stage.refusal(step: "prepare", codes: [CaptureSessionCoordinator.inputLostCode], status: 0) == .ended(.notPrepared)) + #expect(Stage.refusal(step: "prepare", codes: [CaptureSessionCoordinator.inputLostCode], status: 500) == .ended(.refused(step: "prepare"))) + #expect(Stage.refusal(step: "put", codes: [CaptureSessionCoordinator.inputLostCode], status: 0) == .ended(.refused(step: "put"))) } } @@ -287,6 +292,34 @@ final class HeldResultHTTP: CaptureHTTP, Sendable { controller.endScan(recording: fixture.recording) } + /// A photo the scan accepted is lost before the phone could stage it, while the upload runs: + /// the coordinator stops the upload with its lost-input end, and the scan ends as not prepared, + /// not as the service refusing it. The capture is never sealed or finalized. + @Test func aPhotoThePhoneLostWhileSendingEndsTheScanNotPrepared() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let http = FixtureCaptureHTTP(answer: .candidate) + let controller = controller(http) + defer { controller.endScan(recording: fixture.recording) } + controller.beginScan(context(controller), recording: fixture.recording) + controller.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + controller.meterTapped(tap, hit: hit) + controller.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + await controller.settle() + // A photo the scan accepted whose JPEG is gone before the coordinator could stage it. + var lost = try fixture.photo(at: fixture.start + 4) + lost.jpeg = root.appending(path: "missing.jpg") + controller.kept(lost) + try #require(try await until { controller.status?.isFinal == true }, "no final stage: \(String(describing: controller.status))") + #expect(controller.status?.stage == .ended(.notPrepared)) + #expect(controller.status?.consent == .granted) + // Ended before the send: the capture is never sealed or finalized. + controller.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + await controller.settle() + #expect(controller.status?.stage == .ended(.notPrepared)) + #expect(!http.routes.contains("POST captures/finalize")) + } + /// While the service processes, the phone can no longer save the upload's state (its folder /// turned read-only): the real uploader ends and the scan says sending stopped on this phone, /// not that the service refused it, and nothing more is sent. A withdrawal that couldn't be From 766bd8fc3849749a0167bf98487e702edad1ffc7 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 13:41:19 -0500 Subject: [PATCH 64/71] Say the live meter tap is recorded but unchecked on a phone, not missing Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/Runtime/ScanEngine+Processing.swift | 7 ++++--- ios/README.md | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ios/HouseScan/Runtime/ScanEngine+Processing.swift b/ios/HouseScan/Runtime/ScanEngine+Processing.swift index 68f007b36..9c5967029 100644 --- a/ios/HouseScan/Runtime/ScanEngine+Processing.swift +++ b/ios/HouseScan/Runtime/ScanEngine+Processing.swift @@ -230,9 +230,10 @@ extension ScanEngine { // MARK: Setup /// What photo processing can do in this build. Sending a capture off the phone is off in this - /// slice: the 0.4 clock and privacy contract, a scoped credential, the live meter tap, and - /// the deployment aren't settled; per-frame intrinsics are recorded but unchecked on a phone. The one way to run it is DEBUG's fixture on a - /// replay, which answers on the phone (`CaptureFixtureLaunch`); a release build can't, since + /// slice: the 0.4 clock and privacy contract, a scoped credential and the deployment aren't + /// settled. Per-frame intrinsics and the meter tap from the accepted mark's own frame are + /// recorded but unchecked on a phone. The one way to run it is DEBUG's fixture on a replay, + /// which answers on the phone (`CaptureFixtureLaunch`); a release build can't, since /// `debugBuild` is decided at compile time and the launch options aren't even parsed there. static func photoProcessingSetup(_ options: LaunchOptions) -> (setup: PhotoProcessingController.Setup, synthetic: SyntheticCapture?) { #if DEBUG diff --git a/ios/README.md b/ios/README.md index a21abbceb..a456eb6c5 100644 --- a/ios/README.md +++ b/ios/README.md @@ -35,7 +35,7 @@ Native iPhone app for the AR capture walk. The homeowner marks the electric mete **Processing (beta).** Developer options offer two backends, both remote: **Legacy (remote checker)** sends scene.json to the placement server, and **Photo processing (beta)** sends the scan's photos and camera path to the capture API. Legacy is the default. A scan reads the choice once, when it starts, and keeps it with its endpoint in its `ScanContext` (HouseScanKit); a change applies to the next scan, and an App Store install always uses Legacy. A photo-processing scan asks before sending (`PhotoConsentCard`), sends photos as they are kept, and after the review goes to `STATE=processing` instead of the upload: its progress, then the service's answer quoted as written (`PhotoProcessingScreen`). It never falls back to the Legacy checker. An answer is shown only when the run it names, and the phone's own record of what it sent (the recording session and world epoch, the coordinator session, packet and capture), match this scan; that is a local association, since the service returns no digest of its input and no attestation. AR is withheld for every answer. Sending off the phone is off in every build for now, so choosing photo processing ends the scan saying it isn't set up. -Two debug-only switches run the path on a replay against `FixtureCaptureHTTP`, a stand-in for the capture API inside the app (the type is in every build; only its activation is debug-only): `-photoProcessingFixture` alone sends the replay's photos, whose packet always fails the phone's 0.4 checks because a replay has no motion data and too few poses, so the scan ends "couldn't be prepared"; adding `-photoProcessingSyntheticCapture` sends HouseScanKit's `SyntheticCapture` instead, labelled `source.kind: synthetic`, so the real coordinator and uploader carry it to the fixture's answer. Either way the answer is the fixture's, never a service's or a GPU reconstruction's, and the screens say so. Live captures now record each frame's intrinsics with its pose for the 0.4 pose stream, not yet checked on a phone. Still missing before live use: a scoped server-issued credential, the meter tap in the 0.4 packet, the clock and privacy contract, and an input digest or attestation from the service. The code is in `Runtime/ScanEngine+Processing.swift` and HouseScanKit's `Processing/`. +Two debug-only switches run the path on a replay against `FixtureCaptureHTTP`, a stand-in for the capture API inside the app (the type is in every build; only its activation is debug-only): `-photoProcessingFixture` alone sends the replay's photos, whose packet always fails the phone's 0.4 checks because a replay has no motion data and too few poses, so the scan ends "couldn't be prepared"; adding `-photoProcessingSyntheticCapture` sends HouseScanKit's `SyntheticCapture` instead, labelled `source.kind: synthetic`, so the real coordinator and uploader carry it to the fixture's answer. Either way the answer is the fixture's, never a service's or a GPU reconstruction's, and the screens say so. Live captures now record each frame's intrinsics with its pose for the 0.4 pose stream, and an accepted meter mark adds the packet's meter tap from that mark's own ARKit frame (`Runtime/LiveCapture+MeterTap.swift`); a mark that can't give one ends the capture as not prepared. Neither is checked on a phone yet. Still missing before live use: a scoped server-issued credential, the clock and privacy contract, and an input digest or attestation from the service. The code is in `Runtime/ScanEngine+Processing.swift` and HouseScanKit's `Processing/`. **Practice meter.** For trying the whole scan where there is no electric meter, such as indoors. Debug and TestFlight builds show a wrench button at the top of the first screen. Tap it, turn on **Practice meter**, tap Done, then start the scan. At "find the meter", tap any spot on a wall: the usual raycast and refusals apply, and a drawn sample meter appears there, fixed to the meter's anchor, on every camera screen after that. The close-up keeps the live camera's pose and depth, but stores a photo of the sample (made-up maker SAMPLEWORKS, number 12345678), and the real reader reads it. The walk, fog, marks, requests, spot check, upload, result and AR all run for real. Every screen shows a "Practice meter" badge, and `scan-stamp.json` has `"practice": true`; the packet and scene.json are unchanged. The switch is kept until turned off, and the button reads "Practice meter on" while it is. App Store installs never show the button and ignore a stored switch: the app asks StoreKit's `AppTransaction` which server signed the install (sandbox means TestFlight) and treats no answer as the App Store. It is not in the iOS Settings app, because a Settings.bundle ships in the same binary that TestFlight later promotes to the App Store. The code is in `Runtime/Practice/`, `UI/Practice/` and HouseScanKit's `Practice/PracticeMeter.swift`. From 3499e65adc0459f00d789ee30cba33df1721ac25 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 14:32:52 -0500 Subject: [PATCH 65/71] Split the iOS UI tests into three jobs and keep a cancelled job's results With ScreenStatesUITests in the journey job, run 37221532164 (#216 at 6fa43b9b) hit the 90-minute cap: 12 min of package tests and build, then a 75.3 min UI step that stopped after 107 of 109 tests. That run's per-class times put ScreenStatesUITests at 1892 s and the other journey classes at about 2475 s, so ScreenStatesUITests now runs in its own job, screenStates. Every one of the 119 UI tests still runs once under full-ui (32 + 77 + 10), the every-state audit is still the only test pull requests skip without the label, and each job keeps its 90-minute cap. The same run's journey artifact kept nothing: upload-artifact stopped at the first file under the half-written bundle's Staging/ that vanished while it was read. A new step copies the bundle first, lets only Staging/ entries vanish, and writes SNAPSHOT.txt saying whether the copy is complete and listing what vanished; any other error fails the step. The job's own result is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/ios.yml | 42 +++++++--- CONTRIBUTING.md | 2 +- ios/Tools/snapshot-xcresult.py | 88 ++++++++++++++++++++ ios/Tools/test_ci_tools.py | 148 +++++++++++++++++++++++++++++++++ ios/Tools/ui-test-groups.py | 47 +++++++---- 5 files changed, 297 insertions(+), 30 deletions(-) create mode 100644 ios/Tools/snapshot-xcresult.py create mode 100644 ios/Tools/test_ci_tools.py diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 07fc42123..c85f17de0 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -28,17 +28,20 @@ concurrency: jobs: ios-build: - # Two groups of the UI tests, one job each (ios/Tools/ui-test-groups.py holds their selectors - # and checks that together they run every test once). One job risks exceeding the cap below: - # #214's hosted UI step took 72.3 min, and photo processing's tests add about 7.7 min measured - # locally (BRIEF-19), before setup and the build. No combined run has been seen to fail. journey runs the package tests and the Debug - # device build; photoProcessing runs the unsigned Release build. fail-fast is off so a failure - # in one group still leaves the other's results. + # Three groups of the UI tests, one job each (ios/Tools/ui-test-groups.py holds their selectors + # and checks that together they run every test once). With two groups, journey hit the cap + # below in run 37221532164 (#216 at 6fa43b9b): 12 min of package tests and build, then a + # 75.3 min UI step that stopped after 107 of its 109 tests. That run's per-class times put + # ScreenStatesUITests at 1892 s (its every-state audit 905 s) and the other journey classes at + # about 2475 s, so ScreenStatesUITests has its own job. journey runs the package tests, the + # project check and the Debug device build; photoProcessing runs the unsigned Release build; + # screenStates runs only its UI tests. fail-fast is off so a failure in one group still leaves + # the others' results. name: iOS build (${{ matrix.group }}) strategy: fail-fast: false matrix: - group: [journey, photoProcessing] + group: [screenStates, journey, photoProcessing] # A free account runs 5 macOS jobs at once, shared by every open pull request, so drafts and # labels other than full-ui don't take a slot. if: >- @@ -53,7 +56,8 @@ jobs: # every-state audit short. 75 left room until the 2026-10-03 candidate's 86 UI cases: # run 37121918073 attempt 2 spent 10 min 34 s before the UI step and 64 min 39 s in it, # and the cap stopped it after 76 cases with 0 failures. 90 ran all of them; since photo - # processing's tests moved to their own job, 90 is the cap for each group. + # processing's tests moved to their own job, 90 is the cap for each group, and the groups are + # split so that each fits it rather than raising it. timeout-minutes: 90 env: # Newest stable Xcode on macos-26 per images/macos/macos-26-arm64-Readme.md (image 20260907). @@ -67,7 +71,7 @@ jobs: XCODEGEN_SHA256: 4d9e34b62172d645eed6457cac13fc222569974098ef4ee9c3368bedf0196806 # The every-state accessibility audit relaunches the app 65 times (about 10 min). Pull # requests skip it unless labelled full-ui; pushes and manual runs always include it. It is - # in the journey group; photoProcessing always runs its own audits. + # in the screenStates group; photoProcessing always runs its own audits. GROUP: ${{ matrix.group }} FULL_UI: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'full-ui') }} steps: @@ -84,9 +88,12 @@ jobs: xcodebuild -version swift --version - # Before anything slow: the two groups' selectors must cover every UI test exactly once. + # Before anything slow: the groups' selectors must cover every UI test exactly once, and the + # CI tools' own tests (the groups' check and the result snapshot) must pass. - name: Check the UI test groups - run: python3 ios/Tools/ui-test-groups.py check + run: | + python3 -m unittest discover -s ios/Tools -p 'test_*.py' + python3 ios/Tools/ui-test-groups.py check # Booting takes 2 to 3 minutes and needs nothing from the build, so it runs alongside the # steps below; the UI test step waits for it. @@ -188,7 +195,7 @@ jobs: # Replays the committed synthetic wall (ios/HouseScanUITests/Fixtures/synthetic-wall) with # -autopilot and the bundled sample result; photoProcessing adds its in-app fixture. run: | - if [ "$GROUP" = journey ] && [ "$FULL_UI" != "true" ]; then + if [ "$GROUP" = screenStates ] && [ "$FULL_UI" != "true" ]; then echo "::notice::Skipped the every-state accessibility audit (about 10 min). Add the full-ui label to run it on this pull request." fi # Read without mapfile: the runner's /bin/bash is 3.2. @@ -202,11 +209,20 @@ jobs: "${selectors[@]}" \ COMPILER_INDEX_STORE_ENABLE=NO test + # A run the cap cancels leaves the bundle half-written, and its Staging/ files vanish while + # they are read: run 37221532164's journey upload stopped at the first one (ENOENT) and kept + # nothing. This copies the bundle first, lets only Staging/ entries vanish, and lists them in + # SNAPSHOT.txt beside it ("complete" or "incomplete"); any other error fails this step. The + # job's own result, a cancellation included, stays as it was. + - name: Snapshot the UI test results + if: always() + run: python3 ios/Tools/snapshot-xcresult.py "$RUNNER_TEMP/HouseScanUITests.xcresult" "$RUNNER_TEMP/ui-test-results" + - name: Keep UI test screenshots if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ui-test-results-${{ matrix.group }} - path: ${{ runner.temp }}/HouseScanUITests.xcresult + path: ${{ runner.temp }}/ui-test-results if-no-files-found: ignore retention-days: 7 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d434dc5bc..784df22b6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,7 +38,7 @@ Greptile reviews new commits through [`.greptile/config.json`](.greptile/config. The iOS UI tests skip the every-state accessibility audit on pull requests, because it adds about 10 minutes and macOS runners are scarce. Add the `full-ui` label when a pull request changes screens or copy; pushes to `t3/ios-mvf` and `main` always run it. -The iOS check runs as two jobs, each under its own 90-minute cap: `journey` runs the package tests, the Debug build and every UI test class but `PhotoProcessingUITests`, and `photoProcessing` runs an unsigned Release build and that class with its own audits. `ios/Tools/ui-test-groups.py` holds both groups' test selectors; `python3 ios/Tools/ui-test-groups.py check` lists the UI tests from source and fails unless the two groups run each exactly once. Run it after adding or renaming a UI test class. +The iOS check runs as three jobs, each under its own 90-minute cap: `screenStates` runs `ScreenStatesUITests`, the every-state audit included under `full-ui`; `photoProcessing` runs an unsigned Release build and `PhotoProcessingUITests` with its own audits; and `journey` runs the package tests, the Debug build and every other UI test class. `ios/Tools/ui-test-groups.py` holds the groups' test selectors; `python3 ios/Tools/ui-test-groups.py check` lists the UI tests from source and fails unless the groups run each exactly once. Run it after adding or renaming a UI test class. Each job keeps its result bundle as the artifact `ui-test-results-`, with `SNAPSHOT.txt` saying whether the copy is complete; a job the cap cancelled can leave it incomplete. `make check` runs the server, web and iOS suites, then `make scoring`, `make measure-lab`, `make evals`, `make recon` and `make meter-closeup` for each of those folders the branch has. They need uv, Node 24 with pnpm, and Xcode 26 or newer; each directory's README has details. No check is required by branch rules yet. Don't call one required until the rules require its status. diff --git a/ios/Tools/snapshot-xcresult.py b/ios/Tools/snapshot-xcresult.py new file mode 100644 index 000000000..493cf308e --- /dev/null +++ b/ios/Tools/snapshot-xcresult.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Copy a UI test result bundle to a stable folder before CI uploads it, and say what is missing. + + snapshot-xcresult.py + +Why: when the job's time cap cancels xcodebuild, the bundle is left half-written, and files under +its `Staging/` folder go away while they are being read. Run 37221532164 (#216 at 6fa43b9b) lost +its whole journey artifact that way: upload-artifact stopped at the first vanished file (ENOENT on +`Staging/1_Test/Diagnostics/...`) and kept nothing, screenshots of the failed audit included. + +What it does: copies every file of `` into `/`, then writes +`/SNAPSHOT.txt` saying "complete" or "incomplete" and listing each path that vanished. +Only a file or folder under the bundle's top-level `Staging/` may vanish; that is the transient +state of a cancelled run. Any other error stops with exit 1 and names the path, so a broken bundle +is never passed off as a partial one. A missing bundle (the UI step never started) writes nothing +and exits 0, as the upload's `if-no-files-found: ignore` did. The job's own result, a cancellation +included, is not changed by this step. +""" +import os +import pathlib +import shutil +import sys + +EPHEMERAL = "Staging" + + +def is_ephemeral(relative): + parts = pathlib.PurePath(relative).parts + return bool(parts) and parts[0] == EPHEMERAL + + +def snapshot(bundle, out, copy=shutil.copy2): + """Copies `bundle` into `out`. Returns (copied, vanished): a file count and the relative paths + under Staging/ that disappeared mid-copy. Raises on any other error.""" + bundle = pathlib.Path(bundle) + out = pathlib.Path(out) + copied = 0 + vanished = [] + walk_errors = [] + for root, dirs, files in os.walk(bundle, onerror=walk_errors.append): + rel_root = pathlib.Path(root).relative_to(bundle) + (out / rel_root).mkdir(parents=True, exist_ok=True) + for name in sorted(dirs): + if os.path.islink(os.path.join(root, name)): + os.symlink(os.readlink(os.path.join(root, name)), out / rel_root / name) + for name in sorted(files): + relative = rel_root / name + try: + copy(os.path.join(root, name), out / relative, follow_symlinks=False) + copied += 1 + except FileNotFoundError: + if not is_ephemeral(relative): + raise + vanished.append(str(relative)) + for error in walk_errors: + relative = pathlib.Path(error.filename).relative_to(bundle) + if not (isinstance(error, FileNotFoundError) and is_ephemeral(relative)): + raise error + vanished.append(f"{relative}/") + return copied, sorted(vanished) + + +def main(argv): + if len(argv) != 3: + sys.exit(__doc__) + bundle, folder = pathlib.Path(argv[1]), pathlib.Path(argv[2]) + if not bundle.exists(): + print(f"no result bundle at {bundle}; nothing to keep") + return 0 + try: + copied, vanished = snapshot(bundle, folder / bundle.name) + except OSError as error: + print(f"::error::copying {bundle} failed outside its transient {EPHEMERAL}/ folder: {error}") + return 1 + if vanished: + report = [f"incomplete: copied {copied} files; {len(vanished)} entries under {EPHEMERAL}/ vanished while copying"] + report += vanished + print(f"::warning::The UI test results are incomplete: {len(vanished)} entries under {EPHEMERAL}/ vanished while copying (listed in SNAPSHOT.txt).") + else: + report = [f"complete: copied {copied} files"] + folder.mkdir(parents=True, exist_ok=True) + (folder / "SNAPSHOT.txt").write_text("\n".join(report) + "\n", encoding="utf-8") + print(report[0]) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/ios/Tools/test_ci_tools.py b/ios/Tools/test_ci_tools.py new file mode 100644 index 000000000..5be0fe2d1 --- /dev/null +++ b/ios/Tools/test_ci_tools.py @@ -0,0 +1,148 @@ +"""Tests for the CI tools in this folder: the UI test groups' check and the result-bundle snapshot. + + python3 -m unittest discover -s ios/Tools -p 'test_*.py' +""" +import contextlib +import importlib.util +import io +import pathlib +import shutil +import tempfile +import unittest + +HERE = pathlib.Path(__file__).resolve().parent + + +def load(filename, name): + spec = importlib.util.spec_from_file_location(name, HERE / filename) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +groups = load("ui-test-groups.py", "ui_test_groups") +snap = load("snapshot-xcresult.py", "snapshot_xcresult") + + +def quiet(function, *args): + with contextlib.redirect_stdout(io.StringIO()) as out: + result = function(*args) + return result, out.getvalue() + + +class UITestGroupsTests(unittest.TestCase): + def test_the_committed_groups_pass(self): + result, out = quiet(groups.check) + self.assertEqual(result, 0, out) + self.assertIn("overlap: 0", out) + self.assertIn("missing: 0", out) + + def test_a_class_in_two_groups_fails(self): + original = groups.selectors + def overlapping(group, full_ui): + args = original(group, full_ui) + return [a for a in args if a != f"-skip-testing:{groups.TARGET}/{groups.SCREEN_CLASS}"] if group == "journey" else args + groups.selectors = overlapping + try: + result, out = quiet(groups.check) + finally: + groups.selectors = original + self.assertEqual(result, 1) + self.assertIn("in more than one group", out) + + def test_a_class_in_no_group_fails(self): + original = groups.selectors + def dropping(group, full_ui): + return [f"-only-testing:{groups.TARGET}/{groups.PHOTO_CLASS}"] if group == "screenStates" else original(group, full_ui) + groups.selectors = dropping + try: + result, out = quiet(groups.check) + finally: + groups.selectors = original + self.assertEqual(result, 1) + self.assertIn("in no group", out) + self.assertIn("screenStates must hold ScreenStatesUITests and nothing else", out) + + +class SnapshotTests(unittest.TestCase): + def setUp(self): + self.root = pathlib.Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.root) + self.bundle = self.root / "HouseScanUITests.xcresult" + for path in ["Info.plist", "Data/data.0", "Staging/1_Test/Attachments/shot.png", "Staging/1_Test/Diagnostics/log.txt", "Staging/note.txt"]: + file = self.bundle / path + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(path) + self.out = self.root / "out" + + def test_a_finished_bundle_is_copied_whole(self): + copied, vanished = snap.snapshot(self.bundle, self.out / self.bundle.name) + self.assertEqual((copied, vanished), (5, [])) + self.assertEqual((self.out / self.bundle.name / "Staging/1_Test/Attachments/shot.png").read_text(), "Staging/1_Test/Attachments/shot.png") + + def test_a_staging_file_that_vanishes_is_listed_and_the_rest_kept(self): + def copy(source, target, follow_symlinks): + if source.endswith("log.txt"): + raise FileNotFoundError(source) + shutil.copy2(source, target, follow_symlinks=follow_symlinks) + copied, vanished = snap.snapshot(self.bundle, self.out, copy=copy) + self.assertEqual(copied, 4) + self.assertEqual(vanished, ["Staging/1_Test/Diagnostics/log.txt"]) + self.assertTrue((self.out / "Staging/1_Test/Attachments/shot.png").exists()) + + def test_a_staging_folder_that_vanishes_is_listed(self): + def copy(source, target, follow_symlinks): + shutil.copy2(source, target, follow_symlinks=follow_symlinks) + if source.endswith("note.txt"): + shutil.rmtree(self.bundle / "Staging/1_Test") + copied, vanished = snap.snapshot(self.bundle, self.out, copy=copy) + self.assertEqual(vanished, ["Staging/1_Test/"]) + self.assertEqual(copied, 3) + + def test_a_file_outside_staging_that_vanishes_stops_the_copy(self): + def copy(source, target, follow_symlinks): + if source.endswith("data.0"): + raise FileNotFoundError(source) + shutil.copy2(source, target, follow_symlinks=follow_symlinks) + with self.assertRaises(FileNotFoundError): + snap.snapshot(self.bundle, self.out, copy=copy) + + def test_main_reports_incomplete_and_exits_zero(self): + original = snap.snapshot + snap.snapshot = lambda bundle, out: (4, ["Staging/1_Test/Diagnostics/log.txt"]) + try: + result, out = quiet(snap.main, ["snapshot", str(self.bundle), str(self.out)]) + finally: + snap.snapshot = original + self.assertEqual(result, 0) + self.assertIn("::warning::", out) + report = (self.out / "SNAPSHOT.txt").read_text() + self.assertTrue(report.startswith("incomplete: copied 4 files; 1 entries under Staging/ vanished")) + self.assertIn("Staging/1_Test/Diagnostics/log.txt", report) + + def test_main_reports_complete(self): + result, out = quiet(snap.main, ["snapshot", str(self.bundle), str(self.out)]) + self.assertEqual(result, 0) + self.assertEqual((self.out / "SNAPSHOT.txt").read_text(), "complete: copied 5 files\n") + + def test_main_fails_loudly_outside_staging(self): + original = snap.snapshot + def broken(bundle, out): + raise PermissionError(13, "denied", str(self.bundle / "Data/data.0")) + snap.snapshot = broken + try: + result, out = quiet(snap.main, ["snapshot", str(self.bundle), str(self.out)]) + finally: + snap.snapshot = original + self.assertEqual(result, 1) + self.assertIn("::error::", out) + self.assertFalse((self.out / "SNAPSHOT.txt").exists()) + + def test_no_bundle_keeps_nothing(self): + result, out = quiet(snap.main, ["snapshot", str(self.root / "missing.xcresult"), str(self.out)]) + self.assertEqual(result, 0) + self.assertFalse(self.out.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/ios/Tools/ui-test-groups.py b/ios/Tools/ui-test-groups.py index 66ff7d88f..f190422ae 100755 --- a/ios/Tools/ui-test-groups.py +++ b/ios/Tools/ui-test-groups.py @@ -1,9 +1,14 @@ #!/usr/bin/env python3 -"""The two hosted UI test groups, and a check that together they run every UI test once. +"""The three hosted UI test groups, and a check that together they run every UI test once. -CI splits HouseScanUITests across two jobs so neither passes the 90-minute cap: `journey`, every -class but PhotoProcessingUITests, and `photoProcessing`, that class alone. The workflow takes each -group's xcodebuild selectors from here, so the check below reads the same selectors CI passes. +CI splits HouseScanUITests across three jobs so none passes the 90-minute cap: `screenStates`, +ScreenStatesUITests alone; `photoProcessing`, PhotoProcessingUITests alone; and `journey`, every +other class. With ScreenStatesUITests in `journey`, run 37221532164 (#216 at 6fa43b9b) hit the +cap: 12 min of package tests and build before the UI step, which ran 75.3 min and stopped with +107 of 109 tests run. That run's per-class test times put ScreenStatesUITests at 1892 s (its +every-state audit 905 s) and the remaining journey classes at about 2475 s. The workflow takes +each group's xcodebuild selectors from here, so the check below reads the same selectors CI +passes. ui-test-groups.py args [--full-ui true|false] print the group's selectors, one per line ui-test-groups.py check list the tests from source and check them @@ -12,7 +17,8 @@ class but PhotoProcessingUITests, and `photoProcessing`, that class alone. The w which tests each group's selectors select as xcodebuild does (an -only-testing prefix includes, a -skip-testing prefix excludes), and fails unless, with the full audit on, each test is in exactly one group; with it off, the only test left out must be the every-state audit, the existing -pull-request policy. It also fails on a selector that names no class or test. It reads source +pull-request policy. `screenStates` and `photoProcessing` must each hold their one class and +nothing else. It also fails on a selector that names no class or test. It reads source only, so a test hidden behind `#if` would be listed although the build leaves it out; there is none today. """ @@ -22,14 +28,19 @@ class but PhotoProcessingUITests, and `photoProcessing`, that class alone. The w TARGET = "HouseScanUITests" PHOTO_CLASS = "PhotoProcessingUITests" +SCREEN_CLASS = "ScreenStatesUITests" # Pull requests skip this without the full-ui label (CONTRIBUTING.md, "Checks"). -FULL_UI_ONLY = f"{TARGET}/ScreenStatesUITests/testEveryStatePassesTheAccessibilityAudit" -GROUPS = ("journey", "photoProcessing") +FULL_UI_ONLY = f"{TARGET}/{SCREEN_CLASS}/testEveryStatePassesTheAccessibilityAudit" +GROUPS = ("screenStates", "journey", "photoProcessing") +# The groups that hold exactly one class, and that class. +SINGLE_CLASS = {"screenStates": SCREEN_CLASS, "photoProcessing": PHOTO_CLASS} def selectors(group, full_ui): if group == "journey": - args = [f"-only-testing:{TARGET}", f"-skip-testing:{TARGET}/{PHOTO_CLASS}"] + return [f"-only-testing:{TARGET}", f"-skip-testing:{TARGET}/{PHOTO_CLASS}", f"-skip-testing:{TARGET}/{SCREEN_CLASS}"] + if group == "screenStates": + args = [f"-only-testing:{TARGET}/{SCREEN_CLASS}"] if not full_ui: args.append(f"-skip-testing:{FULL_UI_ONLY}") return args @@ -83,22 +94,26 @@ def check(): if prefix != TARGET and not any(t == prefix or t.startswith(prefix + "/") for t in names): problems.append(f"{group}: {arg} names no test") full = {g: selected(names, selectors(g, True)) for g in GROUPS} - overlap = full["journey"] & full["photoProcessing"] - missing = names - full["journey"] - full["photoProcessing"] pr = {g: selected(names, selectors(g, False)) for g in GROUPS} - left_out = names - pr["journey"] - pr["photoProcessing"] + overlap = {t for t in names if sum(t in full[g] for g in GROUPS) > 1} + missing = names - set().union(*full.values()) + left_out = names - set().union(*pr.values()) print(f"tests in source: {len(names)}") - print(f"journey: {len(full['journey'])} (pull request without full-ui: {len(pr['journey'])})") - print(f"photoProcessing: {len(full['photoProcessing'])} (pull request without full-ui: {len(pr['photoProcessing'])})") + for group in GROUPS: + print(f"{group}: {len(full[group])} (pull request without full-ui: {len(pr[group])})") print(f"overlap: {len(overlap)}") print(f"missing: {len(missing)}") print(f"left out without full-ui: {sorted(left_out)}") - problems += [f"in both groups: {t}" for t in sorted(overlap)] + problems += [f"in more than one group: {t}" for t in sorted(overlap)] problems += [f"in no group: {t}" for t in sorted(missing)] if left_out != {FULL_UI_ONLY}: problems.append(f"without full-ui exactly {FULL_UI_ONLY} should be left out, not {sorted(left_out)}") - if not full["photoProcessing"] or any(not t.startswith(f"{TARGET}/{PHOTO_CLASS}/") for t in full["photoProcessing"]): - problems.append("photoProcessing must hold PhotoProcessingUITests and nothing else") + for group, cls in SINGLE_CLASS.items(): + if not full[group] or any(not t.startswith(f"{TARGET}/{cls}/") for t in full[group]): + problems.append(f"{group} must hold {cls} and nothing else") + for group in GROUPS: + if not pr[group]: + problems.append(f"{group} runs no test on a pull request without full-ui") for problem in problems: print(f"error: {problem}") return 1 if problems else 0 From e564760270d2128857e05bda1107ae79cac203bf Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 14:39:36 -0500 Subject: [PATCH 66/71] Wait for the background Simulator boot before checking it The screenStates job has no build between starting the background boot and waiting for it, so its wait ran 3 s later and a second `simctl bootstatus -b` exited 149 while the first boot was under way (run 37228728303). The wait now lets the background boot finish first, then checks the device as before, and prints that boot's log if the device still isn't booted. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/ios.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index c85f17de0..24cb5a906 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -110,6 +110,7 @@ jobs: fi echo "SIM_UDID=$udid" >> "$GITHUB_ENV" nohup xcrun simctl bootstatus "$udid" -b > "$RUNNER_TEMP/simulator-boot.log" 2>&1 & + echo "SIM_BOOT_PID=$!" >> "$GITHUB_ENV" # The package's drift test compares the vendored server schemas with the server's own copies: # the checkout's files once that branch merges, else the t3/server branch, which the shallow @@ -185,10 +186,18 @@ jobs: xcodebuild -project ios/HouseScan.xcodeproj -scheme HouseScan -configuration Release -destination "generic/platform=iOS" CODE_SIGNING_ALLOWED=NO COMPILER_INDEX_STORE_ENABLE=NO build + # The background boot first: screenStates reaches this step 3 s after starting it, and a + # second `bootstatus -b` while that boot was under way exited 149 (run 37228728303). Once it + # has finished, `-b` returns at once on a booted device, and boots it if the first try failed. - name: Wait for the Simulator timeout-minutes: 10 run: | - xcrun simctl bootstatus "$SIM_UDID" -b + while kill -0 "$SIM_BOOT_PID" 2>/dev/null; do sleep 2; done + if ! xcrun simctl bootstatus "$SIM_UDID" -b; then + echo "::error::$SIM_NAME on iOS $SIM_OS did not boot. The background boot's log follows." + cat "$RUNNER_TEMP/simulator-boot.log" + exit 1 + fi echo "Booted $SIM_NAME on iOS $SIM_OS ($SIM_UDID)." - name: UI test the group in the Simulator From f6231b4fab5dbaf09745f02385282ae3eb8a238a Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 15:08:38 -0500 Subject: [PATCH 67/71] Count the photo-processing suites' waits in their own turns The PhotoProcessing, ScopedPhotoProcessing, MeterTap and SyntheticCapture suites waited on a wall-clock deadline. In run 37229126189 a 10 s one gave up on an upload that takes about 4 s locally, with the package suite running 276 s against about 105 s locally; run 37219668935 had stalled every task for about 42 s. Their until helpers now share countedWait, the semantics CaptureUploaderTests.settles took in 6c1f68ea: a budget of the waiter's own 20 ms turns, a final read after the last wait, and the caller's isolation. Each caller keeps its budget; the fixed sleeps that check nothing more is sent are unchanged. CountedWaitTests covers a condition that never holds, one that holds only at the final read, and one that already holds. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Tests/HouseScanKitTests/CountedWait.swift | 47 +++++++++++++++++++ .../MeterTapObservationTests.swift | 5 +- .../PhotoProcessingTests.swift | 5 +- .../ScopedPhotoProcessingTests.swift | 5 +- .../SyntheticCaptureTests.swift | 5 +- 5 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/CountedWait.swift diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CountedWait.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CountedWait.swift new file mode 100644 index 000000000..db0012ea9 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CountedWait.swift @@ -0,0 +1,47 @@ +import Testing + +/// Waits until `condition` holds, as a hang safeguard and not a timing requirement: the photo +/// processing suites' `until` helpers call this so a capture that never reaches a state fails its +/// test instead of hanging the run. +/// +/// The budget is this waiter's own turns, `seconds * 50` waits of 20 ms or more, not wall time, the +/// semantics `CaptureUploaderTests.settles` took in 6c1f68ea. A stalled test process holds the code +/// under test and this waiter up together, and a wall-clock deadline can expire before either gets +/// to run: hosted run 37219668935 stalled every task for about 42 s, and in run 37229126189 a 10 s +/// wall-clock wait here gave up on an upload that takes about 4 s locally. `condition` is read once +/// more after the last wait, so a state reached during that wait still counts. It runs in the +/// caller's isolation (`#isolation`), so a main-actor suite's condition is read on the main actor +/// and never sent across. +func countedWait( + _ seconds: Double, isolation: isolated (any Actor)? = #isolation, _ condition: () -> Bool +) async throws -> Bool { + for _ in 0..= .milliseconds(200)) + #expect(reads == 11) + } + + @Test func aConditionThatHoldsOnlyAtTheFinalReadCounts() async throws { + var reads = 0 + #expect(try await countedWait(0.2) { reads += 1; return reads == 11 }) + } + + @Test func aConditionThatAlreadyHoldsReturnsWithoutWaiting() async throws { + var reads = 0 + #expect(try await countedWait(30) { reads += 1; return true }) + #expect(reads == 1) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/MeterTapObservationTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/MeterTapObservationTests.swift index 5c8f48ecc..d014a3c58 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/MeterTapObservationTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/MeterTapObservationTests.swift @@ -118,10 +118,9 @@ extension Result { return (try frame.observation(hit: NativeCaptureFixture.meter, jpeg: { jpeg }).get(), frame.tapHit(NativeCaptureFixture.meter, estimatedPlane: true)) } + /// A hang safeguard counted in the waiter's own turns (`countedWait`), not wall time. func until(_ seconds: Double = 30, _ condition: () -> Bool) async throws -> Bool { - let deadline = ContinuousClock.now + .seconds(seconds) - while !condition(), ContinuousClock.now < deadline { try await Task.sleep(for: .milliseconds(20)) } - return condition() + try await countedWait(seconds, condition) } /// The forwarded tap is sealed with a ray, rebuilt from its keyframe's pose and intrinsics, diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift index 8e8bd6193..ddd8d64cb 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/PhotoProcessingTests.swift @@ -181,10 +181,9 @@ final class HeldResultHTTP: CaptureHTTP, Sendable { controller.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) } + /// A hang safeguard counted in the waiter's own turns (`countedWait`), not wall time. func until(_ seconds: Double = 30, _ condition: () -> Bool) async throws -> Bool { - let deadline = ContinuousClock.now + .seconds(seconds) - while !condition(), ContinuousClock.now < deadline { try await Task.sleep(for: .milliseconds(20)) } - return condition() + try await countedWait(seconds, condition) } @Test func theFixtureWritesTheCoordinatorsEpoch() { diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedPhotoProcessingTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedPhotoProcessingTests.swift index f23f5403c..7e1012fd8 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedPhotoProcessingTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/ScopedPhotoProcessingTests.swift @@ -59,10 +59,9 @@ final class AuthorizingStorageHTTP: CaptureHTTP, Sendable { ScopedCaptureHTTP(scope: try CaptureAPIScope(base: base), credential: credential, inner: inner) } + /// A hang safeguard counted in the waiter's own turns (`countedWait`), not wall time. func until(_ seconds: Double = 30, _ condition: () -> Bool) async throws -> Bool { - let deadline = ContinuousClock.now + .seconds(seconds) - while !condition(), ContinuousClock.now < deadline { try await Task.sleep(for: .milliseconds(20)) } - return condition() + try await countedWait(seconds, condition) } func run(_ controller: PhotoProcessingController) async throws { diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/SyntheticCaptureTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/SyntheticCaptureTests.swift index 8f2e4f5f8..ef14c31ce 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/SyntheticCaptureTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/SyntheticCaptureTests.swift @@ -73,10 +73,9 @@ import Testing return PhotoProcessingController(setup: .ready(environment, standIn: true)) } + /// A hang safeguard counted in the waiter's own turns (`countedWait`), not wall time. func until(_ condition: () -> Bool) async throws -> Bool { - let deadline = ContinuousClock.now + .seconds(30) - while !condition(), ContinuousClock.now < deadline { try await Task.sleep(for: .milliseconds(20)) } - return condition() + try await countedWait(30, condition) } /// The capture through the real coordinator and uploader: a packet the phone's checks pass, From e66226cbfc9a092031900543603c7f4b386fbbf7 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 15:29:10 -0500 Subject: [PATCH 68/71] Mark a failed result snapshot as partial, and finish the README's live-use sentence When copying the result bundle failed outside Staging/, the step exited without SNAPSHOT.txt, but the upload after it still kept the partial copy unmarked (Greptile on #216). The step now writes SNAPSHOT.txt saying "failed" and that the copy beside it is partial before exiting 1. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/README.md | 2 +- ios/Tools/snapshot-xcresult.py | 9 +++++++-- ios/Tools/test_ci_tools.py | 5 ++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ios/README.md b/ios/README.md index a456eb6c5..cdd5e143d 100644 --- a/ios/README.md +++ b/ios/README.md @@ -35,7 +35,7 @@ Native iPhone app for the AR capture walk. The homeowner marks the electric mete **Processing (beta).** Developer options offer two backends, both remote: **Legacy (remote checker)** sends scene.json to the placement server, and **Photo processing (beta)** sends the scan's photos and camera path to the capture API. Legacy is the default. A scan reads the choice once, when it starts, and keeps it with its endpoint in its `ScanContext` (HouseScanKit); a change applies to the next scan, and an App Store install always uses Legacy. A photo-processing scan asks before sending (`PhotoConsentCard`), sends photos as they are kept, and after the review goes to `STATE=processing` instead of the upload: its progress, then the service's answer quoted as written (`PhotoProcessingScreen`). It never falls back to the Legacy checker. An answer is shown only when the run it names, and the phone's own record of what it sent (the recording session and world epoch, the coordinator session, packet and capture), match this scan; that is a local association, since the service returns no digest of its input and no attestation. AR is withheld for every answer. Sending off the phone is off in every build for now, so choosing photo processing ends the scan saying it isn't set up. -Two debug-only switches run the path on a replay against `FixtureCaptureHTTP`, a stand-in for the capture API inside the app (the type is in every build; only its activation is debug-only): `-photoProcessingFixture` alone sends the replay's photos, whose packet always fails the phone's 0.4 checks because a replay has no motion data and too few poses, so the scan ends "couldn't be prepared"; adding `-photoProcessingSyntheticCapture` sends HouseScanKit's `SyntheticCapture` instead, labelled `source.kind: synthetic`, so the real coordinator and uploader carry it to the fixture's answer. Either way the answer is the fixture's, never a service's or a GPU reconstruction's, and the screens say so. Live captures now record each frame's intrinsics with its pose for the 0.4 pose stream, and an accepted meter mark adds the packet's meter tap from that mark's own ARKit frame (`Runtime/LiveCapture+MeterTap.swift`); a mark that can't give one ends the capture as not prepared. Neither is checked on a phone yet. Still missing before live use: a scoped server-issued credential, the clock and privacy contract, and an input digest or attestation from the service. The code is in `Runtime/ScanEngine+Processing.swift` and HouseScanKit's `Processing/`. +Two debug-only switches run the path on a replay against `FixtureCaptureHTTP`, a stand-in for the capture API inside the app (the type is in every build; only its activation is debug-only): `-photoProcessingFixture` alone sends the replay's photos, whose packet always fails the phone's 0.4 checks because a replay has no motion data and too few poses, so the scan ends "couldn't be prepared"; adding `-photoProcessingSyntheticCapture` sends HouseScanKit's `SyntheticCapture` instead, labelled `source.kind: synthetic`, so the real coordinator and uploader carry it to the fixture's answer. Either way the answer is the fixture's, never a service's or a GPU reconstruction's, and the screens say so. Live captures now record each frame's intrinsics with its pose for the 0.4 pose stream, and an accepted meter mark adds the packet's meter tap from that mark's own ARKit frame (`Runtime/LiveCapture+MeterTap.swift`); a mark that can't give one ends the capture as not prepared. Neither is checked on a phone yet. Live use still requires a scoped server-issued credential, the clock and privacy contract, and an input digest or attestation from the service. The code is in `Runtime/ScanEngine+Processing.swift` and HouseScanKit's `Processing/`. **Practice meter.** For trying the whole scan where there is no electric meter, such as indoors. Debug and TestFlight builds show a wrench button at the top of the first screen. Tap it, turn on **Practice meter**, tap Done, then start the scan. At "find the meter", tap any spot on a wall: the usual raycast and refusals apply, and a drawn sample meter appears there, fixed to the meter's anchor, on every camera screen after that. The close-up keeps the live camera's pose and depth, but stores a photo of the sample (made-up maker SAMPLEWORKS, number 12345678), and the real reader reads it. The walk, fog, marks, requests, spot check, upload, result and AR all run for real. Every screen shows a "Practice meter" badge, and `scan-stamp.json` has `"practice": true`; the packet and scene.json are unchanged. The switch is kept until turned off, and the button reads "Practice meter on" while it is. App Store installs never show the button and ignore a stored switch: the app asks StoreKit's `AppTransaction` which server signed the install (sandbox means TestFlight) and treats no answer as the App Store. It is not in the iOS Settings app, because a Settings.bundle ships in the same binary that TestFlight later promotes to the App Store. The code is in `Runtime/Practice/`, `UI/Practice/` and HouseScanKit's `Practice/PracticeMeter.swift`. diff --git a/ios/Tools/snapshot-xcresult.py b/ios/Tools/snapshot-xcresult.py index 493cf308e..04027cc23 100644 --- a/ios/Tools/snapshot-xcresult.py +++ b/ios/Tools/snapshot-xcresult.py @@ -11,8 +11,9 @@ What it does: copies every file of `` into `/`, then writes `/SNAPSHOT.txt` saying "complete" or "incomplete" and listing each path that vanished. Only a file or folder under the bundle's top-level `Staging/` may vanish; that is the transient -state of a cancelled run. Any other error stops with exit 1 and names the path, so a broken bundle -is never passed off as a partial one. A missing bundle (the UI step never started) writes nothing +state of a cancelled run. Any other error stops with exit 1 and names the path; SNAPSHOT.txt then +says "failed" and that the copy beside it is partial, since the upload after this step keeps +whatever was copied. A broken bundle is never passed off as a complete or merely incomplete one. A missing bundle (the UI step never started) writes nothing and exits 0, as the upload's `if-no-files-found: ignore` did. The job's own result, a cancellation included, is not changed by this step. """ @@ -71,6 +72,10 @@ def main(argv): copied, vanished = snapshot(bundle, folder / bundle.name) except OSError as error: print(f"::error::copying {bundle} failed outside its transient {EPHEMERAL}/ folder: {error}") + folder.mkdir(parents=True, exist_ok=True) + (folder / "SNAPSHOT.txt").write_text( + f"failed: copying stopped at an error outside {EPHEMERAL}/; the copy beside this file is partial\n{error}\n", + encoding="utf-8") return 1 if vanished: report = [f"incomplete: copied {copied} files; {len(vanished)} entries under {EPHEMERAL}/ vanished while copying"] diff --git a/ios/Tools/test_ci_tools.py b/ios/Tools/test_ci_tools.py index 5be0fe2d1..749a5dc09 100644 --- a/ios/Tools/test_ci_tools.py +++ b/ios/Tools/test_ci_tools.py @@ -136,7 +136,10 @@ def broken(bundle, out): snap.snapshot = original self.assertEqual(result, 1) self.assertIn("::error::", out) - self.assertFalse((self.out / "SNAPSHOT.txt").exists()) + # The upload keeps whatever was copied, so the partial copy is marked as failed. + report = (self.out / "SNAPSHOT.txt").read_text() + self.assertTrue(report.startswith("failed: copying stopped at an error outside Staging/; the copy beside this file is partial")) + self.assertIn("data.0", report) def test_no_bundle_keeps_nothing(self): result, out = quiet(snap.main, ["snapshot", str(self.root / "missing.xcresult"), str(self.out)]) From 94503ed6ea952075c0ce901a484152b0af19cde2 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 16:12:13 -0500 Subject: [PATCH 69/71] Fold a follow-up gap request's card at the largest text sizes At AX5 a request the finished check sent back filled the camera: "One more view to finish", the stretch's two ends and the check's words, with "I can't get there" cut off at the bottom edge. Hosted audits failed its contrast there in 2 of 3 runs (37221532164, 37231825488). It now folds the way the past_end card already folds around the circle. The side to aim at leads ("Show the ground to the right", ScanCopy.serverLead). The follow-up, the stretch and the check's own words open under Details, and "I can't get there" moves to the actions as the same MovedReplyButton, with the same identifier, hint, action and 600 ms lock. The camera keeps CameraChromeLayout.minCameraWindow (180 pt). This applies only to server requests in their own words at accessibility sizes, not past_end requests, which fold around the circle, and not while coaching replaces the request. The default size, other gap requests and other screens are unchanged. No new motion. ScreenStatesUITests gains a gapRequest-followUp-AX5 expectation for the folded lead and Details; the audit itself is unchanged. GapFollowUpFoldUITests asserts the frames: the default card's reply as its trailing pill, the folded reply wholly on screen below the card with at least 180 pt of camera, enabled after its lock, and every word under Details. ui-test-groups.py check: 122 tests, overlap 0, missing 0. Validated: see the Simulator job in the slice report. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/UI/Copy/ScanCopy.swift | 16 ++- .../UI/Screens/GapRequestScreen.swift | 35 +++-- .../GapFollowUpFoldUITests.swift | 127 ++++++++++++++++++ .../ScreenStatesUITests.swift | 2 + 4 files changed, 169 insertions(+), 11 deletions(-) create mode 100644 ios/HouseScanUITests/GapFollowUpFoldUITests.swift diff --git a/ios/HouseScan/UI/Copy/ScanCopy.swift b/ios/HouseScan/UI/Copy/ScanCopy.swift index d815e6720..d75feca3f 100644 --- a/ios/HouseScan/UI/Copy/ScanCopy.swift +++ b/ios/HouseScan/UI/Copy/ScanCopy.swift @@ -507,6 +507,16 @@ enum ScanCopy { /// The card for a gap request. A server request can run along much of the wall, so its /// stretch is named by its two ends, all of it (issue #75, `Distance.range`); the phone's own /// requests are short and named by their middle. + /// A server request's lead when folded: the band and which side of the meter, in a few words + /// that stay a short block at AX5. The stretch's two ends are under Details, and the camera and + /// the map mark the stretch itself. + static func serverLead(_ gap: GapRequest) -> String { + let band = gap.band == .ground ? "ground" : "wall" + if gap.span.lowerBound >= 0 { return "Show the \(band) to the right" } + if gap.span.upperBound <= 0 { return "Show the \(band) to the left" } + return "Show the \(band) by your meter" + } + static func gap(_ gap: GapRequest) -> Instruction { let place = Distance.aroundFromMeter(gap.span) let stretch = Distance.range(gap.span) @@ -519,7 +529,11 @@ enum ScanCopy { let title = gap.band == .ground ? "Show the ground \(stretch)" : "Show the wall \(stretch)" // A past_end request asks to walk on past an end; the wall may really stop before // that, and the screen offers the walk's own "Wall ends here" for it (B-12). - guard let side = gap.pastEndSide else { return Instruction(title: title, detail: detail) } + // Folded at the largest text sizes on a follow-up (`GapRequestScreen.followUpFolds`): + // where to aim leads, and the whole request goes under Details. + guard let side = gap.pastEndSide else { + return Instruction(title: title, detail: detail, folded: Instruction.Folded(title: serverLead(gap), detail: "\(title). \(detail)")) + } // Folded, walking on leads, as the request asks; marking the end is under Details, // with the circle and "Wall ends here" in view. return Instruction( diff --git a/ios/HouseScan/UI/Screens/GapRequestScreen.swift b/ios/HouseScan/UI/Screens/GapRequestScreen.swift index 63577b486..e03f2d9f7 100644 --- a/ios/HouseScan/UI/Screens/GapRequestScreen.swift +++ b/ios/HouseScan/UI/Screens/GapRequestScreen.swift @@ -59,7 +59,7 @@ struct GapRequestScreen: View { CameraChrome( instruction: instruction, tone: tone, - reply: state.gap?.isSatisfied == true || asking || askingEnd || endAimFolds ? nil : InstructionCard.Reply( + reply: state.gap?.isSatisfied == true || asking || askingEnd || folds ? nil : InstructionCard.Reply( title: "I can't get there", identifier: "action.skipGap", hint: skipHint, @@ -68,7 +68,7 @@ struct GapRequestScreen: View { // as the phone moves: neither locks the reply again (InstructionCard.replyLock). task: state.gap.map { ScanCopy.gapTask($0) } ), - eyebrow: followUps > 0 && state.gap?.isSatisfied != true && !endAimFolds ? ScanCopy.followUp(remaining: followUps) : nil, + eyebrow: followUps > 0 && state.gap?.isSatisfied != true && !folds ? ScanCopy.followUp(remaining: followUps) : nil, photoCount: state.captureCount, lastCaptureID: state.lastCapture?.id, isReplay: state.isReplay, @@ -92,7 +92,7 @@ struct GapRequestScreen: View { if offersEndMark { markEndButton } - if endAimFolds { + if folds { declineButton } } @@ -184,8 +184,8 @@ struct GapRequestScreen: View { .transition(.opacity) } - /// "I can't get there" among the actions, under "Wall ends here", while the card folds around - /// the circle (`endAimFolds`): the same words, hint and action as the card's reply. + /// "I can't get there" among the actions, first, or under "Wall ends here", while the card is + /// folded (`folds`): the same words, hint and action as the card's reply. @ViewBuilder private var declineButton: some View { if let gap = state.gap { MovedReplyButton( @@ -206,6 +206,22 @@ struct GapRequestScreen: View { return !(coaching.map(ScanCopy.coachingReplacesTask) ?? false) } + /// At the largest text sizes, a request the finished check sent back folds the same way: its + /// "One more view to finish", the stretch's two ends and the check's own words filled the + /// camera, with "I can't get there" cut off at the bottom edge (CI runs 37221532164 and + /// 37231825488). Folded, the side to aim at leads (`ScanCopy.serverLead`), everything else is + /// under Details, "I can't get there" moves to the actions, and the camera keeps + /// `CameraChromeLayout.minCameraWindow`. Only a server request in its own words: a past_end + /// one folds around the circle instead, and coaching that replaces the request keeps the card. + private var followUpFolds: Bool { + guard typeSize.isAccessibilitySize, followUps > 0, let gap = state.gap, !gap.isSatisfied, gap.pastEndSide == nil, + !asking, !askingEnd, case .server = gap.reason else { return false } + return !(coaching.map(ScanCopy.coachingReplacesTask) ?? false) + } + + /// Whether the card is folded and its reply among the actions. + private var folds: Bool { endAimFolds || followUpFolds } + /// A refusal on "Wall ends here" keeps its red triangle, as on the walk; otherwise coaching /// marks the card. private var tone: InstructionCard.Tone { @@ -249,7 +265,7 @@ struct GapRequestScreen: View { private var aims: Bool { guard let gap = state.gap, !asking, !askingEnd, !gap.isSatisfied else { return false } if let coaching, ScanCopy.coachingReplacesTask(coaching) { return false } - if offersEndMark { return true } + if offersEndMark || followUpFolds { return true } guard refusal == nil else { return false } if case .groundNearCandidate = gap.reason { return true } return false @@ -261,10 +277,9 @@ struct GapRequestScreen: View { /// thanks" for as long as it did (field test 4.1, run 3). private var instruction: Instruction { var card = requestCard - guard endAimFolds else { return card } - // Folded around the circle, ride-along coaching leads or opens Details - // (`ScanCopy.foldedAroundCircle`), and "One more view to finish" leaves the card's top - // and leads the words under Details. + guard folds else { return card } + // Folded, ride-along coaching leads or opens Details (`ScanCopy.foldedAroundCircle`), and + // "One more view to finish" leaves the card's top and leads the words under Details. card = ScanCopy.foldedAroundCircle(card, coaching: coaching, refused: refusal != nil) if followUps > 0, let folded = card.folded { card.folded?.detail = "\(ScanCopy.followUp(remaining: followUps)). \(folded.detail)" diff --git a/ios/HouseScanUITests/GapFollowUpFoldUITests.swift b/ios/HouseScanUITests/GapFollowUpFoldUITests.swift new file mode 100644 index 000000000..9749102db --- /dev/null +++ b/ios/HouseScanUITests/GapFollowUpFoldUITests.swift @@ -0,0 +1,127 @@ +import XCTest + +/// A request the finished check sent back ("One more view to finish"), read from the production +/// layout in demo mode. At the largest text size the card folds (`GapRequestScreen.followUpFolds`): +/// the side to aim at leads, the follow-up, the stretch and the check's words are under Details, +/// "I can't get there" sits among the actions, wholly on screen and usable once its lock passes, +/// and the camera between them stays open. At the default size nothing changes. +final class GapFollowUpFoldUITests: XCTestCase { + private static let largestText = ["-UIPreferredContentSizeCategoryName", "UICTContentSizeCategoryAccessibilityXXXL"] + private static let followUp = ["-uiDemo", "-uiDemoFreeze", "-practiceMeter", "NO", "-uiDemoPhase", "gapRequest", "-uiDemoFollowUp"] + /// `CameraChromeLayout.minCameraWindow`: room for the largest aim ring (64 pt radius, 108 + /// percent at its pulse) with a margin, the minimum every folded aiming card keeps. + private static let minCameraWindow: CGFloat = 180 + /// `InstructionCard`'s `replyLock` (600 ms), with room for a slow runner. + private static let lockWait: TimeInterval = 3 + + override func setUp() { + continueAfterFailure = false + } + + @MainActor + private func launch(_ extra: [String]) -> XCUIApplication { + let app = XCUIApplication() + app.launchArguments = Self.followUp + extra + app.launch() + XCTAssertTrue(app.descendants(matching: .any)["screen.gapRequest"].waitForExistence(timeout: 15)) + return app + } + + @MainActor + private func element(_ app: XCUIApplication, _ identifier: String) -> XCUIElement { + app.descendants(matching: .any)[identifier].firstMatch + } + + @MainActor + private func waitUntilEnabled(_ element: XCUIElement) -> Bool { + let enabled = XCTNSPredicateExpectation(predicate: NSPredicate(format: "isEnabled == true"), object: element) + return XCTWaiter().wait(for: [enabled], timeout: Self.lockWait) == .completed + } + + @MainActor + private func attach(_ app: XCUIApplication, _ name: String) { + let shot = XCTAttachment(screenshot: app.screenshot()) + shot.name = name + shot.lifetime = .keepAlways + add(shot) + } + + /// At the default size the card keeps every word and its reply, as before: the follow-up over + /// the stretch's two ends, no Details, and "I can't get there" as the card's trailing pill, + /// under the words, enabled once its lock passes. + @MainActor + func testAtTheDefaultSizeTheCardKeepsItsWordsAndReply() throws { + let app = launch([]) + defer { app.terminate() } + let window = app.windows.firstMatch.frame + let card = element(app, "instruction") + XCTAssertTrue(card.waitForExistence(timeout: 5)) + XCTAssertTrue(card.label.contains("One more view to finish"), card.label) + XCTAssertTrue(card.label.contains("4 ft to 7 ft right of your meter"), card.label) + XCTAssertFalse(element(app, "instruction.details").exists, "the default size folded the card") + let reply = app.buttons["action.skipGap"] + XCTAssertTrue(reply.waitForExistence(timeout: 5)) + XCTAssertTrue(waitUntilEnabled(reply), "I can't get there stayed locked") + XCTAssertTrue(reply.isHittable) + XCTAssertTrue(window.contains(reply.frame), "I can't get there \(reply.frame) isn't wholly on screen") + XCTAssertGreaterThanOrEqual(reply.frame.minY, card.frame.maxY, "the reply must sit under the card's words") + // The card's pill hugs its words at the card's trailing edge; the moved reply is full width. + XCTAssertLessThan(reply.frame.width, window.width * 0.7, "the reply \(reply.frame) left the card") + XCTAssertGreaterThan(reply.frame.midX, window.midX, "the reply \(reply.frame) isn't the card's trailing pill") + XCTAssertTrue(element(app, "action.showResult").exists) + attach(app, "gapRequest-followUp-default") + } + + /// At the largest text size the card folds: a short lead and Details, "I can't get there" + /// wholly on screen below the card and enabled once its lock passes, and at least + /// `minCameraWindow` of camera between them. + @MainActor + func testAtTheLargestSizeTheCardFoldsAndTheReplyStaysOnScreen() throws { + let app = launch(Self.largestText) + defer { app.terminate() } + let window = app.windows.firstMatch.frame + let card = element(app, "instruction") + XCTAssertTrue(card.waitForExistence(timeout: 5)) + XCTAssertTrue(card.label.contains("Show the ground to the right"), card.label) + XCTAssertFalse(card.label.contains("One more view to finish"), "the follow-up should be under Details: \(card.label)") + XCTAssertFalse(card.label.contains("4 ft to 7 ft"), "the stretch should be under Details: \(card.label)") + let details = element(app, "instruction.details") + XCTAssertTrue(details.exists, "the folded card has no Details") + // The card's scrim ends 8 pt under Details (`assertEndCircleOpen`). + let cardBottom = max(card.frame.maxY, details.frame.maxY) + 8 + let reply = app.buttons["action.skipGap"] + XCTAssertTrue(reply.waitForExistence(timeout: 5)) + XCTAssertTrue(waitUntilEnabled(reply), "I can't get there stayed locked") + XCTAssertTrue(reply.isHittable, "I can't get there can't be tapped where it is") + XCTAssertTrue(window.contains(reply.frame), "I can't get there \(reply.frame) isn't wholly on screen in \(window)") + XCTAssertGreaterThanOrEqual(reply.frame.minY, cardBottom, "I can't get there \(reply.frame) is under the card, which ends at \(cardBottom)") + XCTAssertGreaterThanOrEqual( + reply.frame.minY - cardBottom, Self.minCameraWindow, "only \(reply.frame.minY - cardBottom) pt of camera between the card and the actions") + XCTAssertEqual(reply.label, "I can't get there") + attach(app, "gapRequest-followUp-AX5-folded") + } + + /// Details holds every word the unfolded card had: the follow-up, the stretch's two ends and + /// the check's own words. "I can't get there" stays reachable below the open card. + @MainActor + func testAtTheLargestSizeDetailsHoldsEveryWord() throws { + let app = launch(Self.largestText) + defer { app.terminate() } + let details = element(app, "instruction.details") + XCTAssertTrue(details.waitForExistence(timeout: 5)) + details.tap() + let words = element(app, "instruction.detail") + XCTAssertTrue(words.waitForExistence(timeout: 5), "Details didn't open") + for text in ["One more view to finish", "Show the ground from 4 ft to 7 ft right of your meter", "A second look at the ground"] { + XCTAssertTrue(words.label.contains(text), "Details lacks \"\(text)\": \(words.label)") + } + attach(app, "gapRequest-followUp-AX5-details") + let reply = app.buttons["action.skipGap"] + XCTAssertTrue(reply.exists) + for _ in 0..<6 where !reply.isHittable { app.swipeUp(velocity: .slow) } + XCTAssertTrue(reply.isHittable, "I can't get there can't be reached with Details open") + XCTAssertTrue(app.windows.firstMatch.frame.contains(reply.frame)) + XCTAssertGreaterThanOrEqual(reply.frame.minY, words.frame.maxY, "I can't get there is under the open card") + attach(app, "gapRequest-followUp-AX5-details-reply") + } +} diff --git a/ios/HouseScanUITests/ScreenStatesUITests.swift b/ios/HouseScanUITests/ScreenStatesUITests.swift index f1cc52f85..e0f3f3b3f 100644 --- a/ios/HouseScanUITests/ScreenStatesUITests.swift +++ b/ios/HouseScanUITests/ScreenStatesUITests.swift @@ -108,6 +108,8 @@ final class ScreenStatesUITests: XCTestCase { // At AX5 the meter's description folds under Details, and the camera stays open. "findMeter-AX5": [("instruction", "Find your electric meter"), ("instruction.details", "Details")], "gapRequest-followUp": [("instruction", "One more view to finish")], + // At AX5 the side to aim at leads, and the follow-up and the stretch fold under Details. + "gapRequest-followUp-AX5": [("instruction", "Show the ground to the right"), ("instruction.details", "Details")], // #75: a server request's stretch by its two ends, not its middle. "gapRequest-groundOut": [("instruction", "From 4 ft to 7 ft right of your meter.")], "uploading-followUp": [(nil, "One more view to finish")], From 07be16e42621c0b67883f518d60782900fa521a6 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 16:13:44 -0500 Subject: [PATCH 70/71] Name the folded lead's side against the meter, and keep gap(_:)'s doc comment on it Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/UI/Copy/ScanCopy.swift | 12 ++++++------ ios/HouseScanUITests/GapFollowUpFoldUITests.swift | 2 +- ios/HouseScanUITests/ScreenStatesUITests.swift | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ios/HouseScan/UI/Copy/ScanCopy.swift b/ios/HouseScan/UI/Copy/ScanCopy.swift index d75feca3f..a898a6eb0 100644 --- a/ios/HouseScan/UI/Copy/ScanCopy.swift +++ b/ios/HouseScan/UI/Copy/ScanCopy.swift @@ -504,19 +504,19 @@ enum ScanCopy { // MARK: Gap - /// The card for a gap request. A server request can run along much of the wall, so its - /// stretch is named by its two ends, all of it (issue #75, `Distance.range`); the phone's own - /// requests are short and named by their middle. /// A server request's lead when folded: the band and which side of the meter, in a few words /// that stay a short block at AX5. The stretch's two ends are under Details, and the camera and - /// the map mark the stretch itself. + /// the map mark the stretch itself. Named against the meter, as the full card names it. static func serverLead(_ gap: GapRequest) -> String { let band = gap.band == .ground ? "ground" : "wall" - if gap.span.lowerBound >= 0 { return "Show the \(band) to the right" } - if gap.span.upperBound <= 0 { return "Show the \(band) to the left" } + if gap.span.lowerBound >= 0 { return "Show the \(band) right of your meter" } + if gap.span.upperBound <= 0 { return "Show the \(band) left of your meter" } return "Show the \(band) by your meter" } + /// The card for a gap request. A server request can run along much of the wall, so its + /// stretch is named by its two ends, all of it (issue #75, `Distance.range`); the phone's own + /// requests are short and named by their middle. static func gap(_ gap: GapRequest) -> Instruction { let place = Distance.aroundFromMeter(gap.span) let stretch = Distance.range(gap.span) diff --git a/ios/HouseScanUITests/GapFollowUpFoldUITests.swift b/ios/HouseScanUITests/GapFollowUpFoldUITests.swift index 9749102db..548d978b2 100644 --- a/ios/HouseScanUITests/GapFollowUpFoldUITests.swift +++ b/ios/HouseScanUITests/GapFollowUpFoldUITests.swift @@ -82,7 +82,7 @@ final class GapFollowUpFoldUITests: XCTestCase { let window = app.windows.firstMatch.frame let card = element(app, "instruction") XCTAssertTrue(card.waitForExistence(timeout: 5)) - XCTAssertTrue(card.label.contains("Show the ground to the right"), card.label) + XCTAssertTrue(card.label.contains("Show the ground right of your meter"), card.label) XCTAssertFalse(card.label.contains("One more view to finish"), "the follow-up should be under Details: \(card.label)") XCTAssertFalse(card.label.contains("4 ft to 7 ft"), "the stretch should be under Details: \(card.label)") let details = element(app, "instruction.details") diff --git a/ios/HouseScanUITests/ScreenStatesUITests.swift b/ios/HouseScanUITests/ScreenStatesUITests.swift index e0f3f3b3f..0f08148a0 100644 --- a/ios/HouseScanUITests/ScreenStatesUITests.swift +++ b/ios/HouseScanUITests/ScreenStatesUITests.swift @@ -109,7 +109,7 @@ final class ScreenStatesUITests: XCTestCase { "findMeter-AX5": [("instruction", "Find your electric meter"), ("instruction.details", "Details")], "gapRequest-followUp": [("instruction", "One more view to finish")], // At AX5 the side to aim at leads, and the follow-up and the stretch fold under Details. - "gapRequest-followUp-AX5": [("instruction", "Show the ground to the right"), ("instruction.details", "Details")], + "gapRequest-followUp-AX5": [("instruction", "Show the ground right of your meter"), ("instruction.details", "Details")], // #75: a server request's stretch by its two ends, not its middle. "gapRequest-groundOut": [("instruction", "From 4 ft to 7 ft right of your meter.")], "uploading-followUp": [(nil, "One more view to finish")], From 263827efaa5d600ecd7ed0443b6e7dd05fbef807 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 4 Oct 2026 16:46:48 -0500 Subject: [PATCH 71/71] Scroll the Details test until "I can't get there" is wholly on screen Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScanUITests/GapFollowUpFoldUITests.swift | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ios/HouseScanUITests/GapFollowUpFoldUITests.swift b/ios/HouseScanUITests/GapFollowUpFoldUITests.swift index 548d978b2..6be0228b9 100644 --- a/ios/HouseScanUITests/GapFollowUpFoldUITests.swift +++ b/ios/HouseScanUITests/GapFollowUpFoldUITests.swift @@ -118,9 +118,12 @@ final class GapFollowUpFoldUITests: XCTestCase { attach(app, "gapRequest-followUp-AX5-details") let reply = app.buttons["action.skipGap"] XCTAssertTrue(reply.exists) - for _ in 0..<6 where !reply.isHittable { app.swipeUp(velocity: .slow) } + // Scroll until the whole button is in view, not just until it is hittable: a sliver at the + // bottom edge is already hittable (local run ax5-fold-07be16e4-ui stopped there). + let window = app.windows.firstMatch.frame + for _ in 0..<6 where !(reply.isHittable && window.contains(reply.frame)) { app.swipeUp(velocity: .slow) } XCTAssertTrue(reply.isHittable, "I can't get there can't be reached with Details open") - XCTAssertTrue(app.windows.firstMatch.frame.contains(reply.frame)) + XCTAssertTrue(window.contains(reply.frame), "I can't get there \(reply.frame) isn't wholly on screen with Details open") XCTAssertGreaterThanOrEqual(reply.frame.minY, words.frame.maxY, "I can't get there is under the open card") attach(app, "gapRequest-followUp-AX5-details-reply") }