From ec5ad932653b03a3c52417a1821f60ff6e217ea2 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Thu, 1 Oct 2026 00:35:51 -0500 Subject: [PATCH 01/16] ios: carry #123's capture packet, upload and result modules into HouseScanKit Copied unchanged from PR #123 (t3/hunter-integration) at 17b53baa, written there in 02c1e920..cd10f559. Only HouseScanKit library code and its tests come across: the packet 0.4 producer and checks, the capture-API uploader with its saved state and send gate, the session coordinator, and the device-result decoder with its AR gate. The one change to an existing file is the `shift` argument #123 added to the makeJPEG test helper. Nothing calls these modules yet. ScanEngine, the app's UI and the ordinary upload to /v1/placements are untouched, so the shipped app behaves exactly as before. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../CaptureResult/CaptureResult.swift | 166 ++++++ .../CaptureResultPlacement.swift | 182 +++++++ .../CaptureResult/CaptureResultSummary.swift | 42 ++ .../Sources/HouseScanKit/Packet/Gzip.swift | 19 + .../HouseScanKit/Packet/Packet04.swift | 306 +++++++++++ .../HouseScanKit/Packet/Packet04Check.swift | 110 ++++ .../Packet/Packet04Producer.swift | 385 +++++++++++++ .../HouseScanKit/Packet/Packet04Streams.swift | 147 +++++ .../HouseScanKit/Upload/CaptureAPI.swift | 208 +++++++ .../Upload/CaptureSessionCoordinator.swift | 465 ++++++++++++++++ .../Upload/CaptureUploadGate.swift | 72 +++ .../Upload/CaptureUploadState.swift | 167 ++++++ .../HouseScanKit/Upload/CaptureUploader.swift | 510 ++++++++++++++++++ .../CaptureResultSummaryTests.swift | 134 +++++ .../CaptureResultTests.swift | 320 +++++++++++ .../CaptureSessionCoordinatorTests.swift | 393 ++++++++++++++ .../CaptureUploaderTests.swift | 284 ++++++++++ .../LoopbackCaptureAPI.swift | 334 ++++++++++++ .../HouseScanKitTests/Packet04Tests.swift | 253 +++++++++ .../HouseScanKitTests/PacketWriterTests.swift | 4 +- 20 files changed, 4499 insertions(+), 2 deletions(-) create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResult.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultPlacement.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultSummary.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Packet/Gzip.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Check.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Producer.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Streams.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadGate.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultSummaryTests.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultTests.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/Packet04Tests.swift diff --git a/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResult.swift b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResult.swift new file mode 100644 index 000000000..e04617f41 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResult.swift @@ -0,0 +1,166 @@ +import Foundation + +/// The capture API's device result, `GET /v1/captures/{id}/result`, written from the server's +/// documented wire contract. The whole body is camelCase. +/// +/// Only the fields the app shows or gates on are decoded. The verdict and its URL are left out on +/// purpose: the server masks a reject as manual review for the device, and the app must not read +/// around that mask. Unknown keys are ignored, because the server serves a result even when it has +/// drifted from its own schema. A status or outcome kind the app doesn't know decodes as +/// `.unknown`, so a new server value never hides the message and views that came with it. +public enum CaptureResult { + public struct Response: Decodable, Sendable, Equatable { + /// Nil before the server has started a run for the capture. + public var runId: String? + public var status: Status + public var viewsNeeded: [ViewNeeded] + public var memberActions: [String] + /// Nil until the run writes its result: while uploading, awaiting files or processing. + public var outcome: Outcome? + /// Nil when the server sent no criteria. The app shows none rather than inventing rows. + public var criteria: [Criterion]? + /// A signed URL that expires; nil when the run made no preview. + public var previewUrl: String? + /// A path on the API, not a full URL. + public var reviewUrl: String? + } + + public enum Status: Sendable, Equatable, Decodable { + case uploading, awaitingFiles, processing, needsViews, complete, manualReview, failed, expired + case unknown(String) + + public init(rawValue: String) { + self = switch rawValue { + case "uploading": .uploading + case "awaiting_files": .awaitingFiles + case "processing": .processing + case "needs_views": .needsViews + case "complete": .complete + case "manual_review": .manualReview + case "failed": .failed + case "expired": .expired + default: .unknown(rawValue) + } + } + + public init(from decoder: any Decoder) throws { + self.init(rawValue: try decoder.singleValueContainer().decode(String.self)) + } + } + + /// The decision shown to the homeowner. It is already masked for the device. + public struct Outcome: Decodable, Sendable, Equatable { + public var kind: OutcomeKind + public var profile: String + /// Text for the homeowner, shown as written. + public var message: String + public var viewsNeeded: [ViewNeeded] + /// Empty when the server shows a reject as manual review. + public var reasons: [Reason] + /// The ARKit world the spatial fields are in. Nil when the scene was not built in one. + public var arkitEpoch: String? + public var recommendedPlacement: RecommendedPlacement? + } + + public enum OutcomeKind: Sendable, Equatable, Decodable { + case eligible, needsMorePhotos, notEligible, manualReview + case unknown(String) + + public init(rawValue: String) { + self = switch rawValue { + case "eligible": .eligible + case "needs_more_photos": .needsMorePhotos + case "not_eligible": .notEligible + case "manual_review": .manualReview + default: .unknown(rawValue) + } + } + + public init(from decoder: any Decoder) throws { + self.init(rawValue: try decoder.singleValueContainer().decode(String.self)) + } + } + + public struct RecommendedPlacement: Decodable, Sendable, Equatable { + public var wallId: String + public var startSM: Double + public var confidence: Double + /// Nil when the scene was not built in an ARKit world. + public var boxArkitWorld: BoxArkitWorld? + } + + /// The box in ARKit world coordinates, as sent. The numbers are checked only when the app asks + /// for an AR placement (`Record.placement`), so a malformed box withholds AR and leaves the + /// rest of the result readable. + public struct BoxArkitWorld: Decodable, Sendable, Equatable { + /// Box centre to world, 16 numbers column by column. Local +x runs along the wall (left to + /// right seen from outside), +y up, +z out of the wall. + public var pose: [Double] + /// Width, height and depth in ARKit metres. + public var size: [Double] + } + + public struct ViewNeeded: Decodable, Sendable, Equatable { + public var id: String + /// The server's view kind, kept as text so a new kind still shows its prompt. + public var kind: String + public var criteria: [String] + public var why: String + public var promptId: String + /// Rendered text; show it verbatim. + public var prompt: Prompt + public var band: String? + public var wallId: String? + public var target: String? + /// Set in the photo flow instead of ARKit anchors. + public var referenceCrop: ReferenceCrop? + } + + public struct Prompt: Decodable, Sendable, Equatable { + public var title: String + public var body: String + } + + public struct ReferenceCrop: Decodable, Sendable, Equatable { + public var image: String + /// Pixel box in `image`: x0, y0, x1, y1. + public var boxPx: [Double] + } + + public struct Reason: Decodable, Sendable, Equatable { + public var criterion: String + public var text: String + public var measuredFt: Double? + public var plusMinusFt: Double? + public var thresholdFt: Double? + public var comparison: String? + } + + public struct Criterion: Decodable, Sendable, Equatable { + public var id: String + public var outcome: CriterionOutcome + public var unsureCause: String? + public var measuredFt: Double? + public var plusMinusFt: Double? + public var thresholdFt: Double? + public var coverage: String? + } + + public enum CriterionOutcome: Sendable, Equatable, Decodable { + case pass, fail, unsure + case unknown(String) + + public init(rawValue: String) { + self = switch rawValue { + case "pass": .pass + case "fail": .fail + case "unsure": .unsure + default: .unknown(rawValue) + } + } + + public init(from decoder: any Decoder) throws { + self.init(rawValue: try decoder.singleValueContainer().decode(String.self)) + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultPlacement.swift b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultPlacement.swift new file mode 100644 index 000000000..118c246b2 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultPlacement.swift @@ -0,0 +1,182 @@ +import simd + +extension CaptureResult { + /// The local identities a result belongs to. The response carries a run id and an ARKit epoch + /// but not the session or capture, and epochs repeat ("e1" starts every session), so only the + /// caller can say which scan a result is for. + public struct Association: Sendable, Equatable, Hashable { + /// The app's id for one run of the ARKit session. A new session gets a new id, even when it + /// reuses the epoch string. + public var sessionID: String + public var captureID: String + /// The run that finalize started for the capture. + public var runID: String + /// The ARKit epoch the packet was captured in. + public var epoch: String + + public init(sessionID: String, captureID: String, runID: String, epoch: String) { + self.sessionID = sessionID + self.captureID = captureID + self.runID = runID + self.epoch = epoch + } + } + + /// Whether the caller knows the server ran real analysis for this result. A stubbed server + /// returns a result that passes the contract without reconstructing anything, and nothing in + /// the JSON tells the two apart, so only the caller can grant `.verified`. + public enum Analysis: Sendable, Equatable { + case unverified + case verified + } + + /// A decoded result kept with the local scan it was fetched for. + public struct Record: Sendable, Equatable { + public let response: Response + public let association: Association + public let analysis: Analysis + + public init(response: Response, association: Association, analysis: Analysis = .unverified) { + self.response = response + self.association = association + self.analysis = analysis + } + + /// The recommended box in the meter anchor's frame, or the reason there is none. + /// + /// - Parameters: + /// - world: the session, capture, finalized run and epoch on screen now. + /// - meterAnchor: the meter anchor's anchor-to-world pose, in the same world coordinates + /// as the uploaded packet. Attaching the returned box to the anchor then carries it + /// through ARKit's later corrections. + public func placement(in world: Association, meterAnchor: simd_float4x4) -> Placement { + guard let outcome = response.outcome else { return .unavailable(.noOutcome(response.status)) } + guard outcome.kind == .eligible else { return .unavailable(.notEligible(outcome.kind)) } + guard response.status == .complete else { return .unavailable(.statusNotComplete(response.status)) } + guard analysis == .verified else { return .unavailable(.analysisUnverified) } + + guard world.sessionID == association.sessionID else { return .unavailable(.mismatch(.session)) } + guard world.captureID == association.captureID else { return .unavailable(.mismatch(.capture)) } + guard world.runID == association.runID, response.runId == association.runID else { + return .unavailable(.mismatch(.run)) + } + guard let epoch = outcome.arkitEpoch, + let box = outcome.recommendedPlacement?.boxArkitWorld + else { + return .unavailable(outcome.recommendedPlacement == nil ? .noPlacement : .notInARKitWorld) + } + guard world.epoch == association.epoch, epoch == association.epoch else { + return .unavailable(.mismatch(.epoch)) + } + + guard box.pose.count == 16 else { return .unavailable(.malformedBox(.poseCount(box.pose.count))) } + guard box.size.count == 3 else { return .unavailable(.malformedBox(.sizeCount(box.size.count))) } + guard (box.pose + box.size).allSatisfy(\.isFinite) else { return .unavailable(.malformedBox(.nonFinite)) } + guard box.size.allSatisfy({ $0 > 0 }) else { return .unavailable(.malformedBox(.nonPositiveSize)) } + let worldFromBox = Self.matrix(columnMajor: box.pose) + // The server divides the world transform's scale out of the pose's rotation and + // multiplies it into `size` instead. A pose that still carries a scale was built some + // other way, and using it would scale the box a second time. + guard Self.isRigid(worldFromBox) else { return .unavailable(.malformedBox(.notRigid)) } + + let worldFromAnchor = Self.double(meterAnchor) + guard Self.isRigid(worldFromAnchor) else { return .unavailable(.invalidMeterAnchor) } + + let anchorFromBox = Self.float(worldFromAnchor.inverse * worldFromBox) + let size = SIMD3(Float(box.size[0]), Float(box.size[1]), Float(box.size[2])) + // Finite Doubles can overflow Float, and positive sizes can round to zero. + let columns = [anchorFromBox.columns.0, anchorFromBox.columns.1, anchorFromBox.columns.2, anchorFromBox.columns.3] + guard columns.allSatisfy({ c in (0..<4).allSatisfy { c[$0].isFinite } }), + (0..<3).allSatisfy({ size[$0].isFinite }) + else { return .unavailable(.malformedBox(.nonFinite)) } + guard (0..<3).allSatisfy({ size[$0] > 0 }) else { return .unavailable(.malformedBox(.nonPositiveSize)) } + return .box(AnchoredBox(anchorFromBox: anchorFromBox, size: size)) + } + + private static func matrix(columnMajor v: [Double]) -> simd_double4x4 { + simd_double4x4( + SIMD4(v[0], v[1], v[2], v[3]), + SIMD4(v[4], v[5], v[6], v[7]), + SIMD4(v[8], v[9], v[10], v[11]), + SIMD4(v[12], v[13], v[14], v[15])) + } + + private static func double(_ m: simd_float4x4) -> simd_double4x4 { + simd_double4x4( + SIMD4(m.columns.0), SIMD4(m.columns.1), + SIMD4(m.columns.2), SIMD4(m.columns.3)) + } + + private static func float(_ m: simd_double4x4) -> simd_float4x4 { + simd_float4x4( + SIMD4(m.columns.0), SIMD4(m.columns.1), + SIMD4(m.columns.2), SIMD4(m.columns.3)) + } + + /// A finite rotation with no scale or shear, a translation, and a bottom row of 0 0 0 1. + /// The server writes each entry to six decimals, so its rotations are orthonormal to about + /// 1e-6; 1e-3 accepts that and ARKit's float poses, and rejects a 0.1% scale (whose + /// squared column length is off by 2e-3). + private static func isRigid(_ m: simd_double4x4) -> Bool { + let tolerance = 1e-3 + let columns = [m.columns.0, m.columns.1, m.columns.2, m.columns.3] + guard columns.allSatisfy({ c in (0..<4).allSatisfy { c[$0].isFinite } }) else { return false } + let bottom = SIMD4(m.columns.0.w, m.columns.1.w, m.columns.2.w, m.columns.3.w) + guard simd_reduce_max(simd_abs(bottom - SIMD4(0, 0, 0, 1))) <= tolerance else { return false } + let r = simd_double3x3( + SIMD3(m.columns.0.x, m.columns.0.y, m.columns.0.z), + SIMD3(m.columns.1.x, m.columns.1.y, m.columns.1.z), + SIMD3(m.columns.2.x, m.columns.2.y, m.columns.2.z)) + let gram = r.transpose * r + let identity = matrix_identity_double3x3 + for i in 0..<3 where simd_reduce_max(simd_abs(gram[i] - identity[i])) > tolerance { return false } + return r.determinant > 0 + } + } + + public enum Placement: Sendable, Equatable { + case box(AnchoredBox) + case unavailable(Unavailable) + } + + public struct AnchoredBox: Sendable, Equatable { + /// Box centre to meter anchor, rigid. Local +x along the wall, +y up, +z out of the wall. + public var anchorFromBox: simd_float4x4 + /// Width, height and depth in metres, as the server stated them. The server has already + /// applied the scene-to-ARKit scale, so the app uses these as they are. + public var size: SIMD3 + } + + /// Why a result has no AR placement. The rest of the result stays readable in every case. + public enum Unavailable: Sendable, Equatable { + /// The run has not written a result yet. + case noOutcome(Status) + /// Manual review, more photos, not eligible, or a kind this app doesn't know. + case notEligible(OutcomeKind) + case statusNotComplete(Status) + /// The caller has not confirmed that real analysis produced the result. + case analysisUnverified + /// The result belongs to another session, capture, run or ARKit world than the one on + /// screen. + case mismatch(IdentityField) + /// An eligible outcome without a recommended placement, which breaks the contract. + case noPlacement + /// The scene was not built in an ARKit world, so there is no box to anchor. + case notInARKitWorld + case malformedBox(BoxProblem) + /// The caller's meter anchor pose is not finite and rigid. + case invalidMeterAnchor + } + + public enum IdentityField: Sendable, Equatable, CaseIterable { + case session, capture, run, epoch + } + + public enum BoxProblem: Sendable, Equatable { + case poseCount(Int) + case sizeCount(Int) + case nonFinite + case nonPositiveSize + case notRigid + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultSummary.swift b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultSummary.swift new file mode 100644 index 000000000..8a1859073 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultSummary.swift @@ -0,0 +1,42 @@ +import Foundation +import simd + +/// What the integration build shows of a capture-API result: where it came from, what the server +/// decided, its own words, the views it asks for, and why there is no AR box. Built from the +/// decoded response only; nothing is inferred, and an absent part stays absent. +public struct CaptureResultSummary: Sendable, Equatable { + public enum State: Sendable, Equatable { + /// The run has no outcome yet. + case working(CaptureResult.Status) + case decided(CaptureResult.OutcomeKind) + } + + public var state: State + /// The server's message for the homeowner, verbatim; nil before an outcome. + public var message: String? + /// Each requested view's prompt, verbatim, title then body. + public var prompts: [CaptureResult.Prompt] + /// The criteria rows the server sent; nil when it sent none. Never filled in by the app. + public var criteria: [CaptureResult.Criterion]? + /// False until someone has verified the server ran real analysis for this result. + public var analysisVerified: Bool + /// Why AR shows nothing for this result on the scan now on screen. + public var arUnavailable: CaptureResult.Unavailable? + + /// `world` is the scan on screen now; `meterAnchor` its meter anchor's pose in that world, nil + /// when the scan has none to place a box on (AR then says so, after the other checks). + public init(_ record: CaptureResult.Record, world: CaptureResult.Association, meterAnchor: simd_float4x4?) { + let response = record.response + state = response.outcome.map { .decided($0.kind) } ?? .working(response.status) + message = response.outcome?.message + let views = response.viewsNeeded.isEmpty ? response.outcome?.viewsNeeded ?? [] : response.viewsNeeded + prompts = views.map(\.prompt) + criteria = response.criteria + analysisVerified = record.analysis == .verified + // A zero matrix is not a pose, so without an anchor the gate ends on `.invalidMeterAnchor`. + arUnavailable = switch record.placement(in: world, meterAnchor: meterAnchor ?? simd_float4x4()) { + case .box: nil + case .unavailable(let reason): reason + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Gzip.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Gzip.swift new file mode 100644 index 000000000..e4cd01dff --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Gzip.swift @@ -0,0 +1,19 @@ +import Foundation + +/// gzip (RFC 1952) around the raw DEFLATE stream Apple's `.zlib` compression writes. The packet's +/// streams travel as `.csv.gz`, and Foundation has no gzip writer of its own. +public enum Gzip { + public static func compress(_ data: Data) throws -> Data { + // NSData's .zlib is raw DEFLATE (RFC 1951) with no zlib header, which is what gzip wraps. + let deflated = try (data as NSData).compressed(using: .zlib) as Data + var out = Data([0x1F, 0x8B, 0x08, 0x00, 0, 0, 0, 0, 0x00, 0xFF]) + out.append(deflated) + appendLE(ZipCRC32.checksum(data), to: &out) + appendLE(UInt32(truncatingIfNeeded: data.count), to: &out) + return out + } + + private static func appendLE(_ value: UInt32, to data: inout Data) { + withUnsafeBytes(of: value.littleEndian) { data.append(contentsOf: $0) } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04.swift new file mode 100644 index 000000000..f7ce75d4d --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04.swift @@ -0,0 +1,306 @@ +import Foundation + +/// The capture packet 0.4 that the capture API's finalize call takes as its body: `packet.json`, +/// camelCase, in the ARKit world frame of one epoch. This is the subset House Scan produces, +/// written from the published wire contract, not copied from the server's models. Optional +/// fields are left out when nil, which the server reads as absent. +/// +/// Unlike the 1.1 packet (`PacketManifest`), poses here are the raw ARKit camera-to-world +/// transforms, never moved into the meter frame. +public enum Packet04 { + public static let format = "house-capture-packet" + public static let formatVersion = "0.4" + + public struct Packet: Codable, Sendable, Equatable { + public var format = Packet04.format + public var formatVersion = Packet04.formatVersion + public var packetId: String + public var createdAt: String + public var units = Units() + public var conventions = Conventions() + public var source: Source + public var session: Session + public var epochs: [Epoch] + public var keyframes: [Keyframe] + public var stills: [Still]? + public var taps: [Tap]? + public var tracking: [TrackingEvent] + public var streams: Streams + public var scaleReference: ScaleReference + public var files: [FileEntry] + public var ext: [String: String]? + } + + public struct Units: Codable, Sendable, Equatable { + public var length = "meters" + public var angle = "degrees" + public var time = "seconds of device uptime (ARFrame.timestamp and Core Motion timestamp)" + public var image = "pixels of the saved, unrotated landscape JPEG" + public var rotationRate = "radians per second" + public var acceleration = "g (9.80665 m/s^2)" + } + + public struct Conventions: Codable, Sendable, Equatable { + public var world = "ARKit world, worldAlignment .gravity: right-handed, +y up, origin where the epoch started" + public var camera = "ARKit camera: +x right, +y up in the unrotated landscape image, looking along -z" + public var pose = "camera-to-world 4x4, 16 numbers column by column (simd_float4x4 layout), meters" + public var intrinsics = "[fx, fy, cx, cy] in pixels of the saved, unrotated landscape image" + public var pixel = "[u, v] continuous: (0, 0) is the top-left corner of the top-left pixel, v grows down" + } + + public struct Source: Codable, Sendable, Equatable { + public enum Kind: String, Codable, Sendable { case device, replay, synthetic } + public var kind: Kind + public var appBuild: String? + + public init(kind: Kind, appBuild: String?) { + self.kind = kind + self.appBuild = appBuild + } + } + + public enum Tier: String, Codable, Sendable { case arkit, arkitLidar = "arkit_lidar" } + + public struct VideoFormat: Codable, Sendable, Equatable { + public var imageResolution: [Int] + public var framesPerSecond: Double? + + public init(width: Int, height: Int, framesPerSecond: Double?) { + imageResolution = [width, height] + self.framesPerSecond = framesPerSecond + } + } + + public struct Session: Codable, Sendable, Equatable { + public var id: String + public var startedAt: String + public var startedAtUptime: Double + public var endedAtUptime: Double? + public var timeZone: String? + public var appVersion: String + public var deviceModel: String + public var systemVersion: String + public var tier: Tier + public var lidarAvailable: Bool + public var meshReconstructionSupported: Bool? + public var sceneDepthEnabled: Bool + public var sceneReconstruction: String? + public var worldAlignment = "gravity" + public var planeDetection: [String]? + public var videoFormat: VideoFormat + public var motionReferenceFrame: String? + } + + public struct Epoch: Codable, Sendable, Equatable { + public var id: String + public var startTime: Double + public var reason: String + + public init(id: String, startTime: Double, reason: String) { + self.id = id + self.startTime = startTime + self.reason = reason + } + } + + public struct Exposure: Codable, Sendable, Equatable { + public var duration: Double? + public var offset: Double? + public var iso: Double? + public var fNumber: Double? + + public init(duration: Double?, offset: Double?, iso: Double?, fNumber: Double?) { + self.duration = duration + self.offset = offset + self.iso = iso + self.fNumber = fNumber + } + } + + public struct DepthRef: Codable, Sendable, Equatable { + public var file: String + public var confidenceFile: String? + public var w: Int + public var h: Int + } + + public struct Keyframe: Codable, Sendable, Equatable { + public var id: String + public var img: String + public var w: Int + public var h: Int + public var intrinsics: [Double] + public var pose: [Double] + public var timestamp: Double + public var tracking: String + public var epoch: String + public var reason: String + public var exposure: Exposure? + public var depth: DepthRef? + public var purpose: String? + } + + public struct Still: Codable, Sendable, Equatable { + public var id: String + public var purpose: String + public var img: String + public var w: Int + public var h: Int + public var timestamp: Double + /// EXIF orientation of the stored pixels. House Scan stores the sensor image unrotated. + public var orientation: Int + public var keyframe: String? + public var intrinsics: [Double]? + public var intrinsicsSource: String? + } + + public struct TapHit: Codable, Sendable, Equatable { + public var position: [Double] + public var target: String + public var alignment: String? + public var distance: Double? + + public init(position: [Double], target: String, alignment: String?, distance: Double?) { + self.position = position + self.target = target + self.alignment = alignment + self.distance = distance + } + } + + public struct Tap: Codable, Sendable, Equatable { + public var id: String + public var time: Double + public var label: String + public var keyframe: String + public var pixel: [Double] + public var rayOrigin: [Double] + public var rayDirection: [Double] + public var hit: TapHit? + public var epoch: String? + } + + public struct TrackingEvent: Codable, Sendable, Equatable { + public var time: Double + public var state: String + public var epoch: String? + + public init(time: Double, state: String, epoch: String?) { + self.time = time + self.state = state + self.epoch = epoch + } + } + + public struct StreamRef: Codable, Sendable, Equatable { + public var file: String + public var columns: [String] + public var rateHz: Double + } + + public struct Streams: Codable, Sendable, Equatable { + public var imuRaw: StreamRef + public var arkitPoses: StreamRef + } + + public struct CloseUp: Codable, Sendable, Equatable { + public var captured: Bool + public var still: String? + public var arkitDistanceM: Double? + public var side: String? + + public init(captured: Bool, still: String?, arkitDistanceM: Double?, side: String?) { + self.captured = captured + self.still = still + self.arkitDistanceM = arkitDistanceM + self.side = side + } + + public static let notCaptured = CloseUp(captured: false, still: nil, arkitDistanceM: nil, side: nil) + } + + public struct ScaleExt: Codable, Sendable, Equatable { + public var method = "a9-meter-model" + public var obliqueCloseUp: CloseUp + /// Head-on views of doors on the meter side. House Scan does not detect doors yet, so this + /// is empty, which the contract allows. + public var doorViews: [String] = [] + } + + public struct ScaleReference: Codable, Sendable, Equatable { + public var meterCloseUp: CloseUp + public var ext: ScaleExt + } + + public enum Role: String, Codable, Sendable, CaseIterable { + case keyframe, depth, confidence, features, still, stream, other + } + + public struct FileEntry: Codable, Sendable, Equatable { + public var path: String + public var bytes: Int + public var sha256: String + public var role: Role + } + + /// The `meta` a register call sends with an image: exactly one record, the same one the final + /// packet lists. + public enum ImageRecord: Codable, Sendable, Equatable { + case keyframe(Keyframe) + case still(Still) + + public var path: String { + switch self { + case .keyframe(let k): k.img + case .still(let s): s.img + } + } + + private enum CodingKeys: String, CodingKey { case keyframe, still } + + public func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .keyframe(let k): try container.encode(k, forKey: .keyframe) + case .still(let s): try container.encode(s, forKey: .still) + } + } + + public init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + if let k = try container.decodeIfPresent(Keyframe.self, forKey: .keyframe) { + self = .keyframe(k) + } else { + self = .still(try container.decode(Still.self, forKey: .still)) + } + } + } + + // MARK: Id and path patterns the server enforces + + /// `^[A-Za-z0-9_-]{8,64}$`: packetId. + public static func isStorageID(_ s: String) -> Bool { + (8...64).contains(s.count) && s.unicodeScalars.allSatisfy { isAlnum($0) || $0 == "_" || $0 == "-" } + } + + /// `^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$`: keyframe and still ids. + public static func isSegmentID(_ s: String) -> Bool { + guard let first = s.unicodeScalars.first, isAlnum(first), s.count <= 128 else { return false } + return s.unicodeScalars.allSatisfy { isAlnum($0) || $0 == "_" || $0 == "-" } + } + + /// Every segment starts with an alphanumeric, and a dot always starts a new alphanumeric token. + public static func isRelPath(_ s: String) -> Bool { + guard !s.isEmpty else { return false } + return s.split(separator: "/", omittingEmptySubsequences: false).allSatisfy { segment in + segment.split(separator: ".", omittingEmptySubsequences: false).allSatisfy { token in + guard let first = token.unicodeScalars.first, isAlnum(first) else { return false } + return token.unicodeScalars.allSatisfy { isAlnum($0) || $0 == "_" || $0 == "-" } + } + } + } + + private static func isAlnum(_ c: Unicode.Scalar) -> Bool { + ("a"..."z").contains(c) || ("A"..."Z").contains(c) || ("0"..."9").contains(c) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Check.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Check.swift new file mode 100644 index 000000000..d6747e01c --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Check.swift @@ -0,0 +1,110 @@ +import Foundation +import simd + +/// House Scan's own checks of a 0.4 packet against the rules the capture API documents as +/// rejections: id and path patterns, the ARKit tier's required sections, listed files that exist +/// with their stated size and SHA-256, rigid poses, plausible intrinsics, still-to-keyframe links +/// and the tap replay. A packet that passes can still fail a server check this does not repeat. +public enum Packet04Check { + /// The server's tap replay tolerance on origin and direction. + public static let tapReplayTolerance = 1e-3 + + public static func problems(_ p: Packet04.Packet, folder: URL?) -> [String] { + var out: [String] = [] + if !Packet04.isStorageID(p.packetId) { out.append("packetId \(p.packetId) is not a StorageId") } + if p.formatVersion != Packet04.formatVersion { out.append("formatVersion \(p.formatVersion)") } + if p.keyframes.isEmpty { out.append("ARKit tier needs at least one keyframe") } + if p.session.tier == .arkitLidar, !p.session.lidarAvailable { out.append("arkit_lidar needs lidarAvailable") } + if p.session.worldAlignment != "gravity" { out.append("worldAlignment must be gravity") } + if p.streams.imuRaw.rateHz < Packet04Streams.minimumIMUHz { out.append("imuRaw rateHz \(p.streams.imuRaw.rateHz) < 50") } + if p.streams.arkitPoses.rateHz < Packet04Streams.minimumPoseHz { out.append("arkitPoses rateHz \(p.streams.arkitPoses.rateHz) < 10") } + + let epochs = Set(p.epochs.map(\.id)) + var listed: [String: Packet04.FileEntry] = [:] + for entry in p.files { + if !Packet04.isRelPath(entry.path) { out.append("file path \(entry.path) is not a RelPath") } + if listed.updateValue(entry, forKey: entry.path) != nil { out.append("file \(entry.path) listed twice") } + } + func requireListed(_ path: String, _ role: Packet04.Role, _ place: String) { + guard let entry = listed[path] else { return out.append("\(place) \(path) is not in files") } + if entry.role != role { out.append("\(place) \(path) has role \(entry.role.rawValue), not \(role.rawValue)") } + } + + var frames: [String: Packet04.Keyframe] = [:] + var byTime: [Double: Packet04.Keyframe] = [:] + for k in p.keyframes { + if !Packet04.isSegmentID(k.id) { out.append("keyframe id \(k.id) is not a SegmentId") } + if frames.updateValue(k, forKey: k.id) != nil { out.append("keyframe \(k.id) listed twice") } + if !epochs.contains(k.epoch) { out.append("keyframe \(k.id) names unknown epoch \(k.epoch)") } + requireListed(k.img, .keyframe, "keyframe \(k.id) image") + if let depth = k.depth { + requireListed(depth.file, .depth, "keyframe \(k.id) depth") + if let confidence = depth.confidenceFile { requireListed(confidence, .confidence, "keyframe \(k.id) confidence") } + if let entry = listed[depth.file], entry.bytes != depth.w * depth.h * 4 { out.append("keyframe \(k.id) depth size") } + } + guard let pose = matrix(k.pose) else { out.append("keyframe \(k.id) pose is not 16 finite numbers"); continue } + if !PacketPose.isRigid(pose) { out.append("keyframe \(k.id) pose is not rigid") } + if k.intrinsics.count != 4 { + out.append("keyframe \(k.id) intrinsics") + } else if let problem = PacketWriter.intrinsicsProblem( + SIMD4(k.intrinsics.map(Float.init)), width: k.w, height: k.h) { + out.append("keyframe \(k.id) intrinsics: \(problem)") + } + if let other = byTime[k.timestamp], other.epoch == k.epoch, other.pose != k.pose { + out.append("keyframes \(other.id) and \(k.id) share timestamp \(k.timestamp) with different poses") + } + byTime[k.timestamp] = k + } + var stillIDs = Set() + for s in p.stills ?? [] { + if !Packet04.isSegmentID(s.id) { out.append("still id \(s.id) is not a SegmentId") } + stillIDs.insert(s.id) + requireListed(s.img, .still, "still \(s.id) image") + if let link = s.keyframe { + guard let frame = frames[link] else { out.append("still \(s.id) names unknown keyframe \(link)"); continue } + if frame.w != s.w || frame.h != s.h { out.append("still \(s.id) and keyframe \(link) sizes differ") } + } + } + for tap in p.taps ?? [] { + guard let frame = frames[tap.keyframe] else { out.append("tap \(tap.id) names unknown keyframe \(tap.keyframe)"); continue } + if tap.pixel.count != 2 || !(0...Double(frame.w)).contains(tap.pixel[0]) || !(0...Double(frame.h)).contains(tap.pixel[1]) { + out.append("tap \(tap.id) pixel is outside keyframe \(frame.id)") + continue + } + if let epoch = tap.epoch, epoch != frame.epoch { out.append("tap \(tap.id) epoch differs from its keyframe's") } + let replay = ray(pose: frame.pose, intrinsics: frame.intrinsics, pixel: tap.pixel) + let error = zip(replay.origin + replay.direction, tap.rayOrigin + tap.rayDirection).map { abs($0 - $1) }.max() ?? .infinity + if !(error <= tapReplayTolerance) { out.append("tap \(tap.id) ray replays with error \(error)") } + } + for (name, close) in [("meterCloseUp", p.scaleReference.meterCloseUp), ("obliqueCloseUp", p.scaleReference.ext.obliqueCloseUp)] { + if close.captured, !(close.still.map(stillIDs.contains) ?? false) { out.append("\(name) names a still that is not in the packet") } + } + requireListed(p.streams.imuRaw.file, .stream, "imuRaw") + requireListed(p.streams.arkitPoses.file, .stream, "arkitPoses") + + if let folder { + for entry in p.files { + guard let data = try? Data(contentsOf: folder.appending(path: entry.path)) else { out.append("file \(entry.path) is missing"); continue } + if data.count != entry.bytes { out.append("file \(entry.path) is \(data.count) bytes, listed \(entry.bytes)") } + if PacketFiles.sha256(data) != entry.sha256 { out.append("file \(entry.path) SHA-256 differs from its listing") } + } + } + return out + } + + /// The ray through `pixel` as the server replays it: origin the camera position, direction + /// the unit vector of R · ((u − cx)/fx, −(v − cy)/fy, −1). + public static func ray(pose: [Double], intrinsics k: [Double], pixel: [Double]) -> (origin: [Double], direction: [Double]) { + guard pose.count == 16, k.count == 4, pixel.count == 2 else { return ([], []) } + let d = SIMD3((pixel[0] - k[2]) / k[0], -(pixel[1] - k[3]) / k[1], -1) + let c0 = SIMD3(pose[0], pose[1], pose[2]), c1 = SIMD3(pose[4], pose[5], pose[6]), c2 = SIMD3(pose[8], pose[9], pose[10]) + let world = simd_normalize(c0 * d.x + c1 * d.y + c2 * d.z) + return ([pose[12], pose[13], pose[14]], [world.x, world.y, world.z]) + } + + static func matrix(_ values: [Double]) -> simd_float4x4? { + guard values.count == 16, values.allSatisfy(\.isFinite) else { return nil } + let f = values.map(Float.init) + return simd_float4x4(SIMD4(f[0], f[1], f[2], f[3]), SIMD4(f[4], f[5], f[6], f[7]), SIMD4(f[8], f[9], f[10], f[11]), SIMD4(f[12], f[13], f[14], f[15])) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Producer.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Producer.swift new file mode 100644 index 000000000..7691ebe16 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Producer.swift @@ -0,0 +1,385 @@ +import CryptoKit +import Foundation +import ImageIO +import simd + +/// A file of the 0.4 packet that is on disk and will never change: its bytes, digests and, for +/// an image, the record that describes it. The same value registers the file with the capture API +/// and lists it in the final packet, so the two can't disagree. +public struct SealedFile: Codable, Sendable, Equatable { + public var path: String + public var bytes: Int + public var sha256: String + /// Base64 of the 16-byte MD5, as the register call and the storage PUT's Content-MD5 want it. + public var md5: String + public var role: Packet04.Role + public var contentType: String + public var meta: Packet04.ImageRecord? + /// Lower goes first: the meter close-ups before keyframes before streams. + public var priority: Int + + public var entry: Packet04.FileEntry { .init(path: path, bytes: bytes, sha256: sha256, role: role) } + + static func seal(_ data: Data, path: String, role: Packet04.Role, contentType: String, meta: Packet04.ImageRecord?, priority: Int) -> SealedFile { + SealedFile( + path: path, bytes: data.count, sha256: PacketFiles.sha256(data), + md5: Data(Insecure.MD5.hash(data: data)).base64EncodedString(), role: role, contentType: contentType, meta: meta, + priority: priority) + } +} + +/// What the capture observed for one saved camera image, read from its own ARFrame when the +/// pixels were taken: the raw ARKit camera-to-world pose, never one corrected into the meter frame. +public struct Packet04Observation: Sendable, Equatable { + public var t: Double + public var cameraToWorld: simd_float4x4 + /// [fx, fy, cx, cy] in pixels of the stored JPEG. + public var intrinsics: SIMD4 + public var width: Int + public var height: Int + public var tracking: PacketTracking + public var exposure: Packet04.Exposure? + + public init(t: Double, cameraToWorld: simd_float4x4, intrinsics: SIMD4, width: Int, height: Int, tracking: PacketTracking, exposure: Packet04.Exposure? = nil) { + self.t = t + self.cameraToWorld = cameraToWorld + self.intrinsics = intrinsics + self.width = width + self.height = height + self.tracking = tracking + self.exposure = exposure + } +} + +/// ARKit depth for one keyframe: Float32 meters and UInt8 confidence, row by row. +public struct Packet04Depth: Sendable { + public var meters: [Float] + public var confidence: [UInt8] + public var width: Int + public var height: Int + + public init(meters: [Float], confidence: [UInt8], width: Int, height: Int) { + self.meters = meters + self.confidence = confidence + self.width = width + self.height = height + } +} + +/// Fixed facts about the capture, known when the ARKit session starts and the homeowner consents, +/// so the create request can be sent before the first photo. +public struct Packet04SessionInfo: Codable, Sendable, Equatable { + public var packetID: String + public var sessionID: String + public var source: Packet04.Source.Kind + public var appVersion: String + public var deviceModel: String + public var systemVersion: String + public var lidarAvailable: Bool + public var sceneDepthEnabled: Bool + public var meshReconstructionSupported: Bool? + public var sceneReconstruction: String? + public var planeDetection: [String]? + public var videoWidth: Int + public var videoHeight: Int + public var framesPerSecond: Double? + public var timeZone: String? + + public init( + packetID: String, sessionID: String, source: Packet04.Source.Kind, appVersion: String, deviceModel: String, systemVersion: String, + lidarAvailable: Bool, sceneDepthEnabled: Bool, meshReconstructionSupported: Bool?, sceneReconstruction: String?, + planeDetection: [String]?, videoWidth: Int, videoHeight: Int, framesPerSecond: Double?, timeZone: String? + ) { + self.packetID = packetID + self.sessionID = sessionID + self.source = source + self.appVersion = appVersion + self.deviceModel = deviceModel + self.systemVersion = systemVersion + self.lidarAvailable = lidarAvailable + self.sceneDepthEnabled = sceneDepthEnabled + self.meshReconstructionSupported = meshReconstructionSupported + self.sceneReconstruction = sceneReconstruction + self.planeDetection = planeDetection + self.videoWidth = videoWidth + self.videoHeight = videoHeight + self.framesPerSecond = framesPerSecond + self.timeZone = timeZone + } + + /// `arkit_lidar` needs depth the session actually recorded, not a phone that could have. + public var tier: Packet04.Tier { lidarAvailable && sceneDepthEnabled ? .arkitLidar : .arkit } +} + +public enum Packet04Error: Error, Equatable, CustomStringConvertible { + case invalidID(String) + case sealedAfterFinish(String) + case duplicate(String) + case invalidImage(id: String, reason: String) + case notRigid(String) + case unknownKeyframe(String) + case streamTooSlow(stream: String, measuredHz: Double, minimumHz: Double) + case streamEmpty(String) + case noKeyframes + case invalidPacket([String]) + + public var description: String { + switch self { + case .invalidID(let id): "\(id) does not fit the server's id pattern" + case .sealedAfterFinish(let what): "\(what) arrived after the packet was finished" + case .duplicate(let id): "\(id) was sealed twice" + case .invalidImage(let id, let reason): "image \(id): \(reason)" + case .notRigid(let place): "\(place): pose is not a rotation plus a translation" + case .unknownKeyframe(let id): "no keyframe \(id)" + case .streamTooSlow(let stream, let hz, let minimum): "\(stream) measured \(hz) Hz; the contract requires at least \(minimum) Hz" + case .streamEmpty(let stream): "\(stream) has no rows" + case .noKeyframes: "an ARKit packet needs at least one keyframe" + case .invalidPacket(let problems): "packet fails its own checks: \(problems.joined(separator: "; "))" + } + } +} + +/// Builds one 0.4 packet while the capture runs. Each image is written into `folder` and sealed +/// the moment it is observed, so it can be uploaded during the capture; `finish` adds the streams +/// and returns the exact `packet.json` bytes, which are frozen from then on. +/// +/// One producer is one ARKit epoch. A world reset ends the packet: the app starts a new producer +/// with a new packet id rather than mixing two worlds under one epoch. +public actor Packet04Producer { + public nonisolated let folder: URL + public nonisolated let info: Packet04SessionInfo + public nonisolated let epoch = "e1" + private let startedAtUptime: Double + private let startedAt: Date + private var keyframes: [Packet04.Keyframe] = [] + private var stills: [Packet04.Still] = [] + private var taps: [Packet04.Tap] = [] + private var files: [SealedFile] = [] + private var finished: Data? + + /// `startedAtUptime` is the uptime of the session's first ARFrame and `startedAt` the wall clock + /// at that instant. + public init(folder: URL, info: Packet04SessionInfo, startedAtUptime: Double, startedAt: Date) throws { + guard Packet04.isStorageID(info.packetID) else { throw Packet04Error.invalidID(info.packetID) } + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + self.folder = folder + self.info = info + self.startedAtUptime = startedAtUptime + self.startedAt = startedAt + } + + public var sealedFiles: [SealedFile] { files } + public var keyframeCount: Int { keyframes.count } + public var isFinished: Bool { finished != nil } + + /// The keyframe sealed from the frame at `t`, if there is one: a still taken from a frame that + /// was already kept links to it instead of repeating it. + public func keyframeID(at t: Double) -> String? { keyframes.first { $0.timestamp == t }?.id } + + /// "k00001" for 1. + public static func keyframeID(_ number: Int) -> String { + let digits = String(number) + return "k" + String(repeating: "0", count: max(0, 5 - digits.count)) + digits + } + + // MARK: Images + + /// Writes the JPEG as keyframes/.jpg with, when given, its depth and confidence, and seals + /// them. `reason` is the contract's keyframe reason (motion, tap, still, request, auto). + public func sealKeyframe( + jpeg: Data, observation o: Packet04Observation, reason: String, purpose: String? = nil, depth: Packet04Depth? = nil + ) throws -> [SealedFile] { + guard finished == nil else { throw Packet04Error.sealedAfterFinish("keyframe") } + let id = Self.keyframeID(keyframes.count + 1) + try check(o, id: id, jpeg: jpeg) + guard let tracking = Self.trackingText(o.tracking) else { + throw Packet04Error.invalidImage(id: id, reason: "tracking is limited for a reason the contract has no name for") + } + var sealed: [SealedFile] = [] + var depthRef: Packet04.DepthRef? + if let depth, depth.meters.count == depth.width * depth.height, depth.confidence.count == depth.meters.count, depth.width > 0 { + let map = try write(PacketFiles.depthData(meters: depth.meters), "keyframes/\(id).depth.f32", role: .depth, type: "application/octet-stream", priority: 20) + let confidence = try write(Data(depth.confidence), "keyframes/\(id).confidence.u8", role: .confidence, type: "application/octet-stream", priority: 20) + sealed += [map, confidence] + depthRef = .init(file: map.path, confidenceFile: confidence.path, w: depth.width, h: depth.height) + } + let record = Packet04.Keyframe( + id: id, img: "keyframes/\(id).jpg", w: o.width, h: o.height, intrinsics: Self.numbers(o.intrinsics), + pose: PacketPose.columnMajor(o.cameraToWorld), timestamp: o.t, tracking: tracking, epoch: epoch, + reason: reason, exposure: o.exposure, depth: depthRef, purpose: purpose) + let image = try write(jpeg, record.img, role: .keyframe, type: "image/jpeg", meta: .keyframe(record), priority: 10) + keyframes.append(record) + return [image] + sealed + } + + /// A checklist still taken from an ARKit keyframe already sealed: the same pixels under + /// stills/.jpg, linked to the keyframe so it keeps its pose. Stills go first. + /// + /// A retake of the same purpose is a new still ("meter_close-2"): sealed files never change, + /// and `finish` says which one the scale reference names. + public func sealStill(purpose: String, keyframe keyframeID: String) throws -> SealedFile { + guard finished == nil else { throw Packet04Error.sealedAfterFinish("still") } + guard Packet04.isSegmentID(purpose) else { throw Packet04Error.invalidID(purpose) } + guard let frame = keyframes.first(where: { $0.id == keyframeID }) else { throw Packet04Error.unknownKeyframe(keyframeID) } + if let same = stills.last(where: { $0.purpose == purpose }), same.keyframe == keyframeID { + throw Packet04Error.duplicate(same.id) + } + let taken = stills.filter { $0.purpose == purpose }.count + let id = taken == 0 ? purpose : "\(purpose)-\(taken + 1)" + let jpeg = try Data(contentsOf: folder.appending(path: frame.img)) + let still = Packet04.Still( + id: id, purpose: purpose, img: "stills/\(id).jpg", w: frame.w, h: frame.h, timestamp: frame.timestamp, + orientation: 1, keyframe: frame.id, intrinsics: frame.intrinsics, intrinsicsSource: "arkit") + let file = try write(jpeg, still.img, role: .still, type: "image/jpeg", meta: .still(still), priority: 0) + stills.append(still) + return file + } + + /// A tap on a sealed keyframe's pixel, with the ray rebuilt from that keyframe's own pose and + /// intrinsics, so the server's replay reproduces it exactly. + public func addTap(id: String, label: String, keyframe keyframeID: String, pixel: SIMD2, time: Double, hit: Packet04.TapHit?) throws { + guard finished == nil else { throw Packet04Error.sealedAfterFinish("tap") } + guard let frame = keyframes.first(where: { $0.id == keyframeID }) else { throw Packet04Error.unknownKeyframe(keyframeID) } + let ray = Packet04Check.ray(pose: frame.pose, intrinsics: frame.intrinsics, pixel: [pixel.x, pixel.y]) + taps.append(.init( + id: id, time: time, label: label, keyframe: keyframeID, pixel: [pixel.x, pixel.y], rayOrigin: ray.origin, + rayDirection: ray.direction, hit: hit, epoch: epoch)) + } + + /// The frame the homeowner tapped, kept as a keyframe (reason `tap`) unless it already is one, + /// and the tap on it: `pixel` in that frame's stored image, the ray rebuilt from its own pose + /// and intrinsics, and what the phone's raycast hit. + public func sealTap( + id: String, label: String, jpeg: Data, observation: Packet04Observation, pixel: SIMD2, hit: Packet04.TapHit? + ) throws -> [SealedFile] { + var files: [SealedFile] = [] + if keyframeID(at: observation.t) == nil { + files = try sealKeyframe(jpeg: jpeg, observation: observation, reason: "tap") + } + guard let keyframe = keyframeID(at: observation.t) else { throw Packet04Error.unknownKeyframe("at \(observation.t)") } + try addTap(id: id, label: label, keyframe: keyframe, pixel: pixel, time: observation.t, hit: hit) + return files + } + + // MARK: Finish + + /// Writes the two required streams, then returns their sealed files and the exact packet + /// bytes. The packet is checked against the contract's rules first; a failure names each + /// problem instead of sending a packet the server would reject. + /// `acceptedCloseUpAt` is the frame time of the head-on close-up the scan accepted, nil when + /// none was (skipped, or every shot refused): only that still is the scale reference. + public func finish( + poses: [Packet04Streams.PoseRow], accelerometer: [Packet04Streams.MotionRow], gyroscope: [Packet04Streams.MotionRow], + endedAtUptime: Double, acceptedCloseUpAt: Double?, closeUpDistanceM: Double? = nil, createdAt: Date = Date() + ) throws -> (streams: [SealedFile], packet: Data) { + if let finished { return (files.filter { $0.role == .stream }, finished) } + guard !keyframes.isEmpty else { throw Packet04Error.noKeyframes } + let posesCSV = try Packet04Streams.arkitPoses(poses, epoch: epoch) + let imu = try Packet04Streams.imuRaw(accelerometer: accelerometer, gyroscope: gyroscope) + let poseFile = try write(try Gzip.compress(posesCSV.csv), "streams/arkit_poses.csv.gz", role: .stream, type: "application/gzip", priority: 30) + let imuFile = try write(try Gzip.compress(imu.csv), "streams/imu_raw.csv.gz", role: .stream, type: "application/gzip", priority: 30) + // Each sensor as Core Motion timed it, so the server can re-pair them its own way. + let accelFile = try write( + try Gzip.compress(try Packet04Streams.rawMotionCSV(accelerometer, columns: ["t", "ax", "ay", "az"])), + "streams/accelerometer_raw.csv.gz", role: .stream, type: "application/gzip", priority: 31) + let gyroFile = try write( + try Gzip.compress(try Packet04Streams.rawMotionCSV(gyroscope, columns: ["t", "gx", "gy", "gz"])), + "streams/gyroscope_raw.csv.gz", role: .stream, type: "application/gzip", priority: 31) + var ext = imu.pairingNote + ext["rawAccelerometerFile"] = accelFile.path + ext["rawGyroscopeFile"] = gyroFile.path + + let close = acceptedCloseUpAt.flatMap { t in stills.last { $0.purpose == "meter_close" && $0.timestamp == t } } + // The app takes no oblique close-up yet, so none is ever accepted. + let oblique: Packet04.Still? = nil + let packet = Packet04.Packet( + packetId: info.packetID, createdAt: PacketWriter.iso8601(createdAt), + source: .init(kind: info.source, appBuild: info.appVersion), + session: .init( + id: info.sessionID, startedAt: PacketWriter.iso8601(startedAt), startedAtUptime: startedAtUptime, + endedAtUptime: max(endedAtUptime, startedAtUptime), timeZone: info.timeZone, appVersion: info.appVersion, + deviceModel: info.deviceModel, systemVersion: info.systemVersion, tier: info.tier, lidarAvailable: info.lidarAvailable, + meshReconstructionSupported: info.meshReconstructionSupported, sceneDepthEnabled: info.sceneDepthEnabled, + sceneReconstruction: info.sceneReconstruction, planeDetection: info.planeDetection, + videoFormat: .init(width: info.videoWidth, height: info.videoHeight, framesPerSecond: info.framesPerSecond), + motionReferenceFrame: nil), + epochs: [.init(id: epoch, startTime: startedAtUptime, reason: "sessionStart")], + keyframes: keyframes, stills: stills.isEmpty ? nil : stills, taps: taps.isEmpty ? nil : taps, + tracking: Packet04Streams.trackingChanges(poses, epoch: epoch), + streams: .init( + imuRaw: .init(file: imuFile.path, columns: Packet04Streams.imuColumns, rateHz: imu.rateHz), + arkitPoses: .init(file: poseFile.path, columns: Packet04Streams.poseColumns, rateHz: posesCSV.rateHz)), + scaleReference: .init( + meterCloseUp: close.map { .init(captured: true, still: $0.id, arkitDistanceM: closeUpDistanceM, side: nil) } ?? .notCaptured, + ext: .init(obliqueCloseUp: oblique.map { .init(captured: true, still: $0.id, arkitDistanceM: nil, side: nil) } ?? .notCaptured)), + files: files.map(\.entry), + ext: ext) + let problems = Packet04Check.problems(packet, folder: folder) + guard problems.isEmpty else { throw Packet04Error.invalidPacket(problems) } + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys, .withoutEscapingSlashes] + let data = try encoder.encode(packet) + try data.write(to: folder.appending(path: "packet.json"), options: .atomic) + finished = data + return ([poseFile, imuFile, accelFile, gyroFile], data) + } + + // MARK: Helpers + + private func check(_ o: Packet04Observation, id: String, jpeg: Data) throws { + guard PacketPose.isRigid(o.cameraToWorld) else { throw Packet04Error.notRigid("keyframe \(id)") } + guard o.t.isFinite, o.t >= 0 else { throw Packet04Error.invalidImage(id: id, reason: "timestamp \(o.t)") } + if let problem = PacketWriter.intrinsicsProblem(o.intrinsics, width: o.width, height: o.height) { + throw Packet04Error.invalidImage(id: id, reason: problem) + } + if let problem = Self.jpegProblem(jpeg, width: o.width, height: o.height) { + throw Packet04Error.invalidImage(id: id, reason: problem) + } + if let same = keyframes.first(where: { $0.timestamp == o.t }) { + throw Packet04Error.invalidImage(id: id, reason: "keyframe \(same.id) already has timestamp \(o.t)") + } + } + + private func write(_ data: Data, _ path: String, role: Packet04.Role, type: String, meta: Packet04.ImageRecord? = nil, priority: Int) throws -> SealedFile { + guard Packet04.isRelPath(path) else { throw Packet04Error.invalidID(path) } + guard !files.contains(where: { $0.path == path }) else { throw Packet04Error.duplicate(path) } + let url = folder.appending(path: path) + try FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true) + try data.write(to: url, options: .atomic) + let file = SealedFile.seal(data, path: path, role: role, contentType: type, meta: meta, priority: priority) + files.append(file) + return file + } + + /// The bytes must be a JPEG of the stated size, stored unrotated. + static func jpegProblem(_ jpeg: Data, width: Int, height: Int) -> String? { + guard let source = CGImageSourceCreateWithData(jpeg as CFData, nil), CGImageSourceGetType(source) as String? == "public.jpeg", + let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any] + else { return "not a readable JPEG" } + let w = properties[kCGImagePropertyPixelWidth] as? Int, h = properties[kCGImagePropertyPixelHeight] as? Int + guard w == width, h == height else { return "the JPEG is \(w ?? 0)x\(h ?? 0), not \(width)x\(height)" } + if let orientation = properties[kCGImagePropertyOrientation] as? Int, orientation != 1 { + return "EXIF orientation \(orientation); store the unrotated sensor image" + } + return nil + } + + static func numbers(_ k: SIMD4) -> [Double] { [k.x, k.y, k.z, k.w].map(PacketNumber.double) } + + /// The contract's tracking state. Nil for a limited state whose reason the contract has no + /// name for (a reason added after iOS 26): no value there would be an observation. + public static func trackingText(_ tracking: PacketTracking) -> String? { + switch tracking { + case .normal: "normal" + case .notAvailable: "notAvailable" + case .limited(let why): + switch why { + case .initializing?: "limited.initializing" + case .excessiveMotion?: "limited.excessiveMotion" + case .insufficientFeatures?: "limited.insufficientFeatures" + case .relocalizing?: "limited.relocalizing" + case nil: nil + } + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Streams.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Streams.swift new file mode 100644 index 000000000..be5cabecc --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Streams.swift @@ -0,0 +1,147 @@ +import Foundation +import simd + +/// The 0.4 packet's two required streams as CSV, and the tracking changes read from the poses. +public enum Packet04Streams { + public static let poseColumns = ["t", "epoch", "tracking", "tx", "ty", "tz", "qx", "qy", "qz", "qw", "fx", "fy", "cx", "cy"] + public static let imuColumns = ["t", "ax", "ay", "az", "gx", "gy", "gz"] + /// The contract's floors for `rateHz`. + public static let minimumPoseHz = 10.0 + public static let minimumIMUHz = 50.0 + + /// One ARFrame: raw camera-to-world and that frame's intrinsics at the video format's size. + public struct PoseRow: Sendable, Equatable { + public var t: Double + public var tracking: PacketTracking + public var cameraToWorld: simd_float4x4 + public var intrinsics: SIMD4 + + public init(t: Double, tracking: PacketTracking, cameraToWorld: simd_float4x4, intrinsics: SIMD4) { + self.t = t + self.tracking = tracking + self.cameraToWorld = cameraToWorld + self.intrinsics = intrinsics + } + } + + /// One Core Motion sample with its own timestamp: g for the accelerometer, rad/s for the gyro. + public struct MotionRow: Sendable, Equatable { + public var t: Double + public var value: SIMD3 + + public init(t: Double, value: SIMD3) { + self.t = t + self.value = value + } + } + + public static func arkitPoses(_ rows: [PoseRow], epoch: String) throws -> (csv: Data, rateHz: Double) { + guard !rows.isEmpty else { throw Packet04Error.streamEmpty("arkitPoses") } + var text = poseColumns.joined(separator: ",") + "\n" + var times: [Double] = [] + for row in rows where row.t > (times.last ?? -.infinity) { + guard PacketPose.isRigid(row.cameraToWorld) else { continue } + let p = PacketPose.translation(row.cameraToWorld) + let q = PacketPose.quaternion(PacketPose.rotation(row.cameraToWorld)) + let k = row.intrinsics + // The column is free text: a limited state whose reason the contract can't name is + // still limited, and the pose it came with is still a measurement. + let cells = [row.t.description, epoch, Packet04Producer.trackingText(row.tracking) ?? "limited"] + + [p.x, p.y, p.z].map { PacketNumber.double($0).description } + + [q.x, q.y, q.z, q.w].map(\.description) + + [k.x, k.y, k.z, k.w].map { PacketNumber.double($0).description } + text += cells.joined(separator: ",") + "\n" + times.append(row.t) + } + let rate = measuredRate(times) + guard rate >= minimumPoseHz else { throw Packet04Error.streamTooSlow(stream: "arkitPoses", measuredHz: rate, minimumHz: minimumPoseHz) } + return (Data(text.utf8), rate) + } + + /// Accelerometer and gyro rows for `imuRaw`. Core Motion times the two sensors separately, and + /// the contract's row holds both. Each row is an accelerometer sample paired with the gyro + /// sample nearest in time, only when that gyro sample is within `maxPairingOffset`; an + /// accelerometer sample with no gyro sample that close is left out rather than given one. + /// How the server wants the two clocks joined is an open question to the server team, so the + /// rule and its measured worst offset travel in the packet's `ext`. + public static func imuRaw( + accelerometer: [MotionRow], gyroscope: [MotionRow], maxPairingOffset: Double = 0.005 + ) throws -> (csv: Data, rateHz: Double, pairingNote: [String: String]) { + let accel = accelerometer.sorted { $0.t < $1.t } + let gyro = gyroscope.sorted { $0.t < $1.t } + guard !accel.isEmpty, !gyro.isEmpty else { throw Packet04Error.streamEmpty("imuRaw") } + var text = imuColumns.joined(separator: ",") + "\n" + var times: [Double] = [] + var worst = 0.0 + var g = 0 + for a in accel where a.t > (times.last ?? -.infinity) { + while g + 1 < gyro.count, abs(gyro[g + 1].t - a.t) <= abs(gyro[g].t - a.t) { g += 1 } + let offset = abs(gyro[g].t - a.t) + guard offset <= maxPairingOffset else { continue } + worst = max(worst, offset) + let cells = [a.t, a.value.x, a.value.y, a.value.z, gyro[g].value.x, gyro[g].value.y, gyro[g].value.z] + text += cells.map(\.description).joined(separator: ",") + "\n" + times.append(a.t) + } + guard !times.isEmpty else { throw Packet04Error.streamEmpty("imuRaw") } + let rate = measuredRate(times) + guard rate >= minimumIMUHz else { throw Packet04Error.streamTooSlow(stream: "imuRaw", measuredHz: rate, minimumHz: minimumIMUHz) } + let note = [ + "imuRawRowClock": "accelerometer", + "imuRawGyroPairing": "nearest gyro sample within \(maxPairingOffset) s; accelerometer samples without one are left out", + "imuRawGyroOffsetMaxS": worst.description, + "imuRawRowsKept": "\(times.count) of \(accel.count)", + "imuRawPairingStatus": "proposal from the phone; the server has not agreed how accelerometer and gyro samples join into one row", + ] + return (Data(text.utf8), rate, note) + } + + /// The first state and every change. A limited state with no name in the contract is not + /// listed; the rows around it still are. + public static func trackingChanges(_ rows: [PoseRow], epoch: String) -> [Packet04.TrackingEvent] { + var events: [Packet04.TrackingEvent] = [] + for row in rows { + guard let state = Packet04Producer.trackingText(row.tracking), state != events.last?.state else { continue } + events.append(.init(time: row.t, state: state, epoch: epoch)) + } + return events + } + + /// One sensor's samples exactly as Core Motion timed them, `t,x,y,z`: the raw evidence behind + /// `imuRaw`, kept because the pairing rule there is a proposal the server has not agreed to. + public static func rawMotionCSV(_ rows: [MotionRow], columns: [String]) throws -> Data { + guard !rows.isEmpty else { throw Packet04Error.streamEmpty(columns.joined(separator: ",")) } + var text = columns.joined(separator: ",") + "\n" + for row in rows.sorted(by: { $0.t < $1.t }) { + text += [row.t, row.value.x, row.value.y, row.value.z].map(\.description).joined(separator: ",") + "\n" + } + return Data(text.utf8) + } + + /// Pose rows from the recorder's raw rows: trajectory rows `[t, tracking state, tracking + /// reason, 16 camera-to-world values column by column]` and intrinsics rows `[t, fx, fy, cx, + /// cy, width, height]` written for the same frame at the same t. A pose with no intrinsics row + /// at its t is left out rather than given another frame's calibration. + public static func poseRows(trajectory: [[Double]], intrinsics: [[Double]], tracking: (Int, Int) -> PacketTracking) -> [PoseRow] { + var calibration: [Double: SIMD4] = [:] + for row in intrinsics where row.count >= 5 { calibration[row[0]] = SIMD4(Float(row[1]), Float(row[2]), Float(row[3]), Float(row[4])) } + return trajectory.compactMap { row in + guard row.count == 19, let k = calibration[row[0]] else { return nil } + let f = row[3...].map(Float.init) + let m = simd_float4x4(SIMD4(f[0], f[1], f[2], f[3]), SIMD4(f[4], f[5], f[6], f[7]), SIMD4(f[8], f[9], f[10], f[11]), SIMD4(f[12], f[13], f[14], f[15])) + return PoseRow(t: row[0], tracking: tracking(Int(row[1]), Int(row[2])), cameraToWorld: m, intrinsics: k) + } + } + + /// Motion rows from raw `[t, x, y, z]` rows. + public static func motionRows(_ rows: [[Double]]) -> [MotionRow] { + rows.filter { $0.count == 4 }.map { MotionRow(t: $0[0], value: SIMD3($0[1], $0[2], $0[3])) } + } + + /// 1 / the median interval between rows, the rate actually recorded. 0 with fewer than two. + public static func measuredRate(_ times: [Double]) -> Double { + let deltas = zip(times.dropFirst(), times).map { $0 - $1 }.filter { $0 > 0 }.sorted() + guard !deltas.isEmpty else { return 0 } + return 1 / deltas[deltas.count / 2] + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift new file mode 100644 index 000000000..1bc42aa06 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift @@ -0,0 +1,208 @@ +import Foundation + +/// The capture API's request and response bodies that House Scan sends and reads (`/v1/captures` +/// and below), written from the documented wire contract. Unknown response fields are ignored. +public enum CaptureAPI { + public struct Device: Codable, Sendable, Equatable { + public var model: String + public var systemVersion: String + public var appVersion: String + + public init(model: String, systemVersion: String, appVersion: String) { + self.model = model + self.systemVersion = systemVersion + self.appVersion = appVersion + } + } + + /// The create body. Expected counts are left out: they are optional, and a count that grows as + /// photos arrive would change the body the idempotency key is bound to. + public struct CreateRequest: Codable, Sendable, Equatable { + public var packetId: String + public var formatVersion = Packet04.formatVersion + public var tier: Packet04.Tier + public var flow = "guided" + public var device: Device + + public init(packetId: String, tier: Packet04.Tier, device: Device) { + self.packetId = packetId + self.tier = tier + self.device = device + } + } + + public struct Limits: Codable, Sendable, Equatable { + public var maxSinglePutBytes: Int? + public var urlTtlS: Int? + public var maxFiles: Int? + public var maxBatch: Int? + } + + public struct CreateResponse: Codable, Sendable, Equatable { + public var captureId: String + public var status: String + public var upload: Limits? + public var finalizeBy: String? + } + + public struct RegisterFile: Codable, Sendable, Equatable { + public var path: String + public var bytes: Int + public var sha256: String + public var md5: String + public var role: Packet04.Role + public var contentType: String + public var meta: Packet04.ImageRecord? + + public init(_ file: SealedFile) { + path = file.path + bytes = file.bytes + sha256 = file.sha256 + md5 = file.md5 + role = file.role + contentType = file.contentType + meta = file.meta + } + } + + public struct RegisterRequest: Codable, Sendable, Equatable { + public var files: [RegisterFile] + } + + public struct UploadTarget: Codable, Sendable, Equatable { + public var method: String + public var url: String? + public var headers: [String: String]? + public var expiresAt: String? + } + + public struct RegisteredFile: Codable, Sendable, Equatable { + public var path: String + public var state: String + public var upload: UploadTarget? + } + + public struct RegisterResponse: Codable, Sendable, Equatable { + public var files: [RegisteredFile] + } + + public struct CommitFile: Codable, Sendable, Equatable { + public var path: String + public var sha256: String + } + + public struct CommitRequest: Codable, Sendable, Equatable { + public var files: [CommitFile] + } + + public struct CommitResponse: Codable, Sendable, Equatable { + public var committed: [String] + public var notFound: [String] + public var mismatch: [String] + } + + public struct FinalizeResponse: Codable, Sendable, Equatable { + public var status: String + public var missing: [String] + public var runId: String + public var etaS: Int? + } + + public struct Event: Codable, Sendable, Equatable { + public var seq: Int + public var type: String + public var at: String + /// The fields the app reads. Others, which may carry text meant for the homeowner, are + /// not kept. + public var data: EventData? + } + + public struct EventData: Codable, Sendable, Equatable { + public var code: String? + public var kind: String? + public var stage: String? + public var status: String? + public var runId: String? + public var next: String? + } + + public struct EventsResponse: Codable, Sendable, Equatable { + public var status: String + public var next: Int + public var events: [Event] + } + + public struct ErrorItem: Codable, Sendable, Equatable { + public var code: String + public var pointer: String? + public var message: String? + } + + public struct ErrorBody: Codable, Sendable, Equatable { + public var errors: [ErrorItem] + } + + public static func errorCodes(_ body: Data) -> [String] { + (try? JSONDecoder().decode(ErrorBody.self, from: body))?.errors.map(\.code) ?? [] + } + + /// Statuses after which the capture changes no more. + public static let terminalStatuses: Set = ["complete", "manual_review", "failed", "expired", "needs_views"] +} + +/// An HTTP answer: status, body and the headers the uploader reads. +public struct HTTPReply: Sendable, Equatable { + public var status: Int + public var body: Data + public var retryAfter: Double? + + public init(status: Int, body: Data, retryAfter: Double? = nil) { + self.status = status + self.body = body + self.retryAfter = retryAfter + } +} + +/// How the uploader reaches the network. HTTP error statuses come back as replies; only transport +/// failures throw. +public protocol CaptureHTTP: Sendable { + func send(_ request: URLRequest) async throws -> HTTPReply + /// `file` as the whole body of `request`, streamed from disk. + func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply +} + +/// `CaptureHTTP` on one ordinary `URLSession`. +public struct URLSessionCaptureHTTP: CaptureHTTP { + public let session: URLSession + + public init(session: URLSession) { + self.session = session + } + + /// An ephemeral session: nothing cached or stored in cookies, and no credentials offered to + /// the signed storage URLs. + public static func ephemeral(timeout: Double = 60) -> URLSessionCaptureHTTP { + let configuration = URLSessionConfiguration.ephemeral + configuration.timeoutIntervalForRequest = timeout + configuration.httpCookieStorage = nil + configuration.urlCredentialStorage = nil + configuration.httpMaximumConnectionsPerHost = 4 + return URLSessionCaptureHTTP(session: URLSession(configuration: configuration)) + } + + public func send(_ request: URLRequest) async throws -> HTTPReply { + let (data, response) = try await session.data(for: request) + return Self.reply(data, response) + } + + public func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + let (data, response) = try await session.upload(for: request, fromFile: file) + return Self.reply(data, response) + } + + static func reply(_ data: Data, _ response: URLResponse) -> HTTPReply { + let http = response as? HTTPURLResponse + return HTTPReply( + status: http?.statusCode ?? 0, body: data, retryAfter: http?.value(forHTTPHeaderField: "Retry-After").flatMap(Double.init)) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift new file mode 100644 index 000000000..e0a94a0f7 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift @@ -0,0 +1,465 @@ +import Foundation +import simd + +/// A photo the scan kept, as the app stores it: the JPEG on disk and the facts of the ARFrame +/// whose pixels it holds, in that frame's own terms (raw world pose, the camera image's +/// intrinsics and size). +public struct KeptPhoto: Sendable { + public var t: Double + public var cameraToWorld: simd_float4x4 + /// [fx, fy, cx, cy] in pixels of the camera image (`cameraImageSize`). + public var cameraIntrinsics: SIMD4 + public var cameraImageSize: SIMD2 + /// Pixels of the stored JPEG. + public var width: Int + public var height: Int + public var tracking: PacketTracking + public var exposure: Packet04.Exposure? + public var jpeg: URL + /// The still's purpose ("meter_close"); nil for a keyframe. + public var purpose: String? + /// Reads the photo's ARKit depth. Called once, when the photo is kept, because the app writes a + /// retaken close-up's depth over the same files. + public var depth: @Sendable () -> Packet04Depth? + + public init( + t: Double, cameraToWorld: simd_float4x4, cameraIntrinsics: SIMD4, cameraImageSize: SIMD2, width: Int, height: Int, + tracking: PacketTracking, exposure: Packet04.Exposure?, jpeg: URL, purpose: String?, depth: @escaping @Sendable () -> Packet04Depth? = { nil } + ) { + self.t = t + self.cameraToWorld = cameraToWorld + self.cameraIntrinsics = cameraIntrinsics + self.cameraImageSize = cameraImageSize + self.width = width + self.height = height + self.tracking = tracking + self.exposure = exposure + self.jpeg = jpeg + self.purpose = purpose + self.depth = depth + } + + /// The observation in the stored JPEG's pixels: intrinsics scaled when the JPEG is not the + /// camera image's size. + public var observation: Packet04Observation { + let scale = SIMD2(Float(width), Float(height)) / cameraImageSize + let k = cameraIntrinsics + return Packet04Observation( + t: t, cameraToWorld: cameraToWorld, intrinsics: SIMD4(k.x * scale.x, k.y * scale.y, k.z * scale.x, k.w * scale.y), + width: width, height: height, tracking: tracking, exposure: exposure) + } +} + +/// The frame under the homeowner's tap, read when the tap happened. +public struct TapObservation: Sendable { + public var t: Double + public var cameraToWorld: simd_float4x4 + /// [fx, fy, cx, cy] in pixels of the camera image, which the JPEG stores unscaled. + public var intrinsics: SIMD4 + public var width: Int + public var height: Int + public var tracking: PacketTracking + /// The tap in the sensor image's pixels, (0, 0) its top-left corner. + public var pixel: SIMD2 + /// Encodes the frame's image; called off the main actor. + public var jpeg: @Sendable () -> Data? + + /// The same frame with the tap moved to where `world` (the raycast's hit) lands in its image, + /// so pixel, ray and hit agree in this one frame even when it is a frame or two newer than + /// the one the raycast used. Nil when the point is behind the camera or outside the image. + public func pointing(at world: SIMD3) -> TapObservation? { + let p = cameraToWorld.inverse * SIMD4(world, 1) + guard p.z < -0.05 else { return nil } + let u = Double(intrinsics.z + intrinsics.x * p.x / -p.z), v = Double(intrinsics.w - intrinsics.y * p.y / -p.z) + guard (0...Double(width)).contains(u), (0...Double(height)).contains(v) else { return nil } + var aimed = self + aimed.pixel = SIMD2(u, v) + return aimed + } + + public init( + t: Double, cameraToWorld: simd_float4x4, intrinsics: SIMD4, width: Int, height: Int, tracking: PacketTracking, + pixel: SIMD2, jpeg: @escaping @Sendable () -> Data? + ) { + self.t = t + self.cameraToWorld = cameraToWorld + self.intrinsics = intrinsics + self.width = width + self.height = height + self.tracking = tracking + self.pixel = pixel + self.jpeg = jpeg + } +} + +/// Where the coordinator reads the world's recorded streams: `CaptureRecorder` in the app. +public struct RecordingSource: Sendable { + /// Uptime and wall clock of the world's first recorded frame. + public var start: @Sendable () -> (uptime: Double, date: Date)? + /// Every row so far; called off the main actor. + public var rows: @Sendable () -> RecorderRows + + public init(start: @escaping @Sendable () -> (uptime: Double, date: Date)?, rows: @escaping @Sendable () -> RecorderRows) { + self.start = start + self.rows = rows + } +} + +/// The recorder's raw rows: trajectory `[t, state, reason, 16 pose values]`, intrinsics +/// `[t, fx, fy, cx, cy, w, h]`, and Core Motion's `[t, x, y, z]`, each sensor at its own times. +public struct RecorderRows: Sendable { + public var trajectory: [[Double]] + public var intrinsics: [[Double]] + public var accelerometer: [[Double]] + public var gyroscope: [[Double]] + + public init(trajectory: [[Double]], intrinsics: [[Double]], accelerometer: [[Double]], gyroscope: [[Double]]) { + self.trajectory = trajectory + self.intrinsics = intrinsics + self.accelerometer = accelerometer + self.gyroscope = gyroscope + } +} + +/// What the integration build does with a scan: one 0.4 packet per ARKit world, each kept photo +/// sealed into it on the phone as it is kept, the packet frozen when the scan is sent for +/// placement. Sending is separate: only a build allowed to send device data (`sends`) sends, and +/// only after the homeowner's yes for this scan, asked before it starts. From the yes on, every +/// sealed file goes up, and later photos go up as they are kept. A new scan asks again; a world +/// reset within the scan keeps its answer for the scan's next packet. +/// +/// Work for one world runs in order on that world's chain; a world reset or start over ends the +/// session, and nothing queued for it reaches the next one. +@MainActor +public final class CaptureSessionCoordinator { + public struct Environment: Sendable { + public var endpoint: URL + /// Whether device captures may go to `endpoint` at all (`CaptureIntegrationMode.send`). + public var sends: Bool + public var http: any CaptureHTTP + /// Each session writes to `/`. + public var capturesFolder: URL + public var sessionInfo: @Sendable (_ packetID: String, _ video: SIMD2) -> Packet04SessionInfo + public var device: CaptureAPI.Device + public var tier: Packet04.Tier + public var policy: CaptureUploader.Policy + public var log: @Sendable (String) -> Void + + public init( + endpoint: URL, sends: Bool, http: any CaptureHTTP, capturesFolder: URL, + sessionInfo: @escaping @Sendable (_ packetID: String, _ video: SIMD2) -> Packet04SessionInfo, + device: CaptureAPI.Device, tier: Packet04.Tier, policy: CaptureUploader.Policy = .init(), log: @escaping @Sendable (String) -> Void = { _ in } + ) { + self.endpoint = endpoint + self.sends = sends + self.http = http + self.capturesFolder = capturesFolder + self.sessionInfo = sessionInfo + self.device = device + self.tier = tier + self.policy = policy + self.log = log + } + } + + /// Checklist purposes the packet's `stills` take; any other still stays a keyframe only. + public static let stillPurposes: Set = ["meter_close", "meter_oblique"] + + @MainActor + public final class Session { + /// This app's id for this session, new for every session even when two packets both call + /// their ARKit epoch "e1": a result is bound to it, so a late result can't land on a newer scan. + public let localID = UUID().uuidString + public let packetID: String + public let folder: URL + /// Nil until the homeowner says yes on a build that sends. + public fileprivate(set) var uploader: CaptureUploader? + public private(set) var producer: Packet04Producer? + let recording: RecordingSource + var chain: Task? + var sealing = false + var ended = false + /// packet.json as frozen, once the scan was sent for placement. + var frozen: Data? + /// The server's result for this session's capture, decoded and bound to it. + public fileprivate(set) var result: CaptureResult.Record? + + init(packetID: String, folder: URL, recording: RecordingSource) { + self.packetID = packetID + self.folder = folder + self.recording = recording + } + + /// Runs `work` after everything queued before it, unless the session has ended by then. + func enqueue(_ work: @escaping @MainActor (Session) async -> Void) { + let previous = chain + chain = Task { + await previous?.value + guard !self.ended else { return } + await work(self) + } + } + + func producer(first photo: (t: Double, video: SIMD2), info: (String, SIMD2) -> Packet04SessionInfo) throws -> Packet04Producer { + if let producer { return producer } + // The packet starts at the world's first recorded frame; a photo can't be older. + let start = recording.start() ?? (photo.t, Date(timeIntervalSinceNow: photo.t - ProcessInfo.processInfo.systemUptime)) + let made = try Packet04Producer(folder: folder, info: info(packetID, photo.video), startedAtUptime: min(start.uptime, photo.t), startedAt: start.date) + producer = made + return made + } + } + + public let environment: Environment? + /// The homeowner's answer for this scan: nil before they were asked. + public private(set) var consent: Bool? + private var consentedAt: Date? + public private(set) var session: Session? + private var recording: RecordingSource? + /// The current session's status; nil when there is none or nothing is being sent. + public var onStatus: (@MainActor (CaptureUploadStatus?) -> Void)? + /// The current session's result, once the server has one. Never called for an ended session. + public var onResult: (@MainActor (CaptureResult.Record) -> Void)? + /// The packet's ARKit epoch: one packet is one world. + public static let epoch = "e1" + + public init(environment: Environment?) { + self.environment = environment + } + + /// Whether the homeowner should be asked now: a build that sends, a scan under way, and no + /// answer for it yet. + public var needsConsent: Bool { + environment?.sends == true && consent == nil && session != nil + } + + public func answerConsent(_ yes: Bool, at time: Date = Date()) { + consent = yes + consentedAt = yes ? time : nil + if yes, let session { startUploading(session) } + } + + // MARK: Scan events + + /// The live source started recording a world. + public func begin(recording: RecordingSource) { + self.recording = recording + if environment != nil, session == nil { startSession() } + } + + /// The ARKit world was thrown away, or (`newScan`) the scan started over: this packet can't + /// be finished, and the next one belongs to `recording`'s new world. A new scan asks again. + public func newWorld(_ reason: String, recording: RecordingSource, newScan: Bool) { + endSession(reason) + if newScan { + consent = nil + consentedAt = nil + } + self.recording = recording + if environment != nil { startSession() } + } + + public func kept(_ photo: KeptPhoto) { + guard let session, !session.sealing, let environment else { return } + // The app writes a retaken close-up (JPEG and depth) over the same files, so both are taken + // now, the JPEG as an APFS clone, not when the queued work gets to them. + let depth = photo.depth() + let staged = environment.capturesFolder.appending(path: "staging/\(UUID().uuidString).jpg") + do { + try FileManager.default.createDirectory(at: staged.deletingLastPathComponent(), withIntermediateDirectories: true) + try FileManager.default.copyItem(at: photo.jpeg, to: staged) + } catch { + environment.log("capture packet: a kept photo could not be copied: \(error)") + return + } + session.enqueue { session in + defer { try? FileManager.default.removeItem(at: staged) } + do { + let producer = try session.producer(first: (photo.t, photo.cameraImageSize), info: environment.sessionInfo) + let data = try await Task.detached(priority: .utility) { try Data(contentsOf: staged) }.value + var files: [SealedFile] = [] + if await producer.keyframeID(at: photo.t) == nil { + files += try await producer.sealKeyframe( + jpeg: data, observation: photo.observation, reason: photo.purpose == nil ? "auto" : "still", purpose: photo.purpose, depth: depth) + } + if let purpose = photo.purpose, Self.stillPurposes.contains(purpose), let keyframe = await producer.keyframeID(at: photo.t) { + files.append(try await producer.sealStill(purpose: purpose, keyframe: keyframe)) + } + await session.uploader?.add(files) + } catch { + environment.log("capture packet: a kept photo was not sealed: \(error)") + } + } + } + + /// The homeowner marked the meter: its frame becomes a keyframe and the tap goes on it. + public func meterTapped(_ tap: TapObservation, hit: Packet04.TapHit?) { + guard let session, !session.sealing, let environment else { return } + session.enqueue { session in + do { + let producer = try session.producer(first: (tap.t, SIMD2(Float(tap.width), Float(tap.height))), info: environment.sessionInfo) + guard let jpeg = await Task.detached(priority: .userInitiated, operation: { tap.jpeg() }).value else { + environment.log("capture packet: the meter tap's frame could not be encoded; no tap") + return + } + let observation = Packet04Observation( + t: tap.t, cameraToWorld: tap.cameraToWorld, intrinsics: tap.intrinsics, width: tap.width, height: tap.height, tracking: tap.tracking) + let files = try await producer.sealTap(id: "meter", label: "meter", jpeg: jpeg, observation: observation, pixel: tap.pixel, hit: hit) + await session.uploader?.add(files) + } catch { + environment.log("capture packet: the meter tap was not recorded: \(error)") + } + } + } + + /// The scan was sent for placement: freeze the packet with the streams recorded so far. + /// `acceptedCloseUpAt` is the frame time of the close-up the scan accepted, nil after a skip. + /// Later sends of the same scan (a retry, one more view) change nothing. + public func captureEnded(acceptedCloseUpAt: Double?) { + guard let session, !session.sealing, let environment else { return } + session.sealing = true + session.enqueue { session in + guard let producer = session.producer else { + await session.uploader?.abandon("no photos were kept") + return + } + let recording = session.recording + let rows = await Task.detached(priority: .utility) { recording.rows() }.value + let poses = Packet04Streams.poseRows(trajectory: rows.trajectory, intrinsics: rows.intrinsics, tracking: PacketTracking.init(recorderState:reason:)) + do { + let finished = try await producer.finish( + poses: poses, accelerometer: Packet04Streams.motionRows(rows.accelerometer), + gyroscope: Packet04Streams.motionRows(rows.gyroscope), endedAtUptime: poses.last?.t ?? 0, acceptedCloseUpAt: acceptedCloseUpAt) + session.frozen = finished.packet + await session.uploader?.seal(packet: finished.packet, files: await producer.sealedFiles) + } catch { + environment.log("capture packet not finished: \(error)") + await session.uploader?.abandon("packet refused on the phone") + } + } + } + + /// Waits until the current session's queued work and uploads are idle. For tests and the + /// evidence run. + public func settle() async { + guard let session else { return } + while let chain = session.chain { + await chain.value + if session.chain == chain { break } + } + await session.uploader?.settled() + } + + /// After a relaunch: captures frozen before the app quit finish uploading, each only toward the + /// endpoint it was created on and with the yes it recorded for that endpoint. Returns them. + public func resumeSealedCaptures() -> [CaptureUploader] { + guard let environment, environment.sends, + let folders = try? FileManager.default.contentsOfDirectory(at: environment.capturesFolder, includingPropertiesForKeys: nil) + else { return [] } + var resumed: [CaptureUploader] = [] + for folder in folders { + guard let saved = try? CaptureUploadState.load(from: CaptureUploader.stateURL(in: folder)), saved.end == nil, saved.packet != nil, + let uploader = try? CaptureUploader.resume(folder: folder, base: environment.endpoint, http: environment.http, policy: environment.policy) + else { continue } + resumed.append(uploader) + Task { await uploader.kick() } + } + return resumed + } + + // MARK: Sessions + + private func startSession() { + guard let environment, let recording else { return } + let packetID = UUID().uuidString + let session = Session(packetID: packetID, folder: environment.capturesFolder.appending(path: packetID, directoryHint: .isDirectory), recording: recording) + self.session = session + environment.log("capture packet: new packet for this world") + if consent == true { startUploading(session) } + } + + /// Opens the capture on the server and queues everything sealed so far, then the frozen + /// packet if the scan was already sent. Photos kept later go up as they are sealed. + private func startUploading(_ session: Session) { + guard let environment, environment.sends, consent == true, let consentedAt, session.uploader == nil, !session.ended else { return } + do { + let uploader = try CaptureUploader.start( + folder: session.folder, base: environment.endpoint, http: environment.http, + create: .init(packetId: session.packetID, tier: environment.tier, device: environment.device), consentedAt: consentedAt, + policy: environment.policy) + session.uploader = uploader + // Status and results from an ended session's uploader never reach the screen. + let publish: @Sendable (CaptureUploadStatus) -> Void = { [weak self] status in + Task { @MainActor in + guard let self, self.session === session else { return } + self.onStatus?(status) + if status.resultAvailable, session.result == nil { await self.bindResult(of: session) } + } + } + let log = environment.log + Task { + await uploader.observe(publish, log: log) + await uploader.kick() + } + session.enqueue { session in + guard let producer = session.producer else { return } + let files = await producer.sealedFiles + if let frozen = session.frozen { + await uploader.seal(packet: frozen, files: files) + } else { + await uploader.add(files) + } + } + environment.log("capture upload: sending this world's packet") + } catch { + environment.log("capture upload not started: \(error)") + } + } + + /// Decodes the uploader's result and binds it to the session, capture and run that asked for + /// it. The analysis stays unverified: nothing in the response proves the server reconstructed + /// anything, so the spatial result is withheld (`CaptureResult.Record.placement`). + private func bindResult(of session: Session) async { + guard let uploader = session.uploader else { return } + let state = await uploader.snapshot + guard self.session === session, session.result == nil, let body = state.result, let captureID = state.captureID, + let runID = state.finalized?.runID else { return } + do { + let response = try JSONDecoder().decode(CaptureResult.Response.self, from: body) + let record = CaptureResult.Record( + response: response, + association: .init(sessionID: session.localID, captureID: captureID, runID: runID, epoch: Self.epoch)) + session.result = record + onResult?(record) + } catch { + environment?.log("capture result could not be read: \(error)") + } + } + + private func endSession(_ reason: String) { + guard let session else { return } + self.session = nil + session.ended = true + onStatus?(nil) + if let uploader = session.uploader { Task { await uploader.abandon(reason) } } + } +} + +extension PacketTracking { + /// The recorder's codes: state 0 normal, 1 limited, 2 not available; reason 0 none or + /// unknown, 1 initializing, 2 relocalizing, 3 excessive motion, 4 insufficient features. + public init(recorderState state: Int, reason: Int) { + let why: Reason? = switch reason { + case 1: .initializing + case 2: .relocalizing + case 3: .excessiveMotion + case 4: .insufficientFeatures + default: nil + } + self = switch state { + case 0: .normal + case 1: .limited(why) + default: .notAvailable + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadGate.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadGate.swift new file mode 100644 index 000000000..068ee46e2 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadGate.swift @@ -0,0 +1,72 @@ +import Foundation + +/// Whether this build and this homeowner allow a capture upload. No endpoint means nothing is +/// offered and nothing is sent; an endpoint without the homeowner's yes sends nothing either. +public enum CaptureUploadGate { + public enum Decision: Sendable, Equatable { + case off(String) + case on(URL) + } + + /// `endpoint` is the API base including `/v1`, from the build's Info.plist. An empty build + /// setting, an unexpanded `$(...)`, or anything but https (http only to this machine, for + /// tests) is off. + public static func decide(endpoint: String?, consented: Bool) -> Decision { + let text = endpoint?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard !text.isEmpty, !text.hasPrefix("$(") else { return .off("no capture endpoint in this build") } + guard let url = URL(string: text), let scheme = url.scheme?.lowercased(), let host = url.host(), url.user == nil, url.password == nil else { + return .off("capture endpoint is not a URL") + } + let local = host == "127.0.0.1" || host == "localhost" + guard scheme == "https" || (scheme == "http" && local) else { return .off("capture endpoint must be https") } + guard consented else { return .off("the homeowner has not agreed to send the capture") } + return .on(url) + } +} + +/// What the app does with its captures, from three build settings read once at launch. Only the +/// integration build records a 0.4 capture at all; only a build that also names an endpoint and +/// says device data may go there sends one, and then only after the homeowner's yes. +public enum CaptureIntegrationMode: Sendable, Equatable { + /// The client build, or no usable endpoint: nothing is recorded for the capture API or sent. + case off(String) + /// The capture is recorded on the phone and never sent: the endpoint may not receive device + /// data (an unauthenticated test server, unless the build says otherwise). + case recordOnly(URL) + /// Recorded, and sent after the homeowner's yes. + case send(URL) + + public var endpoint: URL? { + switch self { + case .off: nil + case .recordOnly(let url), .send(let url): url + } + } + + /// Where the scan's pictures come from. + public enum Source: Sendable, Equatable { + /// The phone's camera: sending needs the build's device-data switch. + case device + /// A recorded session played back (`-replay`). Its pictures can be anyone's, so it is sent + /// only to a receiver on this machine, and only when the run asks for that + /// (`sendToLocalReceiver`, the UI tests' switch); never to a remote endpoint. + case replay(sendToLocalReceiver: Bool) + } + + /// `integrationBuild` and `sendDeviceData` are Info.plist values that must read exactly "YES". + /// A launch argument can name `endpoint`, but can't turn a client build into an integration + /// build or allow device data. + public static func resolve(integrationBuild: String?, endpoint: String?, sendDeviceData: String?, source: Source = .device) -> CaptureIntegrationMode { + guard integrationBuild == "YES" else { return .off("not the integration build") } + switch CaptureUploadGate.decide(endpoint: endpoint, consented: true) { + case .off(let reason): return .off(reason) + case .on(let url): + switch source { + case .device: return sendDeviceData == "YES" ? .send(url) : .recordOnly(url) + case .replay(let local): + let loopback = url.host() == "127.0.0.1" || url.host() == "localhost" + return local && loopback ? .send(url) : .recordOnly(url) + } + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift new file mode 100644 index 000000000..da01b1c77 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift @@ -0,0 +1,167 @@ +import Foundation + +/// One capture's upload, saved in its folder after every change so a relaunch resumes the same +/// capture with the same identity. The request bodies that idempotency keys are bound to (create +/// and finalize) are saved as the exact bytes first sent, never re-encoded. +public struct CaptureUploadState: Codable, Sendable, Equatable { + public static let fileName = "capture-upload.json" + + public enum FilePhase: Codable, Sendable, Equatable { + /// Sealed on the phone, not yet registered (or its URL must be fetched again). + case queued + case registered(url: String, headers: [String: String], expiresAt: Date?) + /// The storage PUT answered 2xx. Not yet received: only a commit says that. + case uploaded + /// Acknowledged by files:commit. + case committed + } + + public struct File: Codable, Sendable, Equatable { + public var sealed: SealedFile + public var phase: FilePhase = .queued + public var attempts = 0 + /// Order in which the file was sealed, the tie-break after priority. + public var sequence: Int + + public init(sealed: SealedFile, phase: FilePhase = .queued, attempts: Int = 0, sequence: Int) { + self.sealed = sealed + self.phase = phase + self.attempts = attempts + self.sequence = sequence + } + } + + public enum End: Codable, Sendable, Equatable { + /// The ARKit world was reset or the scan started over: this packet will never be finished. + case abandoned(String) + /// The server refused something a retry can't change. + case failed(step: String, codes: [String], status: Int) + /// The capture reached a status after which it changes no more. + case finished(status: String) + } + + public struct Finalized: Codable, Sendable, Equatable { + public var status: String + public var missing: [String] + public var runID: String + + public init(status: String, missing: [String], runID: String) { + self.status = status + self.missing = missing + self.runID = runID + } + } + + /// Local identity of this upload. Answers that arrive for another attempt are dropped. + public var attemptID: String + public var packetID: String + /// The API base the capture was created on. A resume goes only there: its capture id and + /// signed URLs mean nothing to another server. + public var destination: String? + /// The homeowner's yes for this capture and this destination. A capture without one is + /// never resumed. + public var consent: Consent? + + public struct Consent: Codable, Sendable, Equatable { + public var grantedAt: Date + public var destination: String + + public init(grantedAt: Date, destination: String) { + self.grantedAt = grantedAt + self.destination = destination + } + } + /// The create body exactly as first sent. + public var createBody: Data + public var captureID: String? + public var maxBatch = 50 + public var files: [String: File] = [:] + public var nextSequence = 0 + /// packet.json exactly as frozen when the capture stopped. + public var packet: Data? + public var finalized: Finalized? + public var eventCursor = 0 + /// `retry_finalize` answers followed so far. + public var finalizeRetries = 0 + public var backendStatus: String? + public var lastEvent: String? + public var result: Data? + public var end: End? + /// Wall-clock marks of each stage, for latency: no ids, URLs or photos. + public var marks: [String: Date] = [:] + + public init(attemptID: String, packetID: String, createBody: Data) { + self.attemptID = attemptID + self.packetID = packetID + self.createBody = createBody + } + + public var committedCount: Int { files.values.filter { $0.phase == .committed }.count } + public var committedBytes: Int { files.values.filter { $0.phase == .committed }.reduce(0) { $0 + $1.sealed.bytes } } + public var totalBytes: Int { files.values.reduce(0) { $0 + $1.sealed.bytes } } + + public func save(to url: URL) throws { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + try encoder.encode(self).write(to: url, options: .atomic) + } + + public static func load(from url: URL) throws -> CaptureUploadState { + try JSONDecoder().decode(CaptureUploadState.self, from: Data(contentsOf: url)) + } +} + +/// What the experience layer shows: separate facts, never one percentage. +public struct CaptureUploadStatus: Sendable, Equatable { + public enum Phase: Sendable, Equatable { + case creating, uploading, processing, finished, failed, abandoned + } + + public var phase: Phase + /// Files sealed on the phone for this capture. + public var retained: Int + /// Files the server acknowledged with a commit. + public var committed: Int + public var committedBytes: Int + public var totalBytes: Int + /// Whether packet.json is frozen (the capture stopped). + public var sealed: Bool + /// The capture's status as the server last reported it. + public var backendStatus: String? + public var lastEvent: String? + public var eventCursor: Int + public var resultAvailable: Bool + /// The server's id for the capture, once created: what an operator copies to find this scan. + public var captureID: String? + /// The run finalize started, once accepted. + public var runID: String? + /// Set while the uploader waits out a retryable failure. + public var retryingAt: Date? + public var detail: String? + + public init(_ state: CaptureUploadState, retryingAt: Date? = nil, detail: String? = nil) { + phase = switch state.end { + case .abandoned?: .abandoned + case .failed?: .failed + case .finished?: .finished + case nil: state.captureID == nil ? .creating : state.finalized == nil ? .uploading : .processing + } + retained = state.files.count + committed = state.committedCount + committedBytes = state.committedBytes + totalBytes = state.totalBytes + sealed = state.packet != nil + backendStatus = state.backendStatus + lastEvent = state.lastEvent + eventCursor = state.eventCursor + resultAvailable = state.result != nil + captureID = state.captureID + runID = state.finalized?.runID + self.retryingAt = retryingAt + if case .failed(let step, let codes, let status) = state.end { + self.detail = "\(step) \(status) \(codes.joined(separator: ","))" + } else { + self.detail = detail + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift new file mode 100644 index 000000000..1f586b522 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -0,0 +1,510 @@ +import Foundation + +/// Sends one capture to the capture API while it is being taken: create once, then register, PUT +/// and commit each sealed file as it arrives, then finalize with the frozen packet (possibly +/// before the last commits), then follow the events to a result. +/// +/// Every answer is applied only while the attempt that asked is still current, so a reply that +/// arrives after a world reset or a new capture changes nothing. The saved state +/// (`CaptureUploadState`) is written after every change, which is what resumes a capture after a +/// relaunch with the same packet id, capture id and frozen request bodies. +public actor CaptureUploader { + public struct Policy: Sendable { + public var maxConcurrentPuts = 3 + /// Files per register and commit call: the contract's "about every 10 files". + public var batch = 10 + public var firstDelay = 1.0 + public var maxDelay = 30.0 + /// Seconds the server may hold an events poll. + public var eventsWait = 20 + /// A signed URL this close to expiry is fetched again before the PUT. + public var expiryMargin = 60.0 + /// PUTs of one file the storage refuses (not transient) before the upload stops. + public var maxRefusedPuts = 6 + /// `retry_finalize` answers followed before the upload stops. + public var maxFinalizeRetries = 3 + + public init() {} + + public func delay(afterFailure n: Int, retryAfter: Double?) -> Double { + if let retryAfter, retryAfter > 0 { return min(retryAfter, 120) } + return min(maxDelay, firstDelay * pow(2, Double(max(n, 1) - 1))) + } + } + + /// A failure a later try can fix. + struct Transient: Error { + var step: String + var detail: String + var retryAfter: Double? + } + + /// The server refused something a retry can't change. + struct Refused: Error { + var step: String + var status: Int + var codes: [String] + } + + public nonisolated let folder: URL + private let base: URL + private let http: any CaptureHTTP + private let policy: Policy + private let now: @Sendable () -> Date + private let sleep: @Sendable (Double) async throws -> Void + private var state: CaptureUploadState + private var loop: Task? + private var failures = 0 + private var retryingAt: Date? + private var digestRetried: Set = [] + /// Set when a state change could not be saved: the loop stops rather than send a request the + /// saved state doesn't know about. + private var saveFailed = false + private var observer: (@Sendable (CaptureUploadStatus) -> Void)? + private var log: (@Sendable (String) -> Void)? + + public nonisolated static func stateURL(in folder: URL) -> URL { folder.appending(path: CaptureUploadState.fileName) } + + private init( + folder: URL, base: URL, http: any CaptureHTTP, state: CaptureUploadState, policy: Policy, + now: @escaping @Sendable () -> Date, sleep: @escaping @Sendable (Double) async throws -> Void + ) { + self.folder = folder + self.base = base + self.http = http + self.state = state + self.policy = policy + self.now = now + self.sleep = sleep + } + + /// A new capture. The create body is encoded once here and kept as those bytes for every retry. + /// `consentedAt` is when the homeowner said yes to sending this capture to `base`. + public static func start( + folder: URL, base: URL, http: any CaptureHTTP, create: CaptureAPI.CreateRequest, consentedAt: Date, policy: Policy = .init(), + now: @escaping @Sendable () -> Date = { Date() }, + sleep: @escaping @Sendable (Double) async throws -> Void = { try await Task.sleep(for: .seconds($0)) } + ) throws -> CaptureUploader { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + var state = CaptureUploadState(attemptID: UUID().uuidString, packetID: create.packetId, createBody: try encoder.encode(create)) + state.destination = base.absoluteString + state.consent = .init(grantedAt: consentedAt, destination: base.absoluteString) + state.marks["started"] = now() + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + try state.save(to: stateURL(in: folder)) + return CaptureUploader(folder: folder, base: base, http: http, state: state, policy: policy, now: now, sleep: sleep) + } + + /// The capture saved in `folder`, as a relaunch finds it; nil when there is none, when it was + /// created on another API than `base`, or when it holds no yes for `base`. The resumed upload + /// gets a new attempt id, so nothing from the previous process can land in it. + public static func resume( + folder: URL, base: URL, http: any CaptureHTTP, policy: Policy = .init(), + now: @escaping @Sendable () -> Date = { Date() }, + sleep: @escaping @Sendable (Double) async throws -> Void = { try await Task.sleep(for: .seconds($0)) } + ) throws -> CaptureUploader? { + let url = stateURL(in: folder) + guard FileManager.default.fileExists(atPath: url.path) else { return nil } + var state = try CaptureUploadState.load(from: url) + guard state.destination == base.absoluteString, state.consent?.destination == base.absoluteString else { return nil } + state.attemptID = UUID().uuidString + try state.save(to: url) + return CaptureUploader(folder: folder, base: base, http: http, state: state, policy: policy, now: now, sleep: sleep) + } + + public func observe(_ observer: @escaping @Sendable (CaptureUploadStatus) -> Void, log: (@Sendable (String) -> Void)? = nil) { + self.observer = observer + self.log = log + observer(status) + } + + public var status: CaptureUploadStatus { + CaptureUploadStatus(state, retryingAt: retryingAt, detail: saveFailed ? "upload state could not be saved" : nil) + } + public var snapshot: CaptureUploadState { state } + + // MARK: Input + + /// Sealed files, uploaded as soon as the capture exists. A path already known is ignored: + /// sealed files never change. + public func add(_ files: [SealedFile]) { + guard state.end == nil, state.packet == nil else { return } + var added = false + for file in files where state.files[file.path] == nil { + state.files[file.path] = .init(sealed: file, sequence: state.nextSequence) + state.nextSequence += 1 + added = true + } + guard added else { return } + mark("firstRetained") + persist() + kick() + } + + /// The capture stopped: `packet` is frozen as these bytes, and `files` is every file it lists. + /// Any the uploader has not seen yet (an `add` still on its way) is queued here, so the + /// server never waits for a listed file that nobody sends. + public func seal(packet: Data, files: [SealedFile]) { + guard state.end == nil, state.packet == nil else { return } + for file in files where state.files[file.path] == nil { + state.files[file.path] = .init(sealed: file, sequence: state.nextSequence) + state.nextSequence += 1 + } + state.packet = packet + mark("sealed") + persist() + kick() + } + + /// Ends this upload for good: a world reset or starting over. Replies still in flight are + /// dropped when they arrive. + public func abandon(_ reason: String) { + guard state.end == nil else { return } + state.end = .abandoned(reason) + state.attemptID = UUID().uuidString + loop?.cancel() + loop = nil + persist() + } + + /// Starts the loop if it is idle: after a relaunch, or when the network is back. + public func kick() { + guard loop == nil, state.end == nil else { return } + loop = Task { await self.drive() } + } + + /// Waits for the loop to go idle: nothing to send until more input, or ended. + public func settled() async { + while let running = loop { await running.value; if loop == running { loop = nil } } + } + + // MARK: Loop + + private func drive() async { + while !Task.isCancelled, state.end == nil, !saveFailed { + let attempt = state.attemptID + do { + let progressed = try await step() + guard attempt == state.attemptID, state.end == nil else { break } + failures = 0 + if retryingAt != nil { retryingAt = nil; publish() } + if !progressed { break } + } catch let error as Transient { + guard attempt == state.attemptID else { break } + failures += 1 + let delay = policy.delay(afterFailure: failures, retryAfter: error.retryAfter) + retryingAt = now().addingTimeInterval(delay) + log?("capture-upload retry step=\(error.step) in=\(String(format: "%.1f", delay))s detail=\(error.detail)") + publish() + do { try await sleep(delay) } catch { break } + } catch let error as Refused { + guard attempt == state.attemptID else { break } + state.end = .failed(step: error.step, codes: error.codes, status: error.status) + log?("capture-upload failed step=\(error.step) status=\(error.status) codes=\(error.codes.joined(separator: ","))") + persist() + } catch { + break + } + } + loop = nil + } + + /// One unit of work. False when there is nothing to do until more input arrives. + private func step() async throws -> Bool { + if state.captureID == nil { try await create(); return true } + // The frozen packet goes as soon as the capture stops, before the files still uploading: + // the server answers awaiting_files and starts the run on the last commit. + if state.packet != nil, state.finalized == nil { try await finalize(); return true } + requeueExpiring() + let queued = ordered(where: { $0.phase == .queued }) + if !queued.isEmpty { try await register(Array(queued.prefix(min(policy.batch, state.maxBatch)))); return true } + // Each round of PUTs is committed before the next starts, so the first photos count as + // received after one round trip of uploads rather than a whole register batch. + let uploaded = ordered(where: { $0.phase == .uploaded }) + if !uploaded.isEmpty { try await commit(Array(uploaded.prefix(min(policy.batch, state.maxBatch)))); return true } + let registered = ordered(where: { if case .registered = $0.phase { true } else { false } }) + if !registered.isEmpty { try await put(Array(registered.prefix(policy.maxConcurrentPuts))); return true } + if state.finalized != nil, state.end == nil { + if state.result != nil || CaptureAPI.terminalStatuses.contains(state.backendStatus ?? "") { + try await fetchResult() + return true + } + try await pollEvents() + return true + } + return false + } + + private func ordered(where include: (CaptureUploadState.File) -> Bool) -> [CaptureUploadState.File] { + state.files.values.filter(include).sorted { ($0.sealed.priority, $0.sequence) < ($1.sealed.priority, $1.sequence) } + } + + // MARK: Calls + + private func create() async throws { + var request = jsonRequest("captures", method: "POST", body: state.createBody) + request.setValue(state.packetID, forHTTPHeaderField: "Idempotency-Key") + let (reply, attempt) = try await call("create", request) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let body = try? JSONDecoder().decode(CaptureAPI.CreateResponse.self, from: reply.body) else { + throw Refused(step: "create", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + state.captureID = body.captureId + state.backendStatus = body.status + if let batch = body.upload?.maxBatch, batch > 0 { state.maxBatch = batch } + mark("created") + persist() + } + + private func register(_ files: [CaptureUploadState.File]) async throws { + let body = try JSONEncoder().encode(CaptureAPI.RegisterRequest(files: files.map { .init($0.sealed) })) + let (reply, attempt) = try await call("register", jsonRequest("captures/\(captureID)/files", method: "POST", body: body)) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let answer = try? JSONDecoder().decode(CaptureAPI.RegisterResponse.self, from: reply.body) else { + if try captureGone(reply) { return } + throw Refused(step: "register", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + for file in answer.files where state.files[file.path] != nil { + if file.state == "committed" { + setPhase(file.path, .committed) + } else if let target = file.upload, target.method == "PUT", let url = target.url { + setPhase(file.path, .registered(url: url, headers: target.headers ?? [:], expiresAt: target.expiresAt.flatMap(Self.parseDate))) + } else { + throw Refused(step: "register", status: reply.status, codes: ["upload_method_\(file.upload?.method ?? "none")"]) + } + } + persist() + } + + private func put(_ files: [CaptureUploadState.File]) async throws { + let attempt = state.attemptID + var transient: Transient? + var index = 0 + while index < files.count { + let chunk = files[index..).self) { group in + for file in chunk { + guard case .registered(let url, let headers, _) = file.phase, let target = URL(string: url) else { continue } + var request = URLRequest(url: target) + request.httpMethod = "PUT" + // Exactly the headers the server signed; never the API's own credentials. + for (name, value) in headers { request.setValue(value, forHTTPHeaderField: name) } + let local = folder.appending(path: file.sealed.path) + let http = self.http + group.addTask { + do { return (file.sealed.path, .success(try await http.upload(request, file: local))) } catch { return (file.sealed.path, .failure(error)) } + } + } + var out: [(String, Result)] = [] + for await result in group { out.append(result) } + return out + } + guard attempt == state.attemptID, state.end == nil else { return } + for (path, result) in results { + state.files[path]?.attempts += 1 + switch result { + case .success(let reply) where (200..<300).contains(reply.status): + setPhase(path, .uploaded) + case .success(let reply) where reply.status == 400 && !digestRetried.contains(path): + // BadDigest: the bytes on disk are sealed, so sending them once more is the retry. + digestRetried.insert(path) + case .success(let reply) where reply.status == 400: + // The same sealed bytes refused twice: the file on disk no longer matches its digest. + throw Refused(step: "put", status: 400, codes: ["bad_digest"]) + case .success(let reply) where Self.isTransient(reply.status): + transient = Transient(step: "put", detail: "status \(reply.status)", retryAfter: reply.retryAfter) + case .success(let reply): + // 403 expired signature or a spent token: a fresh URL, a bounded number of times. + guard (state.files[path]?.attempts ?? 0) < policy.maxRefusedPuts else { + throw Refused(step: "put", status: reply.status, codes: ["storage_refused"]) + } + setPhase(path, .queued) + case .failure(let error): + transient = Transient(step: "put", detail: Self.describe(error), retryAfter: nil) + } + } + persist() + } + if let transient { throw transient } + } + + private func commit(_ files: [CaptureUploadState.File]) async throws { + let body = try JSONEncoder().encode(CaptureAPI.CommitRequest(files: files.map { .init(path: $0.sealed.path, sha256: $0.sealed.sha256) })) + let (reply, attempt) = try await call("commit", jsonRequest("captures/\(captureID)/files:commit", method: "POST", body: body)) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let answer = try? JSONDecoder().decode(CaptureAPI.CommitResponse.self, from: reply.body) else { + if try captureGone(reply) { return } + throw Refused(step: "commit", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + let committed = Set(answer.committed) + for file in files { + // Only a commit acknowledges a file; notFound, mismatch or no mention sends it again. + setPhase(file.sealed.path, committed.contains(file.sealed.path) ? .committed : .queued) + } + if !committed.isEmpty { mark("firstCommit") } + if state.packet != nil, state.files.values.allSatisfy({ $0.phase == .committed }) { mark("lastCommit") } + persist() + } + + private func finalize() async throws { + guard let packet = state.packet else { return } + var request = jsonRequest("captures/\(captureID)/finalize", method: "POST", body: packet) + request.setValue(state.packetID, forHTTPHeaderField: "Idempotency-Key") + request.setValue(PacketFiles.sha256(packet), forHTTPHeaderField: "X-Packet-Sha256") + let (reply, attempt) = try await call("finalize", request) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let answer = try? JSONDecoder().decode(CaptureAPI.FinalizeResponse.self, from: reply.body) else { + if try captureGone(reply) { return } + throw Refused(step: "finalize", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + state.finalized = .init(status: answer.status, missing: answer.missing, runID: answer.runId) + state.backendStatus = answer.status + mark("finalizeAccepted") + persist() + } + + private func pollEvents() async throws { + var components = URLComponents(url: base.appending(path: "captures/\(captureID)/events"), resolvingAgainstBaseURL: false)! + components.queryItems = [.init(name: "after", value: String(state.eventCursor)), .init(name: "wait", value: String(policy.eventsWait))] + var request = URLRequest(url: components.url!) + request.setValue("application/json", forHTTPHeaderField: "Accept") + request.timeoutInterval = Double(policy.eventsWait) + 30 + let (reply, attempt) = try await call("events", request) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let answer = try? JSONDecoder().decode(CaptureAPI.EventsResponse.self, from: reply.body) else { + if try captureGone(reply) { return } + throw Refused(step: "events", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + state.backendStatus = answer.status + state.eventCursor = max(state.eventCursor, answer.next) + for event in answer.events { + mark("firstEvent") + state.lastEvent = [event.type, event.data?.stage, event.data?.status, event.data?.kind, event.data?.code].compactMap { $0 }.joined(separator: " ") + // A verdict for another run is not this capture's result. + if event.type == "verdict_ready", event.data?.runId == nil || event.data?.runId == state.finalized?.runID { mark("verdictReady") } + if event.type == "failed", event.data?.next == "retry_finalize" { + // The server lost the finalize to a transient storage failure: send the same frozen + // packet again. + guard state.finalizeRetries < policy.maxFinalizeRetries else { + throw Refused(step: "finalize", status: 0, codes: [event.data?.code ?? "retry_finalize"]) + } + state.finalizeRetries += 1 + state.finalized = nil + } + } + if !answer.events.isEmpty { log?("capture-upload events status=\(answer.status) last=\(state.lastEvent ?? "")") } + persist() + } + + private func fetchResult() async throws { + var request = URLRequest(url: base.appending(path: "captures/\(captureID)/result")) + request.setValue("application/json", forHTTPHeaderField: "Accept") + let (reply, attempt) = try await call("result", request) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status) else { + if try captureGone(reply) { return } + throw Refused(step: "result", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + struct RunOnly: Decodable { var runId: String?; var status: String? } + let run = try? JSONDecoder().decode(RunOnly.self, from: reply.body) + if let runID = run?.runId, runID != state.finalized?.runID { + throw Refused(step: "result", status: reply.status, codes: ["result_for_another_run"]) + } + state.result = reply.body + state.end = .finished(status: run?.status ?? state.backendStatus ?? "unknown") + mark("result") + persist() + } + + // MARK: Plumbing + + private var captureID: String { state.captureID ?? "" } + + /// The server lost the capture (404: a redeploy with in-memory state). The same create body + /// opens it again, and every file goes up again under the new id. + private func captureGone(_ reply: HTTPReply) throws -> Bool { + guard reply.status == 404, CaptureAPI.errorCodes(reply.body).contains("capture_not_found") else { return false } + log?("capture-upload the server no longer has the capture; creating it again") + state.captureID = nil + state.finalized = nil + state.eventCursor = 0 + for path in state.files.keys { state.files[path]?.phase = .queued } + persist() + return true + } + + private func requeueExpiring() { + let limit = now().addingTimeInterval(policy.expiryMargin) + for (path, file) in state.files { + if case .registered(_, _, let expires?) = file.phase, expires <= limit { state.files[path]?.phase = .queued } + } + } + + private func setPhase(_ path: String, _ phase: CaptureUploadState.FilePhase) { + state.files[path]?.phase = phase + } + + private func call(_ step: String, _ request: URLRequest) async throws -> (HTTPReply, String) { + let attempt = state.attemptID + let reply: HTTPReply + do { + reply = try await http.send(request) + } catch { + throw Transient(step: step, detail: Self.describe(error), retryAfter: nil) + } + if Self.isTransient(reply.status), attempt == state.attemptID { + throw Transient(step: step, detail: "status \(reply.status)", retryAfter: reply.retryAfter) + } + return (reply, attempt) + } + + private func jsonRequest(_ path: String, method: String, body: Data) -> URLRequest { + var request = URLRequest(url: base.appending(path: path)) + request.httpMethod = method + request.httpBody = body + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.setValue("application/json", forHTTPHeaderField: "Accept") + return request + } + + private func mark(_ name: String) { + guard state.marks[name] == nil else { return } + let at = now() + state.marks[name] = at + let since = state.marks["started"].map { at.timeIntervalSince($0) } ?? 0 + log?("capture-upload stage=\(name) t=\(String(format: "%.2f", since))s committed=\(state.committedCount)/\(state.files.count)") + } + + private func persist() { + do { + try state.save(to: Self.stateURL(in: folder)) + } catch { + saveFailed = true + log?("capture-upload stopped: the upload state could not be saved (\(Self.describe(error)))") + } + publish() + } + + private func publish() { + observer?(status) + } + + /// 501 is the server saying it doesn't do what was asked (multipart), which waiting won't change. + static func isTransient(_ status: Int) -> Bool { status == 0 || status == 408 || status == 429 || ((500..<600).contains(status) && status != 501) } + + /// The error's domain and code only: a URLError's description can carry the signed URL. + static func describe(_ error: any Error) -> String { + let ns = error as NSError + return "\(ns.domain) \(ns.code)" + } + + static func parseDate(_ text: String) -> Date? { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + if let date = formatter.date(from: text) { return date } + formatter.formatOptions = [.withInternetDateTime] + return formatter.date(from: text) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultSummaryTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultSummaryTests.swift new file mode 100644 index 000000000..99efe248a --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultSummaryTests.swift @@ -0,0 +1,134 @@ +import Foundation +import HouseScanKit +import simd +import Testing + +/// What the integration build shows of a capture-API result, from self-authored responses, and +/// the coordinator binding the result it fetched to its own scan. +@Suite struct CaptureResultSummaryTests { + static let world = CaptureResult.Association(sessionID: "session-A", captureID: "cap_A", runID: "run_A", epoch: "e1") + + static func response(_ json: String) throws -> CaptureResult.Response { + try JSONDecoder().decode(CaptureResult.Response.self, from: Data(json.utf8)) + } + + static func summary(_ json: String, association: CaptureResult.Association = world, analysis: CaptureResult.Analysis = .unverified) throws -> CaptureResultSummary { + CaptureResultSummary(.init(response: try response(json), association: association, analysis: analysis), world: world, meterAnchor: matrix_identity_float4x4) + } + + static let prompt = #"{"id":"vn1","kind":"wall_band","criteria":["C3"],"why":"unseen","promptId":"p1","prompt":{"title":"Step back and show the wall","body":"Include the ground under the meter."}}"# + + @Test func processingHasNoOutcomeAndNoMessage() throws { + let s = try Self.summary(#"{"runId":"run_A","status":"processing","viewsNeeded":[],"memberActions":[]}"#) + #expect(s.state == .working(.processing)) + #expect(s.message == nil && s.prompts.isEmpty && s.criteria == nil) + #expect(s.arUnavailable == .noOutcome(.processing)) + } + + @Test func manualReviewKeepsTheServersWordsAndNoInventedChecks() throws { + let s = try Self.summary(#""" + {"runId":"run_A","status":"manual_review","viewsNeeded":[],"memberActions":[], + "outcome":{"kind":"manual_review","profile":"C","message":"RECEIVER TEST: a person will look at this.","viewsNeeded":[],"reasons":[]}} + """#) + #expect(s.state == .decided(.manualReview)) + #expect(s.message == "RECEIVER TEST: a person will look at this.") + #expect(s.criteria == nil) + #expect(s.arUnavailable == .notEligible(.manualReview)) + } + + @Test func aMissingViewShowsItsPromptVerbatim() throws { + let s = try Self.summary(""" + {"runId":"run_A","status":"needs_views","viewsNeeded":[\(Self.prompt)],"memberActions":[], + "outcome":{"kind":"needs_more_photos","profile":"C","message":"One more view, please.","viewsNeeded":[\(Self.prompt)],"reasons":[]}} + """) + #expect(s.state == .decided(.needsMorePhotos)) + #expect(s.prompts.map(\.title) == ["Step back and show the wall"]) + #expect(s.prompts.map(\.body) == ["Include the ground under the meter."]) + } + + /// An eligible-shaped outcome from an unverified server stays without AR, and without the + /// criteria it didn't send. + @Test func eligibleWithoutCriteriaStaysUnverifiedAndOffAR() throws { + let s = try Self.summary(#""" + {"runId":"run_A","status":"complete","viewsNeeded":[],"memberActions":[], + "outcome":{"kind":"eligible","profile":"C","message":"A spot fits.","viewsNeeded":[],"reasons":[],"arkitEpoch":"e1", + "recommendedPlacement":{"wallId":"w1","startSM":1.2,"confidence":0.8, + "boxArkitWorld":{"pose":[1,0,0,0,0,1,0,0,0,0,1,0,0.5,0,-2,1],"size":[0.79,1.0,0.56]}}}} + """#) + #expect(s.state == .decided(.eligible)) + #expect(s.criteria == nil) + #expect(!s.analysisVerified) + #expect(s.arUnavailable == .analysisUnverified) + } + + @Test func anUnknownStatusStaysReadable() throws { + let s = try Self.summary(#"{"runId":"run_A","status":"paused_for_audit","viewsNeeded":[],"memberActions":[]}"#) + #expect(s.state == .working(.unknown("paused_for_audit"))) + #expect(s.arUnavailable == .noOutcome(.unknown("paused_for_audit"))) + } + + /// Even a verified result doesn't place a box for another session, capture or run. + @Test(arguments: [ + CaptureResult.Association(sessionID: "session-B", captureID: "cap_A", runID: "run_A", epoch: "e1"), + CaptureResult.Association(sessionID: "session-A", captureID: "cap_B", runID: "run_A", epoch: "e1"), + CaptureResult.Association(sessionID: "session-A", captureID: "cap_A", runID: "run_B", epoch: "e1"), + ]) + func aResultForAnotherScanPlacesNothing(_ association: CaptureResult.Association) throws { + let s = try Self.summary(#""" + {"runId":"run_A","status":"complete","viewsNeeded":[],"memberActions":[], + "outcome":{"kind":"eligible","profile":"C","message":"A spot fits.","viewsNeeded":[],"reasons":[],"arkitEpoch":"e1", + "recommendedPlacement":{"wallId":"w1","startSM":1.2,"confidence":0.8, + "boxArkitWorld":{"pose":[1,0,0,0,0,1,0,0,0,0,1,0,0.5,0,-2,1],"size":[0.79,1.0,0.56]}}}} + """#, association: association, analysis: .verified) + guard case .mismatch? = s.arUnavailable else { Issue.record("placed a box for another scan: \(String(describing: s.arUnavailable))"); return } + } +} + +/// The coordinator fetches the result through the real uploader and binds it to its own session. +@Suite(.serialized) @MainActor struct CaptureResultBindingTests { + @Test func theFetchedResultIsBoundToThisScanAndStaysUnverified() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "result-binding-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.resultMessage = "RECEIVER TEST: distinctive message 7f3" } + let coordinator = CaptureSessionCoordinator(environment: NativeCaptureFixture.environment( + endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: root.appending(path: "Captures"))) + var results: [CaptureResult.Record] = [] + coordinator.onResult = { results.append($0) } + try await NativeCaptureFixture(folder: root.appending(path: "store")).run(coordinator) + for _ in 0..<100 where results.isEmpty { try await Task.sleep(for: .milliseconds(20)) } + + let session = try #require(coordinator.session) + let record = try #require(results.first) + #expect(results.count == 1) + let state = await session.uploader!.snapshot + #expect(record.association == .init(sessionID: session.localID, captureID: state.captureID!, runID: state.finalized!.runID, epoch: "e1")) + #expect(record.response.outcome?.message == "RECEIVER TEST: distinctive message 7f3") + #expect(record.analysis == .unverified) + #expect(session.result == record) + } + + /// A result arriving for a session that has ended (a new scan started) is never published. + @Test func aResultForAnEndedSessionIsDropped() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "result-stale-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["GET captures/result"] } + let fixture = NativeCaptureFixture(folder: root.appending(path: "store")) + let coordinator = CaptureSessionCoordinator(environment: NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: root.appending(path: "Captures"))) + var results: [CaptureResult.Record] = [] + coordinator.onResult = { results.append($0) } + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 3)) + coordinator.captureEnded(acceptedCloseUpAt: nil) + for _ in 0..<500 where server.requests("GET captures/result").isEmpty { try await Task.sleep(for: .milliseconds(10)) } + let old = try #require(coordinator.session) + coordinator.newWorld("start over", recording: fixture.recording, newScan: true) + server.release("GET captures/result") + try await Task.sleep(for: .milliseconds(500)) + #expect(results.isEmpty) + #expect(old.result == nil) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultTests.swift new file mode 100644 index 000000000..ed3e9919a --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultTests.swift @@ -0,0 +1,320 @@ +import Foundation +import HouseScanKit +import Testing +import simd + +// Every body here is self-authored to the shape of the server's device result; none is a server +// example or a real capture. +@Suite struct CaptureResultTests { + typealias Association = CaptureResult.Association + + static let scan = Association(sessionID: "session-a", captureID: "cap-a", runID: "run-a", epoch: "e1") + + static let eligibleJSON = """ + {"runId": "run-a", "status": "complete", "viewsNeeded": [], "memberActions": [], + "outcome": {"kind": "eligible", "profile": "C", "message": "A battery fits on this wall.", + "viewsNeeded": [], "reasons": [], "arkitEpoch": "e1", + "frameToArkit": [1,0,0,0, 0,1,0,0, 0,0,1,0, 0,0,0,1], + "recommendedPlacement": {"wallId": "w1", "startSM": 1.1, "confidence": 0.7, "profile": "C", + "footprintM": {"w": 0.8, "h": 1.0, "d": 0.5}, + "boxSceneM": {"centerM": [1.5, 0.5, 0.3], "sizeM": [0.8, 1.0, 0.5], "yawDeg": 0}, + "boxArkitWorld": {"pose": [1,0,0,0, 0,1,0,0, 0,0,1,0, 1.5,0.5,0.3,1], "size": [0.8, 1.0, 0.5]}}, + "cableRoutePolylineArkit": null}, + "verdictUrl": null, "previewUrl": null, "reviewUrl": "/v1/captures/cap-a/review"} + """ + + static func decode(_ json: String) throws -> CaptureResult.Response { + try JSONDecoder().decode(CaptureResult.Response.self, from: Data(json.utf8)) + } + + static func eligible() throws -> CaptureResult.Response { try decode(eligibleJSON) } + + static func verified(_ response: CaptureResult.Response) -> CaptureResult.Record { + CaptureResult.Record(response: response, association: scan, analysis: .verified) + } + + // MARK: Decoding + + @Test func processingHasNoOutcome() throws { + let response = try Self.decode(""" + {"runId": null, "status": "processing", "viewsNeeded": [], "memberActions": [], "outcome": null} + """) + #expect(response.status == .processing) + #expect(response.runId == nil && response.outcome == nil && response.criteria == nil) + #expect(response.previewUrl == nil && response.reviewUrl == nil) + #expect(Self.verified(response).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + == .unavailable(.noOutcome(.processing))) + } + + @Test func eligibleWithoutCriteriaKeepsThemAbsent() throws { + let response = try Self.eligible() + #expect(response.outcome?.kind == .eligible) + #expect(response.criteria == nil) + #expect(response.reviewUrl == "/v1/captures/cap-a/review") + let box = try #require(response.outcome?.recommendedPlacement?.boxArkitWorld) + #expect(box.size == [0.8, 1.0, 0.5]) + guard case .box(let placed) = Self.verified(response).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) else { + Issue.record("expected a box") + return + } + #expect(placed.size == SIMD3(0.8, 1.0, 0.5)) + #expect(nearlyEqual(placed.anchorFromBox, translation(1.5, 0.5, 0.3))) + } + + @Test func eligibleWithCriteriaKeepsEachAsSent() throws { + let json = Self.eligibleJSON.replacingOccurrences( + of: "\"verdictUrl\": null", + with: """ + "criteria": [{"id": "c1", "outcome": "pass", "measuredFt": 4.2, "coverage": "observed"}, + {"id": "c2", "outcome": "unsure", "unsureCause": "unobserved"}, + {"id": "c3", "outcome": "waived"}], + "verdictUrl": null + """) + let criteria = try #require(try Self.decode(json).criteria) + #expect(criteria.map(\.outcome) == [.pass, .unsure, .unknown("waived")]) + #expect(criteria[0].measuredFt == 4.2 && criteria[0].coverage == "observed" && criteria[0].thresholdFt == nil) + #expect(criteria[1].unsureCause == "unobserved" && criteria[1].measuredFt == nil) + } + + @Test func manualReviewKeepsViewsAndPromptsVerbatim() throws { + let body = " Hold the phone at chest height,\nand include 3 ft left of the “meter”. " + let view = """ + {"id": "vn1", "kind": "past_end", "criteria": ["c2"], "why": "The left end was not seen.", + "promptId": "p-left-end", "prompt": {"title": "Show the left end", "body": \(try jsonString(body))}, + "side": "left", "wallId": "w1", "futureKey": {"nested": true}, + "referenceCrop": {"image": "k00002", "boxPx": [10, 20, 300, 400]}} + """ + let response = try Self.decode(""" + {"runId": "run-a", "status": "manual_review", "viewsNeeded": [\(view)], + "memberActions": ["retake_meter_closeups"], + "outcome": {"kind": "manual_review", "profile": "C", "message": "An installer will look at this scan.", + "viewsNeeded": [\(view)], "reasons": [], "arkitEpoch": "e1"}, + "criteria": [{"id": "c2", "outcome": "unsure"}], + "verdictUrl": null, "previewUrl": null, "reviewUrl": "/v1/captures/cap-a/review"} + """) + let outcome = try #require(response.outcome) + #expect(outcome.kind == .manualReview && outcome.reasons.isEmpty) + #expect(outcome.message == "An installer will look at this scan.") + #expect(response.viewsNeeded == outcome.viewsNeeded) + let needed = try #require(outcome.viewsNeeded.first) + #expect(needed.prompt.title == "Show the left end") + #expect(needed.prompt.body == body) + #expect(needed.kind == "past_end" && needed.wallId == "w1" && needed.band == nil) + #expect(needed.referenceCrop?.boxPx == [10, 20, 300, 400]) + #expect(response.memberActions == ["retake_meter_closeups"]) + #expect(Self.verified(response).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + == .unavailable(.notEligible(.manualReview))) + } + + @Test func unknownStatusAndKindDecodeAndWithholdAR() throws { + let unknownKind = try Self.decode(Self.eligibleJSON + .replacingOccurrences(of: "\"status\": \"complete\"", with: "\"status\": \"archived\"") + .replacingOccurrences(of: "\"kind\": \"eligible\"", with: "\"kind\": \"deferred\"")) + #expect(unknownKind.status == .unknown("archived")) + #expect(unknownKind.outcome?.kind == .unknown("deferred")) + #expect(unknownKind.outcome?.message == "A battery fits on this wall.") + #expect(Self.verified(unknownKind).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + == .unavailable(.notEligible(.unknown("deferred")))) + + var unknownStatus = try Self.eligible() + unknownStatus.status = .unknown("archived") + #expect(Self.verified(unknownStatus).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + == .unavailable(.statusNotComplete(.unknown("archived")))) + } + + @Test func missingStatusFailsToDecode() { + #expect(throws: DecodingError.self) { + try Self.decode(#"{"runId": "run-a", "viewsNeeded": [], "memberActions": [], "outcome": null}"#) + } + } + + // MARK: Gating + + @Test func eligibleResultIsUnverifiedByDefault() throws { + let record = CaptureResult.Record(response: try Self.eligible(), association: Self.scan) + #expect(record.analysis == .unverified) + #expect(record.placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) == .unavailable(.analysisUnverified)) + } + + enum Mismatch: CaseIterable, Sendable { + case worldSession, worldCapture, worldRun, worldEpoch, responseRun, responseRunMissing, outcomeEpoch + + var field: CaptureResult.IdentityField { + switch self { + case .worldSession: .session + case .worldCapture: .capture + case .worldRun, .responseRun, .responseRunMissing: .run + case .worldEpoch, .outcomeEpoch: .epoch + } + } + } + + @Test(arguments: Mismatch.allCases) + func everyIdentityMismatchWithholdsAR(_ mismatch: Mismatch) throws { + var response = try Self.eligible() + var world = Self.scan + switch mismatch { + case .worldSession: world.sessionID = "session-b" + case .worldCapture: world.captureID = "cap-b" + case .worldRun: world.runID = "run-b" + case .worldEpoch: world.epoch = "e2" + case .responseRun: response.runId = "run-b" + case .responseRunMissing: response.runId = nil + case .outcomeEpoch: response.outcome?.arkitEpoch = "e2" + } + let record = Self.verified(response) + #expect(record.placement(in: world, meterAnchor: matrix_identity_float4x4) == .unavailable(.mismatch(mismatch.field))) + #expect(record.response.outcome?.message == "A battery fits on this wall.") + } + + @Test func epochReusedByANewSessionIsNotAMatch() throws { + let newSession = Association(sessionID: "session-b", captureID: "cap-a", runID: "run-a", epoch: "e1") + #expect(newSession.epoch == Self.scan.epoch) + #expect(Self.verified(try Self.eligible()).placement(in: newSession, meterAnchor: matrix_identity_float4x4) + == .unavailable(.mismatch(.session))) + } + + enum BadBox: CaseIterable, Sendable { + case shortPose, twoSizes, nanPose, infiniteSize, zeroSize, scaledPose, shearedPose, mirroredPose, badBottomRow + case noBox, noEpoch, noPlacement + case floatOverflowSize, floatUnderflowSize, floatOverflowTranslation + + var expected: CaptureResult.Unavailable { + switch self { + case .shortPose: .malformedBox(.poseCount(15)) + case .twoSizes: .malformedBox(.sizeCount(2)) + case .nanPose, .infiniteSize, .floatOverflowSize, .floatOverflowTranslation: .malformedBox(.nonFinite) + case .zeroSize, .floatUnderflowSize: .malformedBox(.nonPositiveSize) + case .scaledPose, .shearedPose, .mirroredPose, .badBottomRow: .malformedBox(.notRigid) + case .noBox, .noEpoch: .notInARKitWorld + case .noPlacement: .noPlacement + } + } + + func apply(to outcome: inout CaptureResult.Outcome) { + switch self { + case .shortPose: outcome.recommendedPlacement?.boxArkitWorld?.pose.removeLast() + case .twoSizes: outcome.recommendedPlacement?.boxArkitWorld?.size = [0.8, 1.0] + case .nanPose: outcome.recommendedPlacement?.boxArkitWorld?.pose[13] = .nan + case .infiniteSize: outcome.recommendedPlacement?.boxArkitWorld?.size[1] = .infinity + case .zeroSize: outcome.recommendedPlacement?.boxArkitWorld?.size[2] = 0 + case .floatOverflowSize: outcome.recommendedPlacement?.boxArkitWorld?.size[0] = 1e40 + case .floatUnderflowSize: outcome.recommendedPlacement?.boxArkitWorld?.size[0] = 1e-50 + case .floatOverflowTranslation: outcome.recommendedPlacement?.boxArkitWorld?.pose[12] = 1e40 + case .scaledPose: + for i in 0..<11 { outcome.recommendedPlacement?.boxArkitWorld?.pose[i] *= 1.25 } + case .shearedPose: outcome.recommendedPlacement?.boxArkitWorld?.pose[4] = 0.3 + case .mirroredPose: outcome.recommendedPlacement?.boxArkitWorld?.pose[0] = -1 + case .badBottomRow: outcome.recommendedPlacement?.boxArkitWorld?.pose[3] = 0.5 + case .noBox: outcome.recommendedPlacement?.boxArkitWorld = nil + case .noEpoch: outcome.arkitEpoch = nil + case .noPlacement: outcome.recommendedPlacement = nil + } + } + } + + @Test(arguments: BadBox.allCases) + func malformedGeometryWithholdsARAndKeepsTheResult(_ bad: BadBox) throws { + var response = try Self.eligible() + var outcome = try #require(response.outcome) + bad.apply(to: &outcome) + response.outcome = outcome + let record = Self.verified(response) + #expect(record.placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) == .unavailable(bad.expected)) + #expect(record.response.outcome?.kind == .eligible) + #expect(record.response.outcome?.message == "A battery fits on this wall.") + } + + @Test func nonFiniteMeterAnchorWithholdsAR() throws { + var anchor = matrix_identity_float4x4 + anchor.columns.3.x = .nan + #expect(Self.verified(try Self.eligible()).placement(in: Self.scan, meterAnchor: anchor) == .unavailable(.invalidMeterAnchor)) + } + + // MARK: Transform + + @Test func boxIsExpressedInATranslatedRotatedMeterAnchor() throws { + let worldFromAnchor = pose(rotationY(degrees: 90), SIMD3(2, 0, -1)) + let anchorFromBox = pose(rotationY(degrees: 30), SIMD3(0.5, 0.4, 0.3)) + var response = try Self.eligible() + response.outcome?.recommendedPlacement?.boxArkitWorld?.pose = columnMajor(worldFromAnchor * anchorFromBox) + + let placement = Self.verified(response).placement(in: Self.scan, meterAnchor: float(worldFromAnchor)) + guard case .box(let placed) = placement else { + Issue.record("expected a box, got \(placement)") + return + } + #expect(nearlyEqual(placed.anchorFromBox, float(anchorFromBox))) + #expect(nearlyEqual(SIMD3(placed.anchorFromBox.columns.3.x, placed.anchorFromBox.columns.3.y, placed.anchorFromBox.columns.3.z), + SIMD3(0.5, 0.4, 0.3), 1e-5)) + #expect(placed.size == SIMD3(0.8, 1.0, 0.5)) + } + + /// The server builds the box's world pose from the scene box and the scene-to-ARKit similarity: + /// the similarity's scale is divided out of the rotation and multiplied into the size. The app + /// must take that size as it is, not scale it again by `frameToArkit`. + @Test func sizeCarriesTheServersScaleOnce() throws { + let scale = 1.25 + let frameToArkit = pose(rotationY(degrees: 40) * scale, SIMD3(1, 0, 2)) + let centerM = SIMD3(1.5, 0.5, 0.3), sizeM = SIMD3(0.8, 1.0, 0.56), yawDeg = 10.0 + + let rotation = rotation3(frameToArkit) * (1 / scale) * rotationY(degrees: yawDeg) + let center = frameToArkit * SIMD4(centerM, 1) + let serverPose = pose(rotation, SIMD3(center.x, center.y, center.z)) + let serverSize = sizeM * scale + + var response = try Self.eligible() + response.outcome?.recommendedPlacement?.boxArkitWorld?.pose = columnMajor(serverPose) + response.outcome?.recommendedPlacement?.boxArkitWorld?.size = [serverSize.x, serverSize.y, serverSize.z] + + let placement = Self.verified(response).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + guard case .box(let placed) = placement else { + Issue.record("expected a box, got \(placement)") + return + } + #expect(nearlyEqual(placed.size, SIMD3(1.0, 1.25, 0.7), 1e-6)) + #expect(nearlyEqual(placed.anchorFromBox, float(serverPose))) + for column in [placed.anchorFromBox.columns.0, placed.anchorFromBox.columns.1, placed.anchorFromBox.columns.2] { + #expect(abs(simd_length(SIMD3(column.x, column.y, column.z)) - 1) < 1e-5) + } + } +} + +private func jsonString(_ text: String) throws -> String { + String(decoding: try JSONEncoder().encode(text), as: UTF8.self) +} + +private func rotationY(degrees: Double) -> simd_double3x3 { + let a = degrees * .pi / 180 + return simd_double3x3(SIMD3(cos(a), 0, -sin(a)), SIMD3(0, 1, 0), SIMD3(sin(a), 0, cos(a))) +} + +private func rotation3(_ m: simd_double4x4) -> simd_double3x3 { + simd_double3x3( + SIMD3(m.columns.0.x, m.columns.0.y, m.columns.0.z), + SIMD3(m.columns.1.x, m.columns.1.y, m.columns.1.z), + SIMD3(m.columns.2.x, m.columns.2.y, m.columns.2.z)) +} + +private func pose(_ r: simd_double3x3, _ t: SIMD3) -> simd_double4x4 { + simd_double4x4(SIMD4(r.columns.0, 0), SIMD4(r.columns.1, 0), SIMD4(r.columns.2, 0), SIMD4(t, 1)) +} + +private func translation(_ x: Float, _ y: Float, _ z: Float) -> simd_float4x4 { + var m = matrix_identity_float4x4 + m.columns.3 = SIMD4(x, y, z, 1) + return m +} + +private func columnMajor(_ m: simd_double4x4) -> [Double] { + [m.columns.0, m.columns.1, m.columns.2, m.columns.3].flatMap { [$0.x, $0.y, $0.z, $0.w] } +} + +private func float(_ m: simd_double4x4) -> simd_float4x4 { + simd_float4x4(SIMD4(m.columns.0), SIMD4(m.columns.1), SIMD4(m.columns.2), SIMD4(m.columns.3)) +} + +private func nearlyEqual(_ a: simd_float4x4, _ b: simd_float4x4, _ tolerance: Float = 1e-5) -> Bool { + [(a.columns.0, b.columns.0), (a.columns.1, b.columns.1), (a.columns.2, b.columns.2), (a.columns.3, b.columns.3)] + .allSatisfy { simd_reduce_max(simd_abs($0 - $1)) <= tolerance } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift new file mode 100644 index 000000000..98ce33e0b --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift @@ -0,0 +1,393 @@ +import Foundation +import HouseScanKit +import simd +import Synchronization +import Testing + +/// A scan shaped like the app's live capture, for the coordinator: 1920 x 1440 sensor JPEGs, the +/// camera 2 m from a wall at 1.5 m height walking +x, the recorder's raw rows (a 60 Hz trajectory +/// with each frame's intrinsics beside it, and Core Motion's accelerometer and gyro at about +/// 100 Hz on their own jittered clocks), the meter tap's frame and pixel, and the meter close-up +/// from its own frame. Pattern images, no house; the packet says `source.kind: synthetic`. +struct NativeCaptureFixture: Sendable { + static let width = 1920, height = 1440 + static let k = SIMD4(1445, 1445, 960, 720) + static let meter = SIMD3(0.4, 1.2, -2) + let start = 5000.0 + let end = 5012.0 + let folder: URL + + func pose(_ t: Double) -> simd_float4x4 { + var m = matrix_identity_float4x4 + m.columns.3 = SIMD4(Float(t - start) * 0.1, 1.5, 0, 1) + return m + } + + /// Where `world` lands in the sensor image of the frame at `t`. + func pixel(of world: SIMD3, at t: Double) -> SIMD2 { + let p = world - SIMD3(pose(t).columns.3.x, pose(t).columns.3.y, pose(t).columns.3.z) + let depth = -p.z + return SIMD2(Double(Self.k.z + Self.k.x * p.x / depth), Double(Self.k.w - Self.k.y * p.y / depth)) + } + + var recording: RecordingSource { + let fixture = self + return RecordingSource(start: { (fixture.start, Date(timeIntervalSince1970: 1_790_000_000)) }, rows: { fixture.rows }) + } + + var rows: RecorderRows { + var trajectory: [[Double]] = [], intrinsics: [[Double]] = [] + for i in 0...Int((end - start) * 60) { + let t = start + Double(i) / 60 + let m = pose(t) + let columns: [SIMD4] = [m.columns.0, m.columns.1, m.columns.2, m.columns.3] + let values: [Double] = columns.flatMap { (c: SIMD4) -> [Double] in [Double(c.x), Double(c.y), Double(c.z), Double(c.w)] } + trajectory.append([t, 0, 0] + values) + intrinsics.append([t, 1445, 1445, 960, 720, 1920, 1440]) + } + // Two sensors on their own clocks: neither shares a timestamp with the other. + var accel: [[Double]] = [], gyro: [[Double]] = [] + for i in 0.. URL { + let url = folder.appending(path: "\(name).jpg") + if !FileManager.default.fileExists(atPath: url.path) { + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + try makeJPEG(width: Self.width, height: Self.height, at: url) + } + return url + } + + func photo(at t: Double, purpose: String? = nil) throws -> KeptPhoto { + KeptPhoto( + t: t, cameraToWorld: pose(t), cameraIntrinsics: Self.k, cameraImageSize: SIMD2(1920, 1440), width: Self.width, height: Self.height, + tracking: .normal, exposure: .init(duration: 0.004, offset: 0, iso: 64, fNumber: 1.6), jpeg: try jpeg(named: "frame-\(t)"), purpose: purpose) + } + + /// The raycast ran on the frame at `t`; the frame the app records is two frames newer, as + /// ARKit's current frame can be, and the tap is re-aimed at the hit in that frame. + func tap(at t: Double) throws -> (TapObservation, Packet04.TapHit) { + let data = try Data(contentsOf: try jpeg(named: "tap-\(t)")) + let recorded = t + 2.0 / 60 + let camera = pose(recorded).columns.3 + let stale = TapObservation( + t: recorded, cameraToWorld: pose(recorded), intrinsics: Self.k, width: Self.width, height: Self.height, tracking: .normal, + pixel: pixel(of: Self.meter, at: t), jpeg: { data }) + let tap = try #require(stale.pointing(at: Self.meter)) + let hit = Packet04.TapHit( + position: [Self.meter.x, Self.meter.y, Self.meter.z].map(Double.init), target: "estimatedPlane", alignment: "vertical", + distance: Double(simd_distance(Self.meter, SIMD3(camera.x, camera.y, camera.z)))) + return (tap, hit) + } + + static func environment( + endpoint: URL, http: any CaptureHTTP, captures: URL, sends: Bool = true, eventsWait: Int = 0, log: @escaping @Sendable (String) -> Void = { _ in } + ) -> CaptureSessionCoordinator.Environment { + var policy = CaptureUploader.Policy() + policy.eventsWait = eventsWait + let device = CaptureAPI.Device(model: "iPhone15,4", systemVersion: "26.0", appVersion: "0.1.0 (1)") + return .init( + endpoint: endpoint, sends: sends, http: http, capturesFolder: captures, + sessionInfo: { packetID, video in + Packet04SessionInfo( + packetID: packetID, sessionID: packetID, source: .synthetic, appVersion: device.appVersion, deviceModel: device.model, + systemVersion: device.systemVersion, lidarAvailable: false, sceneDepthEnabled: false, meshReconstructionSupported: nil, + sceneReconstruction: nil, planeDetection: ["horizontal", "vertical"], videoWidth: Int(video.x), videoHeight: Int(video.y), + framesPerSecond: nil, timeZone: "America/Chicago") + }, + device: device, tier: .arkit, policy: policy, log: log) + } + + /// The whole scan through `coordinator`: the tap, the close-up, keyframes (one of them the + /// close-up's own frame again), then the capture ends. + @MainActor + func run(_ coordinator: CaptureSessionCoordinator, consent: Bool? = true, beforeEnd: @MainActor () async -> Void = {}) async throws { + coordinator.begin(recording: recording) + // The integration build asks before the scan starts. + if let consent { coordinator.answerConsent(consent) } + let (tap, hit) = try self.tap(at: start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.kept(try photo(at: start + 2.5, purpose: "meter_close")) + for i in 0..<5 { coordinator.kept(try photo(at: start + 3 + Double(i) * 1.5)) } + // The app keeps the close-up's frame twice when it is also a walk frame. + coordinator.kept(try photo(at: start + 2.5)) + await coordinator.settle() + await beforeEnd() + coordinator.captureEnded(acceptedCloseUpAt: start + 2.5) + coordinator.captureEnded(acceptedCloseUpAt: start + 2.5) + await coordinator.settle() + } +} + +@Suite(.serialized) @MainActor struct CaptureSessionCoordinatorTests { + let root = FileManager.default.temporaryDirectory.appending(path: "coordinator-\(UUID().uuidString)") + var fixture: NativeCaptureFixture { NativeCaptureFixture(folder: root.appending(path: "store")) } + var captures: URL { root.appending(path: "Captures") } + + @Test func nativeScanExportsAValidPacketAndUploadsItDuringTheScan() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures)) + let committedBeforeEnd = Mutex(0) + try await fixture.run(coordinator) { + let uploader = coordinator.session!.uploader! + committedBeforeEnd.withLock { $0 = 0 } + let count = await uploader.snapshot.committedCount + committedBeforeEnd.withLock { $0 = count } + } + let session = try #require(coordinator.session) + let state = await session.uploader!.snapshot + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: try #require(state.packet)) + + #expect(Packet04Check.problems(packet, folder: session.folder).isEmpty) + // Tap frame, close-up frame and five walk frames; the repeated close-up frame is not a new keyframe. + #expect(packet.keyframes.count == 7) + #expect(packet.keyframes.map(\.reason) == ["tap", "still", "auto", "auto", "auto", "auto", "auto"]) + #expect(packet.stills?.map(\.purpose) == ["meter_close"]) + #expect(packet.scaleReference.meterCloseUp.still == "meter_close") + let tap = try #require(packet.taps?.first) + #expect(tap.label == "meter" && tap.keyframe == "k00001" && tap.epoch == "e1") + // The ray through the tap pixel reaches the meter the raycast hit. + let toMeter = simd_normalize(SIMD3(tap.hit!.position[0] - tap.rayOrigin[0], tap.hit!.position[1] - tap.rayOrigin[1], tap.hit!.position[2] - tap.rayOrigin[2])) + #expect(simd_distance(toMeter, SIMD3(tap.rayDirection[0], tap.rayDirection[1], tap.rayDirection[2])) < 1e-5) + // Raw world poses, straight from the frames. + #expect(packet.keyframes[2].pose[12...14].map { $0 } == [0.3, 1.5, 0]) + // Both sensors' own times travel beside the proposed pairing. + #expect(packet.ext?["imuRawPairingStatus"]?.hasPrefix("proposal") == true) + #expect(Set(packet.files.filter { $0.role == .stream }.map(\.path)) + == ["streams/arkit_poses.csv.gz", "streams/imu_raw.csv.gz", "streams/accelerometer_raw.csv.gz", "streams/gyroscope_raw.csv.gz"]) + + #expect(committedBeforeEnd.withLock { $0 } == packet.files.count - 4, "every image was received before the scan ended") + #expect(state.end == .finished(status: "manual_review")) + #expect(server.requests("POST captures").count == 1) + #expect(server.requests("POST captures/finalize").count == 1) + + if let dir = ProcessInfo.processInfo.environment["HOUSESCAN_EXPORT_EVIDENCE_DIR"] { + let target = URL(fileURLWithPath: dir).appending(path: "native-export-local") + try? FileManager.default.removeItem(at: target) + try FileManager.default.createDirectory(at: target.deletingLastPathComponent(), withIntermediateDirectories: true) + try FileManager.default.copyItem(at: session.folder, to: target) + } + } + + @Test func onlyTheIntegrationBuildRecordsAndOnlyAnAllowedBuildSends() { + let url = "https://capture.example.com/v1" + #expect(CaptureIntegrationMode.resolve(integrationBuild: "NO", endpoint: url, sendDeviceData: "YES") == .off("not the integration build")) + #expect(CaptureIntegrationMode.resolve(integrationBuild: nil, endpoint: url, sendDeviceData: "YES") == .off("not the integration build")) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "$(HOUSESCAN_INTEGRATION_BUILD)", endpoint: url, sendDeviceData: "YES") == .off("not the integration build")) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: "", sendDeviceData: "YES") == .off("no capture endpoint in this build")) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: url, sendDeviceData: "NO") == .recordOnly(URL(string: url)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: url, sendDeviceData: nil) == .recordOnly(URL(string: url)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: url, sendDeviceData: "YES") == .send(URL(string: url)!)) + // A replay goes only to a receiver on this machine, only when asked, whatever the device switch says. + let local = "http://127.0.0.1:8765/v1" + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: local, sendDeviceData: "NO", source: .replay(sendToLocalReceiver: true)) == .send(URL(string: local)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: local, sendDeviceData: "YES", source: .replay(sendToLocalReceiver: false)) == .recordOnly(URL(string: local)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: url, sendDeviceData: "YES", source: .replay(sendToLocalReceiver: true)) == .recordOnly(URL(string: url)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "NO", endpoint: local, sendDeviceData: "YES", source: .replay(sendToLocalReceiver: true)) == .off("not the integration build")) + } + + /// A build that may not send device data records the capture on the phone and never sends it, + /// even with a yes. + @Test func aRecordOnlyBuildKeepsTheCaptureOnThePhone() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures, sends: false)) + try await fixture.run(coordinator) + #expect(coordinator.needsConsent == false) + coordinator.answerConsent(true) + await coordinator.settle() + let session = try #require(coordinator.session) + #expect(session.uploader == nil) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: Data(contentsOf: session.folder.appending(path: "packet.json"))) + #expect(Packet04Check.problems(packet, folder: session.folder).isEmpty) + #expect(server.state.withLock { $0.log.isEmpty }) + } + + /// No answer or a no sends nothing; the next scan asks again. A yes before the scan sends + /// during it, and a world reset inside the scan keeps that yes for the new world's packet. + @Test func consentIsPerScanAndOnlyAYesSends() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let environment = NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures) + + let unanswered = CaptureSessionCoordinator(environment: environment) + try await fixture.run(unanswered, consent: nil) + #expect(unanswered.needsConsent) + #expect(unanswered.session?.uploader == nil) + + let declining = CaptureSessionCoordinator(environment: environment) + try await fixture.run(declining, consent: false) + #expect(declining.session?.uploader == nil) + #expect(server.state.withLock { $0.log.isEmpty }) + declining.newWorld("start over", recording: fixture.recording, newScan: true) + #expect(declining.consent == nil && declining.needsConsent) + + let agreeing = CaptureSessionCoordinator(environment: environment) + agreeing.begin(recording: fixture.recording) + agreeing.answerConsent(true) + let firstWorld = try #require(agreeing.session) + agreeing.newWorld("world reset", recording: fixture.recording, newScan: false) + #expect(agreeing.consent == true) + let secondWorld = try #require(agreeing.session) + #expect(secondWorld.uploader != nil && secondWorld.packetID != firstWorld.packetID) + agreeing.newWorld("start over", recording: fixture.recording, newScan: true) + #expect(agreeing.consent == nil) + #expect(agreeing.session?.uploader == nil) + } + + /// After a relaunch a frozen capture resumes only with the yes it recorded and only toward the + /// endpoint it was created on. + @Test func resumeNeedsTheCapturesYesAndTheSameEndpoint() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let original = try LoopbackCaptureAPI() + let other = try LoopbackCaptureAPI() + // A capture frozen on the phone for `original` whose upload never started. + let folder = captures.appending(path: "frozen") + let capture = try SyntheticCapture04(folder: folder) + let images = try await capture.sealImages() + let (streams, packet) = try await capture.finish() + var saved = CaptureUploadState( + attemptID: "old-process", packetID: capture.producer.info.packetID, + createBody: try JSONEncoder().encode(CaptureAPI.CreateRequest(packetId: capture.producer.info.packetID, tier: .arkit, device: .init(model: "m", systemVersion: "s", appVersion: "a")))) + saved.destination = original.base.absoluteString + // First without a recorded yes: never resumed. + for (i, file) in (images + streams).enumerated() { saved.files[file.path] = .init(sealed: file, sequence: i) } + saved.packet = packet + try saved.save(to: CaptureUploader.stateURL(in: folder)) + let http = URLSessionCaptureHTTP.ephemeral(timeout: 10) + + #expect(CaptureSessionCoordinator(environment: NativeCaptureFixture.environment(endpoint: original.base, http: http, captures: captures)) + .resumeSealedCaptures().isEmpty) + saved.consent = .init(grantedAt: Date(), destination: original.base.absoluteString) + try saved.save(to: CaptureUploader.stateURL(in: folder)) + #expect(CaptureSessionCoordinator(environment: NativeCaptureFixture.environment(endpoint: other.base, http: http, captures: captures)) + .resumeSealedCaptures().isEmpty) + #expect(CaptureSessionCoordinator(environment: NativeCaptureFixture.environment(endpoint: original.base, http: http, captures: captures, sends: false)) + .resumeSealedCaptures().isEmpty) + try await Task.sleep(for: .milliseconds(200)) + #expect(original.state.withLock { $0.log.isEmpty } && other.state.withLock { $0.log.isEmpty }) + + let resumed = CaptureSessionCoordinator(environment: NativeCaptureFixture.environment(endpoint: original.base, http: http, captures: captures)) + .resumeSealedCaptures() + #expect(resumed.count == 1) + for uploader in resumed { await uploader.kick(); await uploader.settled() } + #expect(await resumed.first?.snapshot.end == .finished(status: "manual_review")) + #expect(other.state.withLock { $0.log.isEmpty }) + } + + /// A refused close-up and its accepted retake, written over the same file as the app does: + /// both stay sealed with their own pixels, and the scale reference names the accepted one, + /// with its pixels and pose. A skipped close-up names none. + @Test func onlyTheAcceptedCloseUpIsTheScaleReference() async throws { + defer { try? FileManager.default.removeItem(at: root) } + for accepted in [true, false] { + let coordinator = CaptureSessionCoordinator(environment: NativeCaptureFixture.environment( + endpoint: URL(string: "https://capture.invalid/v1")!, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures, sends: false)) + coordinator.begin(recording: fixture.recording) + let source = fixture.folder.appending(path: "meter_close.jpg") + try FileManager.default.createDirectory(at: fixture.folder, withIntermediateDirectories: true) + try? FileManager.default.removeItem(at: source) + try makeJPEG(width: NativeCaptureFixture.width, height: NativeCaptureFixture.height, at: source, shift: 7) + let refusedBytes = try Data(contentsOf: source) + // Its depth too lives in files the retake overwrites: 4 x 3 meters and confidence. + let depthFile = fixture.folder.appending(path: "meter_close.f32"), confidenceFile = fixture.folder.appending(path: "meter_close.conf.u8") + func writeDepth(_ meters: Float, _ confidence: UInt8) throws { + try PacketFiles.depthData(meters: [Float](repeating: meters, count: 12)).write(to: depthFile) + try Data(repeating: confidence, count: 12).write(to: confidenceFile) + } + try writeDepth(1.5, 2) + func closeUp(at t: Double) -> KeptPhoto { + KeptPhoto( + t: t, cameraToWorld: fixture.pose(t), cameraIntrinsics: NativeCaptureFixture.k, cameraImageSize: SIMD2(1920, 1440), + width: NativeCaptureFixture.width, height: NativeCaptureFixture.height, tracking: .normal, exposure: nil, jpeg: source, purpose: "meter_close", + depth: { + guard let map = try? Data(contentsOf: depthFile), let confidence = try? Data(contentsOf: confidenceFile) else { return nil } + return Packet04Depth(meters: PacketFiles.floats(littleEndian: map), confidence: [UInt8](confidence), width: 4, height: 3) + }) + } + coordinator.kept(closeUp(at: fixture.start + 2)) + // The retake overwrites the same files before the first shot's sealing has run. + try FileManager.default.removeItem(at: source) + try makeJPEG(width: NativeCaptureFixture.width, height: NativeCaptureFixture.height, at: source, shift: 101) + try writeDepth(0.9, 1) + let retakeBytes = try Data(contentsOf: source) + coordinator.kept(closeUp(at: fixture.start + 4)) + coordinator.captureEnded(acceptedCloseUpAt: accepted ? fixture.start + 4 : nil) + await coordinator.settle() + + let session = try #require(coordinator.session) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: Data(contentsOf: session.folder.appending(path: "packet.json"))) + #expect(packet.stills?.map(\.id) == ["meter_close", "meter_close-2"]) + #expect(try Data(contentsOf: session.folder.appending(path: "stills/meter_close.jpg")) == refusedBytes) + #expect(try Data(contentsOf: session.folder.appending(path: "stills/meter_close-2.jpg")) == retakeBytes) + // Each shot keeps its own depth and confidence. + #expect(try Data(contentsOf: session.folder.appending(path: "keyframes/k00001.depth.f32")) == PacketFiles.depthData(meters: [Float](repeating: 1.5, count: 12))) + #expect(try Data(contentsOf: session.folder.appending(path: "keyframes/k00001.confidence.u8")) == Data(repeating: 2, count: 12)) + #expect(try Data(contentsOf: session.folder.appending(path: "keyframes/k00002.depth.f32")) == PacketFiles.depthData(meters: [Float](repeating: 0.9, count: 12))) + #expect(Packet04Check.problems(packet, folder: session.folder).isEmpty) + if accepted { + #expect(packet.scaleReference.meterCloseUp.still == "meter_close-2") + let frame = try #require(packet.keyframes.first { $0.id == packet.stills?.last?.keyframe }) + #expect(frame.timestamp == fixture.start + 4 && frame.pose[12] == 0.4) + } else { + #expect(packet.scaleReference.meterCloseUp == .notCaptured) + } + } + } + + /// A world reset while the first world's photos are still going up: that packet is abandoned, + /// the new world gets its own packet id, and nothing kept after the reset joins the old one. + @Test func aWorldResetStartsANewPacketAndDropsTheOld() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/files"] } + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures)) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + let first = try #require(coordinator.session) + for _ in 0..<500 where server.requests("POST captures/files").isEmpty { try await Task.sleep(for: .milliseconds(10)) } + + coordinator.newWorld("world reset", recording: fixture.recording, newScan: false) + coordinator.kept(try fixture.photo(at: fixture.start + 4)) + server.release("POST captures/files") + server.state.withLock { $0.held = [] } + await coordinator.settle() + await first.uploader!.settled() + + let second = try #require(coordinator.session) + #expect(second.packetID != first.packetID) + let old = await first.uploader!.snapshot + #expect(old.end == .abandoned("world reset")) + #expect(old.files.values.allSatisfy { $0.phase == .queued }) + #expect(await second.uploader!.snapshot.files.keys.sorted() == ["keyframes/k00001.jpg"]) + #expect(await second.uploader!.snapshot.committedCount == 1) + let creates = server.requests("POST captures").compactMap { try? JSONDecoder().decode(CaptureAPI.CreateRequest.self, from: $0.body).packetId } + #expect(Set(creates) == [first.packetID, second.packetID]) + } + + /// Photos kept after the scan was sent don't change the frozen packet or start more uploads. + @Test func photosAfterTheScanWasSentAreNotAdded() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures)) + try await fixture.run(coordinator) + let before = await coordinator.session!.uploader!.snapshot + coordinator.kept(try fixture.photo(at: fixture.end - 0.5)) + await coordinator.settle() + let after = await coordinator.session!.uploader!.snapshot + #expect(after.packet == before.packet) + #expect(after.files.count == before.files.count) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift new file mode 100644 index 000000000..d999ff85e --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift @@ -0,0 +1,284 @@ +import Foundation +import HouseScanKit +import Synchronization +import Testing + +/// The uploader against `LoopbackCaptureAPI` over real HTTP, through the same `URLSession` +/// transport the app uses. Retries wait 10 ms instead of seconds. +@Suite(.serialized) struct CaptureUploaderTests { + struct Rig { + let server: LoopbackCaptureAPI + let root: URL + let capture: SyntheticCapture04 + let uploader: CaptureUploader + let http: any CaptureHTTP + + init( + packetID: String = UUID().uuidString, appVersion: String = "test (1)", server: LoopbackCaptureAPI? = nil, + http: any CaptureHTTP = URLSessionCaptureHTTP.ephemeral(timeout: 10) + ) throws { + self.http = http + self.server = try server ?? LoopbackCaptureAPI() + root = FileManager.default.temporaryDirectory.appending(path: "uploader-\(UUID().uuidString)") + capture = try SyntheticCapture04(folder: root.appending(path: "packet"), packetID: packetID) + uploader = try CaptureUploader.start( + folder: capture.folder, base: self.server.base, http: http, + create: .init(packetId: packetID, tier: .arkit, device: .init(model: "iPhone15,4", systemVersion: "26.0", appVersion: appVersion)), + consentedAt: Date(), policy: Self.fast, sleep: { _ in try await Task.sleep(for: .milliseconds(10)) }) + } + + static var fast: CaptureUploader.Policy { + var policy = CaptureUploader.Policy() + policy.eventsWait = 0 + return policy + } + + func finishAndSeal() async throws { + let (streams, packet) = try await capture.finish() + await uploader.seal(packet: packet, files: streams) + } + + func cleanUp() { try? FileManager.default.removeItem(at: root) } + } + + /// Photos go up during the capture; the frozen packet finalizes early, the streams follow, and + /// the capture ends with a result for its own run. + @Test func joinedPathCommitsPhotosBeforeTheCaptureEnds() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + let images = try await rig.capture.sealImages() + await rig.uploader.add(images) + await rig.uploader.settled() + + let during = await rig.uploader.snapshot + #expect(during.committedCount == images.count) + #expect(during.packet == nil) + #expect(rig.server.requests("POST captures/finalize").isEmpty) + + try await rig.finishAndSeal() + await rig.uploader.settled() + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "manual_review")) + #expect(done.finalized?.status == "awaiting_files") + #expect(done.finalized?.missing.sorted() == [ + "streams/accelerometer_raw.csv.gz", "streams/arkit_poses.csv.gz", "streams/gyroscope_raw.csv.gz", "streams/imu_raw.csv.gz", + ]) + #expect(done.committedCount == images.count + 4) + #expect(done.result != nil) + let marks = done.marks + #expect(try #require(marks["firstCommit"]) < #require(marks["sealed"])) + #expect(try #require(marks["sealed"]) <= #require(marks["lastCommit"])) + + let puts = rig.server.requests("PUT upload") + #expect(puts.count == images.count + 4) + #expect(puts.allSatisfy { $0.headers["authorization"] == nil && $0.headers["content-md5"] != nil }) + #expect(rig.server.requests("POST captures").count == 1) + } + + /// A create and a finalize whose answers are lost are sent again as the same bytes, and the + /// server keeps one capture. + @Test func lostAnswersAreReplayedWithTheFrozenBodies() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.dropNext = ["POST captures", "POST captures/finalize"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 2)) + await rig.uploader.settled() + try await rig.finishAndSeal() + await rig.uploader.settled() + + let creates = rig.server.requests("POST captures") + let finals = rig.server.requests("POST captures/finalize") + #expect(creates.count == 2) + #expect(Set(creates.map(\.body)).count == 1) + #expect(finals.count == 2) + #expect(Set(finals.map(\.body)).count == 1) + #expect(rig.server.state.withLock { $0.captures.count } == 1) + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + } + + /// A relaunch resumes the saved capture: same packet and capture ids, no second create, and the + /// old process's attempt id is replaced. + @Test func relaunchResumesTheSameCapture() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + await rig.uploader.add(try await rig.capture.sealImages(count: 2)) + await rig.uploader.settled() + let before = await rig.uploader.snapshot + + let resumed = try #require(try CaptureUploader.resume( + folder: rig.capture.folder, base: rig.server.base, http: rig.http, policy: Rig.fast, sleep: { _ in })) + let after = await resumed.snapshot + #expect(after.packetID == before.packetID) + #expect(after.captureID == before.captureID) + #expect(after.createBody == before.createBody) + #expect(after.attemptID != before.attemptID) + + let (streams, packet) = try await rig.capture.finish() + await resumed.seal(packet: packet, files: streams) + await resumed.settled() + #expect(await resumed.snapshot.end == .finished(status: "manual_review")) + #expect(rig.server.requests("POST captures").count == 1) + } + + /// The same packet id with a different create body is the server's idempotency conflict; the + /// uploader stops instead of retrying it. + @Test func aChangedCreateBodyIsRefusedNotRetried() async throws { + let packetID = UUID().uuidString + let first = try Rig(packetID: packetID) + defer { first.cleanUp() } + await first.uploader.kick() + await first.uploader.settled() + #expect(await first.uploader.snapshot.captureID != nil) + + let second = try Rig(packetID: packetID, appVersion: "test (2)", server: first.server) + defer { second.cleanUp() } + await second.uploader.kick() + await second.uploader.settled() + #expect(await second.uploader.snapshot.end == .failed(step: "create", codes: ["idempotency_conflict"], status: 409)) + #expect(first.server.requests("POST captures").count == 2) + } + + /// A storage URL answered 403 (expired) is registered again and the file sent to the new URL. + @Test func anExpiredURLIsFetchedAgain() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.expireNextPuts = 1 } + let images = try await rig.capture.sealImages(count: 1) + await rig.uploader.add(images) + await rig.uploader.settled() + let snapshot = await rig.uploader.snapshot + #expect(snapshot.committedCount == images.count) + #expect(rig.server.requests("PUT upload").count == images.count + 1) + let registered = rig.server.requests("POST captures/files").flatMap { request -> [String] in + let body = try? JSONDecoder().decode(CaptureAPI.RegisterRequest.self, from: request.body) + return body?.files.map(\.path) ?? [] + } + #expect(registered.count == images.count + 1) + } + + /// Only paths the commit names as committed advance; notFound and mismatch go up again. + @Test func mixedCommitAnswersAdvanceOnlyCommittedFiles() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + let images = try await rig.capture.sealImages(count: 2) + // Priority order: the still, then k00001, then k00002. + let paths = ["stills/meter_close.jpg", "keyframes/k00001.jpg", "keyframes/k00002.jpg"] + #expect(Set(images.map(\.path)) == Set(paths)) + rig.server.state.withLock { $0.commitOverride = (committed: [paths[0]], notFound: [paths[1]], mismatch: [paths[2]]) } + let seen = Mutex<[Int]>([]) + await rig.uploader.observe { status in seen.withLock { $0.append(status.committed) } } + await rig.uploader.add(images) + await rig.uploader.settled() + + #expect(await rig.uploader.snapshot.committedCount == 3) + // After the first commit exactly one file counted as received. + #expect(seen.withLock { $0.first { $0 > 0 } } == 1) + let puts = rig.server.requests("PUT upload") + #expect(puts.count == 5) + } + + /// A world reset while a register answer is in flight: the late answer changes nothing, and the + /// next capture is a new packet that never reuses the old one. + @Test func resetDuringAPendingAnswerDropsTheLateReply() async throws { + // A background session's transfer outlives the task that asked for it, so the reply + // still arrives after the reset; only the attempt check keeps it out. + let rig = try Rig(http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10))) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.held = ["POST captures/files"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 2)) + for _ in 0..<500 where rig.server.requests("POST captures/files").isEmpty { try await Task.sleep(for: .milliseconds(10)) } + #expect(rig.server.requests("POST captures/files").count == 1) + + await rig.uploader.abandon("world reset") + rig.server.release("POST captures/files") + try await Task.sleep(for: .milliseconds(300)) + await rig.uploader.settled() + + let after = await rig.uploader.snapshot + #expect(after.end == .abandoned("world reset")) + #expect(after.files.values.allSatisfy { $0.phase == .queued }) + #expect(rig.server.requests("PUT upload").isEmpty) + // Nothing more is sent for the abandoned capture. + let late = try await rig.capture.producer.sealKeyframe( + jpeg: try SyntheticCapture04.jpeg(in: rig.root), observation: rig.capture.observation(SyntheticCapture04.start + 1.9), reason: "motion") + await rig.uploader.add(late) + await rig.uploader.settled() + #expect(rig.server.requests("POST captures/files").count == 1) + + let next = try Rig(server: rig.server) + defer { next.cleanUp() } + await next.uploader.add(try await next.capture.sealImages(count: 1)) + await next.uploader.settled() + #expect(await next.uploader.snapshot.captureID != after.captureID) + #expect(await next.uploader.snapshot.committedCount == 2) + } + + /// A finalize the server accepted and then lost (`failed`, next `retry_finalize`) is sent + /// again as the same bytes, and the run then completes. + @Test func aLostFinalizeIsSentAgainOnRetryFinalize() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.loseNextFinalize = true } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + let finals = rig.server.requests("POST captures/finalize") + #expect(finals.count == 2) + #expect(Set(finals.map(\.body)).count == 1) + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + #expect(await rig.uploader.snapshot.finalizeRetries == 1) + } + + /// Storage that keeps refusing the sealed bytes' digest stops the upload after one retry + /// instead of cycling through register and PUT. + @Test func aDigestRefusedTwiceStopsTheUpload() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.badDigestPuts = 100 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + await rig.uploader.settled() + #expect(await rig.uploader.snapshot.end == .failed(step: "put", codes: ["bad_digest"], status: 400)) + #expect(rig.server.requests("PUT upload").count <= 4) + } + + /// A file listed in the frozen packet that reached `seal` before its own `add` is still sent. + @Test func sealQueuesListedFilesItHasNotSeen() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + let images = try await rig.capture.sealImages(count: 2) + await rig.uploader.add(Array(images.prefix(1))) + await rig.uploader.settled() + let (streams, packet) = try await rig.capture.finish() + await rig.uploader.seal(packet: packet, files: images + streams) + await rig.uploader.add(Array(images.suffix(1))) + await rig.uploader.settled() + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "manual_review")) + #expect(done.committedCount == images.count + streams.count) + } + + @Test func noEndpointOrNoConsentMeansNoUpload() { + #expect(CaptureUploadGate.decide(endpoint: nil, consented: true) == .off("no capture endpoint in this build")) + #expect(CaptureUploadGate.decide(endpoint: "", consented: true) == .off("no capture endpoint in this build")) + #expect(CaptureUploadGate.decide(endpoint: "$(HOUSESCAN_CAPTURE_API_URL)", consented: true) == .off("no capture endpoint in this build")) + #expect(CaptureUploadGate.decide(endpoint: "http://example.com/v1", consented: true) == .off("capture endpoint must be https")) + #expect(CaptureUploadGate.decide(endpoint: "https://user:secret@example.com/v1", consented: true) == .off("capture endpoint is not a URL")) + #expect(CaptureUploadGate.decide(endpoint: "https://example.com/v1", consented: false) == .off("the homeowner has not agreed to send the capture")) + #expect(CaptureUploadGate.decide(endpoint: "https://example.com/v1", consented: true) == .on(URL(string: "https://example.com/v1")!)) + } +} + +/// Runs each request in a detached task, so cancelling the caller doesn't cancel the request: +/// the way a background URLSession behaves. +struct UncancellableHTTP: CaptureHTTP { + let inner: any CaptureHTTP + init(_ inner: any CaptureHTTP) { self.inner = inner } + + func send(_ request: URLRequest) async throws -> HTTPReply { + try await Task.detached { try await inner.send(request) }.value + } + + func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + try await Task.detached { try await inner.upload(request, file: file) }.value + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift new file mode 100644 index 000000000..d4c9166e1 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift @@ -0,0 +1,334 @@ +import CryptoKit +import Foundation +import Network +import Synchronization +import HouseScanKit + +/// A local stand-in for the capture API over real HTTP on 127.0.0.1, so the tests drive the +/// actual `URLSession` transport. It implements the documented behaviour the uploader relies on: +/// idempotent create and finalize bound to the exact body, register with signed PUT URLs, +/// Content-MD5 checked on the PUT, commit as the only acknowledgement, early finalize with +/// `awaiting_files`, and events leading to a result. Faults are injected per test. +/// +/// Written for these tests from the public description of the API; it is not the server's code. +final class LoopbackCaptureAPI: Sendable { + struct Request: Sendable { + var method: String + var path: String + var query: [String: String] + var headers: [String: String] + var body: Data + } + + struct Registration { + var sha256: String + var bytes: Int + var md5: String + var contentType: String + } + + struct Capture { + var id: String + var packetID: String + var createBody: Data + var registered: [String: Registration] = [:] + var stored: [String: Data] = [:] + var committed: Set = [] + var packetSHA: String? + var listed: [String] = [] + var runID: String? + var events: [(type: String, data: [String: String])] = [] + var status = "uploading" + /// Accepted a finalize, then lost it to a storage failure: no run until it is sent again. + var finalizeLost = false + } + + struct State { + var captures: [String: Capture] = [:] + var byPacket: [String: String] = [:] + var tokens: [String: (capture: String, path: String)] = [:] + var log: [Request] = [] + var nextID = 1 + /// Close the connection without answering the next request whose route matches. + var dropNext: Set = [] + /// Answer the next N storage PUTs 403, as an expired signature. + var expireNextPuts = 0 + /// Answer the next N storage PUTs 400 BadDigest. + var badDigestPuts = 0 + /// Accept the next finalize, then report it lost with `failed` / `retry_finalize`. + var loseNextFinalize = false + /// The homeowner message the result carries. + var resultMessage = "The loopback receiver's test result." + /// Answer the next commit with this split instead of the truth. + var commitOverride: ((committed: [String], notFound: [String], mismatch: [String]))? + /// Routes whose answers wait until `release` is called. + var held: Set = [] + var parked: [String: [@Sendable () -> Void]] = [:] + } + + let state = Mutex(State()) + private let listener: NWListener + private let queue = DispatchQueue(label: "loopback-capture-api") + var port: UInt16 { listener.port?.rawValue ?? 0 } + + init() throws { + let parameters = NWParameters.tcp + parameters.acceptLocalOnly = true + parameters.requiredLocalEndpoint = .hostPort(host: "127.0.0.1", port: .any) + listener = try NWListener(using: parameters) + let ready = DispatchSemaphore(value: 0) + listener.stateUpdateHandler = { if case .ready = $0 { ready.signal() } } + listener.newConnectionHandler = { [queue] connection in + connection.start(queue: queue) + Self.receive(connection, buffer: Data()) { request in self.handle(request, connection) } + } + listener.start(queue: queue) + guard ready.wait(timeout: .now() + 5) == .success, listener.port != nil else { throw URLError(.cannotConnectToHost) } + } + + deinit { listener.cancel() } + + var base: URL { URL(string: "http://127.0.0.1:\(port)/v1")! } + + func requests(_ route: String) -> [Request] { state.withLock { $0.log.filter { Self.route($0) == route } } } + + func release(_ route: String) { + let waiting = state.withLock { s -> [@Sendable () -> Void] in + s.held.remove(route) + return s.parked.removeValue(forKey: route) ?? [] + } + waiting.forEach { $0() } + } + + /// "POST /v1/captures/{id}/files" style names, ids replaced. + static func route(_ r: Request) -> String { + let parts = r.path.split(separator: "/").map(String.init) + if parts.first == "upload" { return "\(r.method) upload" } + guard parts.count >= 2 else { return "\(r.method) \(r.path)" } + if parts.count == 2 { return "\(r.method) captures" } + return "\(r.method) captures/\(parts[2...].joined(separator: "/"))" + } + + // MARK: HTTP over the connection + + private static func receive(_ connection: NWConnection, buffer: Data, handler: @escaping @Sendable (Request) -> Void) { + connection.receive(minimumIncompleteLength: 1, maximumLength: 1 << 20) { data, _, done, error in + var buffer = buffer + if let data { buffer.append(data) } + if let request = parse(buffer) { return handler(request) } + if done || error != nil { return connection.cancel() } + receive(connection, buffer: buffer, handler: handler) + } + } + + private static func parse(_ data: Data) -> Request? { + guard let end = data.range(of: Data("\r\n\r\n".utf8)) else { return nil } + let head = String(decoding: data[..= 2 else { return nil } + var headers: [String: String] = [:] + for field in head.dropFirst() { + guard let colon = field.firstIndex(of: ":") else { continue } + headers[field[..= length else { return nil } + let target = URLComponents(string: String(line[1])) + var query: [String: String] = [:] + for item in target?.queryItems ?? [] { query[item.name] = item.value } + var path = target?.path ?? "" + if path.hasPrefix("/v1/") { path.removeFirst(4) } else if path.hasPrefix("/") { path.removeFirst() } + return Request(method: String(line[0]), path: path, query: query, headers: headers, body: Data(body.prefix(length))) + } + + private func handle(_ request: Request, _ connection: NWConnection) { + let route = Self.route(request) + let (drop, hold) = state.withLock { s -> (Bool, Bool) in + s.log.append(request) + return (s.dropNext.remove(route) != nil, s.held.contains(route)) + } + let (status, body) = answer(request) + let send: @Sendable () -> Void = { + if drop { return connection.cancel() } + let head = "HTTP/1.1 \(status) X\r\nContent-Type: application/json\r\nContent-Length: \(body.count)\r\nConnection: close\r\n\r\n" + connection.send(content: Data(head.utf8) + body, completion: .contentProcessed { _ in connection.cancel() }) + } + if hold { + state.withLock { $0.parked[route, default: []].append(send) } + } else { + send() + } + } + + // MARK: API behaviour + + private func json(_ object: Any) -> Data { try! JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]) } + private func error(_ status: Int, _ code: String) -> (Int, Data) { (status, json(["errors": [["code": code, "pointer": "/", "message": code]]])) } + + private func answer(_ r: Request) -> (Int, Data) { + state.withLock { s in + let parts = r.path.split(separator: "/").map(String.init) + if parts.first == "upload" { return put(r, token: parts.last ?? "", &s) } + if r.headers["authorization"] != nil { return error(400, "unexpected_authorization") } + if r.method == "POST", parts == ["captures"] { return create(r, &s) } + guard parts.count >= 2, var capture = s.captures[parts[1]] else { return error(404, "capture_not_found") } + defer { s.captures[capture.id] = capture } + switch (r.method, parts.dropFirst(2).joined(separator: "/")) { + case ("POST", "files"): return register(r, &capture, &s) + case ("POST", "files:commit"): return commit(r, &capture, &s) + case ("POST", "finalize"): return finalize(r, &capture, &s) + case ("GET", "events"): return events(r, &capture) + case ("GET", "result"): + let outcome: Any = capture.status == "manual_review" + ? ["kind": "manual_review", "profile": "C", "message": s.resultMessage, "viewsNeeded": [], "reasons": []] as [String: Any] + : NSNull() + return (200, json(["runId": capture.runID ?? "", "status": capture.status, "viewsNeeded": [], "memberActions": [], "outcome": outcome])) + default: return error(404, "route") + } + } + } + + private func create(_ r: Request, _ s: inout State) -> (Int, Data) { + guard let body = try? JSONSerialization.jsonObject(with: r.body) as? [String: Any], let packetID = body["packetId"] as? String else { + return error(422, "schema") + } + guard let key = r.headers["idempotency-key"] else { return error(422, "idempotency_key_missing") } + guard key == packetID else { return error(422, "idempotency_key_mismatch") } + guard body["formatVersion"] as? String == "0.4" else { return error(422, "format_version_unsupported") } + let status: Int + let capture: Capture + if let existing = s.byPacket[packetID].flatMap({ s.captures[$0] }) { + guard existing.createBody == r.body else { return error(409, "idempotency_conflict") } + (status, capture) = (200, existing) + } else { + capture = Capture(id: "cap_\(s.nextID)", packetID: packetID, createBody: r.body) + s.nextID += 1 + s.captures[capture.id] = capture + s.byPacket[packetID] = capture.id + status = 201 + } + return (status, json([ + "captureId": capture.id, "status": "uploading", "eventsUrl": "/v1/captures/\(capture.id)/events", + "finalizeBy": "2099-01-01T00:00:00Z", "upload": ["maxSinglePutBytes": 16_777_216, "maxBatch": 50, "urlTtlS": 3600, "maxFiles": 2000, "partBytes": 8_388_608], + ])) + } + + private func register(_ r: Request, _ c: inout Capture, _ s: inout State) -> (Int, Data) { + guard let body = try? JSONSerialization.jsonObject(with: r.body) as? [String: Any], let files = body["files"] as? [[String: Any]], + (1...50).contains(files.count) else { return error(422, "batch_too_large") } + var out: [[String: Any]] = [] + for file in files { + guard let path = file["path"] as? String, Packet04.isRelPath(path), let sha = file["sha256"] as? String, sha.count == 64, + let bytes = file["bytes"] as? Int, let md5 = file["md5"] as? String, Data(base64Encoded: md5)?.count == 16, + let role = file["role"] as? String, Packet04.Role(rawValue: role) != nil, let type = file["contentType"] as? String + else { return error(422, "schema") } + let meta = file["meta"] as? [String: Any] + let image = role == "keyframe" || role == "still" + if image != (meta != nil) || (meta.map { $0.count != 1 } ?? false) { return error(422, "meta_mismatch") } + if let previous = c.registered[path], previous.sha256 != sha { return error(409, "sha256_conflict") } + if c.packetSHA != nil, !c.listed.contains(path) { return error(409, "not_in_manifest") } + c.registered[path] = Registration(sha256: sha, bytes: bytes, md5: md5, contentType: type) + if c.committed.contains(path) { + out.append(["path": path, "state": "committed"]) + continue + } + let token = UUID().uuidString + s.tokens[token] = (c.id, path) + out.append(["path": path, "state": "pending", "upload": [ + "method": "PUT", "url": "http://127.0.0.1:\(port)/upload/\(token)", + "headers": ["Content-Type": type, "Content-MD5": md5], "expiresAt": "2099-01-01T00:00:00Z", + ]]) + } + return (200, json(["files": out])) + } + + private func put(_ r: Request, token: String, _ s: inout State) -> (Int, Data) { + guard r.method == "PUT", let (captureID, path) = s.tokens[token], var c = s.captures[captureID], let reg = c.registered[path] else { + return (403, Data("token_invalid".utf8)) + } + if r.headers["authorization"] != nil { return (400, Data("authorization sent to storage".utf8)) } + if s.badDigestPuts > 0 { + s.badDigestPuts -= 1 + return (400, Data("BadDigest".utf8)) + } + if s.expireNextPuts > 0 { + s.expireNextPuts -= 1 + s.tokens[token] = nil + return (403, Data("AccessDeniedRequest has expired".utf8)) + } + guard r.headers["content-md5"] == reg.md5, Data(Insecure.MD5.hash(data: r.body)).base64EncodedString() == reg.md5, + r.headers["content-type"] == reg.contentType + else { return (400, Data("BadDigest".utf8)) } + c.stored[path] = r.body + s.captures[captureID] = c + return (200, Data()) + } + + private func commit(_ r: Request, _ c: inout Capture, _ s: inout State) -> (Int, Data) { + guard let body = try? JSONSerialization.jsonObject(with: r.body) as? [String: Any], let files = body["files"] as? [[String: Any]] else { + return error(422, "schema") + } + var committed: [String] = [], notFound: [String] = [], mismatch: [String] = [] + for file in files { + guard let path = file["path"] as? String, let sha = file["sha256"] as? String else { return error(422, "schema") } + guard let data = c.stored[path] else { notFound.append(path); continue } + if PacketFiles.sha256(data) != sha || c.registered[path]?.sha256 != sha { mismatch.append(path); continue } + committed.append(path) + } + if let forced = s.commitOverride { + s.commitOverride = nil + (committed, notFound, mismatch) = forced + } + c.committed.formUnion(committed) + if !committed.isEmpty { c.events.append(("files_committed", ["committed": String(c.committed.count)])) } + startIfComplete(&c) + return (200, json(["committed": committed, "notFound": notFound, "mismatch": mismatch])) + } + + private func finalize(_ r: Request, _ c: inout Capture, _ s: inout State) -> (Int, Data) { + guard r.headers["idempotency-key"] == c.packetID else { return error(422, "idempotency_key_mismatch") } + let sha = PacketFiles.sha256(r.body) + if let declared = r.headers["x-packet-sha256"], declared != sha { return error(422, "packet_sha_mismatch") } + if let previous = c.packetSHA, previous != sha { return error(409, "packet_sha_conflict") } + guard let packet = try? JSONDecoder().decode(Packet04.Packet.self, from: r.body) else { return error(422, "schema") } + guard packet.packetId == c.packetID else { return error(422, "packet_id_mismatch") } + let problems = Packet04Check.problems(packet, folder: nil) + if !problems.isEmpty { return error(422, "schema") } + for entry in packet.files { + if let reg = c.registered[entry.path], reg.sha256 != entry.sha256 || reg.bytes != entry.bytes { return error(422, "sha256_mismatch") } + } + if c.packetSHA == nil { + c.packetSHA = sha + c.listed = packet.files.map(\.path) + c.runID = "run_" + sha.prefix(16) + c.status = "awaiting_files" + if s.loseNextFinalize { + s.loseNextFinalize = false + c.finalizeLost = true + c.events.append(("failed", ["code": "storage_unavailable", "next": "retry_finalize"])) + } + startIfComplete(&c) + } else if c.finalizeLost { + c.finalizeLost = false + startIfComplete(&c) + } + let missing = c.listed.filter { !c.committed.contains($0) } + return (202, json(["status": missing.isEmpty ? "processing" : "awaiting_files", "missing": missing, "runId": c.runID!, "etaS": 1])) + } + + private func startIfComplete(_ c: inout Capture) { + guard c.packetSHA != nil, !c.finalizeLost, c.status == "awaiting_files", c.listed.allSatisfy(c.committed.contains) else { return } + c.status = "manual_review" + c.events.append(("stage", ["stage": "ingest", "status": "ok", "runId": c.runID!])) + c.events.append(("verdict_ready", ["kind": "manual_review", "runId": c.runID!])) + } + + private func events(_ r: Request, _ c: inout Capture) -> (Int, Data) { + let after = Int(r.query["after"] ?? "0") ?? 0 + let list = c.events.enumerated().filter { $0.offset + 1 > after }.map { index, event in + ["seq": index + 1, "type": event.type, "at": "2026-09-27T00:00:00Z", "data": event.data] as [String: Any] + } + return (200, json(["status": c.status, "next": c.events.count, "events": list])) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/Packet04Tests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/Packet04Tests.swift new file mode 100644 index 000000000..8d0c06138 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/Packet04Tests.swift @@ -0,0 +1,253 @@ +import Foundation +import HouseScanKit +import simd +import Testing + +/// A synthetic ARKit capture for the 0.4 tests: a camera walking along +x at 1.5 m height, +/// 96 x 72 JPEGs with intrinsics that fit them, 60 Hz poses and 100 Hz IMU with the gyro 1 ms +/// behind the accelerometer. `source` is synthetic, so no packet from it is evidence. +struct SyntheticCapture04 { + static let start = 1000.0 + let folder: URL + let producer: Packet04Producer + + init(folder: URL, packetID: String = UUID().uuidString) throws { + self.folder = folder + producer = try Packet04Producer(folder: folder, info: Self.info(packetID: packetID), startedAtUptime: Self.start, startedAt: Date(timeIntervalSince1970: 1_790_000_000)) + } + + static func info(packetID: String) -> Packet04SessionInfo { + Packet04SessionInfo( + packetID: packetID, sessionID: "S-1", source: .synthetic, appVersion: "test (1)", deviceModel: "iPhone15,4", systemVersion: "26.0", + lidarAvailable: false, sceneDepthEnabled: false, meshReconstructionSupported: false, sceneReconstruction: "none", + planeDetection: ["horizontal", "vertical"], videoWidth: 1920, videoHeight: 1440, framesPerSecond: 60, timeZone: "America/Chicago") + } + + static func pose(_ t: Double) -> simd_float4x4 { + var m = matrix_identity_float4x4 + m.columns.3 = SIMD4(Float(t - start) * 0.5, 1.5, -2, 1) + return m + } + + static let intrinsics = SIMD4(80, 80, 48, 36) + + func observation(_ t: Double) -> Packet04Observation { + Packet04Observation(t: t, cameraToWorld: Self.pose(t), intrinsics: Self.intrinsics, width: 96, height: 72, tracking: .normal) + } + + static func jpeg(in folder: URL) throws -> Data { + let url = folder.appending(path: "source-\(UUID().uuidString).jpg") + try makeJPEG(width: 96, height: 72, at: url) + defer { try? FileManager.default.removeItem(at: url) } + return try Data(contentsOf: url) + } + + /// Seals `count` keyframes and the head-on meter still on the first one; returns every file. + func sealImages(count: Int = 3) async throws -> [SealedFile] { + let scratch = folder.deletingLastPathComponent() + var files: [SealedFile] = [] + for i in 0.. [Packet04Streams.PoseRow] { + stride(from: start, through: end, by: 1.0 / 60).map { .init(t: $0, tracking: .normal, cameraToWorld: pose($0), intrinsics: SIMD4(1440, 1440, 960, 720)) } + } + + static func imu(until end: Double = start + 2) -> (accel: [Packet04Streams.MotionRow], gyro: [Packet04Streams.MotionRow]) { + let times = Array(stride(from: start, through: end, by: 0.01)) + return (times.map { .init(t: $0, value: SIMD3(0, -1, 0)) }, times.map { .init(t: $0 + 0.001, value: SIMD3(0.01, 0, 0)) }) + } + + func finish() async throws -> (streams: [SealedFile], packet: Data) { + let imu = Self.imu() + return try await producer.finish(poses: Self.poses(), accelerometer: imu.accel, gyroscope: imu.gyro, endedAtUptime: Self.start + 2, acceptedCloseUpAt: Self.start + 0.2) + } +} + +func temporaryPacketFolder(_ name: String) -> URL { + FileManager.default.temporaryDirectory.appending(path: "\(name)-\(UUID().uuidString)/packet") +} + +@Suite struct Packet04Tests { + @Test func finishedPacketPassesItsChecksAndListsEverySealedFile() async throws { + let folder = temporaryPacketFolder("p04-normal") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + let images = try await capture.sealImages() + let (streams, data) = try await capture.finish() + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: data) + + #expect(Packet04Check.problems(packet, folder: folder).isEmpty) + #expect(Set(packet.files.map(\.path)) == Set((images + streams).map(\.path))) + #expect(packet.formatVersion == "0.4") + #expect(packet.session.tier == .arkit) + #expect(packet.epochs == [.init(id: "e1", startTime: SyntheticCapture04.start, reason: "sessionStart")]) + #expect(packet.keyframes.allSatisfy { $0.epoch == "e1" }) + #expect(packet.scaleReference.meterCloseUp == .init(captured: true, still: "meter_close", arkitDistanceM: nil, side: nil)) + #expect(packet.scaleReference.ext.obliqueCloseUp == .notCaptured) + #expect(packet.tracking == [.init(time: SyntheticCapture04.start, state: "normal", epoch: "e1")]) + // The exact bytes finish returned are what is on disk. + #expect(try Data(contentsOf: folder.appending(path: "packet.json")) == data) + } + + @Test func keyframeKeepsTheRawWorldPoseAndItsOwnIntrinsics() async throws { + let folder = temporaryPacketFolder("p04-pose") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + let t = SyntheticCapture04.start + 0.7 + let files = try await capture.producer.sealKeyframe(jpeg: try SyntheticCapture04.jpeg(in: folder.deletingLastPathComponent()), observation: capture.observation(t), reason: "motion") + guard case .keyframe(let record)? = files.first?.meta else { Issue.record("no keyframe record"); return } + #expect(record.pose == [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0.35, 1.5, -2, 1]) + #expect(record.intrinsics == [80, 80, 48, 36]) + #expect(record.timestamp == t) + #expect(files.first?.path == "keyframes/k00001.jpg") + #expect(files.first?.role == .keyframe) + } + + @Test func sealedDigestsMatchTheBytesOnDisk() async throws { + let folder = temporaryPacketFolder("p04-digest") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + for file in try await capture.sealImages() { + let data = try Data(contentsOf: folder.appending(path: file.path)) + #expect(file.bytes == data.count) + #expect(file.sha256 == PacketFiles.sha256(data)) + #expect(Data(base64Encoded: file.md5)?.count == 16) + } + } + + @Test func refusesARotatedImageAndIntrinsicsOfAnotherSize() async throws { + let folder = temporaryPacketFolder("p04-refuse") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + var portrait = capture.observation(SyntheticCapture04.start + 1) + (portrait.width, portrait.height) = (72, 96) + await #expect(throws: Packet04Error.self) { + _ = try await capture.producer.sealKeyframe(jpeg: try SyntheticCapture04.jpeg(in: folder.deletingLastPathComponent()), observation: portrait, reason: "motion") + } + var wrong = capture.observation(SyntheticCapture04.start + 1) + wrong.intrinsics = SIMD4(1440, 1440, 960, 720) + await #expect(throws: Packet04Error.self) { + _ = try await capture.producer.sealKeyframe(jpeg: try SyntheticCapture04.jpeg(in: folder.deletingLastPathComponent()), observation: wrong, reason: "motion") + } + } + + @Test func nothingIsSealedAfterFinishAndFinishReturnsTheSameBytes() async throws { + let folder = temporaryPacketFolder("p04-frozen") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + _ = try await capture.sealImages() + let first = try await capture.finish() + let second = try await capture.finish() + #expect(first.packet == second.packet) + await #expect(throws: Packet04Error.sealedAfterFinish("keyframe")) { + _ = try await capture.producer.sealKeyframe(jpeg: try SyntheticCapture04.jpeg(in: folder.deletingLastPathComponent()), observation: capture.observation(SyntheticCapture04.start + 1.9), reason: "motion") + } + } + + @Test func tapRayReplaysFromItsKeyframe() async throws { + let folder = temporaryPacketFolder("p04-tap") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + _ = try await capture.sealImages(count: 1) + try await capture.producer.addTap(id: "tap1", label: "meter", keyframe: "k00001", pixel: SIMD2(48, 36), time: SyntheticCapture04.start + 0.2, hit: nil) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: try await capture.finish().packet) + let tap = try #require(packet.taps?.first) + // The principal point looks straight down -z from the camera. + #expect(tap.rayOrigin == [0.1, 1.5, -2]) + #expect(tap.rayDirection == [0, 0, -1]) + + var broken = packet + broken.taps?[0].rayDirection = [0, 0.01, -1] + #expect(Packet04Check.problems(broken, folder: nil).contains { $0.hasPrefix("tap tap1 ray replays with error") }) + } + + @Test func checksCatchAMalformedPacket() async throws { + let folder = temporaryPacketFolder("p04-malformed") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + _ = try await capture.sealImages() + let good = try JSONDecoder().decode(Packet04.Packet.self, from: try await capture.finish().packet) + + var unlisted = good + unlisted.files.removeAll { $0.path == "keyframes/k00002.jpg" } + #expect(Packet04Check.problems(unlisted, folder: folder) == ["keyframe k00002 image keyframes/k00002.jpg is not in files"]) + + var badPath = good + badPath.files[0].path = "../escape.jpg" + #expect(Packet04Check.problems(badPath, folder: nil).contains("file path ../escape.jpg is not a RelPath")) + + var badStill = good + badStill.stills?[0].keyframe = "k00009" + #expect(Packet04Check.problems(badStill, folder: nil).contains("still meter_close names unknown keyframe k00009")) + + try Data("changed".utf8).write(to: folder.appending(path: "keyframes/k00003.jpg")) + #expect(Packet04Check.problems(good, folder: folder).contains { $0.hasPrefix("file keyframes/k00003.jpg is 7 bytes") }) + } + + @Test func imuRowsPairOnlyCloseGyroSamplesAndReportTheMeasuredRate() throws { + let accel = (0..<100).map { Packet04Streams.MotionRow(t: 10 + Double($0) * 0.01, value: SIMD3(0, -1, 0)) } + // A gyro sample 1 ms after each accelerometer sample, except the 50th's, so the nearest to + // that accelerometer sample is 9 ms away: it is left out, not given a far reading. + var gyro = accel.map { Packet04Streams.MotionRow(t: $0.t + 0.001, value: SIMD3(0.5, 0, 0)) } + gyro.remove(at: 50) + let out = try Packet04Streams.imuRaw(accelerometer: accel, gyroscope: gyro) + let lines = String(decoding: out.csv, as: UTF8.self).split(separator: "\n") + #expect(lines.first == "t,ax,ay,az,gx,gy,gz") + #expect(lines.count == 1 + 99) + #expect(!lines.contains { $0.hasPrefix("10.5,") }) + #expect(abs(out.rateHz - 100) < 0.01) + #expect(out.pairingNote["imuRawRowsKept"] == "99 of 100") + } + + @Test func poseRowsJoinEachFrameToItsOwnIntrinsics() { + func trajectory(_ t: Double, x: Double) -> [Double] { [t, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, x, 1.5, -2, 1] } + let rows = Packet04Streams.poseRows( + trajectory: [trajectory(1, x: 0.1), trajectory(2, x: 0.2), trajectory(3, x: 0.3)], + // Frame 2 wrote no intrinsics row: it is left out, not given frame 1's or frame 3's. + intrinsics: [[1, 1440, 1441, 960, 720, 1920, 1440], [3, 1500, 1501, 961, 721, 1920, 1440]], + tracking: { state, _ in state == 0 ? .normal : .notAvailable }) + #expect(rows.map(\.t) == [1, 3]) + #expect(rows[1].intrinsics == SIMD4(1500, 1501, 961, 721)) + #expect(rows[1].cameraToWorld.columns.3 == SIMD4(0.3, 1.5, -2, 1)) + #expect(rows[0].tracking == .normal) + } + + @Test func aSlowIMUIsRefusedNotRelabelled() { + let accel = (0..<20).map { Packet04Streams.MotionRow(t: Double($0) * 0.05, value: SIMD3(0, -1, 0)) } + #expect(throws: Packet04Error.streamTooSlow(stream: "imuRaw", measuredHz: 1 / 0.05, minimumHz: 50)) { + _ = try Packet04Streams.imuRaw(accelerometer: accel, gyroscope: accel) + } + } + + @Test func gzipStreamsDecompressWithTheSystemGzip() throws { + let text = Data((0..<2000).map { "\($0),0.5,-1\n" }.joined().utf8) + let url = FileManager.default.temporaryDirectory.appending(path: "p04-\(UUID().uuidString).csv.gz") + defer { try? FileManager.default.removeItem(at: url) } + try Gzip.compress(text).write(to: url) + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/gzip") + process.arguments = ["-dc", url.path] + let pipe = Pipe() + process.standardOutput = pipe + try process.run() + let out = pipe.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + #expect(process.terminationStatus == 0) + #expect(out == text) + } + + @Test func idAndPathPatternsMatchTheServer() { + #expect(Packet04.isStorageID(UUID().uuidString)) + #expect(!Packet04.isStorageID("short")) + #expect(!Packet04.isStorageID("has.dot-in-it")) + #expect(Packet04.isSegmentID("k00001") && Packet04.isSegmentID("meter_close")) + #expect(!Packet04.isSegmentID("-lead") && !Packet04.isSegmentID("a.b")) + #expect(Packet04.isRelPath("keyframes/k00001.depth.f32") && Packet04.isRelPath("streams/imu_raw.csv.gz")) + #expect(!Packet04.isRelPath("a/../b") && !Packet04.isRelPath("a//b") && !Packet04.isRelPath("foo..bar") && !Packet04.isRelPath(".hidden")) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift index 0904fda17..120801e37 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift @@ -19,7 +19,7 @@ enum PacketSchema { /// A JPEG of `width` x `height` with a colour pattern, written by ImageIO. `orientation` sets the /// EXIF orientation tag. -func makeJPEG(width: Int, height: Int, at url: URL, orientation: Int? = nil) throws { +func makeJPEG(width: Int, height: Int, at url: URL, orientation: Int? = nil, shift: Int = 0) throws { let space = CGColorSpaceCreateDeviceRGB() let context = try #require(CGContext( data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 4 * width, space: space, @@ -28,7 +28,7 @@ func makeJPEG(width: Int, height: Int, at url: URL, orientation: Int? = nil) thr for y in 0.. Date: Thu, 1 Oct 2026 00:52:32 -0500 Subject: [PATCH 02/16] ios: a capture result counts only once it carries the run's answer The capture API can say a run is finished before its answer is readable. It answers a result read 200 with the capture's status and a null outcome until the run writes its answer, and 404 result_not_ready when the run reads as done before the answer is stored. The uploader took the first as the final result whenever the status already looked terminal, and failed the capture for good on the second. A result read is now final only when it has an outcome, or when its status is failed or expired, which can end a capture with no answer. A not-ready read is retried with the uploader's usual backoff, at most maxNotReadyResults times (10, about three minutes), and then the upload ends as failed(result, result_not_ready). No measurement sets that bound; it only keeps an answer that never appears from holding the upload open. A 200 that is not JSON now fails as result_unreadable instead of being kept as the answer. The loopback API gains faults for each case, and five tests cover them. The two not-ready tests failed against the carried #123 code before this change. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../HouseScanKit/Upload/CaptureAPI.swift | 4 + .../HouseScanKit/Upload/CaptureUploader.swift | 44 +++++++++- .../CaptureUploaderTests.swift | 88 ++++++++++++++++++- .../LoopbackCaptureAPI.swift | 39 ++++++-- 4 files changed, 162 insertions(+), 13 deletions(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift index 1bc42aa06..930940e24 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift @@ -148,6 +148,10 @@ public enum CaptureAPI { /// Statuses after which the capture changes no more. public static let terminalStatuses: Set = ["complete", "manual_review", "failed", "expired", "needs_views"] + + /// Terminal statuses that can end a capture with no answer, so a result read carrying one and + /// a null outcome is final. The others come with the run's outcome. + public static let statusesWithoutOutcome: Set = ["failed", "expired"] } /// An HTTP answer: status, body and the headers the uploader reads. diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift index 1f586b522..853aff4d2 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -23,6 +23,11 @@ public actor CaptureUploader { public var maxRefusedPuts = 6 /// `retry_finalize` answers followed before the upload stops. public var maxFinalizeRetries = 3 + /// Result reads that find the run's answer not readable yet before the upload stops. Each + /// waits the retry delay first, so the default allows about three minutes. No measurement + /// of how long a finished run's answer takes to become readable sets this number; it only + /// keeps an answer that never appears from holding the upload open. + public var maxNotReadyResults = 10 public init() {} @@ -57,6 +62,7 @@ public actor CaptureUploader { private var failures = 0 private var retryingAt: Date? private var digestRetried: Set = [] + private var notReadyReads = 0 /// Set when a state change could not be saved: the loop stops rather than send a request the /// saved state doesn't know about. private var saveFailed = false @@ -405,19 +411,49 @@ public actor CaptureUploader { guard attempt == state.attemptID else { return } guard (200..<300).contains(reply.status) else { if try captureGone(reply) { return } + // The run reads as done before its answer is stored. + if reply.status == 404, CaptureAPI.errorCodes(reply.body).contains("result_not_ready") { try notReady(reply) } throw Refused(step: "result", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) } - struct RunOnly: Decodable { var runId: String?; var status: String? } - let run = try? JSONDecoder().decode(RunOnly.self, from: reply.body) - if let runID = run?.runId, runID != state.finalized?.runID { + guard let head = try? JSONDecoder().decode(ResultHead.self, from: reply.body) else { + throw Refused(step: "result", status: reply.status, codes: ["result_unreadable"]) + } + if let runID = head.runId, runID != state.finalized?.runID { throw Refused(step: "result", status: reply.status, codes: ["result_for_another_run"]) } + let status = head.status ?? state.backendStatus ?? "unknown" + // Until the run has written its answer the API serves the capture's status with a null + // outcome, and that status can already be one the capture ends in. + if head.outcome == nil, !CaptureAPI.statusesWithoutOutcome.contains(status) { try notReady(reply) } state.result = reply.body - state.end = .finished(status: run?.status ?? state.backendStatus ?? "unknown") + state.end = .finished(status: status) mark("result") persist() } + /// The parts of a result read that say whether it is this run's answer. The rest is decoded + /// later from the saved bytes. + private struct ResultHead: Decodable { + var runId: String? + var status: String? + var outcome: Present? + } + + /// Any JSON value. Decoding keeps only that the key held something other than null. + private struct Present: Decodable { + init(from decoder: any Decoder) throws {} + } + + /// Always throws: a retry after the usual wait, or the end of the upload once + /// `maxNotReadyResults` reads have found no answer. + private func notReady(_ reply: HTTPReply) throws -> Never { + notReadyReads += 1 + guard notReadyReads <= policy.maxNotReadyResults else { + throw Refused(step: "result", status: reply.status, codes: ["result_not_ready"]) + } + throw Transient(step: "result", detail: "result not ready (\(notReadyReads))", retryAfter: reply.retryAfter) + } + // MARK: Plumbing private var captureID: String { state.captureID ?? "" } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift index d999ff85e..9075309f1 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift @@ -15,7 +15,7 @@ import Testing init( packetID: String = UUID().uuidString, appVersion: String = "test (1)", server: LoopbackCaptureAPI? = nil, - http: any CaptureHTTP = URLSessionCaptureHTTP.ephemeral(timeout: 10) + http: any CaptureHTTP = URLSessionCaptureHTTP.ephemeral(timeout: 10), policy: CaptureUploader.Policy = Rig.fast ) throws { self.http = http self.server = try server ?? LoopbackCaptureAPI() @@ -24,7 +24,7 @@ import Testing uploader = try CaptureUploader.start( folder: capture.folder, base: self.server.base, http: http, create: .init(packetId: packetID, tier: .arkit, device: .init(model: "iPhone15,4", systemVersion: "26.0", appVersion: appVersion)), - consentedAt: Date(), policy: Self.fast, sleep: { _ in try await Task.sleep(for: .milliseconds(10)) }) + consentedAt: Date(), policy: policy, sleep: { _ in try await Task.sleep(for: .milliseconds(10)) }) } static var fast: CaptureUploader.Policy { @@ -229,6 +229,90 @@ import Testing #expect(await rig.uploader.snapshot.finalizeRetries == 1) } + /// The run is done but its answer is not readable yet (404 `result_not_ready`): the uploader + /// reads again after a wait instead of failing the capture. + @Test func aResultNotReadableYetIsReadAgain() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.notReadyResults = 2 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "manual_review")) + #expect(try Self.outcomeKind(done.result) == .manualReview) + #expect(rig.server.requests("GET captures/result").count == 3) + } + + /// The API answers a result read with the capture's status and a null outcome until the run + /// has written its answer. A terminal status with that body is not the answer. + @Test func aResultWithoutItsOutcomeIsReadAgain() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.outcomelessResults = 2 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "manual_review")) + #expect(try Self.outcomeKind(done.result) == .manualReview) + #expect(rig.server.requests("GET captures/result").count == 3) + } + + /// An answer that never becomes readable ends the upload visibly after the policy's reads, + /// rather than waiting forever. + @Test func aResultThatNeverArrivesEndsTheUpload() async throws { + var policy = Rig.fast + policy.maxNotReadyResults = 3 + let rig = try Rig(policy: policy) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.outcomelessResults = 100 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .failed(step: "result", codes: ["result_not_ready"], status: 200)) + #expect(done.result == nil) + #expect(rig.server.requests("GET captures/result").count == 4) + } + + /// A 200 whose body is not a result is refused by name instead of being kept as the answer. + @Test func anUnreadableResultStopsTheUpload() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.unreadableResults = 1 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .failed(step: "result", codes: ["result_unreadable"], status: 200)) + #expect(done.result == nil) + #expect(rig.server.requests("GET captures/result").count == 1) + } + + /// `failed` ends a capture without an answer, so its outcomeless result is final at once. + @Test func aFailedCaptureEndsWithoutWaitingForAnOutcome() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.endStatus = "failed" } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "failed")) + #expect(try Self.outcomeKind(done.result) == nil) + #expect(rig.server.requests("GET captures/result").count == 1) + } + + static func outcomeKind(_ body: Data?) throws -> CaptureResult.OutcomeKind? { + try JSONDecoder().decode(CaptureResult.Response.self, from: #require(body)).outcome?.kind + } + /// Storage that keeps refusing the sealed bytes' digest stops the upload after one retry /// instead of cycling through register and PUT. @Test func aDigestRefusedTwiceStopsTheUpload() async throws { diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift index d4c9166e1..02abc05be 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift @@ -59,6 +59,17 @@ final class LoopbackCaptureAPI: Sendable { var loseNextFinalize = false /// The homeowner message the result carries. var resultMessage = "The loopback receiver's test result." + /// Answer the next N result reads 404 `result_not_ready`: the run is done but its answer + /// is not readable yet. + var notReadyResults = 0 + /// Answer the next N result reads 200 with the capture's status and run but a null + /// outcome, the body the API serves before the run has written its answer. + var outcomelessResults = 0 + /// Answer the next N result reads 200 with a body that is not JSON. + var unreadableResults = 0 + /// The status a capture ends in once every listed file is committed. Only `manual_review` + /// ends with an outcome; `failed` ends with a `failed` event and none. + var endStatus = "manual_review" /// Answer the next commit with this split instead of the truth. var commitOverride: ((committed: [String], notFound: [String], mismatch: [String]))? /// Routes whose answers wait until `release` is called. @@ -180,7 +191,17 @@ final class LoopbackCaptureAPI: Sendable { case ("POST", "finalize"): return finalize(r, &capture, &s) case ("GET", "events"): return events(r, &capture) case ("GET", "result"): - let outcome: Any = capture.status == "manual_review" + if s.notReadyResults > 0 { + s.notReadyResults -= 1 + return error(404, "result_not_ready") + } + if s.unreadableResults > 0 { + s.unreadableResults -= 1 + return (200, Data("not a result".utf8)) + } + let withheld = s.outcomelessResults > 0 + if withheld { s.outcomelessResults -= 1 } + let outcome: Any = capture.status == "manual_review" && !withheld ? ["kind": "manual_review", "profile": "C", "message": s.resultMessage, "viewsNeeded": [], "reasons": []] as [String: Any] : NSNull() return (200, json(["runId": capture.runID ?? "", "status": capture.status, "viewsNeeded": [], "memberActions": [], "outcome": outcome])) @@ -282,7 +303,7 @@ final class LoopbackCaptureAPI: Sendable { } c.committed.formUnion(committed) if !committed.isEmpty { c.events.append(("files_committed", ["committed": String(c.committed.count)])) } - startIfComplete(&c) + startIfComplete(&c, endStatus: s.endStatus) return (200, json(["committed": committed, "notFound": notFound, "mismatch": mismatch])) } @@ -308,20 +329,24 @@ final class LoopbackCaptureAPI: Sendable { c.finalizeLost = true c.events.append(("failed", ["code": "storage_unavailable", "next": "retry_finalize"])) } - startIfComplete(&c) + startIfComplete(&c, endStatus: s.endStatus) } else if c.finalizeLost { c.finalizeLost = false - startIfComplete(&c) + startIfComplete(&c, endStatus: s.endStatus) } let missing = c.listed.filter { !c.committed.contains($0) } return (202, json(["status": missing.isEmpty ? "processing" : "awaiting_files", "missing": missing, "runId": c.runID!, "etaS": 1])) } - private func startIfComplete(_ c: inout Capture) { + private func startIfComplete(_ c: inout Capture, endStatus: String) { guard c.packetSHA != nil, !c.finalizeLost, c.status == "awaiting_files", c.listed.allSatisfy(c.committed.contains) else { return } - c.status = "manual_review" + c.status = endStatus c.events.append(("stage", ["stage": "ingest", "status": "ok", "runId": c.runID!])) - c.events.append(("verdict_ready", ["kind": "manual_review", "runId": c.runID!])) + if endStatus == "manual_review" { + c.events.append(("verdict_ready", ["kind": "manual_review", "runId": c.runID!])) + } else { + c.events.append(("failed", ["code": "packet_invalid", "runId": c.runID!])) + } } private func events(_ r: Request, _ c: inout Capture) -> (Int, Data) { From f96e954330a7c68beadfaa5debbd0e1189e03e16 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Thu, 1 Oct 2026 01:18:06 -0500 Subject: [PATCH 03/16] ios tests: name the loopback's failed-run code as the loopback's own The failed event the loopback API emits for a capture that ends `failed` carried a made-up code that read like a real server code. It is now `loopback_run_failed`, so nobody mistakes it for part of the capture API. The uploader ignores this code; only the status matters to the test. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Tests/HouseScanKitTests/LoopbackCaptureAPI.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift index 02abc05be..922216568 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift @@ -345,7 +345,7 @@ final class LoopbackCaptureAPI: Sendable { if endStatus == "manual_review" { c.events.append(("verdict_ready", ["kind": "manual_review", "runId": c.runID!])) } else { - c.events.append(("failed", ["code": "packet_invalid", "runId": c.runID!])) + c.events.append(("failed", ["code": "loopback_run_failed", "runId": c.runID!])) } } From 63b32956b2e85b02886f49cabad3644d529f5316 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Thu, 1 Oct 2026 14:52:09 -0500 Subject: [PATCH 04/16] ios: a no after a yes stops the capture upload; staged photos and unreadable results are handled Three defects review found in the capture library, each with a test that failed before this change: - A no after a yes kept sending. `answerConsent(false)` changed the recorded answer but left the uploader running, so photos kept afterwards were uploaded and the packet was finalized. The uploader is now abandoned as "consent withdrawn", and a second yes in the same scan does not restart it. What the server already received stays there. - A reset left staged photo copies on disk. `kept` copies each JPEG into `Captures/staging` and deleted it only inside the queued work, which a reset skips. The deletion now runs whether or not the work runs. - A result whose outcome was present but malformed finished the upload with an answer nothing could read. The result read now decodes the whole `CaptureResult.Response`, and a body that doesn't decode ends the upload as `result_unreadable`. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Upload/CaptureSessionCoordinator.swift | 26 +++++++-- .../HouseScanKit/Upload/CaptureUploader.swift | 18 +++--- .../CaptureSessionCoordinatorTests.swift | 55 +++++++++++++++++++ .../CaptureUploaderTests.swift | 16 ++++++ .../LoopbackCaptureAPI.swift | 8 +++ 5 files changed, 107 insertions(+), 16 deletions(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift index e0a94a0f7..0a96fac57 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift @@ -179,6 +179,8 @@ public final class CaptureSessionCoordinator { var chain: Task? var sealing = false var ended = false + /// The homeowner said no after a yes: this capture is never sent again. + var consentWithdrawn = false /// packet.json as frozen, once the scan was sent for placement. var frozen: Data? /// The server's result for this session's capture, decoded and bound to it. @@ -191,10 +193,12 @@ public final class CaptureSessionCoordinator { } /// Runs `work` after everything queued before it, unless the session has ended by then. - func enqueue(_ work: @escaping @MainActor (Session) async -> Void) { + /// `cleanup` runs either way, after `work` when it ran. + func enqueue(_ work: @escaping @MainActor (Session) async -> Void, cleanup: (@MainActor @Sendable () -> Void)? = nil) { let previous = chain chain = Task { await previous?.value + defer { cleanup?() } guard !self.ended else { return } await work(self) } @@ -233,10 +237,19 @@ public final class CaptureSessionCoordinator { environment?.sends == true && consent == nil && session != nil } + /// A no after a yes stops this capture's upload for good: nothing kept later is sent, and a + /// second yes in the same scan does not start it again. What the server already has stays there. public func answerConsent(_ yes: Bool, at time: Date = Date()) { consent = yes consentedAt = yes ? time : nil - if yes, let session { startUploading(session) } + guard let session else { return } + if yes { + startUploading(session) + } else if let uploader = session.uploader { + session.consentWithdrawn = true + session.uploader = nil + Task { await uploader.abandon("consent withdrawn") } + } } // MARK: Scan events @@ -272,8 +285,8 @@ public final class CaptureSessionCoordinator { environment.log("capture packet: a kept photo could not be copied: \(error)") return } - session.enqueue { session in - defer { try? FileManager.default.removeItem(at: staged) } + // The copy is deleted even when a reset ends the session before this work runs. + session.enqueue({ session in do { let producer = try session.producer(first: (photo.t, photo.cameraImageSize), info: environment.sessionInfo) let data = try await Task.detached(priority: .utility) { try Data(contentsOf: staged) }.value @@ -289,7 +302,7 @@ public final class CaptureSessionCoordinator { } catch { environment.log("capture packet: a kept photo was not sealed: \(error)") } - } + }, cleanup: { try? FileManager.default.removeItem(at: staged) }) } /// The homeowner marked the meter: its frame becomes a keyframe and the tap goes on it. @@ -381,7 +394,8 @@ public final class CaptureSessionCoordinator { /// Opens the capture on the server and queues everything sealed so far, then the frozen /// packet if the scan was already sent. Photos kept later go up as they are sealed. private func startUploading(_ session: Session) { - guard let environment, environment.sends, consent == true, let consentedAt, session.uploader == nil, !session.ended else { return } + guard let environment, environment.sends, consent == true, let consentedAt, session.uploader == nil, !session.ended, + !session.consentWithdrawn else { return } do { let uploader = try CaptureUploader.start( folder: session.folder, base: environment.endpoint, http: environment.http, diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift index 853aff4d2..840f5b87e 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -415,7 +415,12 @@ public actor CaptureUploader { if reply.status == 404, CaptureAPI.errorCodes(reply.body).contains("result_not_ready") { try notReady(reply) } throw Refused(step: "result", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) } - guard let head = try? JSONDecoder().decode(ResultHead.self, from: reply.body) else { + // The whole body must decode as the result the app reads, or a finished upload could hold + // an answer nothing can show. + let decoder = JSONDecoder() + guard let response = try? decoder.decode(CaptureResult.Response.self, from: reply.body), + let head = try? decoder.decode(ResultHead.self, from: reply.body) + else { throw Refused(step: "result", status: reply.status, codes: ["result_unreadable"]) } if let runID = head.runId, runID != state.finalized?.runID { @@ -424,24 +429,17 @@ public actor CaptureUploader { let status = head.status ?? state.backendStatus ?? "unknown" // Until the run has written its answer the API serves the capture's status with a null // outcome, and that status can already be one the capture ends in. - if head.outcome == nil, !CaptureAPI.statusesWithoutOutcome.contains(status) { try notReady(reply) } + if response.outcome == nil, !CaptureAPI.statusesWithoutOutcome.contains(status) { try notReady(reply) } state.result = reply.body state.end = .finished(status: status) mark("result") persist() } - /// The parts of a result read that say whether it is this run's answer. The rest is decoded - /// later from the saved bytes. + /// The run and status as the server wrote them; `CaptureResult.Status` keeps no raw text. private struct ResultHead: Decodable { var runId: String? var status: String? - var outcome: Present? - } - - /// Any JSON value. Decoding keeps only that the key held something other than null. - private struct Present: Decodable { - init(from decoder: any Decoder) throws {} } /// Always throws: a retry after the usual wait, or the end of the upload once diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift index 98ce33e0b..321327cff 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift @@ -376,6 +376,61 @@ struct NativeCaptureFixture: Sendable { #expect(Set(creates) == [first.packetID, second.packetID]) } + /// A reset that ends the session before its kept photos are sealed still deletes their staged + /// copies, so no copy of a home photo outlives the packet it was kept for. + @Test func aResetDeletesPhotosStagedForTheEndedSession() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures, sends: false)) + let staging = captures.appending(path: "staging") + func staged() -> [String] { (try? FileManager.default.contentsOfDirectory(atPath: staging.path)) ?? [] } + + coordinator.begin(recording: fixture.recording) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + coordinator.kept(try fixture.photo(at: fixture.start + 3)) + #expect(staged().count == 2) + // No suspension between keeping and the reset, so the queued work finds the session ended. + coordinator.newWorld("start over", recording: fixture.recording, newScan: true) + coordinator.kept(try fixture.photo(at: fixture.start + 4)) + await coordinator.settle() + for _ in 0..<500 where !staged().isEmpty { try await Task.sleep(for: .milliseconds(10)) } + #expect(staged().isEmpty) + } + + /// A no after a yes stops sending: the upload is abandoned, nothing kept later goes up, the + /// packet is never finalized, and a second yes in the same scan does not start it again. + @Test func aNoAfterAYesStopsSending() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures)) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + await coordinator.settle() + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + await uploader.settled() + let putsBefore = server.requests("PUT upload").count + #expect(putsBefore == 1) + + coordinator.answerConsent(false) + coordinator.kept(try fixture.photo(at: fixture.start + 4)) + coordinator.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.captureEnded(acceptedCloseUpAt: nil) + await coordinator.settle() + await uploader.settled() + + #expect(await uploader.snapshot.end == .abandoned("consent withdrawn")) + #expect(coordinator.session === session && session.uploader == nil) + #expect(server.requests("PUT upload").count == putsBefore) + #expect(server.requests("POST captures").count == 1) + #expect(server.requests("POST captures/finalize").isEmpty) + } + /// Photos kept after the scan was sent don't change the frozen packet or start more uploads. @Test func photosAfterTheScanWasSentAreNotAdded() async throws { defer { try? FileManager.default.removeItem(at: root) } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift index 9075309f1..8264d0e81 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift @@ -294,6 +294,22 @@ import Testing #expect(rig.server.requests("GET captures/result").count == 1) } + /// An outcome missing fields the app needs is refused by name instead of finishing the upload + /// with an answer nothing can show. + @Test func aMalformedOutcomeStopsTheUpload() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.malformedOutcomeResults = 1 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .failed(step: "result", codes: ["result_unreadable"], status: 200)) + #expect(done.result == nil) + #expect(rig.server.requests("GET captures/result").count == 1) + } + /// `failed` ends a capture without an answer, so its outcomeless result is final at once. @Test func aFailedCaptureEndsWithoutWaitingForAnOutcome() async throws { let rig = try Rig() diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift index 922216568..b92ffaeb5 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift @@ -67,6 +67,9 @@ final class LoopbackCaptureAPI: Sendable { var outcomelessResults = 0 /// Answer the next N result reads 200 with a body that is not JSON. var unreadableResults = 0 + /// Answer the next N result reads 200 with an outcome that has a kind but none of the + /// other fields an outcome must carry. + var malformedOutcomeResults = 0 /// The status a capture ends in once every listed file is committed. Only `manual_review` /// ends with an outcome; `failed` ends with a `failed` event and none. var endStatus = "manual_review" @@ -199,6 +202,11 @@ final class LoopbackCaptureAPI: Sendable { s.unreadableResults -= 1 return (200, Data("not a result".utf8)) } + if s.malformedOutcomeResults > 0 { + s.malformedOutcomeResults -= 1 + return (200, json(["runId": capture.runID ?? "", "status": capture.status, "viewsNeeded": [], "memberActions": [], + "outcome": ["kind": "manual_review"]])) + } let withheld = s.outcomelessResults > 0 if withheld { s.outcomelessResults -= 1 } let outcome: Any = capture.status == "manual_review" && !withheld From 7ec7770ffda2251b00450a66f3ceea83940d3d4c Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Thu, 1 Oct 2026 14:58:03 -0500 Subject: [PATCH 05/16] ios: a withdrawn yes is saved before answerConsent returns, so a relaunch can't resume the capture Review of 63b32956 found that withdrawal lived only in memory until the uploader's asynchronous abandon saved its end. If the app quit in that window with a sealed capture, a relaunch's resumeSealedCaptures would find the saved yes and send the remaining files. answerConsent(false) now writes a `consent-withdrawn` marker into the capture's folder before it returns, and resumeSealedCaptures skips any folder that has one. The new test seals a capture while its finalize is held, withdraws, and relaunches in the same main-actor turn. Before this change that relaunch resumed the capture. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Upload/CaptureSessionCoordinator.swift | 13 +++++++- .../CaptureSessionCoordinatorTests.swift | 32 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift index 0a96fac57..dd54a562e 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift @@ -226,6 +226,9 @@ public final class CaptureSessionCoordinator { public var onResult: (@MainActor (CaptureResult.Record) -> Void)? /// The packet's ARKit epoch: one packet is one world. public static let epoch = "e1" + /// Written into a capture's folder when the homeowner takes back their yes. A relaunch never + /// resumes a folder that has it. + public static let withdrawnFileName = "consent-withdrawn" public init(environment: Environment?) { self.environment = environment @@ -248,6 +251,13 @@ public final class CaptureSessionCoordinator { } else if let uploader = session.uploader { session.consentWithdrawn = true session.uploader = nil + // Abandoning is asynchronous, so the marker is what keeps a relaunch from resuming + // this capture if the app quits before the uploader saves its end. + do { + try Data().write(to: session.folder.appending(path: Self.withdrawnFileName), options: .atomic) + } catch { + environment?.log("capture upload: the withdrawal could not be saved: \(error)") + } Task { await uploader.abandon("consent withdrawn") } } } @@ -371,7 +381,8 @@ public final class CaptureSessionCoordinator { else { return [] } var resumed: [CaptureUploader] = [] for folder in folders { - guard let saved = try? CaptureUploadState.load(from: CaptureUploader.stateURL(in: folder)), saved.end == nil, saved.packet != nil, + guard !FileManager.default.fileExists(atPath: folder.appending(path: Self.withdrawnFileName).path), + let saved = try? CaptureUploadState.load(from: CaptureUploader.stateURL(in: folder)), saved.end == nil, saved.packet != nil, let uploader = try? CaptureUploader.resume(folder: folder, base: environment.endpoint, http: environment.http, policy: environment.policy) else { continue } resumed.append(uploader) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift index 321327cff..186bd82ff 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift @@ -431,6 +431,38 @@ struct NativeCaptureFixture: Sendable { #expect(server.requests("POST captures/finalize").isEmpty) } + /// A withdrawal is saved before `answerConsent(false)` returns: a relaunch straight after it + /// does not resume the sealed capture, even though the uploader has not saved its end yet. + @Test func aWithdrawnCaptureIsNotResumedAfterARelaunch() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/finalize"] } + let environment = NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + for i in 0..<3 { coordinator.kept(try fixture.photo(at: fixture.start + 3 + Double(i) * 1.5)) } + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + let stateURL = CaptureUploader.stateURL(in: session.folder) + for _ in 0..<500 where (try? CaptureUploadState.load(from: stateURL))?.packet == nil { try await Task.sleep(for: .milliseconds(10)) } + #expect(try CaptureUploadState.load(from: stateURL).packet != nil) + + coordinator.answerConsent(false) + // The same main-actor turn: the abandon task has not run yet. + #expect(try CaptureUploadState.load(from: stateURL).end == nil) + #expect(CaptureSessionCoordinator(environment: environment).resumeSealedCaptures().isEmpty) + + server.release("POST captures/finalize") + server.state.withLock { $0.held = [] } + await uploader.settled() + #expect(await uploader.snapshot.end == .abandoned("consent withdrawn")) + } + /// Photos kept after the scan was sent don't change the frozen packet or start more uploads. @Test func photosAfterTheScanWasSentAreNotAdded() async throws { defer { try? FileManager.default.removeItem(at: root) } From bc7d8686bb31ad54a31b48135eacbd504540df49 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Thu, 1 Oct 2026 15:05:06 -0500 Subject: [PATCH 06/16] ios: the uploader itself refuses to resume a withdrawn capture, and a failed marker write removes the saved upload Review of 7ec7770f found two ways a withdrawn capture could still be sent: - Only the coordinator checked the withdrawal marker. Calling CaptureUploader.resume directly would resume the capture before the asynchronous abandon had saved its end. The check now lives in CaptureUploader.resume, which the coordinator's relaunch path also uses, and the marker's name moves to CaptureUploadState. - If the marker couldn't be written, for example on a full disk, withdrawal was only logged. The coordinator now removes the saved upload state instead; resume needs that file, so nothing is resumed. The relaunch test now also calls CaptureUploader.resume directly, and a new test blocks the marker's path so the write fails. Both failed against 7ec7770f. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Upload/CaptureSessionCoordinator.swift | 14 +++++------ .../Upload/CaptureUploadState.swift | 5 ++++ .../HouseScanKit/Upload/CaptureUploader.swift | 9 ++++--- .../CaptureSessionCoordinatorTests.swift | 25 +++++++++++++++++++ 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift index dd54a562e..902497170 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift @@ -226,9 +226,6 @@ public final class CaptureSessionCoordinator { public var onResult: (@MainActor (CaptureResult.Record) -> Void)? /// The packet's ARKit epoch: one packet is one world. public static let epoch = "e1" - /// Written into a capture's folder when the homeowner takes back their yes. A relaunch never - /// resumes a folder that has it. - public static let withdrawnFileName = "consent-withdrawn" public init(environment: Environment?) { self.environment = environment @@ -252,11 +249,13 @@ public final class CaptureSessionCoordinator { session.consentWithdrawn = true session.uploader = nil // Abandoning is asynchronous, so the marker is what keeps a relaunch from resuming - // this capture if the app quits before the uploader saves its end. + // this capture if the app quits before the uploader saves its end. If the marker can't + // be written, the saved upload state goes instead: without it nothing resumes. do { - try Data().write(to: session.folder.appending(path: Self.withdrawnFileName), options: .atomic) + try Data().write(to: CaptureUploadState.withdrawnURL(in: session.folder), options: .atomic) } catch { - environment?.log("capture upload: the withdrawal could not be saved: \(error)") + environment?.log("capture upload: the withdrawal marker could not be written (\(error)); removing the saved upload") + try? FileManager.default.removeItem(at: CaptureUploader.stateURL(in: session.folder)) } Task { await uploader.abandon("consent withdrawn") } } @@ -381,8 +380,7 @@ public final class CaptureSessionCoordinator { else { return [] } var resumed: [CaptureUploader] = [] for folder in folders { - guard !FileManager.default.fileExists(atPath: folder.appending(path: Self.withdrawnFileName).path), - let saved = try? CaptureUploadState.load(from: CaptureUploader.stateURL(in: folder)), saved.end == nil, saved.packet != nil, + guard let saved = try? CaptureUploadState.load(from: CaptureUploader.stateURL(in: folder)), saved.end == nil, saved.packet != nil, let uploader = try? CaptureUploader.resume(folder: folder, base: environment.endpoint, http: environment.http, policy: environment.policy) else { continue } resumed.append(uploader) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift index da01b1c77..de0784dc0 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift @@ -5,6 +5,11 @@ import Foundation /// and finalize) are saved as the exact bytes first sent, never re-encoded. public struct CaptureUploadState: Codable, Sendable, Equatable { public static let fileName = "capture-upload.json" + /// Written into a capture's folder when the homeowner takes back their yes. `CaptureUploader.resume` + /// never resumes a folder that has it. + public static let withdrawnFileName = "consent-withdrawn" + + public static func withdrawnURL(in folder: URL) -> URL { folder.appending(path: withdrawnFileName) } public enum FilePhase: Codable, Sendable, Equatable { /// Sealed on the phone, not yet registered (or its URL must be fetched again). diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift index 840f5b87e..001a19e46 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -102,8 +102,9 @@ public actor CaptureUploader { return CaptureUploader(folder: folder, base: base, http: http, state: state, policy: policy, now: now, sleep: sleep) } - /// The capture saved in `folder`, as a relaunch finds it; nil when there is none, when it was - /// created on another API than `base`, or when it holds no yes for `base`. The resumed upload + /// The capture saved in `folder`, as a relaunch finds it; nil when there is none, when the + /// homeowner withdrew their yes, when it was created on another API than `base`, or when it + /// holds no yes for `base`. The resumed upload /// gets a new attempt id, so nothing from the previous process can land in it. public static func resume( folder: URL, base: URL, http: any CaptureHTTP, policy: Policy = .init(), @@ -111,7 +112,9 @@ public actor CaptureUploader { sleep: @escaping @Sendable (Double) async throws -> Void = { try await Task.sleep(for: .seconds($0)) } ) throws -> CaptureUploader? { let url = stateURL(in: folder) - guard FileManager.default.fileExists(atPath: url.path) else { return nil } + guard FileManager.default.fileExists(atPath: url.path), + !FileManager.default.fileExists(atPath: CaptureUploadState.withdrawnURL(in: folder).path) + else { return nil } var state = try CaptureUploadState.load(from: url) guard state.destination == base.absoluteString, state.consent?.destination == base.absoluteString else { return nil } state.attemptID = UUID().uuidString diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift index 186bd82ff..4cdfc5f93 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift @@ -456,6 +456,7 @@ struct NativeCaptureFixture: Sendable { // The same main-actor turn: the abandon task has not run yet. #expect(try CaptureUploadState.load(from: stateURL).end == nil) #expect(CaptureSessionCoordinator(environment: environment).resumeSealedCaptures().isEmpty) + #expect(try CaptureUploader.resume(folder: session.folder, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) == nil) server.release("POST captures/finalize") server.state.withLock { $0.held = [] } @@ -463,6 +464,30 @@ struct NativeCaptureFixture: Sendable { #expect(await uploader.snapshot.end == .abandoned("consent withdrawn")) } + /// When the withdrawal marker can't be written, the saved upload is removed instead, so a + /// relaunch still finds nothing to resume. + @Test func aWithdrawalThatCannotBeMarkedRemovesTheSavedUpload() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let environment = NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + let stateURL = CaptureUploader.stateURL(in: session.folder) + #expect(FileManager.default.fileExists(atPath: stateURL.path)) + // A directory where the marker goes makes writing it fail. + try FileManager.default.createDirectory(at: CaptureUploadState.withdrawnURL(in: session.folder).appending(path: "blocked"), withIntermediateDirectories: true) + + coordinator.answerConsent(false) + #expect(!FileManager.default.fileExists(atPath: stateURL.path)) + + // Abandoning runs in its own task after this turn. + for _ in 0..<500 where await uploader.snapshot.end == nil { try await Task.sleep(for: .milliseconds(10)) } + #expect(await uploader.snapshot.end == .abandoned("consent withdrawn")) + } + /// Photos kept after the scan was sent don't change the frozen packet or start more uploads. @Test func photosAfterTheScanWasSentAreNotAdded() async throws { defer { try? FileManager.default.removeItem(at: root) } From 9d6665f9c0ba1883c3d86922bf7bb392a6ee9f6d Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Thu, 1 Oct 2026 15:26:11 -0500 Subject: [PATCH 07/16] ios: withdrawing consent goes through one lock that every upload save also takes After a no, a reply already in flight could still save the upload as active before the asynchronous abandon ran. When the withdrawal marker couldn't be written, that save recreated the state file a relaunch resumes, and in every case the loop went on to send the next request. CaptureUploader now owns a Mutex. withdrawConsent(), synchronous and nonisolated, sets it and, under the same lock, writes the consent-withdrawn marker or, if that fails, removes the saved state. persist() saves while holding the lock, and once withdrawn it saves the upload as abandoned with a new attempt id. Whichever runs first, no save after the withdrawal can leave a resumable capture. Every API request and every batch of PUTs checks the flag before it is sent, so nothing new goes out after withdrawConsent returns. A request already on the wire finishes, and its reply can only save the upload as abandoned. The coordinator calls withdrawConsent() before it schedules abandon. aReplyAfterWithdrawalCannotSaveAResumableCapture holds the register reply, withdraws, and keeps the main actor busy so the abandon task cannot run, then releases the reply. It runs with the marker written and with it blocked. Against bc7d8686, both cases saved an active upload and then uploaded the photo. It replaces the fallback test, which checked only that the file was deleted. The same-turn assertion that the saved end was still nil is gone, because a gated save may now record the abandon first. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Upload/CaptureSessionCoordinator.swift | 14 ++---- .../HouseScanKit/Upload/CaptureUploader.swift | 49 ++++++++++++++++++- .../CaptureSessionCoordinatorTests.swift | 39 ++++++++++----- 3 files changed, 81 insertions(+), 21 deletions(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift index 902497170..309ac0eef 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift @@ -248,16 +248,12 @@ public final class CaptureSessionCoordinator { } else if let uploader = session.uploader { session.consentWithdrawn = true session.uploader = nil - // Abandoning is asynchronous, so the marker is what keeps a relaunch from resuming - // this capture if the app quits before the uploader saves its end. If the marker can't - // be written, the saved upload state goes instead: without it nothing resumes. - do { - try Data().write(to: CaptureUploadState.withdrawnURL(in: session.folder), options: .atomic) - } catch { - environment?.log("capture upload: the withdrawal marker could not be written (\(error)); removing the saved upload") - try? FileManager.default.removeItem(at: CaptureUploader.stateURL(in: session.folder)) + // Recorded before returning, so a relaunch can't resume this capture even if the app + // quits before the abandon below runs. + if !uploader.withdrawConsent() { + environment?.log("capture upload: the withdrawal marker could not be written; the saved upload was removed instead") } - Task { await uploader.abandon("consent withdrawn") } + Task { await uploader.abandon(CaptureUploader.withdrawnReason) } } } diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift index 001a19e46..98286973e 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -1,4 +1,5 @@ import Foundation +import Synchronization /// Sends one capture to the capture API while it is being taken: create once, then register, PUT /// and commit each sealed file as it arrives, then finalize with the frozen packet (possibly @@ -44,6 +45,9 @@ public actor CaptureUploader { var retryAfter: Double? } + /// The homeowner took back their yes: nothing more is sent. + struct Withdrawn: Error {} + /// The server refused something a retry can't change. struct Refused: Error { var step: String @@ -63,6 +67,9 @@ public actor CaptureUploader { private var retryingAt: Date? private var digestRetried: Set = [] private var notReadyReads = 0 + /// True once the homeowner took back their yes. Every save happens while holding it, so no + /// save can land between the withdrawal and the abandoned state it leads to. + private nonisolated let withdrawal = Mutex(false) /// Set when a state change could not be saved: the loop stops rather than send a request the /// saved state doesn't know about. private var saveFailed = false @@ -166,6 +173,27 @@ public actor CaptureUploader { kick() } + public static let withdrawnReason = "consent withdrawn" + + /// The homeowner took back their yes. Before returning, this writes the marker `resume` + /// refuses, or removes the saved state if the marker can't be written, and from then on every + /// save records the upload as abandoned, so no reply still in flight can leave a capture a + /// relaunch would resume. Returns whether the marker was written. Call `abandon` afterwards to + /// stop the loop. + @discardableResult + public nonisolated func withdrawConsent() -> Bool { + withdrawal.withLock { withdrawn in + withdrawn = true + do { + try Data().write(to: CaptureUploadState.withdrawnURL(in: folder), options: .atomic) + return true + } catch { + try? FileManager.default.removeItem(at: Self.stateURL(in: folder)) + return false + } + } + } + /// Ends this upload for good: a world reset or starting over. Replies still in flight are /// dropped when they arrive. public func abandon(_ reason: String) { @@ -192,6 +220,8 @@ public actor CaptureUploader { private func drive() async { while !Task.isCancelled, state.end == nil, !saveFailed { + // A withdrawal sends nothing more; the save records the upload as abandoned. + if isWithdrawn { persist(); break } let attempt = state.attemptID do { let progressed = try await step() @@ -207,6 +237,9 @@ public actor CaptureUploader { log?("capture-upload retry step=\(error.step) in=\(String(format: "%.1f", delay))s detail=\(error.detail)") publish() do { try await sleep(delay) } catch { break } + } catch is Withdrawn { + persist() + break } catch let error as Refused { guard attempt == state.attemptID else { break } state.end = .failed(step: error.step, codes: error.codes, status: error.status) @@ -291,6 +324,8 @@ public actor CaptureUploader { var transient: Transient? var index = 0 while index < files.count { + // Each batch of PUTs is admitted only while the homeowner's yes stands. + if isWithdrawn { throw Withdrawn() } let chunk = files[index..).self) { group in @@ -484,6 +519,9 @@ public actor CaptureUploader { } private func call(_ step: String, _ request: URLRequest) async throws -> (HTTPReply, String) { + // Every API request is admitted only while the homeowner's yes stands. One already sent + // finishes, but its reply can only save the upload as abandoned. + if isWithdrawn { throw Withdrawn() } let attempt = state.attemptID let reply: HTTPReply do { @@ -514,9 +552,18 @@ public actor CaptureUploader { log?("capture-upload stage=\(name) t=\(String(format: "%.2f", since))s committed=\(state.committedCount)/\(state.files.count)") } + private nonisolated var isWithdrawn: Bool { withdrawal.withLock { $0 } } + private func persist() { do { - try state.save(to: Self.stateURL(in: folder)) + try withdrawal.withLock { withdrawn in + if withdrawn, state.end == nil { + state.end = .abandoned(Self.withdrawnReason) + // Replies still in flight belong to the old attempt and are dropped. + state.attemptID = UUID().uuidString + } + try state.save(to: Self.stateURL(in: folder)) + } } catch { saveFailed = true log?("capture-upload stopped: the upload state could not be saved (\(Self.describe(error)))") diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift index 4cdfc5f93..ba708610d 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift @@ -454,7 +454,6 @@ struct NativeCaptureFixture: Sendable { coordinator.answerConsent(false) // The same main-actor turn: the abandon task has not run yet. - #expect(try CaptureUploadState.load(from: stateURL).end == nil) #expect(CaptureSessionCoordinator(environment: environment).resumeSealedCaptures().isEmpty) #expect(try CaptureUploader.resume(folder: session.folder, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) == nil) @@ -464,28 +463,46 @@ struct NativeCaptureFixture: Sendable { #expect(await uploader.snapshot.end == .abandoned("consent withdrawn")) } - /// When the withdrawal marker can't be written, the saved upload is removed instead, so a - /// relaunch still finds nothing to resume. - @Test func aWithdrawalThatCannotBeMarkedRemovesTheSavedUpload() async throws { + /// A reply that lands after the homeowner withdrew, but before the uploader is abandoned, + /// can't save a capture a relaunch would resume, whether or not the withdrawal marker could + /// be written. The main actor is held from the withdrawal to the checks, so the abandon task + /// the withdrawal schedules can't run first; only the withdrawal itself can stop the save. + @Test(arguments: [false, true]) func aReplyAfterWithdrawalCannotSaveAResumableCapture(markerBlocked: Bool) async throws { defer { try? FileManager.default.removeItem(at: root) } let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/files"] } let environment = NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures) let coordinator = CaptureSessionCoordinator(environment: environment) coordinator.begin(recording: fixture.recording) coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) let session = try #require(coordinator.session) let uploader = try #require(session.uploader) let stateURL = CaptureUploader.stateURL(in: session.folder) - #expect(FileManager.default.fileExists(atPath: stateURL.path)) - // A directory where the marker goes makes writing it fail. - try FileManager.default.createDirectory(at: CaptureUploadState.withdrawnURL(in: session.folder).appending(path: "blocked"), withIntermediateDirectories: true) + for _ in 0..<500 where server.requests("POST captures/files").isEmpty { try await Task.sleep(for: .milliseconds(10)) } + #expect(server.requests("POST captures/files").count == 1) + if markerBlocked { + // A directory where the marker goes makes writing it fail. + try FileManager.default.createDirectory( + at: CaptureUploadState.withdrawnURL(in: session.folder).appending(path: "blocked"), withIntermediateDirectories: true) + } coordinator.answerConsent(false) - #expect(!FileManager.default.fileExists(atPath: stateURL.path)) + // No suspension from here to the checks. + let before = try? Data(contentsOf: stateURL) + #expect((before == nil) == markerBlocked) + server.release("POST captures/files") + // The uploader saves the register reply on its own executor. + let deadline = Date().addingTimeInterval(5) + while Date() < deadline, (try? Data(contentsOf: stateURL)) == before { usleep(10_000) } + let saved = try CaptureUploadState.load(from: stateURL) + #expect(saved.end == .abandoned("consent withdrawn")) + #expect(try CaptureUploader.resume(folder: session.folder, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) == nil) + #expect(server.requests("PUT upload").isEmpty) - // Abandoning runs in its own task after this turn. - for _ in 0..<500 where await uploader.snapshot.end == nil { try await Task.sleep(for: .milliseconds(10)) } - #expect(await uploader.snapshot.end == .abandoned("consent withdrawn")) + server.state.withLock { $0.held = [] } + await uploader.settled() + #expect(server.requests("PUT upload").isEmpty) } /// Photos kept after the scan was sent don't change the frozen packet or start more uploads. From 37e864030dfc8d61261af8f2019824aa0c42d0c0 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Thu, 1 Oct 2026 15:42:43 -0500 Subject: [PATCH 08/16] ios: withdrawing consent cancels the upload loop, and a late result can't end the upload as finished Review of the withdrawal lock (9d6665f9) found three remaining gaps, fixed here: - A request past its admission check, or a queued PUT child, could still start after withdrawConsent returned. withdrawConsent now cancels the running loop task synchronously. The task is kept in a Mutex so no actor hop is needed, and cancelling it cancels the requests and PUT children it hasn't finished. Each PUT child also checks the flag just before it starts, and kick() won't start a loop once consent is withdrawn. - A late result or refusal set .finished or .failed before saving, and the save kept it. Once consent is withdrawn, persist() now always saves the upload as abandoned. - withdrawConsent returns how the withdrawal was recorded: marked, saved state removed, or neither. The coordinator logs the last case instead of claiming the removal worked. A missing state file counts as removed. Two new tests failed against 9d6665f9. aResultAfterWithdrawalSavesTheUploadAsAbandoned holds a result, withdraws and releases it. withdrawalCancelsTheRequestInFlight holds a register request and shows that the uploader gives it up while the main actor stays busy. The reply-save witness now uses a transport whose replies outlive cancellation, as a background session's do, and it waits until the response is parked before releasing it. The relaunch test waits for the abandon task instead of the loop. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Upload/CaptureSessionCoordinator.swift | 6 +- .../HouseScanKit/Upload/CaptureUploader.swift | 52 +++++++--- .../CaptureSessionCoordinatorTests.swift | 94 +++++++++++++++++-- 3 files changed, 130 insertions(+), 22 deletions(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift index 309ac0eef..c8e04ba4a 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift @@ -250,8 +250,12 @@ public final class CaptureSessionCoordinator { session.uploader = nil // Recorded before returning, so a relaunch can't resume this capture even if the app // quits before the abandon below runs. - if !uploader.withdrawConsent() { + switch uploader.withdrawConsent() { + case .marked: break + case .savedStateRemoved: environment?.log("capture upload: the withdrawal marker could not be written; the saved upload was removed instead") + case .notRecorded(let error): + environment?.log("capture upload: the withdrawal could not be saved (\(error)); a relaunch could still find this capture until the upload is saved as abandoned") } Task { await uploader.abandon(CaptureUploader.withdrawnReason) } } diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift index 98286973e..da0da60ec 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -70,6 +70,8 @@ public actor CaptureUploader { /// True once the homeowner took back their yes. Every save happens while holding it, so no /// save can land between the withdrawal and the abandoned state it leads to. private nonisolated let withdrawal = Mutex(false) + /// The running loop, reachable without the actor so a withdrawal can cancel it at once. + private nonisolated let running = Mutex?>(nil) /// Set when a state change could not be saved: the loop stops rather than send a request the /// saved state doesn't know about. private var saveFailed = false @@ -175,23 +177,42 @@ public actor CaptureUploader { public static let withdrawnReason = "consent withdrawn" - /// The homeowner took back their yes. Before returning, this writes the marker `resume` - /// refuses, or removes the saved state if the marker can't be written, and from then on every - /// save records the upload as abandoned, so no reply still in flight can leave a capture a - /// relaunch would resume. Returns whether the marker was written. Call `abandon` afterwards to - /// stop the loop. + /// How a withdrawal was recorded on disk. + public enum WithdrawalRecord: Sendable, Equatable { + /// The marker `resume` refuses was written. + case marked + /// The marker couldn't be written, so the saved state was removed instead. + case savedStateRemoved + /// Neither the marker nor the removal succeeded: a relaunch could still find the saved + /// yes until a later save records the upload as abandoned. + case notRecorded(String) + } + + /// The homeowner took back their yes. Before returning, this records the withdrawal on disk + /// (see `WithdrawalRecord`) and cancels the running loop, which cancels requests it has not + /// finished. From then on no request is admitted and every save of this uploader records the + /// upload as abandoned, so a reply that still arrives can't leave a capture a relaunch would + /// resume. Call `abandon` afterwards to end the upload in memory. @discardableResult - public nonisolated func withdrawConsent() -> Bool { - withdrawal.withLock { withdrawn in + public nonisolated func withdrawConsent() -> WithdrawalRecord { + let record = withdrawal.withLock { withdrawn -> WithdrawalRecord in withdrawn = true do { try Data().write(to: CaptureUploadState.withdrawnURL(in: folder), options: .atomic) - return true + return .marked } catch { - try? FileManager.default.removeItem(at: Self.stateURL(in: folder)) - return false + do { + try FileManager.default.removeItem(at: Self.stateURL(in: folder)) + return .savedStateRemoved + } catch let removal as NSError where removal.domain == NSCocoaErrorDomain && removal.code == NSFileNoSuchFileError { + return .savedStateRemoved + } catch { + return .notRecorded(Self.describe(error)) + } } } + running.withLock { $0?.cancel() } + return record } /// Ends this upload for good: a world reset or starting over. Replies still in flight are @@ -207,8 +228,10 @@ public actor CaptureUploader { /// Starts the loop if it is idle: after a relaunch, or when the network is back. public func kick() { - guard loop == nil, state.end == nil else { return } - loop = Task { await self.drive() } + guard loop == nil, state.end == nil, !isWithdrawn else { return } + let task = Task { await self.drive() } + loop = task + running.withLock { $0 = task } } /// Waits for the loop to go idle: nothing to send until more input, or ended. @@ -338,6 +361,7 @@ public actor CaptureUploader { let local = folder.appending(path: file.sealed.path) let http = self.http group.addTask { + if self.isWithdrawn { return (file.sealed.path, .failure(Withdrawn())) } do { return (file.sealed.path, .success(try await http.upload(request, file: local))) } catch { return (file.sealed.path, .failure(error)) } } } @@ -345,6 +369,7 @@ public actor CaptureUploader { for await result in group { out.append(result) } return out } + if isWithdrawn { throw Withdrawn() } guard attempt == state.attemptID, state.end == nil else { return } for (path, result) in results { state.files[path]?.attempts += 1 @@ -557,7 +582,8 @@ public actor CaptureUploader { private func persist() { do { try withdrawal.withLock { withdrawn in - if withdrawn, state.end == nil { + // A withdrawal outranks any end a late reply set (finished, failed). + if withdrawn, state.end != .abandoned(Self.withdrawnReason) { state.end = .abandoned(Self.withdrawnReason) // Replies still in flight belong to the old attempt and are dropped. state.attemptID = UUID().uuidString diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift index ba708610d..010498987 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift @@ -459,19 +459,35 @@ struct NativeCaptureFixture: Sendable { server.release("POST captures/finalize") server.state.withLock { $0.held = [] } - await uploader.settled() + // The withdrawal cancelled the loop; the abandon task ends the upload in its own time. + for _ in 0..<500 where await uploader.snapshot.end == nil { try await Task.sleep(for: .milliseconds(10)) } #expect(await uploader.snapshot.end == .abandoned("consent withdrawn")) } + /// Waits, suspending, until the loopback has parked an answer for `route`. + func waitUntilParked(_ server: LoopbackCaptureAPI, _ route: String) async throws { + for _ in 0..<1000 where server.state.withLock({ $0.parked[route]?.isEmpty ?? true }) { try await Task.sleep(for: .milliseconds(10)) } + #expect(server.state.withLock { !($0.parked[route]?.isEmpty ?? true) }) + } + + /// Blocks the main actor until `stateURL` holds something other than `before`, so nothing + /// queued on the main actor (the abandon task a withdrawal schedules) can run meanwhile. + func blockUntilSaved(_ stateURL: URL, differentFrom before: Data?) { + let deadline = Date().addingTimeInterval(5) + while Date() < deadline, (try? Data(contentsOf: stateURL)) == before { usleep(10_000) } + } + /// A reply that lands after the homeowner withdrew, but before the uploader is abandoned, /// can't save a capture a relaunch would resume, whether or not the withdrawal marker could - /// be written. The main actor is held from the withdrawal to the checks, so the abandon task - /// the withdrawal schedules can't run first; only the withdrawal itself can stop the save. + /// be written. The transport lets the reply arrive even though the request's task was + /// cancelled, as a background session does. The main actor is held from the withdrawal to the + /// checks, so the abandon task the withdrawal schedules can't run first. @Test(arguments: [false, true]) func aReplyAfterWithdrawalCannotSaveAResumableCapture(markerBlocked: Bool) async throws { defer { try? FileManager.default.removeItem(at: root) } let server = try LoopbackCaptureAPI() server.state.withLock { $0.held = ["POST captures/files"] } - let environment = NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures) + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures) let coordinator = CaptureSessionCoordinator(environment: environment) coordinator.begin(recording: fixture.recording) coordinator.answerConsent(true) @@ -479,8 +495,7 @@ struct NativeCaptureFixture: Sendable { let session = try #require(coordinator.session) let uploader = try #require(session.uploader) let stateURL = CaptureUploader.stateURL(in: session.folder) - for _ in 0..<500 where server.requests("POST captures/files").isEmpty { try await Task.sleep(for: .milliseconds(10)) } - #expect(server.requests("POST captures/files").count == 1) + try await waitUntilParked(server, "POST captures/files") if markerBlocked { // A directory where the marker goes makes writing it fail. try FileManager.default.createDirectory( @@ -493,8 +508,7 @@ struct NativeCaptureFixture: Sendable { #expect((before == nil) == markerBlocked) server.release("POST captures/files") // The uploader saves the register reply on its own executor. - let deadline = Date().addingTimeInterval(5) - while Date() < deadline, (try? Data(contentsOf: stateURL)) == before { usleep(10_000) } + blockUntilSaved(stateURL, differentFrom: before) let saved = try CaptureUploadState.load(from: stateURL) #expect(saved.end == .abandoned("consent withdrawn")) #expect(try CaptureUploader.resume(folder: session.folder, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) == nil) @@ -505,6 +519,70 @@ struct NativeCaptureFixture: Sendable { #expect(server.requests("PUT upload").isEmpty) } + /// A result that arrives after the withdrawal can't save the upload as finished: the + /// withdrawal outranks the end a late reply sets. + @Test func aResultAfterWithdrawalSavesTheUploadAsAbandoned() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["GET captures/result"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + for i in 0..<3 { coordinator.kept(try fixture.photo(at: fixture.start + 3 + Double(i) * 1.5)) } + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + let stateURL = CaptureUploader.stateURL(in: session.folder) + try await waitUntilParked(server, "GET captures/result") + + coordinator.answerConsent(false) + // No suspension from here to the check. + let before = try? Data(contentsOf: stateURL) + server.release("GET captures/result") + blockUntilSaved(stateURL, differentFrom: before) + #expect(try CaptureUploadState.load(from: stateURL).end == .abandoned("consent withdrawn")) + + server.state.withLock { $0.held = [] } + await uploader.settled() + #expect(await uploader.snapshot.end == .abandoned("consent withdrawn")) + } + + /// Withdrawing cancels the request the uploader is waiting on before the abandon task runs: + /// a held register request is given up at once instead of waiting for its answer, and no + /// photo is uploaded. + @Test func withdrawalCancelsTheRequestInFlight() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let logs = Mutex<[String]>([]) + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/files"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures, + log: { line in logs.withLock { $0.append(line) } }) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + let uploader = try #require(coordinator.session?.uploader) + try await waitUntilParked(server, "POST captures/files") + + coordinator.answerConsent(false) + // The main actor stays busy, so only the withdrawal itself can end the wait. + let deadline = Date().addingTimeInterval(5) + func gaveUp() -> Bool { logs.withLock { $0.contains { $0.contains("capture-upload retry step=register") } } } + while Date() < deadline, !gaveUp() { usleep(10_000) } + #expect(gaveUp()) + + server.release("POST captures/files") + server.state.withLock { $0.held = [] } + await uploader.settled() + #expect(server.requests("PUT upload").isEmpty) + } + /// Photos kept after the scan was sent don't change the frozen packet or start more uploads. @Test func photosAfterTheScanWasSentAreNotAdded() async throws { defer { try? FileManager.default.removeItem(at: root) } From 89eec0238b2b6e4fea3c6b6195b56439f262a878 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Thu, 1 Oct 2026 21:01:36 -0500 Subject: [PATCH 09/16] ios tests: in the blocked-marker case, check resumability without the blocking directory CodeRabbit (review 5387568688) noted that when the test blocks the withdrawal marker, the directory it leaves at the marker's path makes resume refuse by itself. The test now removes that directory first and checks resumeSealedCaptures(), which refuses a saved upload that already has an end, so only the abandoned save can stop the resume. The direct CaptureUploader.resume check stays for the case where the marker was written. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../CaptureSessionCoordinatorTests.swift | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift index 010498987..fdbd84713 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift @@ -511,7 +511,14 @@ struct NativeCaptureFixture: Sendable { blockUntilSaved(stateURL, differentFrom: before) let saved = try CaptureUploadState.load(from: stateURL) #expect(saved.end == .abandoned("consent withdrawn")) - #expect(try CaptureUploader.resume(folder: session.folder, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) == nil) + if markerBlocked { + // The blocking directory sits at the marker's path and would stop a resume by itself; + // without it only the saved end can. + try FileManager.default.removeItem(at: CaptureUploadState.withdrawnURL(in: session.folder)) + } else { + #expect(try CaptureUploader.resume(folder: session.folder, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) == nil) + } + #expect(CaptureSessionCoordinator(environment: environment).resumeSealedCaptures().isEmpty) #expect(server.requests("PUT upload").isEmpty) server.state.withLock { $0.held = [] } From 9e0db8eb12c749edb50dd9b695b0e28004a292c9 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Fri, 2 Oct 2026 05:39:15 -0500 Subject: [PATCH 10/16] ios: a capture that failed or expired without an answer reads as ended, not working CaptureResultSummary turned every result without an outcome into .working(status), so a capture that had failed or expired looked as though it were still running. The new .ended(status) case covers those two statuses. Nothing in the app reads the summary yet. aCaptureThatEndedWithoutAnOutcomeIsNotWorking failed for both statuses at 89eec023. Co-Authored-By: Claude Code --- .../CaptureResult/CaptureResultSummary.swift | 11 ++++++++++- .../HouseScanKitTests/CaptureResultSummaryTests.swift | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultSummary.swift b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultSummary.swift index 8a1859073..65e156101 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultSummary.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultSummary.swift @@ -9,6 +9,8 @@ public struct CaptureResultSummary: Sendable, Equatable { /// The run has no outcome yet. case working(CaptureResult.Status) case decided(CaptureResult.OutcomeKind) + /// The capture is over without an outcome: it failed or expired. + case ended(CaptureResult.Status) } public var state: State @@ -27,7 +29,14 @@ public struct CaptureResultSummary: Sendable, Equatable { /// when the scan has none to place a box on (AR then says so, after the other checks). public init(_ record: CaptureResult.Record, world: CaptureResult.Association, meterAnchor: simd_float4x4?) { let response = record.response - state = response.outcome.map { .decided($0.kind) } ?? .working(response.status) + state = if let outcome = response.outcome { + .decided(outcome.kind) + } else { + switch response.status { + case .failed, .expired: .ended(response.status) + default: .working(response.status) + } + } message = response.outcome?.message let views = response.viewsNeeded.isEmpty ? response.outcome?.viewsNeeded ?? [] : response.viewsNeeded prompts = views.map(\.prompt) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultSummaryTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultSummaryTests.swift index 99efe248a..e8c83cb24 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultSummaryTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultSummaryTests.swift @@ -25,6 +25,14 @@ import Testing #expect(s.arUnavailable == .noOutcome(.processing)) } + /// A capture that failed or expired has no outcome, and it is over: it is not shown as + /// still working. + @Test(arguments: ["failed", "expired"]) func aCaptureThatEndedWithoutAnOutcomeIsNotWorking(status: String) throws { + let s = try Self.summary(#"{"runId":"run_A","status":"\#(status)","viewsNeeded":[],"memberActions":[]}"#) + #expect(s.state == .ended(CaptureResult.Status(rawValue: status))) + #expect(s.message == nil && s.prompts.isEmpty) + } + @Test func manualReviewKeepsTheServersWordsAndNoInventedChecks() throws { let s = try Self.summary(#""" {"runId":"run_A","status":"manual_review","viewsNeeded":[],"memberActions":[], From a8c4fb301a0916bb59d076b278780b6fa11f32c1 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Fri, 2 Oct 2026 05:39:15 -0500 Subject: [PATCH 11/16] ios: the capture uploader checks storage URLs, limits files the server never takes, saves its not-ready count, and paces empty polls Four review findings on #192, each with a test that failed at 89eec023: - Storage URLs (CodeRabbit 4158007398, Greptile 4115041167 on #123). A register answer's PUT URL must now be https, or http to this machine. Otherwise registration fails as upload_url_insecure, or as upload_url_invalid when the URL doesn't parse. Before, an unparseable URL left the file registered and the loop spinning, and a public http URL was used as given. A saved URL that fails the check goes back to register. - Files the server never takes (CodeRabbit 4158007439). A file left out of a register answer, or not acknowledged by a commit, was queued again with no delay and no limit. Each file now counts these in `unacknowledged`. Each time waits the retry delay, and the upload stops as not_acknowledged after maxUnacknowledged (5). - Not-ready reads across relaunch (Codex 4152312613). The count is saved in CaptureUploadState.notReadyReads, so a relaunch continues the same limit instead of starting over. - Empty polls answered at once (Greptile 4159739264). An events answer that comes back with nothing new while the capture is still running, sooner than minimumPollInterval (2 s), now waits out the rest before the next poll. The new state fields are optional, so state saved by earlier builds still loads. No measurement sets the 5-try limit or the 2 s interval. The uploader's new tests wait for it to go idle with a deadline, so a loop that never stops fails the test instead of hanging the run. The HouseScanKit package passes 681/681 locally. Co-Authored-By: Claude Code --- .../Upload/CaptureUploadState.swift | 6 + .../HouseScanKit/Upload/CaptureUploader.swift | 55 ++++++++- .../CaptureUploaderTests.swift | 113 +++++++++++++++++- .../LoopbackCaptureAPI.swift | 26 +++- 4 files changed, 191 insertions(+), 9 deletions(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift index de0784dc0..8024c9938 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift @@ -25,6 +25,9 @@ public struct CaptureUploadState: Codable, Sendable, Equatable { public var sealed: SealedFile public var phase: FilePhase = .queued public var attempts = 0 + /// Register answers that left this file out, plus commit answers that didn't acknowledge + /// it. Nil in state saved before this was counted. + public var unacknowledged: Int? /// Order in which the file was sealed, the tie-break after priority. public var sequence: Int @@ -88,6 +91,9 @@ public struct CaptureUploadState: Codable, Sendable, Equatable { public var eventCursor = 0 /// `retry_finalize` answers followed so far. public var finalizeRetries = 0 + /// Result reads that found no answer yet. Saved so a relaunch continues the same limit; nil in + /// state saved before this was counted. + public var notReadyReads: Int? public var backendStatus: String? public var lastEvent: String? public var result: Data? diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift index da0da60ec..6d284bfa1 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -29,6 +29,13 @@ public actor CaptureUploader { /// of how long a finished run's answer takes to become readable sets this number; it only /// keeps an answer that never appears from holding the upload open. public var maxNotReadyResults = 10 + /// Times one file may be left out of a register answer or a commit acknowledgement before + /// the upload stops. Each time waits the retry delay first. No measurement sets this; it + /// keeps a server that never takes a file from getting the same photo forever. + public var maxUnacknowledged = 5 + /// Seconds between events polls that come back at once with nothing new and the capture + /// still running, for a server or proxy that doesn't hold the poll. Not measured. + public var minimumPollInterval = 2.0 public init() {} @@ -66,7 +73,6 @@ public actor CaptureUploader { private var failures = 0 private var retryingAt: Date? private var digestRetried: Set = [] - private var notReadyReads = 0 /// True once the homeowner took back their yes. Every save happens while holding it, so no /// save can land between the withdrawal and the abandoned state it leads to. private nonisolated let withdrawal = Mutex(false) @@ -334,12 +340,40 @@ public actor CaptureUploader { if file.state == "committed" { setPhase(file.path, .committed) } else if let target = file.upload, target.method == "PUT", let url = target.url { + // Photos go only over https, or http to this machine for tests, and only to a URL + // the PUT can use: a URL that doesn't parse would leave the file registered forever. + if let problem = Self.uploadURLProblem(url) { throw Refused(step: "register", status: reply.status, codes: [problem]) } setPhase(file.path, .registered(url: url, headers: target.headers ?? [:], expiresAt: target.expiresAt.flatMap(Self.parseDate))) } else { throw Refused(step: "register", status: reply.status, codes: ["upload_method_\(file.upload?.method ?? "none")"]) } } + let answered = Set(answer.files.map(\.path)) + try unacknowledged(files.map(\.sealed.path).filter { !answered.contains($0) }, step: "register", status: reply.status) + } + + /// Counts files the server didn't take, saves, and backs off (or stops once one has been left + /// out more than `maxUnacknowledged` times). Saves and returns when there are none. + private func unacknowledged(_ paths: [String], step: String, status: Int) throws { + for path in paths { + let count = (state.files[path]?.unacknowledged ?? 0) + 1 + state.files[path]?.unacknowledged = count + } persist() + guard !paths.isEmpty else { return } + if paths.contains(where: { (state.files[$0]?.unacknowledged ?? 0) > policy.maxUnacknowledged }) { + throw Refused(step: step, status: status, codes: ["not_acknowledged"]) + } + throw Transient(step: step, detail: "\(paths.count) not acknowledged", retryAfter: nil) + } + + /// Why `text` can't receive a photo: nil when it is an https URL, or http to this machine. + static func uploadURLProblem(_ text: String) -> String? { + guard let url = URL(string: text), let scheme = url.scheme?.lowercased(), let host = url.host(), !host.isEmpty else { + return "upload_url_invalid" + } + let local = host == "127.0.0.1" || host == "localhost" + return scheme == "https" || (scheme == "http" && local) ? nil : "upload_url_insecure" } private func put(_ files: [CaptureUploadState.File]) async throws { @@ -351,9 +385,13 @@ public actor CaptureUploader { if isWithdrawn { throw Withdrawn() } let chunk = files[index..).self) { group in for file in chunk { - guard case .registered(let url, let headers, _) = file.phase, let target = URL(string: url) else { continue } + guard case .registered(let url, let headers, _) = file.phase, Self.uploadURLProblem(url) == nil, let target = URL(string: url) else { continue } var request = URLRequest(url: target) request.httpMethod = "PUT" // Exactly the headers the server signed; never the API's own credentials. @@ -414,7 +452,7 @@ public actor CaptureUploader { } if !committed.isEmpty { mark("firstCommit") } if state.packet != nil, state.files.values.allSatisfy({ $0.phase == .committed }) { mark("lastCommit") } - persist() + try unacknowledged(files.map(\.sealed.path).filter { !committed.contains($0) }, step: "commit", status: reply.status) } private func finalize() async throws { @@ -440,6 +478,7 @@ public actor CaptureUploader { var request = URLRequest(url: components.url!) request.setValue("application/json", forHTTPHeaderField: "Accept") request.timeoutInterval = Double(policy.eventsWait) + 30 + let sent = now() let (reply, attempt) = try await call("events", request) guard attempt == state.attemptID else { return } guard (200..<300).contains(reply.status), let answer = try? JSONDecoder().decode(CaptureAPI.EventsResponse.self, from: reply.body) else { @@ -465,6 +504,12 @@ public actor CaptureUploader { } if !answer.events.isEmpty { log?("capture-upload events status=\(answer.status) last=\(state.lastEvent ?? "")") } persist() + // An answer that came back at once with nothing new means the server didn't hold the + // poll; waiting here keeps the loop from asking again immediately. + let waited = now().timeIntervalSince(sent) + if answer.events.isEmpty, !CaptureAPI.terminalStatuses.contains(answer.status), waited < policy.minimumPollInterval { + try await sleep(policy.minimumPollInterval - waited) + } } private func fetchResult() async throws { @@ -508,7 +553,9 @@ public actor CaptureUploader { /// Always throws: a retry after the usual wait, or the end of the upload once /// `maxNotReadyResults` reads have found no answer. private func notReady(_ reply: HTTPReply) throws -> Never { - notReadyReads += 1 + let notReadyReads = (state.notReadyReads ?? 0) + 1 + state.notReadyReads = notReadyReads + persist() guard notReadyReads <= policy.maxNotReadyResults else { throw Refused(step: "result", status: reply.status, codes: ["result_not_ready"]) } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift index 8264d0e81..3713170f3 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift @@ -15,7 +15,8 @@ import Testing init( packetID: String = UUID().uuidString, appVersion: String = "test (1)", server: LoopbackCaptureAPI? = nil, - http: any CaptureHTTP = URLSessionCaptureHTTP.ephemeral(timeout: 10), policy: CaptureUploader.Policy = Rig.fast + http: any CaptureHTTP = URLSessionCaptureHTTP.ephemeral(timeout: 10), policy: CaptureUploader.Policy = Rig.fast, + sleep: @escaping @Sendable (Double) async throws -> Void = { _ in try await Task.sleep(for: .milliseconds(10)) } ) throws { self.http = http self.server = try server ?? LoopbackCaptureAPI() @@ -24,7 +25,7 @@ import Testing uploader = try CaptureUploader.start( folder: capture.folder, base: self.server.base, http: http, create: .init(packetId: packetID, tier: .arkit, device: .init(model: "iPhone15,4", systemVersion: "26.0", appVersion: appVersion)), - consentedAt: Date(), policy: policy, sleep: { _ in try await Task.sleep(for: .milliseconds(10)) }) + consentedAt: Date(), policy: policy, sleep: sleep) } static var fast: CaptureUploader.Policy { @@ -310,6 +311,114 @@ import Testing #expect(rig.server.requests("GET captures/result").count == 1) } + /// Whether `uploader` goes idle within `seconds`. A loop that never goes idle fails the test + /// here instead of hanging the run; the process ending stops it. + static func settles(_ uploader: CaptureUploader, within seconds: Int = 20) async -> Bool { + let done = Mutex(false) + Task.detached { await uploader.settled(); done.withLock { $0 = true } } + let deadline = ContinuousClock.now + .seconds(seconds) + while ContinuousClock.now < deadline { + if done.withLock({ $0 }) { return true } + try? await Task.sleep(for: .milliseconds(20)) + } + return false + } + + /// A storage URL that is not https (or http to this machine) is refused when it is + /// registered, before any bytes go to it; one that doesn't parse is refused the same way + /// instead of leaving the file registered and the loop spinning. + @Test(arguments: [("http://storage.example.com/upload/x", "upload_url_insecure"), ("", "upload_url_invalid")]) + func aStorageURLThatIsNotSafeIsRefusedBeforeAnyPut(url: String, code: String) async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.uploadURLOverride = url } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "register", codes: [code], status: 200)) + #expect(rig.server.requests("POST captures/files").count == 1) + #expect(rig.server.requests("PUT upload").isEmpty) + } + + /// Commits that never acknowledge a file back off between tries and stop after the policy's + /// limit, instead of sending the same photo again and again. + @Test func aCommitThatNeverAcknowledgesBacksOffAndStops() async throws { + var policy = Rig.fast + policy.maxUnacknowledged = 3 + let sleeps = Mutex<[Double]>([]) + let rig = try Rig(policy: policy, sleep: { seconds in sleeps.withLock { $0.append(seconds) } }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.commitNeverAcknowledges = true } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "commit", codes: ["not_acknowledged"], status: 200)) + // The images are the close-up still and its keyframe. + #expect(rig.server.requests("POST captures/files:commit").count == 4) + #expect(sleeps.withLock { $0.count } == 3) + } + + /// Register answers that leave a file out back off and stop the same way. + @Test func aRegisterThatOmitsFilesBacksOffAndStops() async throws { + var policy = Rig.fast + policy.maxUnacknowledged = 3 + let sleeps = Mutex<[Double]>([]) + let rig = try Rig(policy: policy, sleep: { seconds in sleeps.withLock { $0.append(seconds) } }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.registerOmitsFiles = true } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "register", codes: ["not_acknowledged"], status: 200)) + #expect(rig.server.requests("POST captures/files").count == 4) + #expect(rig.server.requests("PUT upload").isEmpty) + #expect(sleeps.withLock { $0.count } == 3) + } + + /// The limit on reads that find no answer is saved with the upload, so relaunching doesn't + /// start it again: across two processes the reads still stop at `maxNotReadyResults + 1`. + @Test func theNotReadyLimitSurvivesARelaunch() async throws { + var policy = Rig.fast + policy.maxNotReadyResults = 3 + let waits = Mutex(0) + // The first process quits during its second wait. + let rig = try Rig(policy: policy, sleep: { _ in + let n = waits.withLock { $0 += 1; return $0 } + if n >= 2 { throw CancellationError() } + }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.outcomelessResults = 100 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + try #require(await Self.settles(rig.uploader)) + #expect(rig.server.requests("GET captures/result").count == 2) + #expect(await rig.uploader.snapshot.end == nil) + + let resumed = try #require(try CaptureUploader.resume( + folder: rig.capture.folder, base: rig.server.base, http: rig.http, policy: policy, sleep: { _ in })) + await resumed.kick() + try #require(await Self.settles(resumed)) + #expect(await resumed.snapshot.end == .failed(step: "result", codes: ["result_not_ready"], status: 200)) + #expect(rig.server.requests("GET captures/result").count == 4) + } + + /// A server that answers an events poll at once with nothing new is not polled again at once: + /// the uploader waits out `minimumPollInterval` first. + @Test func emptyEventsAnsweredAtOnceArePaced() async throws { + let sleeps = Mutex<[Double]>([]) + let rig = try Rig(sleep: { seconds in sleeps.withLock { $0.append(seconds) } }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.processingPolls = 3 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + let waited = sleeps.withLock { $0 } + #expect(waited.count == 3) + #expect(waited.allSatisfy { $0 > 0 && $0 <= CaptureUploader.Policy().minimumPollInterval }) + } + /// `failed` ends a capture without an answer, so its outcomeless result is final at once. @Test func aFailedCaptureEndsWithoutWaitingForAnOutcome() async throws { let rig = try Rig() diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift index b92ffaeb5..f89a42382 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift @@ -73,6 +73,15 @@ final class LoopbackCaptureAPI: Sendable { /// The status a capture ends in once every listed file is committed. Only `manual_review` /// ends with an outcome; `failed` ends with a `failed` event and none. var endStatus = "manual_review" + /// Every register answer gives this URL for its PUTs instead of the loopback's own. + var uploadURLOverride: String? + /// Register answers leave out every file, as if the server never saw the request's list. + var registerOmitsFiles = false + /// Commits never acknowledge a file: every path comes back in `notFound`. + var commitNeverAcknowledges = false + /// Once every listed file is committed, the next N event polls answer at once with status + /// `processing` and no events, as a server that doesn't hold the poll would; then the run ends. + var processingPolls = 0 /// Answer the next commit with this split instead of the truth. var commitOverride: ((committed: [String], notFound: [String], mismatch: [String]))? /// Routes whose answers wait until `release` is called. @@ -192,7 +201,7 @@ final class LoopbackCaptureAPI: Sendable { case ("POST", "files"): return register(r, &capture, &s) case ("POST", "files:commit"): return commit(r, &capture, &s) case ("POST", "finalize"): return finalize(r, &capture, &s) - case ("GET", "events"): return events(r, &capture) + case ("GET", "events"): return events(r, &capture, &s) case ("GET", "result"): if s.notReadyResults > 0 { s.notReadyResults -= 1 @@ -262,10 +271,11 @@ final class LoopbackCaptureAPI: Sendable { out.append(["path": path, "state": "committed"]) continue } + if s.registerOmitsFiles { continue } let token = UUID().uuidString s.tokens[token] = (c.id, path) out.append(["path": path, "state": "pending", "upload": [ - "method": "PUT", "url": "http://127.0.0.1:\(port)/upload/\(token)", + "method": "PUT", "url": s.uploadURLOverride ?? "http://127.0.0.1:\(port)/upload/\(token)", "headers": ["Content-Type": type, "Content-MD5": md5], "expiresAt": "2099-01-01T00:00:00Z", ]]) } @@ -309,6 +319,11 @@ final class LoopbackCaptureAPI: Sendable { s.commitOverride = nil (committed, notFound, mismatch) = forced } + if s.commitNeverAcknowledges { + notFound += committed + mismatch + committed = [] + mismatch = [] + } c.committed.formUnion(committed) if !committed.isEmpty { c.events.append(("files_committed", ["committed": String(c.committed.count)])) } startIfComplete(&c, endStatus: s.endStatus) @@ -357,7 +372,12 @@ final class LoopbackCaptureAPI: Sendable { } } - private func events(_ r: Request, _ c: inout Capture) -> (Int, Data) { + private func events(_ r: Request, _ c: inout Capture, _ s: inout State) -> (Int, Data) { + if c.status == "manual_review" || c.status == "failed", s.processingPolls > 0, c.runID != nil { + s.processingPolls -= 1 + let after = Int(r.query["after"] ?? "0") ?? 0 + return (200, json(["status": "processing", "next": after, "events": [] as [Any]])) + } let after = Int(r.query["after"] ?? "0") ?? 0 let list = c.events.enumerated().filter { $0.offset + 1 > after }.map { index, event in ["seq": index + 1, "type": event.type, "at": "2026-09-27T00:00:00Z", "data": event.data] as [String: Any] From aac6c9097a9fd0b32f52587608c9246de9dc40bc Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Fri, 2 Oct 2026 07:20:42 -0500 Subject: [PATCH 12/16] ios: a capture the server opens again gets fresh retry limits; the poll-pacing test uses a controlled clock Review of a8c4fb30 found that when the server lost a capture and the uploader opened it again, the new run kept the old run's saved not-ready count. With a limit of 3, an old count of 3 ended the new run on its first not-ready read. captureGone now resets notReadyReads and every file's unacknowledged count, because a new server capture is a new run. Relaunches of the same capture still keep both counts. aRecreatedCaptureStartsANewNotReadyLimit failed at a8c4fb30: the recreated run ended failed instead of finished. The pacing test now moves its clock only when the uploader sleeps, and it checks three waits of exactly the 2 s interval plus a terminal fetch with no wait. A new test checks that an answer held longer than the interval gets no extra wait. Co-Authored-By: Claude Code --- .../HouseScanKit/Upload/CaptureUploader.swift | 7 +- .../CaptureUploaderTests.swift | 68 +++++++++++++++++-- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift index 6d284bfa1..e3118f1dc 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -574,7 +574,12 @@ public actor CaptureUploader { state.captureID = nil state.finalized = nil state.eventCursor = 0 - for path in state.files.keys { state.files[path]?.phase = .queued } + // A new server capture is a new run: the old one's limits don't carry over. + state.notReadyReads = nil + for path in state.files.keys { + state.files[path]?.phase = .queued + state.files[path]?.unacknowledged = nil + } persist() return true } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift index 3713170f3..29560e9f1 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift @@ -16,7 +16,8 @@ import Testing init( packetID: String = UUID().uuidString, appVersion: String = "test (1)", server: LoopbackCaptureAPI? = nil, http: any CaptureHTTP = URLSessionCaptureHTTP.ephemeral(timeout: 10), policy: CaptureUploader.Policy = Rig.fast, - sleep: @escaping @Sendable (Double) async throws -> Void = { _ in try await Task.sleep(for: .milliseconds(10)) } + sleep: @escaping @Sendable (Double) async throws -> Void = { _ in try await Task.sleep(for: .milliseconds(10)) }, + now: @escaping @Sendable () -> Date = { Date() } ) throws { self.http = http self.server = try server ?? LoopbackCaptureAPI() @@ -25,7 +26,7 @@ import Testing uploader = try CaptureUploader.start( folder: capture.folder, base: self.server.base, http: http, create: .init(packetId: packetID, tier: .arkit, device: .init(model: "iPhone15,4", systemVersion: "26.0", appVersion: appVersion)), - consentedAt: Date(), policy: policy, sleep: sleep) + consentedAt: Date(), policy: policy, now: now, sleep: sleep) } static var fast: CaptureUploader.Policy { @@ -403,10 +404,15 @@ import Testing } /// A server that answers an events poll at once with nothing new is not polled again at once: - /// the uploader waits out `minimumPollInterval` first. + /// the uploader waits out `minimumPollInterval` first. The clock moves only when the uploader + /// sleeps, so every empty answer arrives at once and each wait is the whole interval. The + /// terminal answer is fetched without a wait. @Test func emptyEventsAnsweredAtOnceArePaced() async throws { + let clock = Mutex(Date(timeIntervalSince1970: 1_000_000)) let sleeps = Mutex<[Double]>([]) - let rig = try Rig(sleep: { seconds in sleeps.withLock { $0.append(seconds) } }) + let rig = try Rig( + sleep: { seconds in sleeps.withLock { $0.append(seconds) }; clock.withLock { $0 += seconds } }, + now: { clock.withLock { $0 } }) defer { rig.cleanUp() } rig.server.state.withLock { $0.processingPolls = 3 } await rig.uploader.add(try await rig.capture.sealImages(count: 1)) @@ -414,9 +420,57 @@ import Testing try #require(await Self.settles(rig.uploader)) #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) - let waited = sleeps.withLock { $0 } - #expect(waited.count == 3) - #expect(waited.allSatisfy { $0 > 0 && $0 <= CaptureUploader.Policy().minimumPollInterval }) + let interval = CaptureUploader.Policy().minimumPollInterval + #expect(sleeps.withLock { $0 } == [interval, interval, interval]) + #expect(rig.server.requests("GET captures/events").count == 4) + } + + /// An empty answer the server held for the whole interval needs no further wait. + @Test func emptyEventsHeldPastTheIntervalAreNotPaced() async throws { + // Every reading of the clock is 3 s after the last, longer than the 2 s interval. + let clock = Mutex(Date(timeIntervalSince1970: 1_000_000)) + let sleeps = Mutex<[Double]>([]) + let rig = try Rig( + sleep: { seconds in sleeps.withLock { $0.append(seconds) } }, + now: { clock.withLock { $0 += 3; return $0 } }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.processingPolls = 3 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + #expect(sleeps.withLock { $0 }.isEmpty) + #expect(rig.server.requests("GET captures/events").count == 4) + } + + /// A capture the server lost and the uploader opened again is a new run, so it gets a fresh + /// not-ready limit: the old run's reads don't end it. + @Test func aRecreatedCaptureStartsANewNotReadyLimit() async throws { + var policy = Rig.fast + policy.maxNotReadyResults = 3 + let waits = Mutex(0) + // The first process quits during its third wait, after three not-ready reads. + let rig = try Rig(policy: policy, sleep: { _ in + let n = waits.withLock { $0 += 1; return $0 } + if n >= 3 { throw CancellationError() } + }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.outcomelessResults = 100 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + try #require(await Self.settles(rig.uploader)) + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: rig.capture.folder)).notReadyReads == 3) + + // The server forgets every capture (a redeploy with in-memory state). The next run has + // one not-ready read before its answer. + rig.server.state.withLock { $0.captures = [:]; $0.byPacket = [:]; $0.outcomelessResults = 1 } + let resumed = try #require(try CaptureUploader.resume( + folder: rig.capture.folder, base: rig.server.base, http: rig.http, policy: policy, sleep: { _ in })) + await resumed.kick() + try #require(await Self.settles(resumed)) + #expect(await resumed.snapshot.end == .finished(status: "manual_review")) + #expect(rig.server.requests("POST captures").count == 2) } /// `failed` ends a capture without an answer, so its outcomeless result is final at once. From 997f266dde97d679ef5db2ed2fac06a5a951ca1c Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Fri, 2 Oct 2026 08:18:45 -0500 Subject: [PATCH 13/16] ios: the capture transport refuses redirects, so a photo or API request goes only to the URL that was checked URLSessionCaptureHTTP used an ordinary URLSession with no delegate. A URLSession follows HTTP redirects by default, and the uploader checks only the storage URL the server first gives. So a 307 or 308 from storage would have carried the photo to a URL nobody checked. App Transport Security would still block cleartext to public hosts. But it allows any https host, and the app's NSAllowsLocalNetworking permits plain http on the local network. Both send and upload now pass a task delegate, NoRedirects, that refuses every redirect. URLSession then returns the 3xx response itself, so callers classify it like any other refusal: a redirected PUT counts as a storage refusal, and a redirected API request fails on its status. The CaptureHTTP protocol now says implementations must not follow redirects. Two tests drive the real URLSession against two local loopback servers. aStorageRedirectIsNotFollowed redirects every photo PUT, and anAPIRedirectIsNotFollowed redirects create. Both failed at aac6c909: the second server received the redirected request. The maxUnacknowledged comment now says what the tests pin: 5 retries, with the upload stopping on the 6th unacknowledged answer. Co-Authored-By: Claude Code --- .../HouseScanKit/Upload/CaptureAPI.swift | 21 ++++++++++-- .../HouseScanKit/Upload/CaptureUploader.swift | 14 +++++--- .../CaptureUploaderTests.swift | 34 +++++++++++++++++++ .../LoopbackCaptureAPI.swift | 12 ++++--- 4 files changed, 69 insertions(+), 12 deletions(-) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift index 930940e24..dc33d562d 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift @@ -168,7 +168,9 @@ public struct HTTPReply: Sendable, Equatable { } /// How the uploader reaches the network. HTTP error statuses come back as replies; only transport -/// failures throw. +/// failures throw. An implementation must not follow redirects: the uploader checks each URL +/// before it sends a photo there, and a redirect would send it somewhere unchecked. A redirect +/// comes back as its own 3xx reply. public protocol CaptureHTTP: Sendable { func send(_ request: URLRequest) async throws -> HTTPReply /// `file` as the whole body of `request`, streamed from disk. @@ -195,15 +197,28 @@ public struct URLSessionCaptureHTTP: CaptureHTTP { } public func send(_ request: URLRequest) async throws -> HTTPReply { - let (data, response) = try await session.data(for: request) + let (data, response) = try await session.data(for: request, delegate: NoRedirects.shared) return Self.reply(data, response) } public func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { - let (data, response) = try await session.upload(for: request, fromFile: file) + let (data, response) = try await session.upload(for: request, fromFile: file, delegate: NoRedirects.shared) return Self.reply(data, response) } + /// Refuses every redirect, so the task returns the 3xx response itself. URLSession follows + /// redirects by default, and App Transport Security alone would still allow one to another + /// https host, or to plain http on the local network, which the app's Info.plist permits. + final class NoRedirects: NSObject, URLSessionTaskDelegate, Sendable { + static let shared = NoRedirects() + + func urlSession( + _ session: URLSession, task: URLSessionTask, willPerformHTTPRedirection response: HTTPURLResponse, newRequest request: URLRequest + ) async -> URLRequest? { + nil + } + } + static func reply(_ data: Data, _ response: URLResponse) -> HTTPReply { let http = response as? HTTPURLResponse return HTTPReply( diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift index e3118f1dc..58625e433 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -29,9 +29,11 @@ public actor CaptureUploader { /// of how long a finished run's answer takes to become readable sets this number; it only /// keeps an answer that never appears from holding the upload open. public var maxNotReadyResults = 10 - /// Times one file may be left out of a register answer or a commit acknowledgement before - /// the upload stops. Each time waits the retry delay first. No measurement sets this; it - /// keeps a server that never takes a file from getting the same photo forever. + /// Retries after a register answer leaves a file out or a commit doesn't acknowledge it, + /// counted per file per server capture. Each retry waits the retry delay; the upload stops + /// on the next unacknowledged answer, so a file is sent at most `maxUnacknowledged + 1` + /// times. No measurement sets this; it keeps a server that never takes a file from getting + /// the same photo forever. public var maxUnacknowledged = 5 /// Seconds between events polls that come back at once with nothing new and the capture /// still running, for a server or proxy that doesn't hold the poll. Not measured. @@ -340,8 +342,10 @@ public actor CaptureUploader { if file.state == "committed" { setPhase(file.path, .committed) } else if let target = file.upload, target.method == "PUT", let url = target.url { - // Photos go only over https, or http to this machine for tests, and only to a URL - // the PUT can use: a URL that doesn't parse would leave the file registered forever. + // Photos go only to an https URL, or http to this machine for tests, and only to a + // URL the PUT can use: a URL that doesn't parse would leave the file registered + // forever. This checks the URL as given; `CaptureHTTP` refuses redirects, so a + // storage server can't send the photo on to a URL this never checked. if let problem = Self.uploadURLProblem(url) { throw Refused(step: "register", status: reply.status, codes: [problem]) } setPhase(file.path, .registered(url: url, headers: target.headers ?? [:], expiresAt: target.expiresAt.flatMap(Self.parseDate))) } else { diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift index 29560e9f1..a956552af 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift @@ -341,6 +341,40 @@ import Testing #expect(rig.server.requests("PUT upload").isEmpty) } + /// Storage that answers a PUT with a redirect doesn't get the photo sent on: the real + /// `URLSession` transport refuses every redirect, so the photo reaches only the URL that passed + /// the check, and the 307 comes back as a storage refusal. The redirect goes to a second local + /// server, which must see nothing. + @Test func aStorageRedirectIsNotFollowed() async throws { + let elsewhere = try LoopbackCaptureAPI() + var policy = Rig.fast + policy.maxRefusedPuts = 2 + let rig = try Rig(policy: policy) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.redirects["PUT upload"] = "http://127.0.0.1:\(elsewhere.port)/upload/elsewhere" } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "put", codes: ["storage_refused"], status: 307)) + #expect(!rig.server.requests("PUT upload").isEmpty) + #expect(elsewhere.state.withLock { $0.log.isEmpty }) + } + + /// The API's own requests aren't redirected either: a 307 on create comes back as the reply, + /// which the uploader refuses by its status, and the second server sees nothing. + @Test func anAPIRedirectIsNotFollowed() async throws { + let elsewhere = try LoopbackCaptureAPI() + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.redirects["POST captures"] = "\(elsewhere.base.absoluteString)/captures" } + await rig.uploader.kick() + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "create", codes: [], status: 307)) + #expect(rig.server.requests("POST captures").count == 1) + #expect(elsewhere.state.withLock { $0.log.isEmpty }) + } + /// Commits that never acknowledge a file back off between tries and stop after the policy's /// limit, instead of sending the same photo again and again. @Test func aCommitThatNeverAcknowledgesBacksOffAndStops() async throws { diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift index f89a42382..c10744ee1 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift @@ -82,6 +82,9 @@ final class LoopbackCaptureAPI: Sendable { /// Once every listed file is committed, the next N event polls answer at once with status /// `processing` and no events, as a server that doesn't hold the poll would; then the run ends. var processingPolls = 0 + /// Routes ("PUT upload", "POST captures", ...) answered `307` with the given `Location`, as + /// a server sending the request somewhere else would. + var redirects: [String: String] = [:] /// Answer the next commit with this split instead of the truth. var commitOverride: ((committed: [String], notFound: [String], mismatch: [String]))? /// Routes whose answers wait until `release` is called. @@ -167,14 +170,15 @@ final class LoopbackCaptureAPI: Sendable { private func handle(_ request: Request, _ connection: NWConnection) { let route = Self.route(request) - let (drop, hold) = state.withLock { s -> (Bool, Bool) in + let (drop, hold, redirect) = state.withLock { s -> (Bool, Bool, String?) in s.log.append(request) - return (s.dropNext.remove(route) != nil, s.held.contains(route)) + return (s.dropNext.remove(route) != nil, s.held.contains(route), s.redirects[route]) } - let (status, body) = answer(request) + let (status, body) = redirect == nil ? answer(request) : (307, Data()) + let location = redirect.map { "Location: \($0)\r\n" } ?? "" let send: @Sendable () -> Void = { if drop { return connection.cancel() } - let head = "HTTP/1.1 \(status) X\r\nContent-Type: application/json\r\nContent-Length: \(body.count)\r\nConnection: close\r\n\r\n" + let head = "HTTP/1.1 \(status) X\r\n\(location)Content-Type: application/json\r\nContent-Length: \(body.count)\r\nConnection: close\r\n\r\n" connection.send(content: Data(head.utf8) + body, completion: .contentProcessed { _ in connection.cancel() }) } if hold { From f5422be8c6d9ff5a92b41bca8b96f07187398a49 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Fri, 2 Oct 2026 12:25:35 -0500 Subject: [PATCH 14/16] ios: the packet check refuses a tap ray that isn't two finite 3-vectors Packet04Check compared a tap's submitted ray with its replay by zipping the two origin+direction lists and taking the largest difference. zip stops at the shorter list, and max() can skip a NaN. So an empty direction, a 2+4 split of the same six numbers, an extra component or a NaN all passed the check that claims to validate tap replay. The producer isn't known to emit such rays; the check simply didn't catch them. The check now requires exactly three finite values in each of rayOrigin and rayDirection before it compares, and reports "tap ray is not two finite 3-vectors". The replay tolerance is unchanged. aTapRayThatIsNotTwoFinite3VectorsIsRefused covers five mutations of a real producer packet. Four of them passed the check at 997f266d. The infinite origin was already refused through its infinite error and stays as a guard. Co-Authored-By: Claude Code --- .../HouseScanKit/Packet/Packet04Check.swift | 5 ++++ .../HouseScanKitTests/Packet04Tests.swift | 24 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Check.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Check.swift index d6747e01c..3ea995149 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Check.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Check.swift @@ -72,6 +72,11 @@ public enum Packet04Check { continue } if let epoch = tap.epoch, epoch != frame.epoch { out.append("tap \(tap.id) epoch differs from its keyframe's") } + // Compared pairwise below, a missing or extra component, or a NaN, would go unseen. + guard tap.rayOrigin.count == 3, tap.rayDirection.count == 3, (tap.rayOrigin + tap.rayDirection).allSatisfy(\.isFinite) else { + out.append("tap \(tap.id) ray is not two finite 3-vectors") + continue + } let replay = ray(pose: frame.pose, intrinsics: frame.intrinsics, pixel: tap.pixel) let error = zip(replay.origin + replay.direction, tap.rayOrigin + tap.rayDirection).map { abs($0 - $1) }.max() ?? .infinity if !(error <= tapReplayTolerance) { out.append("tap \(tap.id) ray replays with error \(error)") } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/Packet04Tests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/Packet04Tests.swift index 8d0c06138..909bb67b2 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/Packet04Tests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/Packet04Tests.swift @@ -166,6 +166,30 @@ func temporaryPacketFolder(_ name: String) -> URL { #expect(Packet04Check.problems(broken, folder: nil).contains { $0.hasPrefix("tap tap1 ray replays with error") }) } + /// A tap ray must be two 3-vectors of finite numbers. Joined and compared pairwise, a missing + /// or misshaped vector, or a NaN, could otherwise pass the replay check unseen. + @Test(arguments: [ + ([0.1, 1.5, -2], []), + ([0.1, 1.5], [-2, 0, 0, -1]), + ([0.1, 1.5, -2], [0, 0, -1, 0]), + ([0.1, 1.5, -2], [0, Double.nan, -1]), + ([0.1, .infinity, -2], [0, 0, -1]), + ] as [([Double], [Double])]) + func aTapRayThatIsNotTwoFinite3VectorsIsRefused(origin: [Double], direction: [Double]) async throws { + let folder = temporaryPacketFolder("p04-tap-shape") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + _ = try await capture.sealImages(count: 1) + try await capture.producer.addTap(id: "tap1", label: "meter", keyframe: "k00001", pixel: SIMD2(48, 36), time: SyntheticCapture04.start + 0.2, hit: nil) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: try await capture.finish().packet) + #expect(!Packet04Check.problems(packet, folder: nil).contains { $0.hasPrefix("tap tap1") }) + + var broken = packet + broken.taps?[0].rayOrigin = origin + broken.taps?[0].rayDirection = direction + #expect(Packet04Check.problems(broken, folder: nil).contains { $0.hasPrefix("tap tap1") }) + } + @Test func checksCatchAMalformedPacket() async throws { let folder = temporaryPacketFolder("p04-malformed") defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } From ff8c0ed7d7df9053a33babf70809220ffe2fdd04 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Fri, 2 Oct 2026 13:43:15 -0500 Subject: [PATCH 15/16] fix(ios): apply consent withdrawal to the capture folder Keep one live uploader per canonical folder and serialize ownership, saves, and withdrawal. Refuse reuse after withdrawal even when both persistence operations fail, and return that failure to the coordinator caller. Co-Authored-By: Claude Code (cherry picked from commit 86a2633d34d5a987bf1e4d74eb2ff4c64ca2a933) --- .../Upload/CaptureSessionCoordinator.swift | 30 ++- .../Upload/CaptureUploadState.swift | 19 ++ .../HouseScanKit/Upload/CaptureUploader.swift | 174 +++++++++----- .../CaptureFolderOwnershipTests.swift | 224 ++++++++++++++++++ .../CaptureSessionCoordinatorTests.swift | 104 ++++++++ .../CaptureUploaderTests.swift | 31 ++- 6 files changed, 511 insertions(+), 71 deletions(-) create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/CaptureFolderOwnershipTests.swift diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift index c8e04ba4a..d35fea631 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift @@ -181,6 +181,7 @@ public final class CaptureSessionCoordinator { var ended = false /// The homeowner said no after a yes: this capture is never sent again. var consentWithdrawn = false + var withdrawalFailure: ConsentWithdrawalError? /// packet.json as frozen, once the scan was sent for placement. var frozen: Data? /// The server's result for this session's capture, decoded and bound to it. @@ -237,28 +238,38 @@ public final class CaptureSessionCoordinator { environment?.sends == true && consent == nil && session != nil } + public enum ConsentWithdrawalError: Error, Sendable, Equatable { + /// Sending stopped in this process, but a relaunch may still find the old saved yes. + case notRecorded(String) + } + /// A no after a yes stops this capture's upload for good: nothing kept later is sent, and a /// second yes in the same scan does not start it again. What the server already has stays there. - public func answerConsent(_ yes: Bool, at time: Date = Date()) { + /// Failure means sending stopped here but the withdrawal could not be saved for a relaunch. + @discardableResult + public func answerConsent(_ yes: Bool, at time: Date = Date()) -> Result { consent = yes consentedAt = yes ? time : nil - guard let session else { return } + guard let session else { return .success(()) } + if let failure = session.withdrawalFailure { return .failure(failure) } if yes { startUploading(session) } else if let uploader = session.uploader { session.consentWithdrawn = true session.uploader = nil - // Recorded before returning, so a relaunch can't resume this capture even if the app - // quits before the abandon below runs. + // The result distinguishes durable withdrawal from stopping only this live process. switch uploader.withdrawConsent() { case .marked: break case .savedStateRemoved: environment?.log("capture upload: the withdrawal marker could not be written; the saved upload was removed instead") case .notRecorded(let error): - environment?.log("capture upload: the withdrawal could not be saved (\(error)); a relaunch could still find this capture until the upload is saved as abandoned") + session.withdrawalFailure = .notRecorded(error) + environment?.log("capture upload: the withdrawal could not be saved (\(error)); sending stopped in this process, but a relaunch could still find the saved consent") } Task { await uploader.abandon(CaptureUploader.withdrawnReason) } } + if let failure = session.withdrawalFailure { return .failure(failure) } + return .success(()) } // MARK: Scan events @@ -412,9 +423,9 @@ public final class CaptureSessionCoordinator { policy: environment.policy) session.uploader = uploader // Status and results from an ended session's uploader never reach the screen. - let publish: @Sendable (CaptureUploadStatus) -> Void = { [weak self] status in + let publish: @Sendable (CaptureUploadStatus) -> Void = { [weak self, weak session] status in Task { @MainActor in - guard let self, self.session === session else { return } + guard let self, let session, self.session === session else { return } self.onStatus?(status) if status.resultAvailable, session.result == nil { await self.bindResult(of: session) } } @@ -464,7 +475,10 @@ public final class CaptureSessionCoordinator { self.session = nil session.ended = true onStatus?(nil) - if let uploader = session.uploader { Task { await uploader.abandon(reason) } } + if let uploader = session.uploader { + uploader.stopSending() + Task { await uploader.abandon(reason) } + } } } diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift index 8024c9938..f27754c2c 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift @@ -11,6 +11,25 @@ public struct CaptureUploadState: Codable, Sendable, Equatable { public static func withdrawnURL(in folder: URL) -> URL { folder.appending(path: withdrawnFileName) } + /// Called with the uploader's folder registry locked so no initial save or reply can restore + /// a yes between these writes. Neither failing proves anything durable after a process exit. + static func recordWithdrawal(in folder: URL) -> CaptureUploader.WithdrawalRecord { + do { + try Data().write(to: withdrawnURL(in: folder), options: .atomic) + return .marked + } catch { + let markerError = CaptureUploader.describe(error) + do { + try FileManager.default.removeItem(at: CaptureUploader.stateURL(in: folder)) + return .savedStateRemoved + } catch let removal as NSError where removal.domain == NSCocoaErrorDomain && removal.code == NSFileNoSuchFileError { + return .savedStateRemoved + } catch { + return .notRecorded("marker: \(markerError); state removal: \(CaptureUploader.describe(error))") + } + } + } + public enum FilePhase: Codable, Sendable, Equatable { /// Sealed on the phone, not yet registered (or its URL must be fetched again). case queued diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift index 58625e433..6fbf75319 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -75,11 +75,26 @@ public actor CaptureUploader { private var failures = 0 private var retryingAt: Date? private var digestRetried: Set = [] - /// True once the homeowner took back their yes. Every save happens while holding it, so no - /// save can land between the withdrawal and the abandoned state it leads to. - private nonisolated let withdrawal = Mutex(false) - /// The running loop, reachable without the actor so a withdrawal can cancel it at once. - private nonisolated let running = Mutex?>(nil) + /// The registry covers public start/resume as well as coordinator discovery. An idle live + /// uploader still owns its folder, since it can receive more sealed files. Weak ownership + /// releases only after the uploader and its running task are gone. Withdrawn entries stay + /// for this process's lifetime, including when every disk write failed. + private struct FolderEntry { + weak var owner: CaptureUploader? + var withdrawn = false + var stopped = false + var running: Task? + } + private static let folders = Mutex<[String: FolderEntry]>([:]) + private nonisolated let folderKey: String + + public enum OwnershipError: Error, Sendable, Equatable { + case alreadyOwned + case consentWithdrawn + /// Use resume for an existing saved upload; start must never overwrite its consent. + case savedUploadExists + case folderIdentityUnavailable + } /// Set when a state change could not be saved: the loop stops rather than send a request the /// saved state doesn't know about. private var saveFailed = false @@ -88,11 +103,23 @@ public actor CaptureUploader { public nonisolated static func stateURL(in folder: URL) -> URL { folder.appending(path: CaptureUploadState.fileName) } + /// Resolve symbolic links before asking for the filesystem's case-correct path. The + /// canonical-path resource alone preserves a final-component symlink on macOS. + /// Refuse an unknown identity instead of allowing a second registry key for one folder. + private static func canonicalFolder(_ folder: URL) throws -> URL { + let fresh = URL(fileURLWithPath: folder.path, isDirectory: true).standardizedFileURL.resolvingSymlinksInPath() + guard let path = try fresh.resourceValues(forKeys: [.canonicalPathKey]).canonicalPath else { + throw OwnershipError.folderIdentityUnavailable + } + return URL(fileURLWithPath: path, isDirectory: true) + } + private init( folder: URL, base: URL, http: any CaptureHTTP, state: CaptureUploadState, policy: Policy, now: @escaping @Sendable () -> Date, sleep: @escaping @Sendable (Double) async throws -> Void ) { self.folder = folder + self.folderKey = folder.path self.base = base self.http = http self.state = state @@ -108,35 +135,54 @@ public actor CaptureUploader { now: @escaping @Sendable () -> Date = { Date() }, sleep: @escaping @Sendable (Double) async throws -> Void = { try await Task.sleep(for: .seconds($0)) } ) throws -> CaptureUploader { - let encoder = JSONEncoder() - encoder.outputFormatting = [.sortedKeys] - var state = CaptureUploadState(attemptID: UUID().uuidString, packetID: create.packetId, createBody: try encoder.encode(create)) - state.destination = base.absoluteString - state.consent = .init(grantedAt: consentedAt, destination: base.absoluteString) - state.marks["started"] = now() try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) - try state.save(to: stateURL(in: folder)) - return CaptureUploader(folder: folder, base: base, http: http, state: state, policy: policy, now: now, sleep: sleep) + let folder = try canonicalFolder(folder) + return try folders.withLock { entries in + entries = entries.filter { $0.value.owner != nil || $0.value.withdrawn } + guard entries[folder.path]?.withdrawn != true, + !FileManager.default.fileExists(atPath: CaptureUploadState.withdrawnURL(in: folder).path) + else { throw OwnershipError.consentWithdrawn } + guard entries[folder.path]?.owner == nil else { throw OwnershipError.alreadyOwned } + guard !FileManager.default.fileExists(atPath: stateURL(in: folder).path) else { throw OwnershipError.savedUploadExists } + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + var state = CaptureUploadState(attemptID: UUID().uuidString, packetID: create.packetId, createBody: try encoder.encode(create)) + state.destination = base.absoluteString + state.consent = .init(grantedAt: consentedAt, destination: base.absoluteString) + state.marks["started"] = now() + try state.save(to: stateURL(in: folder)) + let uploader = CaptureUploader(folder: folder, base: base, http: http, state: state, policy: policy, now: now, sleep: sleep) + entries[folder.path] = FolderEntry(owner: uploader) + return uploader + } } - /// The capture saved in `folder`, as a relaunch finds it; nil when there is none, when the - /// homeowner withdrew their yes, when it was created on another API than `base`, or when it - /// holds no yes for `base`. The resumed upload - /// gets a new attempt id, so nothing from the previous process can land in it. + /// Resumes a saved upload with its recorded consent and endpoint. Returns nil when absent, + /// ended or withdrawn; throws `alreadyOwned` while another live uploader owns the folder. + /// The new attempt id and ownership are saved together, before any request can start. public static func resume( folder: URL, base: URL, http: any CaptureHTTP, policy: Policy = .init(), now: @escaping @Sendable () -> Date = { Date() }, sleep: @escaping @Sendable (Double) async throws -> Void = { try await Task.sleep(for: .seconds($0)) } ) throws -> CaptureUploader? { - let url = stateURL(in: folder) - guard FileManager.default.fileExists(atPath: url.path), - !FileManager.default.fileExists(atPath: CaptureUploadState.withdrawnURL(in: folder).path) - else { return nil } - var state = try CaptureUploadState.load(from: url) - guard state.destination == base.absoluteString, state.consent?.destination == base.absoluteString else { return nil } - state.attemptID = UUID().uuidString - try state.save(to: url) - return CaptureUploader(folder: folder, base: base, http: http, state: state, policy: policy, now: now, sleep: sleep) + guard FileManager.default.fileExists(atPath: folder.path) else { return nil } + let folder = try canonicalFolder(folder) + return try folders.withLock { entries in + entries = entries.filter { $0.value.owner != nil || $0.value.withdrawn } + let url = stateURL(in: folder) + guard entries[folder.path]?.withdrawn != true, + FileManager.default.fileExists(atPath: url.path), + !FileManager.default.fileExists(atPath: CaptureUploadState.withdrawnURL(in: folder).path) + else { return nil } + guard entries[folder.path]?.owner == nil else { throw OwnershipError.alreadyOwned } + var state = try CaptureUploadState.load(from: url) + guard state.end == nil, state.destination == base.absoluteString, state.consent?.destination == base.absoluteString else { return nil } + state.attemptID = UUID().uuidString + try state.save(to: url) + let uploader = CaptureUploader(folder: folder, base: base, http: http, state: state, policy: policy, now: now, sleep: sleep) + entries[folder.path] = FolderEntry(owner: uploader) + return uploader + } } public func observe(_ observer: @escaping @Sendable (CaptureUploadStatus) -> Void, log: (@Sendable (String) -> Void)? = nil) { @@ -196,50 +242,52 @@ public actor CaptureUploader { case notRecorded(String) } - /// The homeowner took back their yes. Before returning, this records the withdrawal on disk - /// (see `WithdrawalRecord`) and cancels the running loop, which cancels requests it has not - /// finished. From then on no request is admitted and every save of this uploader records the - /// upload as abandoned, so a reply that still arrives can't leave a capture a relaunch would - /// resume. Call `abandon` afterwards to end the upload in memory. + /// Revokes the folder before returning and cancels its running task. A request already + /// admitted to a transport that ignores cancellation may finish; its reply cannot admit + /// another request or save resumable consent. `.notRecorded` means only the live process + /// is protected: after a crash the old saved yes may still exist. @discardableResult public nonisolated func withdrawConsent() -> WithdrawalRecord { - let record = withdrawal.withLock { withdrawn -> WithdrawalRecord in - withdrawn = true - do { - try Data().write(to: CaptureUploadState.withdrawnURL(in: folder), options: .atomic) - return .marked - } catch { - do { - try FileManager.default.removeItem(at: Self.stateURL(in: folder)) - return .savedStateRemoved - } catch let removal as NSError where removal.domain == NSCocoaErrorDomain && removal.code == NSFileNoSuchFileError { - return .savedStateRemoved - } catch { - return .notRecorded(Self.describe(error)) - } - } + let (record, task) = Self.folders.withLock { entries in + entries[folderKey, default: FolderEntry()].withdrawn = true + let record = CaptureUploadState.recordWithdrawal(in: folder) + return (record, entries[folderKey]?.running) } - running.withLock { $0?.cancel() } + // Cancellation handlers belong to the transport; never call them while holding our lock. + task?.cancel() return record } + /// Closes admission synchronously when a coordinator ends a world, before its queued actor + /// work can kick the upload. Ownership stays until the old uploader and its requests die. + nonisolated func stopSending() { + let task = Self.folders.withLock { entries in + entries[folderKey]?.stopped = true + return entries[folderKey]?.running + } + task?.cancel() + } + /// Ends this upload for good: a world reset or starting over. Replies still in flight are /// dropped when they arrive. public func abandon(_ reason: String) { + stopSending() guard state.end == nil else { return } state.end = .abandoned(reason) state.attemptID = UUID().uuidString loop?.cancel() - loop = nil persist() } /// Starts the loop if it is idle: after a relaunch, or when the network is back. public func kick() { - guard loop == nil, state.end == nil, !isWithdrawn else { return } - let task = Task { await self.drive() } - loop = task - running.withLock { $0 = task } + guard loop == nil, state.end == nil, !saveFailed else { return } + Self.folders.withLock { entries in + guard let entry = entries[folderKey], entry.owner === self, !entry.withdrawn, !entry.stopped else { return } + let task = Task { await self.drive() } + loop = task + entries[folderKey]?.running = task + } } /// Waits for the loop to go idle: nothing to send until more input, or ended. @@ -281,6 +329,7 @@ public actor CaptureUploader { } } loop = nil + Self.folders.withLock { $0[folderKey]?.running = nil } } /// One unit of work. False when there is nothing to do until more input arrives. @@ -386,7 +435,7 @@ public actor CaptureUploader { var index = 0 while index < files.count { // Each batch of PUTs is admitted only while the homeowner's yes stands. - if isWithdrawn { throw Withdrawn() } + if !canSend || Task.isCancelled { throw Withdrawn() } let chunk = files[index.. (HTTPReply, String) { // Every API request is admitted only while the homeowner's yes stands. One already sent // finishes, but its reply can only save the upload as abandoned. - if isWithdrawn { throw Withdrawn() } + if !canSend || Task.isCancelled { throw Withdrawn() } let attempt = state.attemptID let reply: HTTPReply do { @@ -633,11 +682,20 @@ public actor CaptureUploader { log?("capture-upload stage=\(name) t=\(String(format: "%.2f", since))s committed=\(state.committedCount)/\(state.files.count)") } - private nonisolated var isWithdrawn: Bool { withdrawal.withLock { $0 } } + private nonisolated var isWithdrawn: Bool { Self.folders.withLock { $0[folderKey]?.withdrawn == true } } + + private nonisolated var canSend: Bool { + Self.folders.withLock { entries in + guard let entry = entries[folderKey] else { return false } + return entry.owner === self && !entry.withdrawn && !entry.stopped + } + } private func persist() { do { - try withdrawal.withLock { withdrawn in + try Self.folders.withLock { entries in + guard entries[folderKey]?.owner === self else { throw OwnershipError.alreadyOwned } + let withdrawn = entries[folderKey]?.withdrawn == true // A withdrawal outranks any end a late reply set (finished, failed). if withdrawn, state.end != .abandoned(Self.withdrawnReason) { state.end = .abandoned(Self.withdrawnReason) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureFolderOwnershipTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureFolderOwnershipTests.swift new file mode 100644 index 000000000..aec7ef934 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureFolderOwnershipTests.swift @@ -0,0 +1,224 @@ +import Foundation +import HouseScanKit +import Synchronization +import Testing + +@Suite(.serialized) struct CaptureFolderOwnershipTests { + private func create(_ folder: URL, _ base: URL) throws -> CaptureUploader { + try CaptureUploader.start( + folder: folder, base: base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), + create: .init(packetId: "ownership-test", tier: .arkit, device: .init(model: "test", systemVersion: "test", appVersion: "test")), + consentedAt: Date()) + } + + @Test func concurrentStartsSaveOnlyTheOwnersAttempt() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "owners-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let owners = await withTaskGroup(of: CaptureUploader?.self, returning: [CaptureUploader].self) { group in + for _ in 0..<12 { + group.addTask { + do { return try create(root, server.base) } + catch { #expect(error as? CaptureUploader.OwnershipError == .alreadyOwned); return nil } + } + } + var owners: [CaptureUploader] = [] + for await owner in group { if let owner { owners.append(owner) } } + return owners + } + try #require(owners.count == 1) + let before = await owners[0].snapshot + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: root)) == before) + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned) { + try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + } + await owners[0].kick() + try #require(await CaptureUploaderTests.settles(owners[0])) + #expect(server.requests("POST captures").count == 1) + } + + @Test func concurrentResumesSaveOnlyTheOwnersAttempt() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "resume-owners-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + var first: CaptureUploader? = try create(root, server.base) + weak var released = first + first = nil + try #require(released == nil) + let owners = await withTaskGroup(of: CaptureUploader?.self, returning: [CaptureUploader].self) { group in + for _ in 0..<12 { + group.addTask { + do { return try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) } + catch { #expect(error as? CaptureUploader.OwnershipError == .alreadyOwned); return nil } + } + } + var owners: [CaptureUploader] = [] + for await owner in group { if let owner { owners.append(owner) } } + return owners + } + try #require(owners.count == 1) + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: root)) == (await owners[0].snapshot)) + await owners[0].kick() + try #require(await CaptureUploaderTests.settles(owners[0])) + #expect(server.requests("POST captures").count == 1) + } + + @Test func aliasesCannotAcquireTheSameFolder() throws { + let root = FileManager.default.temporaryDirectory.appending(path: "alias-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let folder = root.appending(path: "Capture") + let owner = try create(folder, server.base) + let link = root.appending(path: "link") + try FileManager.default.createSymbolicLink(at: link, withDestinationURL: folder) + var aliases = [link, root.appending(path: "Capture/../Capture")] + let lowercased = root.appending(path: "capture") + if FileManager.default.fileExists(atPath: lowercased.path) { aliases.append(lowercased) } + for alias in aliases { + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned, "alias: \(alias.path)") { try create(alias, server.base) } + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned, "alias: \(alias.path)") { + try CaptureUploader.resume(folder: alias, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + } + } + #expect(owner.folder.path == (try folder.resourceValues(forKeys: [.canonicalPathKey]).canonicalPath)) + #expect(server.state.withLock { $0.log.isEmpty }) + } + + @Test func idleOwnershipReleasesOnlyAfterTheUploaderDies() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "lifetime-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + var owner: CaptureUploader? = try create(root, server.base) + await owner?.kick() + try #require(await CaptureUploaderTests.settles(try #require(owner))) + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned) { + try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + } + weak var released = owner + owner = nil + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + #expect(throws: CaptureUploader.OwnershipError.savedUploadExists) { try create(root, server.base) } + let resumed = try #require(try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10))) + await resumed.kick() + try #require(await CaptureUploaderTests.settles(resumed)) + #expect(server.requests("POST captures").count == 1) + } + + @Test func anInFlightRequestKeepsOwnershipAfterTheCallerReleasesIt() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "inflight-owner-\(UUID().uuidString)") + let server = try LoopbackCaptureAPI() + defer { + server.state.withLock { $0.held = [] } + server.release("POST captures") + try? FileManager.default.removeItem(at: root) + } + server.state.withLock { $0.held = ["POST captures"] } + var owner: CaptureUploader? = try create(root, server.base) + await owner?.kick() + for _ in 0..<500 where server.state.withLock({ $0.parked["POST captures"]?.isEmpty ?? true }) { + try await Task.sleep(for: .milliseconds(10)) + } + try #require(server.state.withLock { !($0.parked["POST captures"]?.isEmpty ?? true) }) + weak var runningOwner = owner + owner = nil + #expect(runningOwner != nil) + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned) { + try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + } + server.state.withLock { $0.held = [] } + server.release("POST captures") + for _ in 0..<500 where runningOwner != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(runningOwner == nil) + let resumed = try #require(try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10))) + await resumed.kick() + try #require(await CaptureUploaderTests.settles(resumed)) + #expect(server.requests("POST captures").count == 1) + } + + /// Failure injection uses permissions only on this test's temporary folder. Restoring the + /// stale saved yes after stopping models the disk contents a failed withdrawal can leave. + @Test(arguments: [0, 1, 2]) func withdrawalBlocksReuseEvenWhenDiskStillHasConsent(failedWrites: Int) async throws { + var rig = try CaptureUploaderTests.Rig(http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10))) + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: rig.capture.folder.path) + rig.cleanUp() + } + rig.server.state.withLock { $0.held = ["POST captures/files"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + for _ in 0..<500 where rig.server.state.withLock({ $0.parked["POST captures/files"]?.isEmpty ?? true }) { + try await Task.sleep(for: .milliseconds(10)) + } + try #require(rig.server.state.withLock { !($0.parked["POST captures/files"]?.isEmpty ?? true) }) + let stateURL = CaptureUploader.stateURL(in: rig.capture.folder) + let staleConsent = try Data(contentsOf: stateURL) + let marker = CaptureUploadState.withdrawnURL(in: rig.capture.folder) + if failedWrites == 1 { + try FileManager.default.createDirectory(at: marker.appending(path: "blocked"), withIntermediateDirectories: true) + } else if failedWrites == 2 { + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: rig.capture.folder.path) + } + let result = rig.uploader.withdrawConsent() + switch failedWrites { + case 0: #expect(result == .marked) + case 1: #expect(result == .savedStateRemoved) + default: + guard case .notRecorded = result else { Issue.record("Both filesystem writes must fail"); return } + #expect(try Data(contentsOf: stateURL) == staleConsent) + } + rig.server.state.withLock { $0.held = [] } + rig.server.release("POST captures/files") + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(rig.server.requests("PUT upload").isEmpty) + #expect(rig.server.requests("POST captures/files:commit").isEmpty) + #expect(await rig.uploader.snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: rig.capture.folder.path) + if FileManager.default.fileExists(atPath: marker.path) { try FileManager.default.removeItem(at: marker) } + try staleConsent.write(to: stateURL, options: .atomic) + weak var released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + #expect(try CaptureUploader.resume(folder: rig.capture.folder, base: rig.server.base, http: rig.http) == nil) + #expect(throws: CaptureUploader.OwnershipError.consentWithdrawn) { try create(rig.capture.folder, rig.server.base) } + #expect(try Data(contentsOf: stateURL) == staleConsent) + } + + @Test(arguments: [false, true]) func withdrawalDuringPutsStopsTheNextBatchAndCommit(uncancellable: Bool) async throws { + let transport = URLSessionCaptureHTTP.ephemeral(timeout: 10) + let http: any CaptureHTTP = uncancellable ? UncancellableHTTP(transport) : transport + var policy = CaptureUploaderTests.Rig.fast + policy.maxConcurrentPuts = 2 + let rig = try CaptureUploaderTests.Rig(http: http, policy: policy) + defer { + rig.server.state.withLock { $0.held = [] } + rig.server.release("PUT upload") + rig.cleanUp() + } + rig.server.state.withLock { $0.held = ["PUT upload"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 3)) + for _ in 0..<500 where rig.server.state.withLock({ ($0.parked["PUT upload"]?.count ?? 0) < 2 }) { + try await Task.sleep(for: .milliseconds(10)) + } + try #require(rig.server.state.withLock { ($0.parked["PUT upload"]?.count ?? 0) == 2 }) + #expect(rig.uploader.withdrawConsent() == .marked) + await rig.uploader.kick() + rig.server.state.withLock { $0.held = [] } + rig.server.release("PUT upload") + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(rig.server.requests("PUT upload").count == 2) + #expect(rig.server.requests("POST captures/files:commit").isEmpty) + #expect(await rig.uploader.snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) + } + + @Test func withdrawalBeforeConcurrentKicksAdmitsNoRequests() async throws { + let rig = try CaptureUploaderTests.Rig() + defer { rig.cleanUp() } + #expect(rig.uploader.withdrawConsent() == .marked) + await withTaskGroup(of: Void.self) { group in + for _ in 0..<20 { group.addTask { await rig.uploader.kick() } } + } + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(rig.server.state.withLock { $0.log.isEmpty }) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift index fdbd84713..7657f425e 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift @@ -464,6 +464,110 @@ struct NativeCaptureFixture: Sendable { #expect(await uploader.snapshot.end == .abandoned("consent withdrawn")) } + /// A relaunch scan in the live process must not create a second sender for the sealed + /// folder. On the old code B survives A's withdrawal and sends the remaining files. + @Test func withdrawalCoversTheFolderDuringResumeDiscovery() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/finalize"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + for i in 0..<3 { coordinator.kept(try fixture.photo(at: fixture.start + 3 + Double(i) * 1.5)) } + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + let uploader = try #require(coordinator.session?.uploader) + try await waitUntilParked(server, "POST captures/finalize") + + let duplicates = CaptureSessionCoordinator(environment: environment).resumeSealedCaptures() + for duplicate in duplicates { await duplicate.kick() } + if !duplicates.isEmpty { + for _ in 0..<500 where server.requests("POST captures/finalize").count < 2 { + try await Task.sleep(for: .milliseconds(10)) + } + try #require(server.requests("POST captures/finalize").count == 2) + } + #expect(duplicates.isEmpty) + coordinator.answerConsent(false) + let puts = server.requests("PUT upload").count + let commits = server.requests("POST captures/files:commit").count + server.state.withLock { $0.held = [] } + server.release("POST captures/finalize") + try #require(await CaptureUploaderTests.settles(uploader)) + for duplicate in duplicates { try #require(await CaptureUploaderTests.settles(duplicate)) } + #expect(server.requests("PUT upload").count == puts) + #expect(server.requests("POST captures/files:commit").count == commits) + #expect(server.requests("GET captures/result").isEmpty) + #expect(CaptureSessionCoordinator(environment: environment).resumeSealedCaptures().isEmpty) + } + + @Test func droppingAnIdleCoordinatorReleasesFolderOwnership() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let environment = NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures) + var coordinator: CaptureSessionCoordinator? = CaptureSessionCoordinator(environment: environment) + coordinator?.begin(recording: fixture.recording) + coordinator?.answerConsent(true) + coordinator?.kept(try fixture.photo(at: fixture.start + 2)) + await coordinator?.settle() + let folder = try #require(coordinator?.session?.folder) + weak var oldSession = coordinator?.session + weak var oldUploader = coordinator?.session?.uploader + coordinator = nil + for _ in 0..<500 where oldSession != nil || oldUploader != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(oldSession == nil && oldUploader == nil) + let resumed = try #require(try CaptureUploader.resume(folder: folder, base: server.base, http: environment.http)) + await resumed.kick() + try #require(await CaptureUploaderTests.settles(resumed)) + #expect(server.requests("POST captures").count == 1) + } + + @Test func anUnsavedWithdrawalIsReturnedToTheCaller() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/files"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + try await waitUntilParked(server, "POST captures/files") + let stateURL = CaptureUploader.stateURL(in: session.folder) + let before = try Data(contentsOf: stateURL) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: session.folder.path) + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: session.folder.path) } + + let result = coordinator.answerConsent(false) + guard case .failure(.notRecorded(let detail)) = result else { + Issue.record("The caller must receive the persistence failure") + server.state.withLock { $0.held = [] } + server.release("POST captures/files") + return + } + #expect(detail.contains("marker:") && detail.contains("state removal:")) + #expect(try Data(contentsOf: stateURL) == before) + // Repeated answers must not turn the failed withdrawal into a reported success. + if case .success = coordinator.answerConsent(false) { Issue.record("Repeated no lost the failure") } + if case .success = coordinator.answerConsent(true) { Issue.record("Later yes lost the failure") } + #expect(session.uploader == nil) + #expect(try CaptureUploader.resume(folder: session.folder, base: server.base, http: environment.http) == nil) + #expect(CaptureSessionCoordinator(environment: environment).resumeSealedCaptures().isEmpty) + server.state.withLock { $0.held = [] } + server.release("POST captures/files") + try #require(await CaptureUploaderTests.settles(uploader)) + #expect(server.requests("PUT upload").isEmpty) + #expect(server.requests("POST captures/files:commit").isEmpty) + // All writes still fail, so the app cannot promise the persisted yes was removed. + #expect(try Data(contentsOf: stateURL) == before) + } + /// Waits, suspending, until the loopback has parked an answer for `route`. func waitUntilParked(_ server: LoopbackCaptureAPI, _ route: String) async throws { for _ in 0..<1000 where server.state.withLock({ $0.parked[route]?.isEmpty ?? true }) { try await Task.sleep(for: .milliseconds(10)) } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift index a956552af..464ced87c 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift @@ -10,7 +10,13 @@ import Testing let server: LoopbackCaptureAPI let root: URL let capture: SyntheticCapture04 - let uploader: CaptureUploader + private var ownedUploader: CaptureUploader? + var uploader: CaptureUploader { + guard let ownedUploader else { preconditionFailure("The test released this uploader to simulate process exit") } + return ownedUploader + } + + mutating func releaseUploader() { ownedUploader = nil } let http: any CaptureHTTP init( @@ -23,7 +29,7 @@ import Testing self.server = try server ?? LoopbackCaptureAPI() root = FileManager.default.temporaryDirectory.appending(path: "uploader-\(UUID().uuidString)") capture = try SyntheticCapture04(folder: root.appending(path: "packet"), packetID: packetID) - uploader = try CaptureUploader.start( + ownedUploader = try CaptureUploader.start( folder: capture.folder, base: self.server.base, http: http, create: .init(packetId: packetID, tier: .arkit, device: .init(model: "iPhone15,4", systemVersion: "26.0", appVersion: appVersion)), consentedAt: Date(), policy: policy, now: now, sleep: sleep) @@ -101,12 +107,17 @@ import Testing /// A relaunch resumes the saved capture: same packet and capture ids, no second create, and the /// old process's attempt id is replaced. @Test func relaunchResumesTheSameCapture() async throws { - let rig = try Rig() + var rig = try Rig() defer { rig.cleanUp() } await rig.uploader.add(try await rig.capture.sealImages(count: 2)) await rig.uploader.settled() let before = await rig.uploader.snapshot + // The old process releases every uploader before the new one resumes its saved state. + weak var released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) let resumed = try #require(try CaptureUploader.resume( folder: rig.capture.folder, base: rig.server.base, http: rig.http, policy: Rig.fast, sleep: { _ in })) let after = await resumed.snapshot @@ -417,7 +428,7 @@ import Testing policy.maxNotReadyResults = 3 let waits = Mutex(0) // The first process quits during its second wait. - let rig = try Rig(policy: policy, sleep: { _ in + var rig = try Rig(policy: policy, sleep: { _ in let n = waits.withLock { $0 += 1; return $0 } if n >= 2 { throw CancellationError() } }) @@ -429,6 +440,11 @@ import Testing #expect(rig.server.requests("GET captures/result").count == 2) #expect(await rig.uploader.snapshot.end == nil) + // The old process releases every uploader before the new one resumes its saved state. + weak var released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) let resumed = try #require(try CaptureUploader.resume( folder: rig.capture.folder, base: rig.server.base, http: rig.http, policy: policy, sleep: { _ in })) await resumed.kick() @@ -485,7 +501,7 @@ import Testing policy.maxNotReadyResults = 3 let waits = Mutex(0) // The first process quits during its third wait, after three not-ready reads. - let rig = try Rig(policy: policy, sleep: { _ in + var rig = try Rig(policy: policy, sleep: { _ in let n = waits.withLock { $0 += 1; return $0 } if n >= 3 { throw CancellationError() } }) @@ -499,6 +515,11 @@ import Testing // The server forgets every capture (a redeploy with in-memory state). The next run has // one not-ready read before its answer. rig.server.state.withLock { $0.captures = [:]; $0.byPacket = [:]; $0.outcomelessResults = 1 } + // The old process releases every uploader before the new one resumes its saved state. + weak var released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) let resumed = try #require(try CaptureUploader.resume( folder: rig.capture.folder, base: rig.server.base, http: rig.http, policy: policy, sleep: { _ in })) await resumed.kick() From c46457711e9d56ae02c8d0ffa58a5b9267f7cb99 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Fri, 2 Oct 2026 14:04:56 -0500 Subject: [PATCH 16/16] ios tests: declare never-reassigned weak test references with weak let CI builds the package tests with -warnings-as-errors (ios.yml:137). Swift 6.3 warns when a weak var is never reassigned, so hosted run 37050869900 failed to compile ff8c0ed7's tests at nine such declarations: the ownership, coordinator and uploader tests that check an uploader or session is released. Local runs passed because they didn't use the flag. The nine become weak let, which Swift has accepted since 6.2. Each is only read, to check that the object was released, so the tests mean the same. No source code changes. Co-Authored-By: Claude Code --- .../HouseScanKitTests/CaptureFolderOwnershipTests.swift | 8 ++++---- .../CaptureSessionCoordinatorTests.swift | 4 ++-- .../Tests/HouseScanKitTests/CaptureUploaderTests.swift | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureFolderOwnershipTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureFolderOwnershipTests.swift index aec7ef934..ea16aeffa 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureFolderOwnershipTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureFolderOwnershipTests.swift @@ -42,7 +42,7 @@ import Testing defer { try? FileManager.default.removeItem(at: root) } let server = try LoopbackCaptureAPI() var first: CaptureUploader? = try create(root, server.base) - weak var released = first + weak let released = first first = nil try #require(released == nil) let owners = await withTaskGroup(of: CaptureUploader?.self, returning: [CaptureUploader].self) { group in @@ -94,7 +94,7 @@ import Testing #expect(throws: CaptureUploader.OwnershipError.alreadyOwned) { try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) } - weak var released = owner + weak let released = owner owner = nil for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } try #require(released == nil) @@ -120,7 +120,7 @@ import Testing try await Task.sleep(for: .milliseconds(10)) } try #require(server.state.withLock { !($0.parked["POST captures"]?.isEmpty ?? true) }) - weak var runningOwner = owner + weak let runningOwner = owner owner = nil #expect(runningOwner != nil) #expect(throws: CaptureUploader.OwnershipError.alreadyOwned) { @@ -175,7 +175,7 @@ import Testing try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: rig.capture.folder.path) if FileManager.default.fileExists(atPath: marker.path) { try FileManager.default.removeItem(at: marker) } try staleConsent.write(to: stateURL, options: .atomic) - weak var released = rig.uploader + weak let released = rig.uploader rig.releaseUploader() for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } try #require(released == nil) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift index 7657f425e..1e1bfb85a 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift @@ -515,8 +515,8 @@ struct NativeCaptureFixture: Sendable { coordinator?.kept(try fixture.photo(at: fixture.start + 2)) await coordinator?.settle() let folder = try #require(coordinator?.session?.folder) - weak var oldSession = coordinator?.session - weak var oldUploader = coordinator?.session?.uploader + weak let oldSession = coordinator?.session + weak let oldUploader = coordinator?.session?.uploader coordinator = nil for _ in 0..<500 where oldSession != nil || oldUploader != nil { try await Task.sleep(for: .milliseconds(10)) } try #require(oldSession == nil && oldUploader == nil) diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift index 464ced87c..664d5564b 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift @@ -114,7 +114,7 @@ import Testing let before = await rig.uploader.snapshot // The old process releases every uploader before the new one resumes its saved state. - weak var released = rig.uploader + weak let released = rig.uploader rig.releaseUploader() for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } try #require(released == nil) @@ -441,7 +441,7 @@ import Testing #expect(await rig.uploader.snapshot.end == nil) // The old process releases every uploader before the new one resumes its saved state. - weak var released = rig.uploader + weak let released = rig.uploader rig.releaseUploader() for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } try #require(released == nil) @@ -516,7 +516,7 @@ import Testing // one not-ready read before its answer. rig.server.state.withLock { $0.captures = [:]; $0.byPacket = [:]; $0.outcomelessResults = 1 } // The old process releases every uploader before the new one resumes its saved state. - weak var released = rig.uploader + weak let released = rig.uploader rig.releaseUploader() for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } try #require(released == nil)