diff --git a/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResult.swift b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResult.swift new file mode 100644 index 000000000..e04617f41 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResult.swift @@ -0,0 +1,166 @@ +import Foundation + +/// The capture API's device result, `GET /v1/captures/{id}/result`, written from the server's +/// documented wire contract. The whole body is camelCase. +/// +/// Only the fields the app shows or gates on are decoded. The verdict and its URL are left out on +/// purpose: the server masks a reject as manual review for the device, and the app must not read +/// around that mask. Unknown keys are ignored, because the server serves a result even when it has +/// drifted from its own schema. A status or outcome kind the app doesn't know decodes as +/// `.unknown`, so a new server value never hides the message and views that came with it. +public enum CaptureResult { + public struct Response: Decodable, Sendable, Equatable { + /// Nil before the server has started a run for the capture. + public var runId: String? + public var status: Status + public var viewsNeeded: [ViewNeeded] + public var memberActions: [String] + /// Nil until the run writes its result: while uploading, awaiting files or processing. + public var outcome: Outcome? + /// Nil when the server sent no criteria. The app shows none rather than inventing rows. + public var criteria: [Criterion]? + /// A signed URL that expires; nil when the run made no preview. + public var previewUrl: String? + /// A path on the API, not a full URL. + public var reviewUrl: String? + } + + public enum Status: Sendable, Equatable, Decodable { + case uploading, awaitingFiles, processing, needsViews, complete, manualReview, failed, expired + case unknown(String) + + public init(rawValue: String) { + self = switch rawValue { + case "uploading": .uploading + case "awaiting_files": .awaitingFiles + case "processing": .processing + case "needs_views": .needsViews + case "complete": .complete + case "manual_review": .manualReview + case "failed": .failed + case "expired": .expired + default: .unknown(rawValue) + } + } + + public init(from decoder: any Decoder) throws { + self.init(rawValue: try decoder.singleValueContainer().decode(String.self)) + } + } + + /// The decision shown to the homeowner. It is already masked for the device. + public struct Outcome: Decodable, Sendable, Equatable { + public var kind: OutcomeKind + public var profile: String + /// Text for the homeowner, shown as written. + public var message: String + public var viewsNeeded: [ViewNeeded] + /// Empty when the server shows a reject as manual review. + public var reasons: [Reason] + /// The ARKit world the spatial fields are in. Nil when the scene was not built in one. + public var arkitEpoch: String? + public var recommendedPlacement: RecommendedPlacement? + } + + public enum OutcomeKind: Sendable, Equatable, Decodable { + case eligible, needsMorePhotos, notEligible, manualReview + case unknown(String) + + public init(rawValue: String) { + self = switch rawValue { + case "eligible": .eligible + case "needs_more_photos": .needsMorePhotos + case "not_eligible": .notEligible + case "manual_review": .manualReview + default: .unknown(rawValue) + } + } + + public init(from decoder: any Decoder) throws { + self.init(rawValue: try decoder.singleValueContainer().decode(String.self)) + } + } + + public struct RecommendedPlacement: Decodable, Sendable, Equatable { + public var wallId: String + public var startSM: Double + public var confidence: Double + /// Nil when the scene was not built in an ARKit world. + public var boxArkitWorld: BoxArkitWorld? + } + + /// The box in ARKit world coordinates, as sent. The numbers are checked only when the app asks + /// for an AR placement (`Record.placement`), so a malformed box withholds AR and leaves the + /// rest of the result readable. + public struct BoxArkitWorld: Decodable, Sendable, Equatable { + /// Box centre to world, 16 numbers column by column. Local +x runs along the wall (left to + /// right seen from outside), +y up, +z out of the wall. + public var pose: [Double] + /// Width, height and depth in ARKit metres. + public var size: [Double] + } + + public struct ViewNeeded: Decodable, Sendable, Equatable { + public var id: String + /// The server's view kind, kept as text so a new kind still shows its prompt. + public var kind: String + public var criteria: [String] + public var why: String + public var promptId: String + /// Rendered text; show it verbatim. + public var prompt: Prompt + public var band: String? + public var wallId: String? + public var target: String? + /// Set in the photo flow instead of ARKit anchors. + public var referenceCrop: ReferenceCrop? + } + + public struct Prompt: Decodable, Sendable, Equatable { + public var title: String + public var body: String + } + + public struct ReferenceCrop: Decodable, Sendable, Equatable { + public var image: String + /// Pixel box in `image`: x0, y0, x1, y1. + public var boxPx: [Double] + } + + public struct Reason: Decodable, Sendable, Equatable { + public var criterion: String + public var text: String + public var measuredFt: Double? + public var plusMinusFt: Double? + public var thresholdFt: Double? + public var comparison: String? + } + + public struct Criterion: Decodable, Sendable, Equatable { + public var id: String + public var outcome: CriterionOutcome + public var unsureCause: String? + public var measuredFt: Double? + public var plusMinusFt: Double? + public var thresholdFt: Double? + public var coverage: String? + } + + public enum CriterionOutcome: Sendable, Equatable, Decodable { + case pass, fail, unsure + case unknown(String) + + public init(rawValue: String) { + self = switch rawValue { + case "pass": .pass + case "fail": .fail + case "unsure": .unsure + default: .unknown(rawValue) + } + } + + public init(from decoder: any Decoder) throws { + self.init(rawValue: try decoder.singleValueContainer().decode(String.self)) + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultPlacement.swift b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultPlacement.swift new file mode 100644 index 000000000..118c246b2 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultPlacement.swift @@ -0,0 +1,182 @@ +import simd + +extension CaptureResult { + /// The local identities a result belongs to. The response carries a run id and an ARKit epoch + /// but not the session or capture, and epochs repeat ("e1" starts every session), so only the + /// caller can say which scan a result is for. + public struct Association: Sendable, Equatable, Hashable { + /// The app's id for one run of the ARKit session. A new session gets a new id, even when it + /// reuses the epoch string. + public var sessionID: String + public var captureID: String + /// The run that finalize started for the capture. + public var runID: String + /// The ARKit epoch the packet was captured in. + public var epoch: String + + public init(sessionID: String, captureID: String, runID: String, epoch: String) { + self.sessionID = sessionID + self.captureID = captureID + self.runID = runID + self.epoch = epoch + } + } + + /// Whether the caller knows the server ran real analysis for this result. A stubbed server + /// returns a result that passes the contract without reconstructing anything, and nothing in + /// the JSON tells the two apart, so only the caller can grant `.verified`. + public enum Analysis: Sendable, Equatable { + case unverified + case verified + } + + /// A decoded result kept with the local scan it was fetched for. + public struct Record: Sendable, Equatable { + public let response: Response + public let association: Association + public let analysis: Analysis + + public init(response: Response, association: Association, analysis: Analysis = .unverified) { + self.response = response + self.association = association + self.analysis = analysis + } + + /// The recommended box in the meter anchor's frame, or the reason there is none. + /// + /// - Parameters: + /// - world: the session, capture, finalized run and epoch on screen now. + /// - meterAnchor: the meter anchor's anchor-to-world pose, in the same world coordinates + /// as the uploaded packet. Attaching the returned box to the anchor then carries it + /// through ARKit's later corrections. + public func placement(in world: Association, meterAnchor: simd_float4x4) -> Placement { + guard let outcome = response.outcome else { return .unavailable(.noOutcome(response.status)) } + guard outcome.kind == .eligible else { return .unavailable(.notEligible(outcome.kind)) } + guard response.status == .complete else { return .unavailable(.statusNotComplete(response.status)) } + guard analysis == .verified else { return .unavailable(.analysisUnverified) } + + guard world.sessionID == association.sessionID else { return .unavailable(.mismatch(.session)) } + guard world.captureID == association.captureID else { return .unavailable(.mismatch(.capture)) } + guard world.runID == association.runID, response.runId == association.runID else { + return .unavailable(.mismatch(.run)) + } + guard let epoch = outcome.arkitEpoch, + let box = outcome.recommendedPlacement?.boxArkitWorld + else { + return .unavailable(outcome.recommendedPlacement == nil ? .noPlacement : .notInARKitWorld) + } + guard world.epoch == association.epoch, epoch == association.epoch else { + return .unavailable(.mismatch(.epoch)) + } + + guard box.pose.count == 16 else { return .unavailable(.malformedBox(.poseCount(box.pose.count))) } + guard box.size.count == 3 else { return .unavailable(.malformedBox(.sizeCount(box.size.count))) } + guard (box.pose + box.size).allSatisfy(\.isFinite) else { return .unavailable(.malformedBox(.nonFinite)) } + guard box.size.allSatisfy({ $0 > 0 }) else { return .unavailable(.malformedBox(.nonPositiveSize)) } + let worldFromBox = Self.matrix(columnMajor: box.pose) + // The server divides the world transform's scale out of the pose's rotation and + // multiplies it into `size` instead. A pose that still carries a scale was built some + // other way, and using it would scale the box a second time. + guard Self.isRigid(worldFromBox) else { return .unavailable(.malformedBox(.notRigid)) } + + let worldFromAnchor = Self.double(meterAnchor) + guard Self.isRigid(worldFromAnchor) else { return .unavailable(.invalidMeterAnchor) } + + let anchorFromBox = Self.float(worldFromAnchor.inverse * worldFromBox) + let size = SIMD3(Float(box.size[0]), Float(box.size[1]), Float(box.size[2])) + // Finite Doubles can overflow Float, and positive sizes can round to zero. + let columns = [anchorFromBox.columns.0, anchorFromBox.columns.1, anchorFromBox.columns.2, anchorFromBox.columns.3] + guard columns.allSatisfy({ c in (0..<4).allSatisfy { c[$0].isFinite } }), + (0..<3).allSatisfy({ size[$0].isFinite }) + else { return .unavailable(.malformedBox(.nonFinite)) } + guard (0..<3).allSatisfy({ size[$0] > 0 }) else { return .unavailable(.malformedBox(.nonPositiveSize)) } + return .box(AnchoredBox(anchorFromBox: anchorFromBox, size: size)) + } + + private static func matrix(columnMajor v: [Double]) -> simd_double4x4 { + simd_double4x4( + SIMD4(v[0], v[1], v[2], v[3]), + SIMD4(v[4], v[5], v[6], v[7]), + SIMD4(v[8], v[9], v[10], v[11]), + SIMD4(v[12], v[13], v[14], v[15])) + } + + private static func double(_ m: simd_float4x4) -> simd_double4x4 { + simd_double4x4( + SIMD4(m.columns.0), SIMD4(m.columns.1), + SIMD4(m.columns.2), SIMD4(m.columns.3)) + } + + private static func float(_ m: simd_double4x4) -> simd_float4x4 { + simd_float4x4( + SIMD4(m.columns.0), SIMD4(m.columns.1), + SIMD4(m.columns.2), SIMD4(m.columns.3)) + } + + /// A finite rotation with no scale or shear, a translation, and a bottom row of 0 0 0 1. + /// The server writes each entry to six decimals, so its rotations are orthonormal to about + /// 1e-6; 1e-3 accepts that and ARKit's float poses, and rejects a 0.1% scale (whose + /// squared column length is off by 2e-3). + private static func isRigid(_ m: simd_double4x4) -> Bool { + let tolerance = 1e-3 + let columns = [m.columns.0, m.columns.1, m.columns.2, m.columns.3] + guard columns.allSatisfy({ c in (0..<4).allSatisfy { c[$0].isFinite } }) else { return false } + let bottom = SIMD4(m.columns.0.w, m.columns.1.w, m.columns.2.w, m.columns.3.w) + guard simd_reduce_max(simd_abs(bottom - SIMD4(0, 0, 0, 1))) <= tolerance else { return false } + let r = simd_double3x3( + SIMD3(m.columns.0.x, m.columns.0.y, m.columns.0.z), + SIMD3(m.columns.1.x, m.columns.1.y, m.columns.1.z), + SIMD3(m.columns.2.x, m.columns.2.y, m.columns.2.z)) + let gram = r.transpose * r + let identity = matrix_identity_double3x3 + for i in 0..<3 where simd_reduce_max(simd_abs(gram[i] - identity[i])) > tolerance { return false } + return r.determinant > 0 + } + } + + public enum Placement: Sendable, Equatable { + case box(AnchoredBox) + case unavailable(Unavailable) + } + + public struct AnchoredBox: Sendable, Equatable { + /// Box centre to meter anchor, rigid. Local +x along the wall, +y up, +z out of the wall. + public var anchorFromBox: simd_float4x4 + /// Width, height and depth in metres, as the server stated them. The server has already + /// applied the scene-to-ARKit scale, so the app uses these as they are. + public var size: SIMD3 + } + + /// Why a result has no AR placement. The rest of the result stays readable in every case. + public enum Unavailable: Sendable, Equatable { + /// The run has not written a result yet. + case noOutcome(Status) + /// Manual review, more photos, not eligible, or a kind this app doesn't know. + case notEligible(OutcomeKind) + case statusNotComplete(Status) + /// The caller has not confirmed that real analysis produced the result. + case analysisUnverified + /// The result belongs to another session, capture, run or ARKit world than the one on + /// screen. + case mismatch(IdentityField) + /// An eligible outcome without a recommended placement, which breaks the contract. + case noPlacement + /// The scene was not built in an ARKit world, so there is no box to anchor. + case notInARKitWorld + case malformedBox(BoxProblem) + /// The caller's meter anchor pose is not finite and rigid. + case invalidMeterAnchor + } + + public enum IdentityField: Sendable, Equatable, CaseIterable { + case session, capture, run, epoch + } + + public enum BoxProblem: Sendable, Equatable { + case poseCount(Int) + case sizeCount(Int) + case nonFinite + case nonPositiveSize + case notRigid + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultSummary.swift b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultSummary.swift new file mode 100644 index 000000000..65e156101 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/CaptureResult/CaptureResultSummary.swift @@ -0,0 +1,51 @@ +import Foundation +import simd + +/// What the integration build shows of a capture-API result: where it came from, what the server +/// decided, its own words, the views it asks for, and why there is no AR box. Built from the +/// decoded response only; nothing is inferred, and an absent part stays absent. +public struct CaptureResultSummary: Sendable, Equatable { + public enum State: Sendable, Equatable { + /// The run has no outcome yet. + case working(CaptureResult.Status) + case decided(CaptureResult.OutcomeKind) + /// The capture is over without an outcome: it failed or expired. + case ended(CaptureResult.Status) + } + + public var state: State + /// The server's message for the homeowner, verbatim; nil before an outcome. + public var message: String? + /// Each requested view's prompt, verbatim, title then body. + public var prompts: [CaptureResult.Prompt] + /// The criteria rows the server sent; nil when it sent none. Never filled in by the app. + public var criteria: [CaptureResult.Criterion]? + /// False until someone has verified the server ran real analysis for this result. + public var analysisVerified: Bool + /// Why AR shows nothing for this result on the scan now on screen. + public var arUnavailable: CaptureResult.Unavailable? + + /// `world` is the scan on screen now; `meterAnchor` its meter anchor's pose in that world, nil + /// when the scan has none to place a box on (AR then says so, after the other checks). + public init(_ record: CaptureResult.Record, world: CaptureResult.Association, meterAnchor: simd_float4x4?) { + let response = record.response + state = if let outcome = response.outcome { + .decided(outcome.kind) + } else { + switch response.status { + case .failed, .expired: .ended(response.status) + default: .working(response.status) + } + } + message = response.outcome?.message + let views = response.viewsNeeded.isEmpty ? response.outcome?.viewsNeeded ?? [] : response.viewsNeeded + prompts = views.map(\.prompt) + criteria = response.criteria + analysisVerified = record.analysis == .verified + // A zero matrix is not a pose, so without an anchor the gate ends on `.invalidMeterAnchor`. + arUnavailable = switch record.placement(in: world, meterAnchor: meterAnchor ?? simd_float4x4()) { + case .box: nil + case .unavailable(let reason): reason + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Gzip.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Gzip.swift new file mode 100644 index 000000000..e4cd01dff --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Gzip.swift @@ -0,0 +1,19 @@ +import Foundation + +/// gzip (RFC 1952) around the raw DEFLATE stream Apple's `.zlib` compression writes. The packet's +/// streams travel as `.csv.gz`, and Foundation has no gzip writer of its own. +public enum Gzip { + public static func compress(_ data: Data) throws -> Data { + // NSData's .zlib is raw DEFLATE (RFC 1951) with no zlib header, which is what gzip wraps. + let deflated = try (data as NSData).compressed(using: .zlib) as Data + var out = Data([0x1F, 0x8B, 0x08, 0x00, 0, 0, 0, 0, 0x00, 0xFF]) + out.append(deflated) + appendLE(ZipCRC32.checksum(data), to: &out) + appendLE(UInt32(truncatingIfNeeded: data.count), to: &out) + return out + } + + private static func appendLE(_ value: UInt32, to data: inout Data) { + withUnsafeBytes(of: value.littleEndian) { data.append(contentsOf: $0) } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04.swift new file mode 100644 index 000000000..f7ce75d4d --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04.swift @@ -0,0 +1,306 @@ +import Foundation + +/// The capture packet 0.4 that the capture API's finalize call takes as its body: `packet.json`, +/// camelCase, in the ARKit world frame of one epoch. This is the subset House Scan produces, +/// written from the published wire contract, not copied from the server's models. Optional +/// fields are left out when nil, which the server reads as absent. +/// +/// Unlike the 1.1 packet (`PacketManifest`), poses here are the raw ARKit camera-to-world +/// transforms, never moved into the meter frame. +public enum Packet04 { + public static let format = "house-capture-packet" + public static let formatVersion = "0.4" + + public struct Packet: Codable, Sendable, Equatable { + public var format = Packet04.format + public var formatVersion = Packet04.formatVersion + public var packetId: String + public var createdAt: String + public var units = Units() + public var conventions = Conventions() + public var source: Source + public var session: Session + public var epochs: [Epoch] + public var keyframes: [Keyframe] + public var stills: [Still]? + public var taps: [Tap]? + public var tracking: [TrackingEvent] + public var streams: Streams + public var scaleReference: ScaleReference + public var files: [FileEntry] + public var ext: [String: String]? + } + + public struct Units: Codable, Sendable, Equatable { + public var length = "meters" + public var angle = "degrees" + public var time = "seconds of device uptime (ARFrame.timestamp and Core Motion timestamp)" + public var image = "pixels of the saved, unrotated landscape JPEG" + public var rotationRate = "radians per second" + public var acceleration = "g (9.80665 m/s^2)" + } + + public struct Conventions: Codable, Sendable, Equatable { + public var world = "ARKit world, worldAlignment .gravity: right-handed, +y up, origin where the epoch started" + public var camera = "ARKit camera: +x right, +y up in the unrotated landscape image, looking along -z" + public var pose = "camera-to-world 4x4, 16 numbers column by column (simd_float4x4 layout), meters" + public var intrinsics = "[fx, fy, cx, cy] in pixels of the saved, unrotated landscape image" + public var pixel = "[u, v] continuous: (0, 0) is the top-left corner of the top-left pixel, v grows down" + } + + public struct Source: Codable, Sendable, Equatable { + public enum Kind: String, Codable, Sendable { case device, replay, synthetic } + public var kind: Kind + public var appBuild: String? + + public init(kind: Kind, appBuild: String?) { + self.kind = kind + self.appBuild = appBuild + } + } + + public enum Tier: String, Codable, Sendable { case arkit, arkitLidar = "arkit_lidar" } + + public struct VideoFormat: Codable, Sendable, Equatable { + public var imageResolution: [Int] + public var framesPerSecond: Double? + + public init(width: Int, height: Int, framesPerSecond: Double?) { + imageResolution = [width, height] + self.framesPerSecond = framesPerSecond + } + } + + public struct Session: Codable, Sendable, Equatable { + public var id: String + public var startedAt: String + public var startedAtUptime: Double + public var endedAtUptime: Double? + public var timeZone: String? + public var appVersion: String + public var deviceModel: String + public var systemVersion: String + public var tier: Tier + public var lidarAvailable: Bool + public var meshReconstructionSupported: Bool? + public var sceneDepthEnabled: Bool + public var sceneReconstruction: String? + public var worldAlignment = "gravity" + public var planeDetection: [String]? + public var videoFormat: VideoFormat + public var motionReferenceFrame: String? + } + + public struct Epoch: Codable, Sendable, Equatable { + public var id: String + public var startTime: Double + public var reason: String + + public init(id: String, startTime: Double, reason: String) { + self.id = id + self.startTime = startTime + self.reason = reason + } + } + + public struct Exposure: Codable, Sendable, Equatable { + public var duration: Double? + public var offset: Double? + public var iso: Double? + public var fNumber: Double? + + public init(duration: Double?, offset: Double?, iso: Double?, fNumber: Double?) { + self.duration = duration + self.offset = offset + self.iso = iso + self.fNumber = fNumber + } + } + + public struct DepthRef: Codable, Sendable, Equatable { + public var file: String + public var confidenceFile: String? + public var w: Int + public var h: Int + } + + public struct Keyframe: Codable, Sendable, Equatable { + public var id: String + public var img: String + public var w: Int + public var h: Int + public var intrinsics: [Double] + public var pose: [Double] + public var timestamp: Double + public var tracking: String + public var epoch: String + public var reason: String + public var exposure: Exposure? + public var depth: DepthRef? + public var purpose: String? + } + + public struct Still: Codable, Sendable, Equatable { + public var id: String + public var purpose: String + public var img: String + public var w: Int + public var h: Int + public var timestamp: Double + /// EXIF orientation of the stored pixels. House Scan stores the sensor image unrotated. + public var orientation: Int + public var keyframe: String? + public var intrinsics: [Double]? + public var intrinsicsSource: String? + } + + public struct TapHit: Codable, Sendable, Equatable { + public var position: [Double] + public var target: String + public var alignment: String? + public var distance: Double? + + public init(position: [Double], target: String, alignment: String?, distance: Double?) { + self.position = position + self.target = target + self.alignment = alignment + self.distance = distance + } + } + + public struct Tap: Codable, Sendable, Equatable { + public var id: String + public var time: Double + public var label: String + public var keyframe: String + public var pixel: [Double] + public var rayOrigin: [Double] + public var rayDirection: [Double] + public var hit: TapHit? + public var epoch: String? + } + + public struct TrackingEvent: Codable, Sendable, Equatable { + public var time: Double + public var state: String + public var epoch: String? + + public init(time: Double, state: String, epoch: String?) { + self.time = time + self.state = state + self.epoch = epoch + } + } + + public struct StreamRef: Codable, Sendable, Equatable { + public var file: String + public var columns: [String] + public var rateHz: Double + } + + public struct Streams: Codable, Sendable, Equatable { + public var imuRaw: StreamRef + public var arkitPoses: StreamRef + } + + public struct CloseUp: Codable, Sendable, Equatable { + public var captured: Bool + public var still: String? + public var arkitDistanceM: Double? + public var side: String? + + public init(captured: Bool, still: String?, arkitDistanceM: Double?, side: String?) { + self.captured = captured + self.still = still + self.arkitDistanceM = arkitDistanceM + self.side = side + } + + public static let notCaptured = CloseUp(captured: false, still: nil, arkitDistanceM: nil, side: nil) + } + + public struct ScaleExt: Codable, Sendable, Equatable { + public var method = "a9-meter-model" + public var obliqueCloseUp: CloseUp + /// Head-on views of doors on the meter side. House Scan does not detect doors yet, so this + /// is empty, which the contract allows. + public var doorViews: [String] = [] + } + + public struct ScaleReference: Codable, Sendable, Equatable { + public var meterCloseUp: CloseUp + public var ext: ScaleExt + } + + public enum Role: String, Codable, Sendable, CaseIterable { + case keyframe, depth, confidence, features, still, stream, other + } + + public struct FileEntry: Codable, Sendable, Equatable { + public var path: String + public var bytes: Int + public var sha256: String + public var role: Role + } + + /// The `meta` a register call sends with an image: exactly one record, the same one the final + /// packet lists. + public enum ImageRecord: Codable, Sendable, Equatable { + case keyframe(Keyframe) + case still(Still) + + public var path: String { + switch self { + case .keyframe(let k): k.img + case .still(let s): s.img + } + } + + private enum CodingKeys: String, CodingKey { case keyframe, still } + + public func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .keyframe(let k): try container.encode(k, forKey: .keyframe) + case .still(let s): try container.encode(s, forKey: .still) + } + } + + public init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + if let k = try container.decodeIfPresent(Keyframe.self, forKey: .keyframe) { + self = .keyframe(k) + } else { + self = .still(try container.decode(Still.self, forKey: .still)) + } + } + } + + // MARK: Id and path patterns the server enforces + + /// `^[A-Za-z0-9_-]{8,64}$`: packetId. + public static func isStorageID(_ s: String) -> Bool { + (8...64).contains(s.count) && s.unicodeScalars.allSatisfy { isAlnum($0) || $0 == "_" || $0 == "-" } + } + + /// `^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$`: keyframe and still ids. + public static func isSegmentID(_ s: String) -> Bool { + guard let first = s.unicodeScalars.first, isAlnum(first), s.count <= 128 else { return false } + return s.unicodeScalars.allSatisfy { isAlnum($0) || $0 == "_" || $0 == "-" } + } + + /// Every segment starts with an alphanumeric, and a dot always starts a new alphanumeric token. + public static func isRelPath(_ s: String) -> Bool { + guard !s.isEmpty else { return false } + return s.split(separator: "/", omittingEmptySubsequences: false).allSatisfy { segment in + segment.split(separator: ".", omittingEmptySubsequences: false).allSatisfy { token in + guard let first = token.unicodeScalars.first, isAlnum(first) else { return false } + return token.unicodeScalars.allSatisfy { isAlnum($0) || $0 == "_" || $0 == "-" } + } + } + } + + private static func isAlnum(_ c: Unicode.Scalar) -> Bool { + ("a"..."z").contains(c) || ("A"..."Z").contains(c) || ("0"..."9").contains(c) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Check.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Check.swift new file mode 100644 index 000000000..3ea995149 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Check.swift @@ -0,0 +1,115 @@ +import Foundation +import simd + +/// House Scan's own checks of a 0.4 packet against the rules the capture API documents as +/// rejections: id and path patterns, the ARKit tier's required sections, listed files that exist +/// with their stated size and SHA-256, rigid poses, plausible intrinsics, still-to-keyframe links +/// and the tap replay. A packet that passes can still fail a server check this does not repeat. +public enum Packet04Check { + /// The server's tap replay tolerance on origin and direction. + public static let tapReplayTolerance = 1e-3 + + public static func problems(_ p: Packet04.Packet, folder: URL?) -> [String] { + var out: [String] = [] + if !Packet04.isStorageID(p.packetId) { out.append("packetId \(p.packetId) is not a StorageId") } + if p.formatVersion != Packet04.formatVersion { out.append("formatVersion \(p.formatVersion)") } + if p.keyframes.isEmpty { out.append("ARKit tier needs at least one keyframe") } + if p.session.tier == .arkitLidar, !p.session.lidarAvailable { out.append("arkit_lidar needs lidarAvailable") } + if p.session.worldAlignment != "gravity" { out.append("worldAlignment must be gravity") } + if p.streams.imuRaw.rateHz < Packet04Streams.minimumIMUHz { out.append("imuRaw rateHz \(p.streams.imuRaw.rateHz) < 50") } + if p.streams.arkitPoses.rateHz < Packet04Streams.minimumPoseHz { out.append("arkitPoses rateHz \(p.streams.arkitPoses.rateHz) < 10") } + + let epochs = Set(p.epochs.map(\.id)) + var listed: [String: Packet04.FileEntry] = [:] + for entry in p.files { + if !Packet04.isRelPath(entry.path) { out.append("file path \(entry.path) is not a RelPath") } + if listed.updateValue(entry, forKey: entry.path) != nil { out.append("file \(entry.path) listed twice") } + } + func requireListed(_ path: String, _ role: Packet04.Role, _ place: String) { + guard let entry = listed[path] else { return out.append("\(place) \(path) is not in files") } + if entry.role != role { out.append("\(place) \(path) has role \(entry.role.rawValue), not \(role.rawValue)") } + } + + var frames: [String: Packet04.Keyframe] = [:] + var byTime: [Double: Packet04.Keyframe] = [:] + for k in p.keyframes { + if !Packet04.isSegmentID(k.id) { out.append("keyframe id \(k.id) is not a SegmentId") } + if frames.updateValue(k, forKey: k.id) != nil { out.append("keyframe \(k.id) listed twice") } + if !epochs.contains(k.epoch) { out.append("keyframe \(k.id) names unknown epoch \(k.epoch)") } + requireListed(k.img, .keyframe, "keyframe \(k.id) image") + if let depth = k.depth { + requireListed(depth.file, .depth, "keyframe \(k.id) depth") + if let confidence = depth.confidenceFile { requireListed(confidence, .confidence, "keyframe \(k.id) confidence") } + if let entry = listed[depth.file], entry.bytes != depth.w * depth.h * 4 { out.append("keyframe \(k.id) depth size") } + } + guard let pose = matrix(k.pose) else { out.append("keyframe \(k.id) pose is not 16 finite numbers"); continue } + if !PacketPose.isRigid(pose) { out.append("keyframe \(k.id) pose is not rigid") } + if k.intrinsics.count != 4 { + out.append("keyframe \(k.id) intrinsics") + } else if let problem = PacketWriter.intrinsicsProblem( + SIMD4(k.intrinsics.map(Float.init)), width: k.w, height: k.h) { + out.append("keyframe \(k.id) intrinsics: \(problem)") + } + if let other = byTime[k.timestamp], other.epoch == k.epoch, other.pose != k.pose { + out.append("keyframes \(other.id) and \(k.id) share timestamp \(k.timestamp) with different poses") + } + byTime[k.timestamp] = k + } + var stillIDs = Set() + for s in p.stills ?? [] { + if !Packet04.isSegmentID(s.id) { out.append("still id \(s.id) is not a SegmentId") } + stillIDs.insert(s.id) + requireListed(s.img, .still, "still \(s.id) image") + if let link = s.keyframe { + guard let frame = frames[link] else { out.append("still \(s.id) names unknown keyframe \(link)"); continue } + if frame.w != s.w || frame.h != s.h { out.append("still \(s.id) and keyframe \(link) sizes differ") } + } + } + for tap in p.taps ?? [] { + guard let frame = frames[tap.keyframe] else { out.append("tap \(tap.id) names unknown keyframe \(tap.keyframe)"); continue } + if tap.pixel.count != 2 || !(0...Double(frame.w)).contains(tap.pixel[0]) || !(0...Double(frame.h)).contains(tap.pixel[1]) { + out.append("tap \(tap.id) pixel is outside keyframe \(frame.id)") + continue + } + if let epoch = tap.epoch, epoch != frame.epoch { out.append("tap \(tap.id) epoch differs from its keyframe's") } + // Compared pairwise below, a missing or extra component, or a NaN, would go unseen. + guard tap.rayOrigin.count == 3, tap.rayDirection.count == 3, (tap.rayOrigin + tap.rayDirection).allSatisfy(\.isFinite) else { + out.append("tap \(tap.id) ray is not two finite 3-vectors") + continue + } + let replay = ray(pose: frame.pose, intrinsics: frame.intrinsics, pixel: tap.pixel) + let error = zip(replay.origin + replay.direction, tap.rayOrigin + tap.rayDirection).map { abs($0 - $1) }.max() ?? .infinity + if !(error <= tapReplayTolerance) { out.append("tap \(tap.id) ray replays with error \(error)") } + } + for (name, close) in [("meterCloseUp", p.scaleReference.meterCloseUp), ("obliqueCloseUp", p.scaleReference.ext.obliqueCloseUp)] { + if close.captured, !(close.still.map(stillIDs.contains) ?? false) { out.append("\(name) names a still that is not in the packet") } + } + requireListed(p.streams.imuRaw.file, .stream, "imuRaw") + requireListed(p.streams.arkitPoses.file, .stream, "arkitPoses") + + if let folder { + for entry in p.files { + guard let data = try? Data(contentsOf: folder.appending(path: entry.path)) else { out.append("file \(entry.path) is missing"); continue } + if data.count != entry.bytes { out.append("file \(entry.path) is \(data.count) bytes, listed \(entry.bytes)") } + if PacketFiles.sha256(data) != entry.sha256 { out.append("file \(entry.path) SHA-256 differs from its listing") } + } + } + return out + } + + /// The ray through `pixel` as the server replays it: origin the camera position, direction + /// the unit vector of R · ((u − cx)/fx, −(v − cy)/fy, −1). + public static func ray(pose: [Double], intrinsics k: [Double], pixel: [Double]) -> (origin: [Double], direction: [Double]) { + guard pose.count == 16, k.count == 4, pixel.count == 2 else { return ([], []) } + let d = SIMD3((pixel[0] - k[2]) / k[0], -(pixel[1] - k[3]) / k[1], -1) + let c0 = SIMD3(pose[0], pose[1], pose[2]), c1 = SIMD3(pose[4], pose[5], pose[6]), c2 = SIMD3(pose[8], pose[9], pose[10]) + let world = simd_normalize(c0 * d.x + c1 * d.y + c2 * d.z) + return ([pose[12], pose[13], pose[14]], [world.x, world.y, world.z]) + } + + static func matrix(_ values: [Double]) -> simd_float4x4? { + guard values.count == 16, values.allSatisfy(\.isFinite) else { return nil } + let f = values.map(Float.init) + return simd_float4x4(SIMD4(f[0], f[1], f[2], f[3]), SIMD4(f[4], f[5], f[6], f[7]), SIMD4(f[8], f[9], f[10], f[11]), SIMD4(f[12], f[13], f[14], f[15])) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Producer.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Producer.swift new file mode 100644 index 000000000..7691ebe16 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Producer.swift @@ -0,0 +1,385 @@ +import CryptoKit +import Foundation +import ImageIO +import simd + +/// A file of the 0.4 packet that is on disk and will never change: its bytes, digests and, for +/// an image, the record that describes it. The same value registers the file with the capture API +/// and lists it in the final packet, so the two can't disagree. +public struct SealedFile: Codable, Sendable, Equatable { + public var path: String + public var bytes: Int + public var sha256: String + /// Base64 of the 16-byte MD5, as the register call and the storage PUT's Content-MD5 want it. + public var md5: String + public var role: Packet04.Role + public var contentType: String + public var meta: Packet04.ImageRecord? + /// Lower goes first: the meter close-ups before keyframes before streams. + public var priority: Int + + public var entry: Packet04.FileEntry { .init(path: path, bytes: bytes, sha256: sha256, role: role) } + + static func seal(_ data: Data, path: String, role: Packet04.Role, contentType: String, meta: Packet04.ImageRecord?, priority: Int) -> SealedFile { + SealedFile( + path: path, bytes: data.count, sha256: PacketFiles.sha256(data), + md5: Data(Insecure.MD5.hash(data: data)).base64EncodedString(), role: role, contentType: contentType, meta: meta, + priority: priority) + } +} + +/// What the capture observed for one saved camera image, read from its own ARFrame when the +/// pixels were taken: the raw ARKit camera-to-world pose, never one corrected into the meter frame. +public struct Packet04Observation: Sendable, Equatable { + public var t: Double + public var cameraToWorld: simd_float4x4 + /// [fx, fy, cx, cy] in pixels of the stored JPEG. + public var intrinsics: SIMD4 + public var width: Int + public var height: Int + public var tracking: PacketTracking + public var exposure: Packet04.Exposure? + + public init(t: Double, cameraToWorld: simd_float4x4, intrinsics: SIMD4, width: Int, height: Int, tracking: PacketTracking, exposure: Packet04.Exposure? = nil) { + self.t = t + self.cameraToWorld = cameraToWorld + self.intrinsics = intrinsics + self.width = width + self.height = height + self.tracking = tracking + self.exposure = exposure + } +} + +/// ARKit depth for one keyframe: Float32 meters and UInt8 confidence, row by row. +public struct Packet04Depth: Sendable { + public var meters: [Float] + public var confidence: [UInt8] + public var width: Int + public var height: Int + + public init(meters: [Float], confidence: [UInt8], width: Int, height: Int) { + self.meters = meters + self.confidence = confidence + self.width = width + self.height = height + } +} + +/// Fixed facts about the capture, known when the ARKit session starts and the homeowner consents, +/// so the create request can be sent before the first photo. +public struct Packet04SessionInfo: Codable, Sendable, Equatable { + public var packetID: String + public var sessionID: String + public var source: Packet04.Source.Kind + public var appVersion: String + public var deviceModel: String + public var systemVersion: String + public var lidarAvailable: Bool + public var sceneDepthEnabled: Bool + public var meshReconstructionSupported: Bool? + public var sceneReconstruction: String? + public var planeDetection: [String]? + public var videoWidth: Int + public var videoHeight: Int + public var framesPerSecond: Double? + public var timeZone: String? + + public init( + packetID: String, sessionID: String, source: Packet04.Source.Kind, appVersion: String, deviceModel: String, systemVersion: String, + lidarAvailable: Bool, sceneDepthEnabled: Bool, meshReconstructionSupported: Bool?, sceneReconstruction: String?, + planeDetection: [String]?, videoWidth: Int, videoHeight: Int, framesPerSecond: Double?, timeZone: String? + ) { + self.packetID = packetID + self.sessionID = sessionID + self.source = source + self.appVersion = appVersion + self.deviceModel = deviceModel + self.systemVersion = systemVersion + self.lidarAvailable = lidarAvailable + self.sceneDepthEnabled = sceneDepthEnabled + self.meshReconstructionSupported = meshReconstructionSupported + self.sceneReconstruction = sceneReconstruction + self.planeDetection = planeDetection + self.videoWidth = videoWidth + self.videoHeight = videoHeight + self.framesPerSecond = framesPerSecond + self.timeZone = timeZone + } + + /// `arkit_lidar` needs depth the session actually recorded, not a phone that could have. + public var tier: Packet04.Tier { lidarAvailable && sceneDepthEnabled ? .arkitLidar : .arkit } +} + +public enum Packet04Error: Error, Equatable, CustomStringConvertible { + case invalidID(String) + case sealedAfterFinish(String) + case duplicate(String) + case invalidImage(id: String, reason: String) + case notRigid(String) + case unknownKeyframe(String) + case streamTooSlow(stream: String, measuredHz: Double, minimumHz: Double) + case streamEmpty(String) + case noKeyframes + case invalidPacket([String]) + + public var description: String { + switch self { + case .invalidID(let id): "\(id) does not fit the server's id pattern" + case .sealedAfterFinish(let what): "\(what) arrived after the packet was finished" + case .duplicate(let id): "\(id) was sealed twice" + case .invalidImage(let id, let reason): "image \(id): \(reason)" + case .notRigid(let place): "\(place): pose is not a rotation plus a translation" + case .unknownKeyframe(let id): "no keyframe \(id)" + case .streamTooSlow(let stream, let hz, let minimum): "\(stream) measured \(hz) Hz; the contract requires at least \(minimum) Hz" + case .streamEmpty(let stream): "\(stream) has no rows" + case .noKeyframes: "an ARKit packet needs at least one keyframe" + case .invalidPacket(let problems): "packet fails its own checks: \(problems.joined(separator: "; "))" + } + } +} + +/// Builds one 0.4 packet while the capture runs. Each image is written into `folder` and sealed +/// the moment it is observed, so it can be uploaded during the capture; `finish` adds the streams +/// and returns the exact `packet.json` bytes, which are frozen from then on. +/// +/// One producer is one ARKit epoch. A world reset ends the packet: the app starts a new producer +/// with a new packet id rather than mixing two worlds under one epoch. +public actor Packet04Producer { + public nonisolated let folder: URL + public nonisolated let info: Packet04SessionInfo + public nonisolated let epoch = "e1" + private let startedAtUptime: Double + private let startedAt: Date + private var keyframes: [Packet04.Keyframe] = [] + private var stills: [Packet04.Still] = [] + private var taps: [Packet04.Tap] = [] + private var files: [SealedFile] = [] + private var finished: Data? + + /// `startedAtUptime` is the uptime of the session's first ARFrame and `startedAt` the wall clock + /// at that instant. + public init(folder: URL, info: Packet04SessionInfo, startedAtUptime: Double, startedAt: Date) throws { + guard Packet04.isStorageID(info.packetID) else { throw Packet04Error.invalidID(info.packetID) } + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + self.folder = folder + self.info = info + self.startedAtUptime = startedAtUptime + self.startedAt = startedAt + } + + public var sealedFiles: [SealedFile] { files } + public var keyframeCount: Int { keyframes.count } + public var isFinished: Bool { finished != nil } + + /// The keyframe sealed from the frame at `t`, if there is one: a still taken from a frame that + /// was already kept links to it instead of repeating it. + public func keyframeID(at t: Double) -> String? { keyframes.first { $0.timestamp == t }?.id } + + /// "k00001" for 1. + public static func keyframeID(_ number: Int) -> String { + let digits = String(number) + return "k" + String(repeating: "0", count: max(0, 5 - digits.count)) + digits + } + + // MARK: Images + + /// Writes the JPEG as keyframes/.jpg with, when given, its depth and confidence, and seals + /// them. `reason` is the contract's keyframe reason (motion, tap, still, request, auto). + public func sealKeyframe( + jpeg: Data, observation o: Packet04Observation, reason: String, purpose: String? = nil, depth: Packet04Depth? = nil + ) throws -> [SealedFile] { + guard finished == nil else { throw Packet04Error.sealedAfterFinish("keyframe") } + let id = Self.keyframeID(keyframes.count + 1) + try check(o, id: id, jpeg: jpeg) + guard let tracking = Self.trackingText(o.tracking) else { + throw Packet04Error.invalidImage(id: id, reason: "tracking is limited for a reason the contract has no name for") + } + var sealed: [SealedFile] = [] + var depthRef: Packet04.DepthRef? + if let depth, depth.meters.count == depth.width * depth.height, depth.confidence.count == depth.meters.count, depth.width > 0 { + let map = try write(PacketFiles.depthData(meters: depth.meters), "keyframes/\(id).depth.f32", role: .depth, type: "application/octet-stream", priority: 20) + let confidence = try write(Data(depth.confidence), "keyframes/\(id).confidence.u8", role: .confidence, type: "application/octet-stream", priority: 20) + sealed += [map, confidence] + depthRef = .init(file: map.path, confidenceFile: confidence.path, w: depth.width, h: depth.height) + } + let record = Packet04.Keyframe( + id: id, img: "keyframes/\(id).jpg", w: o.width, h: o.height, intrinsics: Self.numbers(o.intrinsics), + pose: PacketPose.columnMajor(o.cameraToWorld), timestamp: o.t, tracking: tracking, epoch: epoch, + reason: reason, exposure: o.exposure, depth: depthRef, purpose: purpose) + let image = try write(jpeg, record.img, role: .keyframe, type: "image/jpeg", meta: .keyframe(record), priority: 10) + keyframes.append(record) + return [image] + sealed + } + + /// A checklist still taken from an ARKit keyframe already sealed: the same pixels under + /// stills/.jpg, linked to the keyframe so it keeps its pose. Stills go first. + /// + /// A retake of the same purpose is a new still ("meter_close-2"): sealed files never change, + /// and `finish` says which one the scale reference names. + public func sealStill(purpose: String, keyframe keyframeID: String) throws -> SealedFile { + guard finished == nil else { throw Packet04Error.sealedAfterFinish("still") } + guard Packet04.isSegmentID(purpose) else { throw Packet04Error.invalidID(purpose) } + guard let frame = keyframes.first(where: { $0.id == keyframeID }) else { throw Packet04Error.unknownKeyframe(keyframeID) } + if let same = stills.last(where: { $0.purpose == purpose }), same.keyframe == keyframeID { + throw Packet04Error.duplicate(same.id) + } + let taken = stills.filter { $0.purpose == purpose }.count + let id = taken == 0 ? purpose : "\(purpose)-\(taken + 1)" + let jpeg = try Data(contentsOf: folder.appending(path: frame.img)) + let still = Packet04.Still( + id: id, purpose: purpose, img: "stills/\(id).jpg", w: frame.w, h: frame.h, timestamp: frame.timestamp, + orientation: 1, keyframe: frame.id, intrinsics: frame.intrinsics, intrinsicsSource: "arkit") + let file = try write(jpeg, still.img, role: .still, type: "image/jpeg", meta: .still(still), priority: 0) + stills.append(still) + return file + } + + /// A tap on a sealed keyframe's pixel, with the ray rebuilt from that keyframe's own pose and + /// intrinsics, so the server's replay reproduces it exactly. + public func addTap(id: String, label: String, keyframe keyframeID: String, pixel: SIMD2, time: Double, hit: Packet04.TapHit?) throws { + guard finished == nil else { throw Packet04Error.sealedAfterFinish("tap") } + guard let frame = keyframes.first(where: { $0.id == keyframeID }) else { throw Packet04Error.unknownKeyframe(keyframeID) } + let ray = Packet04Check.ray(pose: frame.pose, intrinsics: frame.intrinsics, pixel: [pixel.x, pixel.y]) + taps.append(.init( + id: id, time: time, label: label, keyframe: keyframeID, pixel: [pixel.x, pixel.y], rayOrigin: ray.origin, + rayDirection: ray.direction, hit: hit, epoch: epoch)) + } + + /// The frame the homeowner tapped, kept as a keyframe (reason `tap`) unless it already is one, + /// and the tap on it: `pixel` in that frame's stored image, the ray rebuilt from its own pose + /// and intrinsics, and what the phone's raycast hit. + public func sealTap( + id: String, label: String, jpeg: Data, observation: Packet04Observation, pixel: SIMD2, hit: Packet04.TapHit? + ) throws -> [SealedFile] { + var files: [SealedFile] = [] + if keyframeID(at: observation.t) == nil { + files = try sealKeyframe(jpeg: jpeg, observation: observation, reason: "tap") + } + guard let keyframe = keyframeID(at: observation.t) else { throw Packet04Error.unknownKeyframe("at \(observation.t)") } + try addTap(id: id, label: label, keyframe: keyframe, pixel: pixel, time: observation.t, hit: hit) + return files + } + + // MARK: Finish + + /// Writes the two required streams, then returns their sealed files and the exact packet + /// bytes. The packet is checked against the contract's rules first; a failure names each + /// problem instead of sending a packet the server would reject. + /// `acceptedCloseUpAt` is the frame time of the head-on close-up the scan accepted, nil when + /// none was (skipped, or every shot refused): only that still is the scale reference. + public func finish( + poses: [Packet04Streams.PoseRow], accelerometer: [Packet04Streams.MotionRow], gyroscope: [Packet04Streams.MotionRow], + endedAtUptime: Double, acceptedCloseUpAt: Double?, closeUpDistanceM: Double? = nil, createdAt: Date = Date() + ) throws -> (streams: [SealedFile], packet: Data) { + if let finished { return (files.filter { $0.role == .stream }, finished) } + guard !keyframes.isEmpty else { throw Packet04Error.noKeyframes } + let posesCSV = try Packet04Streams.arkitPoses(poses, epoch: epoch) + let imu = try Packet04Streams.imuRaw(accelerometer: accelerometer, gyroscope: gyroscope) + let poseFile = try write(try Gzip.compress(posesCSV.csv), "streams/arkit_poses.csv.gz", role: .stream, type: "application/gzip", priority: 30) + let imuFile = try write(try Gzip.compress(imu.csv), "streams/imu_raw.csv.gz", role: .stream, type: "application/gzip", priority: 30) + // Each sensor as Core Motion timed it, so the server can re-pair them its own way. + let accelFile = try write( + try Gzip.compress(try Packet04Streams.rawMotionCSV(accelerometer, columns: ["t", "ax", "ay", "az"])), + "streams/accelerometer_raw.csv.gz", role: .stream, type: "application/gzip", priority: 31) + let gyroFile = try write( + try Gzip.compress(try Packet04Streams.rawMotionCSV(gyroscope, columns: ["t", "gx", "gy", "gz"])), + "streams/gyroscope_raw.csv.gz", role: .stream, type: "application/gzip", priority: 31) + var ext = imu.pairingNote + ext["rawAccelerometerFile"] = accelFile.path + ext["rawGyroscopeFile"] = gyroFile.path + + let close = acceptedCloseUpAt.flatMap { t in stills.last { $0.purpose == "meter_close" && $0.timestamp == t } } + // The app takes no oblique close-up yet, so none is ever accepted. + let oblique: Packet04.Still? = nil + let packet = Packet04.Packet( + packetId: info.packetID, createdAt: PacketWriter.iso8601(createdAt), + source: .init(kind: info.source, appBuild: info.appVersion), + session: .init( + id: info.sessionID, startedAt: PacketWriter.iso8601(startedAt), startedAtUptime: startedAtUptime, + endedAtUptime: max(endedAtUptime, startedAtUptime), timeZone: info.timeZone, appVersion: info.appVersion, + deviceModel: info.deviceModel, systemVersion: info.systemVersion, tier: info.tier, lidarAvailable: info.lidarAvailable, + meshReconstructionSupported: info.meshReconstructionSupported, sceneDepthEnabled: info.sceneDepthEnabled, + sceneReconstruction: info.sceneReconstruction, planeDetection: info.planeDetection, + videoFormat: .init(width: info.videoWidth, height: info.videoHeight, framesPerSecond: info.framesPerSecond), + motionReferenceFrame: nil), + epochs: [.init(id: epoch, startTime: startedAtUptime, reason: "sessionStart")], + keyframes: keyframes, stills: stills.isEmpty ? nil : stills, taps: taps.isEmpty ? nil : taps, + tracking: Packet04Streams.trackingChanges(poses, epoch: epoch), + streams: .init( + imuRaw: .init(file: imuFile.path, columns: Packet04Streams.imuColumns, rateHz: imu.rateHz), + arkitPoses: .init(file: poseFile.path, columns: Packet04Streams.poseColumns, rateHz: posesCSV.rateHz)), + scaleReference: .init( + meterCloseUp: close.map { .init(captured: true, still: $0.id, arkitDistanceM: closeUpDistanceM, side: nil) } ?? .notCaptured, + ext: .init(obliqueCloseUp: oblique.map { .init(captured: true, still: $0.id, arkitDistanceM: nil, side: nil) } ?? .notCaptured)), + files: files.map(\.entry), + ext: ext) + let problems = Packet04Check.problems(packet, folder: folder) + guard problems.isEmpty else { throw Packet04Error.invalidPacket(problems) } + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys, .withoutEscapingSlashes] + let data = try encoder.encode(packet) + try data.write(to: folder.appending(path: "packet.json"), options: .atomic) + finished = data + return ([poseFile, imuFile, accelFile, gyroFile], data) + } + + // MARK: Helpers + + private func check(_ o: Packet04Observation, id: String, jpeg: Data) throws { + guard PacketPose.isRigid(o.cameraToWorld) else { throw Packet04Error.notRigid("keyframe \(id)") } + guard o.t.isFinite, o.t >= 0 else { throw Packet04Error.invalidImage(id: id, reason: "timestamp \(o.t)") } + if let problem = PacketWriter.intrinsicsProblem(o.intrinsics, width: o.width, height: o.height) { + throw Packet04Error.invalidImage(id: id, reason: problem) + } + if let problem = Self.jpegProblem(jpeg, width: o.width, height: o.height) { + throw Packet04Error.invalidImage(id: id, reason: problem) + } + if let same = keyframes.first(where: { $0.timestamp == o.t }) { + throw Packet04Error.invalidImage(id: id, reason: "keyframe \(same.id) already has timestamp \(o.t)") + } + } + + private func write(_ data: Data, _ path: String, role: Packet04.Role, type: String, meta: Packet04.ImageRecord? = nil, priority: Int) throws -> SealedFile { + guard Packet04.isRelPath(path) else { throw Packet04Error.invalidID(path) } + guard !files.contains(where: { $0.path == path }) else { throw Packet04Error.duplicate(path) } + let url = folder.appending(path: path) + try FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true) + try data.write(to: url, options: .atomic) + let file = SealedFile.seal(data, path: path, role: role, contentType: type, meta: meta, priority: priority) + files.append(file) + return file + } + + /// The bytes must be a JPEG of the stated size, stored unrotated. + static func jpegProblem(_ jpeg: Data, width: Int, height: Int) -> String? { + guard let source = CGImageSourceCreateWithData(jpeg as CFData, nil), CGImageSourceGetType(source) as String? == "public.jpeg", + let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any] + else { return "not a readable JPEG" } + let w = properties[kCGImagePropertyPixelWidth] as? Int, h = properties[kCGImagePropertyPixelHeight] as? Int + guard w == width, h == height else { return "the JPEG is \(w ?? 0)x\(h ?? 0), not \(width)x\(height)" } + if let orientation = properties[kCGImagePropertyOrientation] as? Int, orientation != 1 { + return "EXIF orientation \(orientation); store the unrotated sensor image" + } + return nil + } + + static func numbers(_ k: SIMD4) -> [Double] { [k.x, k.y, k.z, k.w].map(PacketNumber.double) } + + /// The contract's tracking state. Nil for a limited state whose reason the contract has no + /// name for (a reason added after iOS 26): no value there would be an observation. + public static func trackingText(_ tracking: PacketTracking) -> String? { + switch tracking { + case .normal: "normal" + case .notAvailable: "notAvailable" + case .limited(let why): + switch why { + case .initializing?: "limited.initializing" + case .excessiveMotion?: "limited.excessiveMotion" + case .insufficientFeatures?: "limited.insufficientFeatures" + case .relocalizing?: "limited.relocalizing" + case nil: nil + } + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Streams.swift b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Streams.swift new file mode 100644 index 000000000..be5cabecc --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Packet/Packet04Streams.swift @@ -0,0 +1,147 @@ +import Foundation +import simd + +/// The 0.4 packet's two required streams as CSV, and the tracking changes read from the poses. +public enum Packet04Streams { + public static let poseColumns = ["t", "epoch", "tracking", "tx", "ty", "tz", "qx", "qy", "qz", "qw", "fx", "fy", "cx", "cy"] + public static let imuColumns = ["t", "ax", "ay", "az", "gx", "gy", "gz"] + /// The contract's floors for `rateHz`. + public static let minimumPoseHz = 10.0 + public static let minimumIMUHz = 50.0 + + /// One ARFrame: raw camera-to-world and that frame's intrinsics at the video format's size. + public struct PoseRow: Sendable, Equatable { + public var t: Double + public var tracking: PacketTracking + public var cameraToWorld: simd_float4x4 + public var intrinsics: SIMD4 + + public init(t: Double, tracking: PacketTracking, cameraToWorld: simd_float4x4, intrinsics: SIMD4) { + self.t = t + self.tracking = tracking + self.cameraToWorld = cameraToWorld + self.intrinsics = intrinsics + } + } + + /// One Core Motion sample with its own timestamp: g for the accelerometer, rad/s for the gyro. + public struct MotionRow: Sendable, Equatable { + public var t: Double + public var value: SIMD3 + + public init(t: Double, value: SIMD3) { + self.t = t + self.value = value + } + } + + public static func arkitPoses(_ rows: [PoseRow], epoch: String) throws -> (csv: Data, rateHz: Double) { + guard !rows.isEmpty else { throw Packet04Error.streamEmpty("arkitPoses") } + var text = poseColumns.joined(separator: ",") + "\n" + var times: [Double] = [] + for row in rows where row.t > (times.last ?? -.infinity) { + guard PacketPose.isRigid(row.cameraToWorld) else { continue } + let p = PacketPose.translation(row.cameraToWorld) + let q = PacketPose.quaternion(PacketPose.rotation(row.cameraToWorld)) + let k = row.intrinsics + // The column is free text: a limited state whose reason the contract can't name is + // still limited, and the pose it came with is still a measurement. + let cells = [row.t.description, epoch, Packet04Producer.trackingText(row.tracking) ?? "limited"] + + [p.x, p.y, p.z].map { PacketNumber.double($0).description } + + [q.x, q.y, q.z, q.w].map(\.description) + + [k.x, k.y, k.z, k.w].map { PacketNumber.double($0).description } + text += cells.joined(separator: ",") + "\n" + times.append(row.t) + } + let rate = measuredRate(times) + guard rate >= minimumPoseHz else { throw Packet04Error.streamTooSlow(stream: "arkitPoses", measuredHz: rate, minimumHz: minimumPoseHz) } + return (Data(text.utf8), rate) + } + + /// Accelerometer and gyro rows for `imuRaw`. Core Motion times the two sensors separately, and + /// the contract's row holds both. Each row is an accelerometer sample paired with the gyro + /// sample nearest in time, only when that gyro sample is within `maxPairingOffset`; an + /// accelerometer sample with no gyro sample that close is left out rather than given one. + /// How the server wants the two clocks joined is an open question to the server team, so the + /// rule and its measured worst offset travel in the packet's `ext`. + public static func imuRaw( + accelerometer: [MotionRow], gyroscope: [MotionRow], maxPairingOffset: Double = 0.005 + ) throws -> (csv: Data, rateHz: Double, pairingNote: [String: String]) { + let accel = accelerometer.sorted { $0.t < $1.t } + let gyro = gyroscope.sorted { $0.t < $1.t } + guard !accel.isEmpty, !gyro.isEmpty else { throw Packet04Error.streamEmpty("imuRaw") } + var text = imuColumns.joined(separator: ",") + "\n" + var times: [Double] = [] + var worst = 0.0 + var g = 0 + for a in accel where a.t > (times.last ?? -.infinity) { + while g + 1 < gyro.count, abs(gyro[g + 1].t - a.t) <= abs(gyro[g].t - a.t) { g += 1 } + let offset = abs(gyro[g].t - a.t) + guard offset <= maxPairingOffset else { continue } + worst = max(worst, offset) + let cells = [a.t, a.value.x, a.value.y, a.value.z, gyro[g].value.x, gyro[g].value.y, gyro[g].value.z] + text += cells.map(\.description).joined(separator: ",") + "\n" + times.append(a.t) + } + guard !times.isEmpty else { throw Packet04Error.streamEmpty("imuRaw") } + let rate = measuredRate(times) + guard rate >= minimumIMUHz else { throw Packet04Error.streamTooSlow(stream: "imuRaw", measuredHz: rate, minimumHz: minimumIMUHz) } + let note = [ + "imuRawRowClock": "accelerometer", + "imuRawGyroPairing": "nearest gyro sample within \(maxPairingOffset) s; accelerometer samples without one are left out", + "imuRawGyroOffsetMaxS": worst.description, + "imuRawRowsKept": "\(times.count) of \(accel.count)", + "imuRawPairingStatus": "proposal from the phone; the server has not agreed how accelerometer and gyro samples join into one row", + ] + return (Data(text.utf8), rate, note) + } + + /// The first state and every change. A limited state with no name in the contract is not + /// listed; the rows around it still are. + public static func trackingChanges(_ rows: [PoseRow], epoch: String) -> [Packet04.TrackingEvent] { + var events: [Packet04.TrackingEvent] = [] + for row in rows { + guard let state = Packet04Producer.trackingText(row.tracking), state != events.last?.state else { continue } + events.append(.init(time: row.t, state: state, epoch: epoch)) + } + return events + } + + /// One sensor's samples exactly as Core Motion timed them, `t,x,y,z`: the raw evidence behind + /// `imuRaw`, kept because the pairing rule there is a proposal the server has not agreed to. + public static func rawMotionCSV(_ rows: [MotionRow], columns: [String]) throws -> Data { + guard !rows.isEmpty else { throw Packet04Error.streamEmpty(columns.joined(separator: ",")) } + var text = columns.joined(separator: ",") + "\n" + for row in rows.sorted(by: { $0.t < $1.t }) { + text += [row.t, row.value.x, row.value.y, row.value.z].map(\.description).joined(separator: ",") + "\n" + } + return Data(text.utf8) + } + + /// Pose rows from the recorder's raw rows: trajectory rows `[t, tracking state, tracking + /// reason, 16 camera-to-world values column by column]` and intrinsics rows `[t, fx, fy, cx, + /// cy, width, height]` written for the same frame at the same t. A pose with no intrinsics row + /// at its t is left out rather than given another frame's calibration. + public static func poseRows(trajectory: [[Double]], intrinsics: [[Double]], tracking: (Int, Int) -> PacketTracking) -> [PoseRow] { + var calibration: [Double: SIMD4] = [:] + for row in intrinsics where row.count >= 5 { calibration[row[0]] = SIMD4(Float(row[1]), Float(row[2]), Float(row[3]), Float(row[4])) } + return trajectory.compactMap { row in + guard row.count == 19, let k = calibration[row[0]] else { return nil } + let f = row[3...].map(Float.init) + let m = simd_float4x4(SIMD4(f[0], f[1], f[2], f[3]), SIMD4(f[4], f[5], f[6], f[7]), SIMD4(f[8], f[9], f[10], f[11]), SIMD4(f[12], f[13], f[14], f[15])) + return PoseRow(t: row[0], tracking: tracking(Int(row[1]), Int(row[2])), cameraToWorld: m, intrinsics: k) + } + } + + /// Motion rows from raw `[t, x, y, z]` rows. + public static func motionRows(_ rows: [[Double]]) -> [MotionRow] { + rows.filter { $0.count == 4 }.map { MotionRow(t: $0[0], value: SIMD3($0[1], $0[2], $0[3])) } + } + + /// 1 / the median interval between rows, the rate actually recorded. 0 with fewer than two. + public static func measuredRate(_ times: [Double]) -> Double { + let deltas = zip(times.dropFirst(), times).map { $0 - $1 }.filter { $0 > 0 }.sorted() + guard !deltas.isEmpty else { return 0 } + return 1 / deltas[deltas.count / 2] + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift new file mode 100644 index 000000000..dc33d562d --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureAPI.swift @@ -0,0 +1,227 @@ +import Foundation + +/// The capture API's request and response bodies that House Scan sends and reads (`/v1/captures` +/// and below), written from the documented wire contract. Unknown response fields are ignored. +public enum CaptureAPI { + public struct Device: Codable, Sendable, Equatable { + public var model: String + public var systemVersion: String + public var appVersion: String + + public init(model: String, systemVersion: String, appVersion: String) { + self.model = model + self.systemVersion = systemVersion + self.appVersion = appVersion + } + } + + /// The create body. Expected counts are left out: they are optional, and a count that grows as + /// photos arrive would change the body the idempotency key is bound to. + public struct CreateRequest: Codable, Sendable, Equatable { + public var packetId: String + public var formatVersion = Packet04.formatVersion + public var tier: Packet04.Tier + public var flow = "guided" + public var device: Device + + public init(packetId: String, tier: Packet04.Tier, device: Device) { + self.packetId = packetId + self.tier = tier + self.device = device + } + } + + public struct Limits: Codable, Sendable, Equatable { + public var maxSinglePutBytes: Int? + public var urlTtlS: Int? + public var maxFiles: Int? + public var maxBatch: Int? + } + + public struct CreateResponse: Codable, Sendable, Equatable { + public var captureId: String + public var status: String + public var upload: Limits? + public var finalizeBy: String? + } + + public struct RegisterFile: Codable, Sendable, Equatable { + public var path: String + public var bytes: Int + public var sha256: String + public var md5: String + public var role: Packet04.Role + public var contentType: String + public var meta: Packet04.ImageRecord? + + public init(_ file: SealedFile) { + path = file.path + bytes = file.bytes + sha256 = file.sha256 + md5 = file.md5 + role = file.role + contentType = file.contentType + meta = file.meta + } + } + + public struct RegisterRequest: Codable, Sendable, Equatable { + public var files: [RegisterFile] + } + + public struct UploadTarget: Codable, Sendable, Equatable { + public var method: String + public var url: String? + public var headers: [String: String]? + public var expiresAt: String? + } + + public struct RegisteredFile: Codable, Sendable, Equatable { + public var path: String + public var state: String + public var upload: UploadTarget? + } + + public struct RegisterResponse: Codable, Sendable, Equatable { + public var files: [RegisteredFile] + } + + public struct CommitFile: Codable, Sendable, Equatable { + public var path: String + public var sha256: String + } + + public struct CommitRequest: Codable, Sendable, Equatable { + public var files: [CommitFile] + } + + public struct CommitResponse: Codable, Sendable, Equatable { + public var committed: [String] + public var notFound: [String] + public var mismatch: [String] + } + + public struct FinalizeResponse: Codable, Sendable, Equatable { + public var status: String + public var missing: [String] + public var runId: String + public var etaS: Int? + } + + public struct Event: Codable, Sendable, Equatable { + public var seq: Int + public var type: String + public var at: String + /// The fields the app reads. Others, which may carry text meant for the homeowner, are + /// not kept. + public var data: EventData? + } + + public struct EventData: Codable, Sendable, Equatable { + public var code: String? + public var kind: String? + public var stage: String? + public var status: String? + public var runId: String? + public var next: String? + } + + public struct EventsResponse: Codable, Sendable, Equatable { + public var status: String + public var next: Int + public var events: [Event] + } + + public struct ErrorItem: Codable, Sendable, Equatable { + public var code: String + public var pointer: String? + public var message: String? + } + + public struct ErrorBody: Codable, Sendable, Equatable { + public var errors: [ErrorItem] + } + + public static func errorCodes(_ body: Data) -> [String] { + (try? JSONDecoder().decode(ErrorBody.self, from: body))?.errors.map(\.code) ?? [] + } + + /// Statuses after which the capture changes no more. + public static let terminalStatuses: Set = ["complete", "manual_review", "failed", "expired", "needs_views"] + + /// Terminal statuses that can end a capture with no answer, so a result read carrying one and + /// a null outcome is final. The others come with the run's outcome. + public static let statusesWithoutOutcome: Set = ["failed", "expired"] +} + +/// An HTTP answer: status, body and the headers the uploader reads. +public struct HTTPReply: Sendable, Equatable { + public var status: Int + public var body: Data + public var retryAfter: Double? + + public init(status: Int, body: Data, retryAfter: Double? = nil) { + self.status = status + self.body = body + self.retryAfter = retryAfter + } +} + +/// How the uploader reaches the network. HTTP error statuses come back as replies; only transport +/// failures throw. An implementation must not follow redirects: the uploader checks each URL +/// before it sends a photo there, and a redirect would send it somewhere unchecked. A redirect +/// comes back as its own 3xx reply. +public protocol CaptureHTTP: Sendable { + func send(_ request: URLRequest) async throws -> HTTPReply + /// `file` as the whole body of `request`, streamed from disk. + func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply +} + +/// `CaptureHTTP` on one ordinary `URLSession`. +public struct URLSessionCaptureHTTP: CaptureHTTP { + public let session: URLSession + + public init(session: URLSession) { + self.session = session + } + + /// An ephemeral session: nothing cached or stored in cookies, and no credentials offered to + /// the signed storage URLs. + public static func ephemeral(timeout: Double = 60) -> URLSessionCaptureHTTP { + let configuration = URLSessionConfiguration.ephemeral + configuration.timeoutIntervalForRequest = timeout + configuration.httpCookieStorage = nil + configuration.urlCredentialStorage = nil + configuration.httpMaximumConnectionsPerHost = 4 + return URLSessionCaptureHTTP(session: URLSession(configuration: configuration)) + } + + public func send(_ request: URLRequest) async throws -> HTTPReply { + let (data, response) = try await session.data(for: request, delegate: NoRedirects.shared) + return Self.reply(data, response) + } + + public func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + let (data, response) = try await session.upload(for: request, fromFile: file, delegate: NoRedirects.shared) + return Self.reply(data, response) + } + + /// Refuses every redirect, so the task returns the 3xx response itself. URLSession follows + /// redirects by default, and App Transport Security alone would still allow one to another + /// https host, or to plain http on the local network, which the app's Info.plist permits. + final class NoRedirects: NSObject, URLSessionTaskDelegate, Sendable { + static let shared = NoRedirects() + + func urlSession( + _ session: URLSession, task: URLSessionTask, willPerformHTTPRedirection response: HTTPURLResponse, newRequest request: URLRequest + ) async -> URLRequest? { + nil + } + } + + static func reply(_ data: Data, _ response: URLResponse) -> HTTPReply { + let http = response as? HTTPURLResponse + return HTTPReply( + status: http?.statusCode ?? 0, body: data, retryAfter: http?.value(forHTTPHeaderField: "Retry-After").flatMap(Double.init)) + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift new file mode 100644 index 000000000..d35fea631 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureSessionCoordinator.swift @@ -0,0 +1,502 @@ +import Foundation +import simd + +/// A photo the scan kept, as the app stores it: the JPEG on disk and the facts of the ARFrame +/// whose pixels it holds, in that frame's own terms (raw world pose, the camera image's +/// intrinsics and size). +public struct KeptPhoto: Sendable { + public var t: Double + public var cameraToWorld: simd_float4x4 + /// [fx, fy, cx, cy] in pixels of the camera image (`cameraImageSize`). + public var cameraIntrinsics: SIMD4 + public var cameraImageSize: SIMD2 + /// Pixels of the stored JPEG. + public var width: Int + public var height: Int + public var tracking: PacketTracking + public var exposure: Packet04.Exposure? + public var jpeg: URL + /// The still's purpose ("meter_close"); nil for a keyframe. + public var purpose: String? + /// Reads the photo's ARKit depth. Called once, when the photo is kept, because the app writes a + /// retaken close-up's depth over the same files. + public var depth: @Sendable () -> Packet04Depth? + + public init( + t: Double, cameraToWorld: simd_float4x4, cameraIntrinsics: SIMD4, cameraImageSize: SIMD2, width: Int, height: Int, + tracking: PacketTracking, exposure: Packet04.Exposure?, jpeg: URL, purpose: String?, depth: @escaping @Sendable () -> Packet04Depth? = { nil } + ) { + self.t = t + self.cameraToWorld = cameraToWorld + self.cameraIntrinsics = cameraIntrinsics + self.cameraImageSize = cameraImageSize + self.width = width + self.height = height + self.tracking = tracking + self.exposure = exposure + self.jpeg = jpeg + self.purpose = purpose + self.depth = depth + } + + /// The observation in the stored JPEG's pixels: intrinsics scaled when the JPEG is not the + /// camera image's size. + public var observation: Packet04Observation { + let scale = SIMD2(Float(width), Float(height)) / cameraImageSize + let k = cameraIntrinsics + return Packet04Observation( + t: t, cameraToWorld: cameraToWorld, intrinsics: SIMD4(k.x * scale.x, k.y * scale.y, k.z * scale.x, k.w * scale.y), + width: width, height: height, tracking: tracking, exposure: exposure) + } +} + +/// The frame under the homeowner's tap, read when the tap happened. +public struct TapObservation: Sendable { + public var t: Double + public var cameraToWorld: simd_float4x4 + /// [fx, fy, cx, cy] in pixels of the camera image, which the JPEG stores unscaled. + public var intrinsics: SIMD4 + public var width: Int + public var height: Int + public var tracking: PacketTracking + /// The tap in the sensor image's pixels, (0, 0) its top-left corner. + public var pixel: SIMD2 + /// Encodes the frame's image; called off the main actor. + public var jpeg: @Sendable () -> Data? + + /// The same frame with the tap moved to where `world` (the raycast's hit) lands in its image, + /// so pixel, ray and hit agree in this one frame even when it is a frame or two newer than + /// the one the raycast used. Nil when the point is behind the camera or outside the image. + public func pointing(at world: SIMD3) -> TapObservation? { + let p = cameraToWorld.inverse * SIMD4(world, 1) + guard p.z < -0.05 else { return nil } + let u = Double(intrinsics.z + intrinsics.x * p.x / -p.z), v = Double(intrinsics.w - intrinsics.y * p.y / -p.z) + guard (0...Double(width)).contains(u), (0...Double(height)).contains(v) else { return nil } + var aimed = self + aimed.pixel = SIMD2(u, v) + return aimed + } + + public init( + t: Double, cameraToWorld: simd_float4x4, intrinsics: SIMD4, width: Int, height: Int, tracking: PacketTracking, + pixel: SIMD2, jpeg: @escaping @Sendable () -> Data? + ) { + self.t = t + self.cameraToWorld = cameraToWorld + self.intrinsics = intrinsics + self.width = width + self.height = height + self.tracking = tracking + self.pixel = pixel + self.jpeg = jpeg + } +} + +/// Where the coordinator reads the world's recorded streams: `CaptureRecorder` in the app. +public struct RecordingSource: Sendable { + /// Uptime and wall clock of the world's first recorded frame. + public var start: @Sendable () -> (uptime: Double, date: Date)? + /// Every row so far; called off the main actor. + public var rows: @Sendable () -> RecorderRows + + public init(start: @escaping @Sendable () -> (uptime: Double, date: Date)?, rows: @escaping @Sendable () -> RecorderRows) { + self.start = start + self.rows = rows + } +} + +/// The recorder's raw rows: trajectory `[t, state, reason, 16 pose values]`, intrinsics +/// `[t, fx, fy, cx, cy, w, h]`, and Core Motion's `[t, x, y, z]`, each sensor at its own times. +public struct RecorderRows: Sendable { + public var trajectory: [[Double]] + public var intrinsics: [[Double]] + public var accelerometer: [[Double]] + public var gyroscope: [[Double]] + + public init(trajectory: [[Double]], intrinsics: [[Double]], accelerometer: [[Double]], gyroscope: [[Double]]) { + self.trajectory = trajectory + self.intrinsics = intrinsics + self.accelerometer = accelerometer + self.gyroscope = gyroscope + } +} + +/// What the integration build does with a scan: one 0.4 packet per ARKit world, each kept photo +/// sealed into it on the phone as it is kept, the packet frozen when the scan is sent for +/// placement. Sending is separate: only a build allowed to send device data (`sends`) sends, and +/// only after the homeowner's yes for this scan, asked before it starts. From the yes on, every +/// sealed file goes up, and later photos go up as they are kept. A new scan asks again; a world +/// reset within the scan keeps its answer for the scan's next packet. +/// +/// Work for one world runs in order on that world's chain; a world reset or start over ends the +/// session, and nothing queued for it reaches the next one. +@MainActor +public final class CaptureSessionCoordinator { + public struct Environment: Sendable { + public var endpoint: URL + /// Whether device captures may go to `endpoint` at all (`CaptureIntegrationMode.send`). + public var sends: Bool + public var http: any CaptureHTTP + /// Each session writes to `/`. + public var capturesFolder: URL + public var sessionInfo: @Sendable (_ packetID: String, _ video: SIMD2) -> Packet04SessionInfo + public var device: CaptureAPI.Device + public var tier: Packet04.Tier + public var policy: CaptureUploader.Policy + public var log: @Sendable (String) -> Void + + public init( + endpoint: URL, sends: Bool, http: any CaptureHTTP, capturesFolder: URL, + sessionInfo: @escaping @Sendable (_ packetID: String, _ video: SIMD2) -> Packet04SessionInfo, + device: CaptureAPI.Device, tier: Packet04.Tier, policy: CaptureUploader.Policy = .init(), log: @escaping @Sendable (String) -> Void = { _ in } + ) { + self.endpoint = endpoint + self.sends = sends + self.http = http + self.capturesFolder = capturesFolder + self.sessionInfo = sessionInfo + self.device = device + self.tier = tier + self.policy = policy + self.log = log + } + } + + /// Checklist purposes the packet's `stills` take; any other still stays a keyframe only. + public static let stillPurposes: Set = ["meter_close", "meter_oblique"] + + @MainActor + public final class Session { + /// This app's id for this session, new for every session even when two packets both call + /// their ARKit epoch "e1": a result is bound to it, so a late result can't land on a newer scan. + public let localID = UUID().uuidString + public let packetID: String + public let folder: URL + /// Nil until the homeowner says yes on a build that sends. + public fileprivate(set) var uploader: CaptureUploader? + public private(set) var producer: Packet04Producer? + let recording: RecordingSource + var chain: Task? + var sealing = false + var ended = false + /// The homeowner said no after a yes: this capture is never sent again. + var consentWithdrawn = false + var withdrawalFailure: ConsentWithdrawalError? + /// packet.json as frozen, once the scan was sent for placement. + var frozen: Data? + /// The server's result for this session's capture, decoded and bound to it. + public fileprivate(set) var result: CaptureResult.Record? + + init(packetID: String, folder: URL, recording: RecordingSource) { + self.packetID = packetID + self.folder = folder + self.recording = recording + } + + /// Runs `work` after everything queued before it, unless the session has ended by then. + /// `cleanup` runs either way, after `work` when it ran. + func enqueue(_ work: @escaping @MainActor (Session) async -> Void, cleanup: (@MainActor @Sendable () -> Void)? = nil) { + let previous = chain + chain = Task { + await previous?.value + defer { cleanup?() } + guard !self.ended else { return } + await work(self) + } + } + + func producer(first photo: (t: Double, video: SIMD2), info: (String, SIMD2) -> Packet04SessionInfo) throws -> Packet04Producer { + if let producer { return producer } + // The packet starts at the world's first recorded frame; a photo can't be older. + let start = recording.start() ?? (photo.t, Date(timeIntervalSinceNow: photo.t - ProcessInfo.processInfo.systemUptime)) + let made = try Packet04Producer(folder: folder, info: info(packetID, photo.video), startedAtUptime: min(start.uptime, photo.t), startedAt: start.date) + producer = made + return made + } + } + + public let environment: Environment? + /// The homeowner's answer for this scan: nil before they were asked. + public private(set) var consent: Bool? + private var consentedAt: Date? + public private(set) var session: Session? + private var recording: RecordingSource? + /// The current session's status; nil when there is none or nothing is being sent. + public var onStatus: (@MainActor (CaptureUploadStatus?) -> Void)? + /// The current session's result, once the server has one. Never called for an ended session. + public var onResult: (@MainActor (CaptureResult.Record) -> Void)? + /// The packet's ARKit epoch: one packet is one world. + public static let epoch = "e1" + + public init(environment: Environment?) { + self.environment = environment + } + + /// Whether the homeowner should be asked now: a build that sends, a scan under way, and no + /// answer for it yet. + public var needsConsent: Bool { + environment?.sends == true && consent == nil && session != nil + } + + public enum ConsentWithdrawalError: Error, Sendable, Equatable { + /// Sending stopped in this process, but a relaunch may still find the old saved yes. + case notRecorded(String) + } + + /// A no after a yes stops this capture's upload for good: nothing kept later is sent, and a + /// second yes in the same scan does not start it again. What the server already has stays there. + /// Failure means sending stopped here but the withdrawal could not be saved for a relaunch. + @discardableResult + public func answerConsent(_ yes: Bool, at time: Date = Date()) -> Result { + consent = yes + consentedAt = yes ? time : nil + guard let session else { return .success(()) } + if let failure = session.withdrawalFailure { return .failure(failure) } + if yes { + startUploading(session) + } else if let uploader = session.uploader { + session.consentWithdrawn = true + session.uploader = nil + // The result distinguishes durable withdrawal from stopping only this live process. + switch uploader.withdrawConsent() { + case .marked: break + case .savedStateRemoved: + environment?.log("capture upload: the withdrawal marker could not be written; the saved upload was removed instead") + case .notRecorded(let error): + session.withdrawalFailure = .notRecorded(error) + environment?.log("capture upload: the withdrawal could not be saved (\(error)); sending stopped in this process, but a relaunch could still find the saved consent") + } + Task { await uploader.abandon(CaptureUploader.withdrawnReason) } + } + if let failure = session.withdrawalFailure { return .failure(failure) } + return .success(()) + } + + // MARK: Scan events + + /// The live source started recording a world. + public func begin(recording: RecordingSource) { + self.recording = recording + if environment != nil, session == nil { startSession() } + } + + /// The ARKit world was thrown away, or (`newScan`) the scan started over: this packet can't + /// be finished, and the next one belongs to `recording`'s new world. A new scan asks again. + public func newWorld(_ reason: String, recording: RecordingSource, newScan: Bool) { + endSession(reason) + if newScan { + consent = nil + consentedAt = nil + } + self.recording = recording + if environment != nil { startSession() } + } + + public func kept(_ photo: KeptPhoto) { + guard let session, !session.sealing, let environment else { return } + // The app writes a retaken close-up (JPEG and depth) over the same files, so both are taken + // now, the JPEG as an APFS clone, not when the queued work gets to them. + let depth = photo.depth() + let staged = environment.capturesFolder.appending(path: "staging/\(UUID().uuidString).jpg") + do { + try FileManager.default.createDirectory(at: staged.deletingLastPathComponent(), withIntermediateDirectories: true) + try FileManager.default.copyItem(at: photo.jpeg, to: staged) + } catch { + environment.log("capture packet: a kept photo could not be copied: \(error)") + return + } + // The copy is deleted even when a reset ends the session before this work runs. + session.enqueue({ session in + do { + let producer = try session.producer(first: (photo.t, photo.cameraImageSize), info: environment.sessionInfo) + let data = try await Task.detached(priority: .utility) { try Data(contentsOf: staged) }.value + var files: [SealedFile] = [] + if await producer.keyframeID(at: photo.t) == nil { + files += try await producer.sealKeyframe( + jpeg: data, observation: photo.observation, reason: photo.purpose == nil ? "auto" : "still", purpose: photo.purpose, depth: depth) + } + if let purpose = photo.purpose, Self.stillPurposes.contains(purpose), let keyframe = await producer.keyframeID(at: photo.t) { + files.append(try await producer.sealStill(purpose: purpose, keyframe: keyframe)) + } + await session.uploader?.add(files) + } catch { + environment.log("capture packet: a kept photo was not sealed: \(error)") + } + }, cleanup: { try? FileManager.default.removeItem(at: staged) }) + } + + /// The homeowner marked the meter: its frame becomes a keyframe and the tap goes on it. + public func meterTapped(_ tap: TapObservation, hit: Packet04.TapHit?) { + guard let session, !session.sealing, let environment else { return } + session.enqueue { session in + do { + let producer = try session.producer(first: (tap.t, SIMD2(Float(tap.width), Float(tap.height))), info: environment.sessionInfo) + guard let jpeg = await Task.detached(priority: .userInitiated, operation: { tap.jpeg() }).value else { + environment.log("capture packet: the meter tap's frame could not be encoded; no tap") + return + } + let observation = Packet04Observation( + t: tap.t, cameraToWorld: tap.cameraToWorld, intrinsics: tap.intrinsics, width: tap.width, height: tap.height, tracking: tap.tracking) + let files = try await producer.sealTap(id: "meter", label: "meter", jpeg: jpeg, observation: observation, pixel: tap.pixel, hit: hit) + await session.uploader?.add(files) + } catch { + environment.log("capture packet: the meter tap was not recorded: \(error)") + } + } + } + + /// The scan was sent for placement: freeze the packet with the streams recorded so far. + /// `acceptedCloseUpAt` is the frame time of the close-up the scan accepted, nil after a skip. + /// Later sends of the same scan (a retry, one more view) change nothing. + public func captureEnded(acceptedCloseUpAt: Double?) { + guard let session, !session.sealing, let environment else { return } + session.sealing = true + session.enqueue { session in + guard let producer = session.producer else { + await session.uploader?.abandon("no photos were kept") + return + } + let recording = session.recording + let rows = await Task.detached(priority: .utility) { recording.rows() }.value + let poses = Packet04Streams.poseRows(trajectory: rows.trajectory, intrinsics: rows.intrinsics, tracking: PacketTracking.init(recorderState:reason:)) + do { + let finished = try await producer.finish( + poses: poses, accelerometer: Packet04Streams.motionRows(rows.accelerometer), + gyroscope: Packet04Streams.motionRows(rows.gyroscope), endedAtUptime: poses.last?.t ?? 0, acceptedCloseUpAt: acceptedCloseUpAt) + session.frozen = finished.packet + await session.uploader?.seal(packet: finished.packet, files: await producer.sealedFiles) + } catch { + environment.log("capture packet not finished: \(error)") + await session.uploader?.abandon("packet refused on the phone") + } + } + } + + /// Waits until the current session's queued work and uploads are idle. For tests and the + /// evidence run. + public func settle() async { + guard let session else { return } + while let chain = session.chain { + await chain.value + if session.chain == chain { break } + } + await session.uploader?.settled() + } + + /// After a relaunch: captures frozen before the app quit finish uploading, each only toward the + /// endpoint it was created on and with the yes it recorded for that endpoint. Returns them. + public func resumeSealedCaptures() -> [CaptureUploader] { + guard let environment, environment.sends, + let folders = try? FileManager.default.contentsOfDirectory(at: environment.capturesFolder, includingPropertiesForKeys: nil) + else { return [] } + var resumed: [CaptureUploader] = [] + for folder in folders { + guard let saved = try? CaptureUploadState.load(from: CaptureUploader.stateURL(in: folder)), saved.end == nil, saved.packet != nil, + let uploader = try? CaptureUploader.resume(folder: folder, base: environment.endpoint, http: environment.http, policy: environment.policy) + else { continue } + resumed.append(uploader) + Task { await uploader.kick() } + } + return resumed + } + + // MARK: Sessions + + private func startSession() { + guard let environment, let recording else { return } + let packetID = UUID().uuidString + let session = Session(packetID: packetID, folder: environment.capturesFolder.appending(path: packetID, directoryHint: .isDirectory), recording: recording) + self.session = session + environment.log("capture packet: new packet for this world") + if consent == true { startUploading(session) } + } + + /// Opens the capture on the server and queues everything sealed so far, then the frozen + /// packet if the scan was already sent. Photos kept later go up as they are sealed. + private func startUploading(_ session: Session) { + guard let environment, environment.sends, consent == true, let consentedAt, session.uploader == nil, !session.ended, + !session.consentWithdrawn else { return } + do { + let uploader = try CaptureUploader.start( + folder: session.folder, base: environment.endpoint, http: environment.http, + create: .init(packetId: session.packetID, tier: environment.tier, device: environment.device), consentedAt: consentedAt, + policy: environment.policy) + session.uploader = uploader + // Status and results from an ended session's uploader never reach the screen. + let publish: @Sendable (CaptureUploadStatus) -> Void = { [weak self, weak session] status in + Task { @MainActor in + guard let self, let session, self.session === session else { return } + self.onStatus?(status) + if status.resultAvailable, session.result == nil { await self.bindResult(of: session) } + } + } + let log = environment.log + Task { + await uploader.observe(publish, log: log) + await uploader.kick() + } + session.enqueue { session in + guard let producer = session.producer else { return } + let files = await producer.sealedFiles + if let frozen = session.frozen { + await uploader.seal(packet: frozen, files: files) + } else { + await uploader.add(files) + } + } + environment.log("capture upload: sending this world's packet") + } catch { + environment.log("capture upload not started: \(error)") + } + } + + /// Decodes the uploader's result and binds it to the session, capture and run that asked for + /// it. The analysis stays unverified: nothing in the response proves the server reconstructed + /// anything, so the spatial result is withheld (`CaptureResult.Record.placement`). + private func bindResult(of session: Session) async { + guard let uploader = session.uploader else { return } + let state = await uploader.snapshot + guard self.session === session, session.result == nil, let body = state.result, let captureID = state.captureID, + let runID = state.finalized?.runID else { return } + do { + let response = try JSONDecoder().decode(CaptureResult.Response.self, from: body) + let record = CaptureResult.Record( + response: response, + association: .init(sessionID: session.localID, captureID: captureID, runID: runID, epoch: Self.epoch)) + session.result = record + onResult?(record) + } catch { + environment?.log("capture result could not be read: \(error)") + } + } + + private func endSession(_ reason: String) { + guard let session else { return } + self.session = nil + session.ended = true + onStatus?(nil) + if let uploader = session.uploader { + uploader.stopSending() + Task { await uploader.abandon(reason) } + } + } +} + +extension PacketTracking { + /// The recorder's codes: state 0 normal, 1 limited, 2 not available; reason 0 none or + /// unknown, 1 initializing, 2 relocalizing, 3 excessive motion, 4 insufficient features. + public init(recorderState state: Int, reason: Int) { + let why: Reason? = switch reason { + case 1: .initializing + case 2: .relocalizing + case 3: .excessiveMotion + case 4: .insufficientFeatures + default: nil + } + self = switch state { + case 0: .normal + case 1: .limited(why) + default: .notAvailable + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadGate.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadGate.swift new file mode 100644 index 000000000..068ee46e2 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadGate.swift @@ -0,0 +1,72 @@ +import Foundation + +/// Whether this build and this homeowner allow a capture upload. No endpoint means nothing is +/// offered and nothing is sent; an endpoint without the homeowner's yes sends nothing either. +public enum CaptureUploadGate { + public enum Decision: Sendable, Equatable { + case off(String) + case on(URL) + } + + /// `endpoint` is the API base including `/v1`, from the build's Info.plist. An empty build + /// setting, an unexpanded `$(...)`, or anything but https (http only to this machine, for + /// tests) is off. + public static func decide(endpoint: String?, consented: Bool) -> Decision { + let text = endpoint?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard !text.isEmpty, !text.hasPrefix("$(") else { return .off("no capture endpoint in this build") } + guard let url = URL(string: text), let scheme = url.scheme?.lowercased(), let host = url.host(), url.user == nil, url.password == nil else { + return .off("capture endpoint is not a URL") + } + let local = host == "127.0.0.1" || host == "localhost" + guard scheme == "https" || (scheme == "http" && local) else { return .off("capture endpoint must be https") } + guard consented else { return .off("the homeowner has not agreed to send the capture") } + return .on(url) + } +} + +/// What the app does with its captures, from three build settings read once at launch. Only the +/// integration build records a 0.4 capture at all; only a build that also names an endpoint and +/// says device data may go there sends one, and then only after the homeowner's yes. +public enum CaptureIntegrationMode: Sendable, Equatable { + /// The client build, or no usable endpoint: nothing is recorded for the capture API or sent. + case off(String) + /// The capture is recorded on the phone and never sent: the endpoint may not receive device + /// data (an unauthenticated test server, unless the build says otherwise). + case recordOnly(URL) + /// Recorded, and sent after the homeowner's yes. + case send(URL) + + public var endpoint: URL? { + switch self { + case .off: nil + case .recordOnly(let url), .send(let url): url + } + } + + /// Where the scan's pictures come from. + public enum Source: Sendable, Equatable { + /// The phone's camera: sending needs the build's device-data switch. + case device + /// A recorded session played back (`-replay`). Its pictures can be anyone's, so it is sent + /// only to a receiver on this machine, and only when the run asks for that + /// (`sendToLocalReceiver`, the UI tests' switch); never to a remote endpoint. + case replay(sendToLocalReceiver: Bool) + } + + /// `integrationBuild` and `sendDeviceData` are Info.plist values that must read exactly "YES". + /// A launch argument can name `endpoint`, but can't turn a client build into an integration + /// build or allow device data. + public static func resolve(integrationBuild: String?, endpoint: String?, sendDeviceData: String?, source: Source = .device) -> CaptureIntegrationMode { + guard integrationBuild == "YES" else { return .off("not the integration build") } + switch CaptureUploadGate.decide(endpoint: endpoint, consented: true) { + case .off(let reason): return .off(reason) + case .on(let url): + switch source { + case .device: return sendDeviceData == "YES" ? .send(url) : .recordOnly(url) + case .replay(let local): + let loopback = url.host() == "127.0.0.1" || url.host() == "localhost" + return local && loopback ? .send(url) : .recordOnly(url) + } + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift new file mode 100644 index 000000000..f27754c2c --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploadState.swift @@ -0,0 +1,197 @@ +import Foundation + +/// One capture's upload, saved in its folder after every change so a relaunch resumes the same +/// capture with the same identity. The request bodies that idempotency keys are bound to (create +/// and finalize) are saved as the exact bytes first sent, never re-encoded. +public struct CaptureUploadState: Codable, Sendable, Equatable { + public static let fileName = "capture-upload.json" + /// Written into a capture's folder when the homeowner takes back their yes. `CaptureUploader.resume` + /// never resumes a folder that has it. + public static let withdrawnFileName = "consent-withdrawn" + + public static func withdrawnURL(in folder: URL) -> URL { folder.appending(path: withdrawnFileName) } + + /// Called with the uploader's folder registry locked so no initial save or reply can restore + /// a yes between these writes. Neither failing proves anything durable after a process exit. + static func recordWithdrawal(in folder: URL) -> CaptureUploader.WithdrawalRecord { + do { + try Data().write(to: withdrawnURL(in: folder), options: .atomic) + return .marked + } catch { + let markerError = CaptureUploader.describe(error) + do { + try FileManager.default.removeItem(at: CaptureUploader.stateURL(in: folder)) + return .savedStateRemoved + } catch let removal as NSError where removal.domain == NSCocoaErrorDomain && removal.code == NSFileNoSuchFileError { + return .savedStateRemoved + } catch { + return .notRecorded("marker: \(markerError); state removal: \(CaptureUploader.describe(error))") + } + } + } + + public enum FilePhase: Codable, Sendable, Equatable { + /// Sealed on the phone, not yet registered (or its URL must be fetched again). + case queued + case registered(url: String, headers: [String: String], expiresAt: Date?) + /// The storage PUT answered 2xx. Not yet received: only a commit says that. + case uploaded + /// Acknowledged by files:commit. + case committed + } + + public struct File: Codable, Sendable, Equatable { + public var sealed: SealedFile + public var phase: FilePhase = .queued + public var attempts = 0 + /// Register answers that left this file out, plus commit answers that didn't acknowledge + /// it. Nil in state saved before this was counted. + public var unacknowledged: Int? + /// Order in which the file was sealed, the tie-break after priority. + public var sequence: Int + + public init(sealed: SealedFile, phase: FilePhase = .queued, attempts: Int = 0, sequence: Int) { + self.sealed = sealed + self.phase = phase + self.attempts = attempts + self.sequence = sequence + } + } + + public enum End: Codable, Sendable, Equatable { + /// The ARKit world was reset or the scan started over: this packet will never be finished. + case abandoned(String) + /// The server refused something a retry can't change. + case failed(step: String, codes: [String], status: Int) + /// The capture reached a status after which it changes no more. + case finished(status: String) + } + + public struct Finalized: Codable, Sendable, Equatable { + public var status: String + public var missing: [String] + public var runID: String + + public init(status: String, missing: [String], runID: String) { + self.status = status + self.missing = missing + self.runID = runID + } + } + + /// Local identity of this upload. Answers that arrive for another attempt are dropped. + public var attemptID: String + public var packetID: String + /// The API base the capture was created on. A resume goes only there: its capture id and + /// signed URLs mean nothing to another server. + public var destination: String? + /// The homeowner's yes for this capture and this destination. A capture without one is + /// never resumed. + public var consent: Consent? + + public struct Consent: Codable, Sendable, Equatable { + public var grantedAt: Date + public var destination: String + + public init(grantedAt: Date, destination: String) { + self.grantedAt = grantedAt + self.destination = destination + } + } + /// The create body exactly as first sent. + public var createBody: Data + public var captureID: String? + public var maxBatch = 50 + public var files: [String: File] = [:] + public var nextSequence = 0 + /// packet.json exactly as frozen when the capture stopped. + public var packet: Data? + public var finalized: Finalized? + public var eventCursor = 0 + /// `retry_finalize` answers followed so far. + public var finalizeRetries = 0 + /// Result reads that found no answer yet. Saved so a relaunch continues the same limit; nil in + /// state saved before this was counted. + public var notReadyReads: Int? + public var backendStatus: String? + public var lastEvent: String? + public var result: Data? + public var end: End? + /// Wall-clock marks of each stage, for latency: no ids, URLs or photos. + public var marks: [String: Date] = [:] + + public init(attemptID: String, packetID: String, createBody: Data) { + self.attemptID = attemptID + self.packetID = packetID + self.createBody = createBody + } + + public var committedCount: Int { files.values.filter { $0.phase == .committed }.count } + public var committedBytes: Int { files.values.filter { $0.phase == .committed }.reduce(0) { $0 + $1.sealed.bytes } } + public var totalBytes: Int { files.values.reduce(0) { $0 + $1.sealed.bytes } } + + public func save(to url: URL) throws { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + try encoder.encode(self).write(to: url, options: .atomic) + } + + public static func load(from url: URL) throws -> CaptureUploadState { + try JSONDecoder().decode(CaptureUploadState.self, from: Data(contentsOf: url)) + } +} + +/// What the experience layer shows: separate facts, never one percentage. +public struct CaptureUploadStatus: Sendable, Equatable { + public enum Phase: Sendable, Equatable { + case creating, uploading, processing, finished, failed, abandoned + } + + public var phase: Phase + /// Files sealed on the phone for this capture. + public var retained: Int + /// Files the server acknowledged with a commit. + public var committed: Int + public var committedBytes: Int + public var totalBytes: Int + /// Whether packet.json is frozen (the capture stopped). + public var sealed: Bool + /// The capture's status as the server last reported it. + public var backendStatus: String? + public var lastEvent: String? + public var eventCursor: Int + public var resultAvailable: Bool + /// The server's id for the capture, once created: what an operator copies to find this scan. + public var captureID: String? + /// The run finalize started, once accepted. + public var runID: String? + /// Set while the uploader waits out a retryable failure. + public var retryingAt: Date? + public var detail: String? + + public init(_ state: CaptureUploadState, retryingAt: Date? = nil, detail: String? = nil) { + phase = switch state.end { + case .abandoned?: .abandoned + case .failed?: .failed + case .finished?: .finished + case nil: state.captureID == nil ? .creating : state.finalized == nil ? .uploading : .processing + } + retained = state.files.count + committed = state.committedCount + committedBytes = state.committedBytes + totalBytes = state.totalBytes + sealed = state.packet != nil + backendStatus = state.backendStatus + lastEvent = state.lastEvent + eventCursor = state.eventCursor + resultAvailable = state.result != nil + captureID = state.captureID + runID = state.finalized?.runID + self.retryingAt = retryingAt + if case .failed(let step, let codes, let status) = state.end { + self.detail = "\(step) \(status) \(codes.joined(separator: ","))" + } else { + self.detail = detail + } + } +} diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift new file mode 100644 index 000000000..6fbf75319 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Upload/CaptureUploader.swift @@ -0,0 +1,734 @@ +import Foundation +import Synchronization + +/// Sends one capture to the capture API while it is being taken: create once, then register, PUT +/// and commit each sealed file as it arrives, then finalize with the frozen packet (possibly +/// before the last commits), then follow the events to a result. +/// +/// Every answer is applied only while the attempt that asked is still current, so a reply that +/// arrives after a world reset or a new capture changes nothing. The saved state +/// (`CaptureUploadState`) is written after every change, which is what resumes a capture after a +/// relaunch with the same packet id, capture id and frozen request bodies. +public actor CaptureUploader { + public struct Policy: Sendable { + public var maxConcurrentPuts = 3 + /// Files per register and commit call: the contract's "about every 10 files". + public var batch = 10 + public var firstDelay = 1.0 + public var maxDelay = 30.0 + /// Seconds the server may hold an events poll. + public var eventsWait = 20 + /// A signed URL this close to expiry is fetched again before the PUT. + public var expiryMargin = 60.0 + /// PUTs of one file the storage refuses (not transient) before the upload stops. + public var maxRefusedPuts = 6 + /// `retry_finalize` answers followed before the upload stops. + public var maxFinalizeRetries = 3 + /// Result reads that find the run's answer not readable yet before the upload stops. Each + /// waits the retry delay first, so the default allows about three minutes. No measurement + /// of how long a finished run's answer takes to become readable sets this number; it only + /// keeps an answer that never appears from holding the upload open. + public var maxNotReadyResults = 10 + /// Retries after a register answer leaves a file out or a commit doesn't acknowledge it, + /// counted per file per server capture. Each retry waits the retry delay; the upload stops + /// on the next unacknowledged answer, so a file is sent at most `maxUnacknowledged + 1` + /// times. No measurement sets this; it keeps a server that never takes a file from getting + /// the same photo forever. + public var maxUnacknowledged = 5 + /// Seconds between events polls that come back at once with nothing new and the capture + /// still running, for a server or proxy that doesn't hold the poll. Not measured. + public var minimumPollInterval = 2.0 + + public init() {} + + public func delay(afterFailure n: Int, retryAfter: Double?) -> Double { + if let retryAfter, retryAfter > 0 { return min(retryAfter, 120) } + return min(maxDelay, firstDelay * pow(2, Double(max(n, 1) - 1))) + } + } + + /// A failure a later try can fix. + struct Transient: Error { + var step: String + var detail: String + var retryAfter: Double? + } + + /// The homeowner took back their yes: nothing more is sent. + struct Withdrawn: Error {} + + /// The server refused something a retry can't change. + struct Refused: Error { + var step: String + var status: Int + var codes: [String] + } + + public nonisolated let folder: URL + private let base: URL + private let http: any CaptureHTTP + private let policy: Policy + private let now: @Sendable () -> Date + private let sleep: @Sendable (Double) async throws -> Void + private var state: CaptureUploadState + private var loop: Task? + private var failures = 0 + private var retryingAt: Date? + private var digestRetried: Set = [] + /// The registry covers public start/resume as well as coordinator discovery. An idle live + /// uploader still owns its folder, since it can receive more sealed files. Weak ownership + /// releases only after the uploader and its running task are gone. Withdrawn entries stay + /// for this process's lifetime, including when every disk write failed. + private struct FolderEntry { + weak var owner: CaptureUploader? + var withdrawn = false + var stopped = false + var running: Task? + } + private static let folders = Mutex<[String: FolderEntry]>([:]) + private nonisolated let folderKey: String + + public enum OwnershipError: Error, Sendable, Equatable { + case alreadyOwned + case consentWithdrawn + /// Use resume for an existing saved upload; start must never overwrite its consent. + case savedUploadExists + case folderIdentityUnavailable + } + /// Set when a state change could not be saved: the loop stops rather than send a request the + /// saved state doesn't know about. + private var saveFailed = false + private var observer: (@Sendable (CaptureUploadStatus) -> Void)? + private var log: (@Sendable (String) -> Void)? + + public nonisolated static func stateURL(in folder: URL) -> URL { folder.appending(path: CaptureUploadState.fileName) } + + /// Resolve symbolic links before asking for the filesystem's case-correct path. The + /// canonical-path resource alone preserves a final-component symlink on macOS. + /// Refuse an unknown identity instead of allowing a second registry key for one folder. + private static func canonicalFolder(_ folder: URL) throws -> URL { + let fresh = URL(fileURLWithPath: folder.path, isDirectory: true).standardizedFileURL.resolvingSymlinksInPath() + guard let path = try fresh.resourceValues(forKeys: [.canonicalPathKey]).canonicalPath else { + throw OwnershipError.folderIdentityUnavailable + } + return URL(fileURLWithPath: path, isDirectory: true) + } + + private init( + folder: URL, base: URL, http: any CaptureHTTP, state: CaptureUploadState, policy: Policy, + now: @escaping @Sendable () -> Date, sleep: @escaping @Sendable (Double) async throws -> Void + ) { + self.folder = folder + self.folderKey = folder.path + self.base = base + self.http = http + self.state = state + self.policy = policy + self.now = now + self.sleep = sleep + } + + /// A new capture. The create body is encoded once here and kept as those bytes for every retry. + /// `consentedAt` is when the homeowner said yes to sending this capture to `base`. + public static func start( + folder: URL, base: URL, http: any CaptureHTTP, create: CaptureAPI.CreateRequest, consentedAt: Date, policy: Policy = .init(), + now: @escaping @Sendable () -> Date = { Date() }, + sleep: @escaping @Sendable (Double) async throws -> Void = { try await Task.sleep(for: .seconds($0)) } + ) throws -> CaptureUploader { + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + let folder = try canonicalFolder(folder) + return try folders.withLock { entries in + entries = entries.filter { $0.value.owner != nil || $0.value.withdrawn } + guard entries[folder.path]?.withdrawn != true, + !FileManager.default.fileExists(atPath: CaptureUploadState.withdrawnURL(in: folder).path) + else { throw OwnershipError.consentWithdrawn } + guard entries[folder.path]?.owner == nil else { throw OwnershipError.alreadyOwned } + guard !FileManager.default.fileExists(atPath: stateURL(in: folder).path) else { throw OwnershipError.savedUploadExists } + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + var state = CaptureUploadState(attemptID: UUID().uuidString, packetID: create.packetId, createBody: try encoder.encode(create)) + state.destination = base.absoluteString + state.consent = .init(grantedAt: consentedAt, destination: base.absoluteString) + state.marks["started"] = now() + try state.save(to: stateURL(in: folder)) + let uploader = CaptureUploader(folder: folder, base: base, http: http, state: state, policy: policy, now: now, sleep: sleep) + entries[folder.path] = FolderEntry(owner: uploader) + return uploader + } + } + + /// Resumes a saved upload with its recorded consent and endpoint. Returns nil when absent, + /// ended or withdrawn; throws `alreadyOwned` while another live uploader owns the folder. + /// The new attempt id and ownership are saved together, before any request can start. + public static func resume( + folder: URL, base: URL, http: any CaptureHTTP, policy: Policy = .init(), + now: @escaping @Sendable () -> Date = { Date() }, + sleep: @escaping @Sendable (Double) async throws -> Void = { try await Task.sleep(for: .seconds($0)) } + ) throws -> CaptureUploader? { + guard FileManager.default.fileExists(atPath: folder.path) else { return nil } + let folder = try canonicalFolder(folder) + return try folders.withLock { entries in + entries = entries.filter { $0.value.owner != nil || $0.value.withdrawn } + let url = stateURL(in: folder) + guard entries[folder.path]?.withdrawn != true, + FileManager.default.fileExists(atPath: url.path), + !FileManager.default.fileExists(atPath: CaptureUploadState.withdrawnURL(in: folder).path) + else { return nil } + guard entries[folder.path]?.owner == nil else { throw OwnershipError.alreadyOwned } + var state = try CaptureUploadState.load(from: url) + guard state.end == nil, state.destination == base.absoluteString, state.consent?.destination == base.absoluteString else { return nil } + state.attemptID = UUID().uuidString + try state.save(to: url) + let uploader = CaptureUploader(folder: folder, base: base, http: http, state: state, policy: policy, now: now, sleep: sleep) + entries[folder.path] = FolderEntry(owner: uploader) + return uploader + } + } + + public func observe(_ observer: @escaping @Sendable (CaptureUploadStatus) -> Void, log: (@Sendable (String) -> Void)? = nil) { + self.observer = observer + self.log = log + observer(status) + } + + public var status: CaptureUploadStatus { + CaptureUploadStatus(state, retryingAt: retryingAt, detail: saveFailed ? "upload state could not be saved" : nil) + } + public var snapshot: CaptureUploadState { state } + + // MARK: Input + + /// Sealed files, uploaded as soon as the capture exists. A path already known is ignored: + /// sealed files never change. + public func add(_ files: [SealedFile]) { + guard state.end == nil, state.packet == nil else { return } + var added = false + for file in files where state.files[file.path] == nil { + state.files[file.path] = .init(sealed: file, sequence: state.nextSequence) + state.nextSequence += 1 + added = true + } + guard added else { return } + mark("firstRetained") + persist() + kick() + } + + /// The capture stopped: `packet` is frozen as these bytes, and `files` is every file it lists. + /// Any the uploader has not seen yet (an `add` still on its way) is queued here, so the + /// server never waits for a listed file that nobody sends. + public func seal(packet: Data, files: [SealedFile]) { + guard state.end == nil, state.packet == nil else { return } + for file in files where state.files[file.path] == nil { + state.files[file.path] = .init(sealed: file, sequence: state.nextSequence) + state.nextSequence += 1 + } + state.packet = packet + mark("sealed") + persist() + kick() + } + + public static let withdrawnReason = "consent withdrawn" + + /// How a withdrawal was recorded on disk. + public enum WithdrawalRecord: Sendable, Equatable { + /// The marker `resume` refuses was written. + case marked + /// The marker couldn't be written, so the saved state was removed instead. + case savedStateRemoved + /// Neither the marker nor the removal succeeded: a relaunch could still find the saved + /// yes until a later save records the upload as abandoned. + case notRecorded(String) + } + + /// Revokes the folder before returning and cancels its running task. A request already + /// admitted to a transport that ignores cancellation may finish; its reply cannot admit + /// another request or save resumable consent. `.notRecorded` means only the live process + /// is protected: after a crash the old saved yes may still exist. + @discardableResult + public nonisolated func withdrawConsent() -> WithdrawalRecord { + let (record, task) = Self.folders.withLock { entries in + entries[folderKey, default: FolderEntry()].withdrawn = true + let record = CaptureUploadState.recordWithdrawal(in: folder) + return (record, entries[folderKey]?.running) + } + // Cancellation handlers belong to the transport; never call them while holding our lock. + task?.cancel() + return record + } + + /// Closes admission synchronously when a coordinator ends a world, before its queued actor + /// work can kick the upload. Ownership stays until the old uploader and its requests die. + nonisolated func stopSending() { + let task = Self.folders.withLock { entries in + entries[folderKey]?.stopped = true + return entries[folderKey]?.running + } + task?.cancel() + } + + /// Ends this upload for good: a world reset or starting over. Replies still in flight are + /// dropped when they arrive. + public func abandon(_ reason: String) { + stopSending() + guard state.end == nil else { return } + state.end = .abandoned(reason) + state.attemptID = UUID().uuidString + loop?.cancel() + persist() + } + + /// Starts the loop if it is idle: after a relaunch, or when the network is back. + public func kick() { + guard loop == nil, state.end == nil, !saveFailed else { return } + Self.folders.withLock { entries in + guard let entry = entries[folderKey], entry.owner === self, !entry.withdrawn, !entry.stopped else { return } + let task = Task { await self.drive() } + loop = task + entries[folderKey]?.running = task + } + } + + /// Waits for the loop to go idle: nothing to send until more input, or ended. + public func settled() async { + while let running = loop { await running.value; if loop == running { loop = nil } } + } + + // MARK: Loop + + private func drive() async { + while !Task.isCancelled, state.end == nil, !saveFailed { + // A withdrawal sends nothing more; the save records the upload as abandoned. + if isWithdrawn { persist(); break } + let attempt = state.attemptID + do { + let progressed = try await step() + guard attempt == state.attemptID, state.end == nil else { break } + failures = 0 + if retryingAt != nil { retryingAt = nil; publish() } + if !progressed { break } + } catch let error as Transient { + guard attempt == state.attemptID else { break } + failures += 1 + let delay = policy.delay(afterFailure: failures, retryAfter: error.retryAfter) + retryingAt = now().addingTimeInterval(delay) + log?("capture-upload retry step=\(error.step) in=\(String(format: "%.1f", delay))s detail=\(error.detail)") + publish() + do { try await sleep(delay) } catch { break } + } catch is Withdrawn { + persist() + break + } catch let error as Refused { + guard attempt == state.attemptID else { break } + state.end = .failed(step: error.step, codes: error.codes, status: error.status) + log?("capture-upload failed step=\(error.step) status=\(error.status) codes=\(error.codes.joined(separator: ","))") + persist() + } catch { + break + } + } + loop = nil + Self.folders.withLock { $0[folderKey]?.running = nil } + } + + /// One unit of work. False when there is nothing to do until more input arrives. + private func step() async throws -> Bool { + if state.captureID == nil { try await create(); return true } + // The frozen packet goes as soon as the capture stops, before the files still uploading: + // the server answers awaiting_files and starts the run on the last commit. + if state.packet != nil, state.finalized == nil { try await finalize(); return true } + requeueExpiring() + let queued = ordered(where: { $0.phase == .queued }) + if !queued.isEmpty { try await register(Array(queued.prefix(min(policy.batch, state.maxBatch)))); return true } + // Each round of PUTs is committed before the next starts, so the first photos count as + // received after one round trip of uploads rather than a whole register batch. + let uploaded = ordered(where: { $0.phase == .uploaded }) + if !uploaded.isEmpty { try await commit(Array(uploaded.prefix(min(policy.batch, state.maxBatch)))); return true } + let registered = ordered(where: { if case .registered = $0.phase { true } else { false } }) + if !registered.isEmpty { try await put(Array(registered.prefix(policy.maxConcurrentPuts))); return true } + if state.finalized != nil, state.end == nil { + if state.result != nil || CaptureAPI.terminalStatuses.contains(state.backendStatus ?? "") { + try await fetchResult() + return true + } + try await pollEvents() + return true + } + return false + } + + private func ordered(where include: (CaptureUploadState.File) -> Bool) -> [CaptureUploadState.File] { + state.files.values.filter(include).sorted { ($0.sealed.priority, $0.sequence) < ($1.sealed.priority, $1.sequence) } + } + + // MARK: Calls + + private func create() async throws { + var request = jsonRequest("captures", method: "POST", body: state.createBody) + request.setValue(state.packetID, forHTTPHeaderField: "Idempotency-Key") + let (reply, attempt) = try await call("create", request) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let body = try? JSONDecoder().decode(CaptureAPI.CreateResponse.self, from: reply.body) else { + throw Refused(step: "create", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + state.captureID = body.captureId + state.backendStatus = body.status + if let batch = body.upload?.maxBatch, batch > 0 { state.maxBatch = batch } + mark("created") + persist() + } + + private func register(_ files: [CaptureUploadState.File]) async throws { + let body = try JSONEncoder().encode(CaptureAPI.RegisterRequest(files: files.map { .init($0.sealed) })) + let (reply, attempt) = try await call("register", jsonRequest("captures/\(captureID)/files", method: "POST", body: body)) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let answer = try? JSONDecoder().decode(CaptureAPI.RegisterResponse.self, from: reply.body) else { + if try captureGone(reply) { return } + throw Refused(step: "register", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + for file in answer.files where state.files[file.path] != nil { + if file.state == "committed" { + setPhase(file.path, .committed) + } else if let target = file.upload, target.method == "PUT", let url = target.url { + // Photos go only to an https URL, or http to this machine for tests, and only to a + // URL the PUT can use: a URL that doesn't parse would leave the file registered + // forever. This checks the URL as given; `CaptureHTTP` refuses redirects, so a + // storage server can't send the photo on to a URL this never checked. + if let problem = Self.uploadURLProblem(url) { throw Refused(step: "register", status: reply.status, codes: [problem]) } + setPhase(file.path, .registered(url: url, headers: target.headers ?? [:], expiresAt: target.expiresAt.flatMap(Self.parseDate))) + } else { + throw Refused(step: "register", status: reply.status, codes: ["upload_method_\(file.upload?.method ?? "none")"]) + } + } + let answered = Set(answer.files.map(\.path)) + try unacknowledged(files.map(\.sealed.path).filter { !answered.contains($0) }, step: "register", status: reply.status) + } + + /// Counts files the server didn't take, saves, and backs off (or stops once one has been left + /// out more than `maxUnacknowledged` times). Saves and returns when there are none. + private func unacknowledged(_ paths: [String], step: String, status: Int) throws { + for path in paths { + let count = (state.files[path]?.unacknowledged ?? 0) + 1 + state.files[path]?.unacknowledged = count + } + persist() + guard !paths.isEmpty else { return } + if paths.contains(where: { (state.files[$0]?.unacknowledged ?? 0) > policy.maxUnacknowledged }) { + throw Refused(step: step, status: status, codes: ["not_acknowledged"]) + } + throw Transient(step: step, detail: "\(paths.count) not acknowledged", retryAfter: nil) + } + + /// Why `text` can't receive a photo: nil when it is an https URL, or http to this machine. + static func uploadURLProblem(_ text: String) -> String? { + guard let url = URL(string: text), let scheme = url.scheme?.lowercased(), let host = url.host(), !host.isEmpty else { + return "upload_url_invalid" + } + let local = host == "127.0.0.1" || host == "localhost" + return scheme == "https" || (scheme == "http" && local) ? nil : "upload_url_insecure" + } + + private func put(_ files: [CaptureUploadState.File]) async throws { + let attempt = state.attemptID + var transient: Transient? + var index = 0 + while index < files.count { + // Each batch of PUTs is admitted only while the homeowner's yes stands. + if !canSend || Task.isCancelled { throw Withdrawn() } + let chunk = files[index..).self) { group in + for file in chunk { + guard case .registered(let url, let headers, _) = file.phase, Self.uploadURLProblem(url) == nil, let target = URL(string: url) else { continue } + var request = URLRequest(url: target) + request.httpMethod = "PUT" + // Exactly the headers the server signed; never the API's own credentials. + for (name, value) in headers { request.setValue(value, forHTTPHeaderField: name) } + let local = folder.appending(path: file.sealed.path) + let http = self.http + group.addTask { + if !self.canSend || Task.isCancelled { return (file.sealed.path, .failure(Withdrawn())) } + do { return (file.sealed.path, .success(try await http.upload(request, file: local))) } catch { return (file.sealed.path, .failure(error)) } + } + } + var out: [(String, Result)] = [] + for await result in group { out.append(result) } + return out + } + if !canSend || Task.isCancelled { throw Withdrawn() } + guard attempt == state.attemptID, state.end == nil else { return } + for (path, result) in results { + state.files[path]?.attempts += 1 + switch result { + case .success(let reply) where (200..<300).contains(reply.status): + setPhase(path, .uploaded) + case .success(let reply) where reply.status == 400 && !digestRetried.contains(path): + // BadDigest: the bytes on disk are sealed, so sending them once more is the retry. + digestRetried.insert(path) + case .success(let reply) where reply.status == 400: + // The same sealed bytes refused twice: the file on disk no longer matches its digest. + throw Refused(step: "put", status: 400, codes: ["bad_digest"]) + case .success(let reply) where Self.isTransient(reply.status): + transient = Transient(step: "put", detail: "status \(reply.status)", retryAfter: reply.retryAfter) + case .success(let reply): + // 403 expired signature or a spent token: a fresh URL, a bounded number of times. + guard (state.files[path]?.attempts ?? 0) < policy.maxRefusedPuts else { + throw Refused(step: "put", status: reply.status, codes: ["storage_refused"]) + } + setPhase(path, .queued) + case .failure(let error): + transient = Transient(step: "put", detail: Self.describe(error), retryAfter: nil) + } + } + persist() + } + if let transient { throw transient } + } + + private func commit(_ files: [CaptureUploadState.File]) async throws { + let body = try JSONEncoder().encode(CaptureAPI.CommitRequest(files: files.map { .init(path: $0.sealed.path, sha256: $0.sealed.sha256) })) + let (reply, attempt) = try await call("commit", jsonRequest("captures/\(captureID)/files:commit", method: "POST", body: body)) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let answer = try? JSONDecoder().decode(CaptureAPI.CommitResponse.self, from: reply.body) else { + if try captureGone(reply) { return } + throw Refused(step: "commit", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + let committed = Set(answer.committed) + for file in files { + // Only a commit acknowledges a file; notFound, mismatch or no mention sends it again. + setPhase(file.sealed.path, committed.contains(file.sealed.path) ? .committed : .queued) + } + if !committed.isEmpty { mark("firstCommit") } + if state.packet != nil, state.files.values.allSatisfy({ $0.phase == .committed }) { mark("lastCommit") } + try unacknowledged(files.map(\.sealed.path).filter { !committed.contains($0) }, step: "commit", status: reply.status) + } + + private func finalize() async throws { + guard let packet = state.packet else { return } + var request = jsonRequest("captures/\(captureID)/finalize", method: "POST", body: packet) + request.setValue(state.packetID, forHTTPHeaderField: "Idempotency-Key") + request.setValue(PacketFiles.sha256(packet), forHTTPHeaderField: "X-Packet-Sha256") + let (reply, attempt) = try await call("finalize", request) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let answer = try? JSONDecoder().decode(CaptureAPI.FinalizeResponse.self, from: reply.body) else { + if try captureGone(reply) { return } + throw Refused(step: "finalize", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + state.finalized = .init(status: answer.status, missing: answer.missing, runID: answer.runId) + state.backendStatus = answer.status + mark("finalizeAccepted") + persist() + } + + private func pollEvents() async throws { + var components = URLComponents(url: base.appending(path: "captures/\(captureID)/events"), resolvingAgainstBaseURL: false)! + components.queryItems = [.init(name: "after", value: String(state.eventCursor)), .init(name: "wait", value: String(policy.eventsWait))] + var request = URLRequest(url: components.url!) + request.setValue("application/json", forHTTPHeaderField: "Accept") + request.timeoutInterval = Double(policy.eventsWait) + 30 + let sent = now() + let (reply, attempt) = try await call("events", request) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status), let answer = try? JSONDecoder().decode(CaptureAPI.EventsResponse.self, from: reply.body) else { + if try captureGone(reply) { return } + throw Refused(step: "events", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + state.backendStatus = answer.status + state.eventCursor = max(state.eventCursor, answer.next) + for event in answer.events { + mark("firstEvent") + state.lastEvent = [event.type, event.data?.stage, event.data?.status, event.data?.kind, event.data?.code].compactMap { $0 }.joined(separator: " ") + // A verdict for another run is not this capture's result. + if event.type == "verdict_ready", event.data?.runId == nil || event.data?.runId == state.finalized?.runID { mark("verdictReady") } + if event.type == "failed", event.data?.next == "retry_finalize" { + // The server lost the finalize to a transient storage failure: send the same frozen + // packet again. + guard state.finalizeRetries < policy.maxFinalizeRetries else { + throw Refused(step: "finalize", status: 0, codes: [event.data?.code ?? "retry_finalize"]) + } + state.finalizeRetries += 1 + state.finalized = nil + } + } + if !answer.events.isEmpty { log?("capture-upload events status=\(answer.status) last=\(state.lastEvent ?? "")") } + persist() + // An answer that came back at once with nothing new means the server didn't hold the + // poll; waiting here keeps the loop from asking again immediately. + let waited = now().timeIntervalSince(sent) + if answer.events.isEmpty, !CaptureAPI.terminalStatuses.contains(answer.status), waited < policy.minimumPollInterval { + try await sleep(policy.minimumPollInterval - waited) + } + } + + private func fetchResult() async throws { + var request = URLRequest(url: base.appending(path: "captures/\(captureID)/result")) + request.setValue("application/json", forHTTPHeaderField: "Accept") + let (reply, attempt) = try await call("result", request) + guard attempt == state.attemptID else { return } + guard (200..<300).contains(reply.status) else { + if try captureGone(reply) { return } + // The run reads as done before its answer is stored. + if reply.status == 404, CaptureAPI.errorCodes(reply.body).contains("result_not_ready") { try notReady(reply) } + throw Refused(step: "result", status: reply.status, codes: CaptureAPI.errorCodes(reply.body)) + } + // The whole body must decode as the result the app reads, or a finished upload could hold + // an answer nothing can show. + let decoder = JSONDecoder() + guard let response = try? decoder.decode(CaptureResult.Response.self, from: reply.body), + let head = try? decoder.decode(ResultHead.self, from: reply.body) + else { + throw Refused(step: "result", status: reply.status, codes: ["result_unreadable"]) + } + if let runID = head.runId, runID != state.finalized?.runID { + throw Refused(step: "result", status: reply.status, codes: ["result_for_another_run"]) + } + let status = head.status ?? state.backendStatus ?? "unknown" + // Until the run has written its answer the API serves the capture's status with a null + // outcome, and that status can already be one the capture ends in. + if response.outcome == nil, !CaptureAPI.statusesWithoutOutcome.contains(status) { try notReady(reply) } + state.result = reply.body + state.end = .finished(status: status) + mark("result") + persist() + } + + /// The run and status as the server wrote them; `CaptureResult.Status` keeps no raw text. + private struct ResultHead: Decodable { + var runId: String? + var status: String? + } + + /// Always throws: a retry after the usual wait, or the end of the upload once + /// `maxNotReadyResults` reads have found no answer. + private func notReady(_ reply: HTTPReply) throws -> Never { + let notReadyReads = (state.notReadyReads ?? 0) + 1 + state.notReadyReads = notReadyReads + persist() + guard notReadyReads <= policy.maxNotReadyResults else { + throw Refused(step: "result", status: reply.status, codes: ["result_not_ready"]) + } + throw Transient(step: "result", detail: "result not ready (\(notReadyReads))", retryAfter: reply.retryAfter) + } + + // MARK: Plumbing + + private var captureID: String { state.captureID ?? "" } + + /// The server lost the capture (404: a redeploy with in-memory state). The same create body + /// opens it again, and every file goes up again under the new id. + private func captureGone(_ reply: HTTPReply) throws -> Bool { + guard reply.status == 404, CaptureAPI.errorCodes(reply.body).contains("capture_not_found") else { return false } + log?("capture-upload the server no longer has the capture; creating it again") + state.captureID = nil + state.finalized = nil + state.eventCursor = 0 + // A new server capture is a new run: the old one's limits don't carry over. + state.notReadyReads = nil + for path in state.files.keys { + state.files[path]?.phase = .queued + state.files[path]?.unacknowledged = nil + } + persist() + return true + } + + private func requeueExpiring() { + let limit = now().addingTimeInterval(policy.expiryMargin) + for (path, file) in state.files { + if case .registered(_, _, let expires?) = file.phase, expires <= limit { state.files[path]?.phase = .queued } + } + } + + private func setPhase(_ path: String, _ phase: CaptureUploadState.FilePhase) { + state.files[path]?.phase = phase + } + + private func call(_ step: String, _ request: URLRequest) async throws -> (HTTPReply, String) { + // Every API request is admitted only while the homeowner's yes stands. One already sent + // finishes, but its reply can only save the upload as abandoned. + if !canSend || Task.isCancelled { throw Withdrawn() } + let attempt = state.attemptID + let reply: HTTPReply + do { + reply = try await http.send(request) + } catch { + throw Transient(step: step, detail: Self.describe(error), retryAfter: nil) + } + if Self.isTransient(reply.status), attempt == state.attemptID { + throw Transient(step: step, detail: "status \(reply.status)", retryAfter: reply.retryAfter) + } + return (reply, attempt) + } + + private func jsonRequest(_ path: String, method: String, body: Data) -> URLRequest { + var request = URLRequest(url: base.appending(path: path)) + request.httpMethod = method + request.httpBody = body + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.setValue("application/json", forHTTPHeaderField: "Accept") + return request + } + + private func mark(_ name: String) { + guard state.marks[name] == nil else { return } + let at = now() + state.marks[name] = at + let since = state.marks["started"].map { at.timeIntervalSince($0) } ?? 0 + log?("capture-upload stage=\(name) t=\(String(format: "%.2f", since))s committed=\(state.committedCount)/\(state.files.count)") + } + + private nonisolated var isWithdrawn: Bool { Self.folders.withLock { $0[folderKey]?.withdrawn == true } } + + private nonisolated var canSend: Bool { + Self.folders.withLock { entries in + guard let entry = entries[folderKey] else { return false } + return entry.owner === self && !entry.withdrawn && !entry.stopped + } + } + + private func persist() { + do { + try Self.folders.withLock { entries in + guard entries[folderKey]?.owner === self else { throw OwnershipError.alreadyOwned } + let withdrawn = entries[folderKey]?.withdrawn == true + // A withdrawal outranks any end a late reply set (finished, failed). + if withdrawn, state.end != .abandoned(Self.withdrawnReason) { + state.end = .abandoned(Self.withdrawnReason) + // Replies still in flight belong to the old attempt and are dropped. + state.attemptID = UUID().uuidString + } + try state.save(to: Self.stateURL(in: folder)) + } + } catch { + saveFailed = true + log?("capture-upload stopped: the upload state could not be saved (\(Self.describe(error)))") + } + publish() + } + + private func publish() { + observer?(status) + } + + /// 501 is the server saying it doesn't do what was asked (multipart), which waiting won't change. + static func isTransient(_ status: Int) -> Bool { status == 0 || status == 408 || status == 429 || ((500..<600).contains(status) && status != 501) } + + /// The error's domain and code only: a URLError's description can carry the signed URL. + static func describe(_ error: any Error) -> String { + let ns = error as NSError + return "\(ns.domain) \(ns.code)" + } + + static func parseDate(_ text: String) -> Date? { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + if let date = formatter.date(from: text) { return date } + formatter.formatOptions = [.withInternetDateTime] + return formatter.date(from: text) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureFolderOwnershipTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureFolderOwnershipTests.swift new file mode 100644 index 000000000..ea16aeffa --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureFolderOwnershipTests.swift @@ -0,0 +1,224 @@ +import Foundation +import HouseScanKit +import Synchronization +import Testing + +@Suite(.serialized) struct CaptureFolderOwnershipTests { + private func create(_ folder: URL, _ base: URL) throws -> CaptureUploader { + try CaptureUploader.start( + folder: folder, base: base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), + create: .init(packetId: "ownership-test", tier: .arkit, device: .init(model: "test", systemVersion: "test", appVersion: "test")), + consentedAt: Date()) + } + + @Test func concurrentStartsSaveOnlyTheOwnersAttempt() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "owners-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let owners = await withTaskGroup(of: CaptureUploader?.self, returning: [CaptureUploader].self) { group in + for _ in 0..<12 { + group.addTask { + do { return try create(root, server.base) } + catch { #expect(error as? CaptureUploader.OwnershipError == .alreadyOwned); return nil } + } + } + var owners: [CaptureUploader] = [] + for await owner in group { if let owner { owners.append(owner) } } + return owners + } + try #require(owners.count == 1) + let before = await owners[0].snapshot + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: root)) == before) + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned) { + try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + } + await owners[0].kick() + try #require(await CaptureUploaderTests.settles(owners[0])) + #expect(server.requests("POST captures").count == 1) + } + + @Test func concurrentResumesSaveOnlyTheOwnersAttempt() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "resume-owners-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + var first: CaptureUploader? = try create(root, server.base) + weak let released = first + first = nil + try #require(released == nil) + let owners = await withTaskGroup(of: CaptureUploader?.self, returning: [CaptureUploader].self) { group in + for _ in 0..<12 { + group.addTask { + do { return try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) } + catch { #expect(error as? CaptureUploader.OwnershipError == .alreadyOwned); return nil } + } + } + var owners: [CaptureUploader] = [] + for await owner in group { if let owner { owners.append(owner) } } + return owners + } + try #require(owners.count == 1) + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: root)) == (await owners[0].snapshot)) + await owners[0].kick() + try #require(await CaptureUploaderTests.settles(owners[0])) + #expect(server.requests("POST captures").count == 1) + } + + @Test func aliasesCannotAcquireTheSameFolder() throws { + let root = FileManager.default.temporaryDirectory.appending(path: "alias-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let folder = root.appending(path: "Capture") + let owner = try create(folder, server.base) + let link = root.appending(path: "link") + try FileManager.default.createSymbolicLink(at: link, withDestinationURL: folder) + var aliases = [link, root.appending(path: "Capture/../Capture")] + let lowercased = root.appending(path: "capture") + if FileManager.default.fileExists(atPath: lowercased.path) { aliases.append(lowercased) } + for alias in aliases { + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned, "alias: \(alias.path)") { try create(alias, server.base) } + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned, "alias: \(alias.path)") { + try CaptureUploader.resume(folder: alias, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + } + } + #expect(owner.folder.path == (try folder.resourceValues(forKeys: [.canonicalPathKey]).canonicalPath)) + #expect(server.state.withLock { $0.log.isEmpty }) + } + + @Test func idleOwnershipReleasesOnlyAfterTheUploaderDies() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "lifetime-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + var owner: CaptureUploader? = try create(root, server.base) + await owner?.kick() + try #require(await CaptureUploaderTests.settles(try #require(owner))) + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned) { + try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + } + weak let released = owner + owner = nil + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + #expect(throws: CaptureUploader.OwnershipError.savedUploadExists) { try create(root, server.base) } + let resumed = try #require(try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10))) + await resumed.kick() + try #require(await CaptureUploaderTests.settles(resumed)) + #expect(server.requests("POST captures").count == 1) + } + + @Test func anInFlightRequestKeepsOwnershipAfterTheCallerReleasesIt() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "inflight-owner-\(UUID().uuidString)") + let server = try LoopbackCaptureAPI() + defer { + server.state.withLock { $0.held = [] } + server.release("POST captures") + try? FileManager.default.removeItem(at: root) + } + server.state.withLock { $0.held = ["POST captures"] } + var owner: CaptureUploader? = try create(root, server.base) + await owner?.kick() + for _ in 0..<500 where server.state.withLock({ $0.parked["POST captures"]?.isEmpty ?? true }) { + try await Task.sleep(for: .milliseconds(10)) + } + try #require(server.state.withLock { !($0.parked["POST captures"]?.isEmpty ?? true) }) + weak let runningOwner = owner + owner = nil + #expect(runningOwner != nil) + #expect(throws: CaptureUploader.OwnershipError.alreadyOwned) { + try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) + } + server.state.withLock { $0.held = [] } + server.release("POST captures") + for _ in 0..<500 where runningOwner != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(runningOwner == nil) + let resumed = try #require(try CaptureUploader.resume(folder: root, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10))) + await resumed.kick() + try #require(await CaptureUploaderTests.settles(resumed)) + #expect(server.requests("POST captures").count == 1) + } + + /// Failure injection uses permissions only on this test's temporary folder. Restoring the + /// stale saved yes after stopping models the disk contents a failed withdrawal can leave. + @Test(arguments: [0, 1, 2]) func withdrawalBlocksReuseEvenWhenDiskStillHasConsent(failedWrites: Int) async throws { + var rig = try CaptureUploaderTests.Rig(http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10))) + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: rig.capture.folder.path) + rig.cleanUp() + } + rig.server.state.withLock { $0.held = ["POST captures/files"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + for _ in 0..<500 where rig.server.state.withLock({ $0.parked["POST captures/files"]?.isEmpty ?? true }) { + try await Task.sleep(for: .milliseconds(10)) + } + try #require(rig.server.state.withLock { !($0.parked["POST captures/files"]?.isEmpty ?? true) }) + let stateURL = CaptureUploader.stateURL(in: rig.capture.folder) + let staleConsent = try Data(contentsOf: stateURL) + let marker = CaptureUploadState.withdrawnURL(in: rig.capture.folder) + if failedWrites == 1 { + try FileManager.default.createDirectory(at: marker.appending(path: "blocked"), withIntermediateDirectories: true) + } else if failedWrites == 2 { + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: rig.capture.folder.path) + } + let result = rig.uploader.withdrawConsent() + switch failedWrites { + case 0: #expect(result == .marked) + case 1: #expect(result == .savedStateRemoved) + default: + guard case .notRecorded = result else { Issue.record("Both filesystem writes must fail"); return } + #expect(try Data(contentsOf: stateURL) == staleConsent) + } + rig.server.state.withLock { $0.held = [] } + rig.server.release("POST captures/files") + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(rig.server.requests("PUT upload").isEmpty) + #expect(rig.server.requests("POST captures/files:commit").isEmpty) + #expect(await rig.uploader.snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: rig.capture.folder.path) + if FileManager.default.fileExists(atPath: marker.path) { try FileManager.default.removeItem(at: marker) } + try staleConsent.write(to: stateURL, options: .atomic) + weak let released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + #expect(try CaptureUploader.resume(folder: rig.capture.folder, base: rig.server.base, http: rig.http) == nil) + #expect(throws: CaptureUploader.OwnershipError.consentWithdrawn) { try create(rig.capture.folder, rig.server.base) } + #expect(try Data(contentsOf: stateURL) == staleConsent) + } + + @Test(arguments: [false, true]) func withdrawalDuringPutsStopsTheNextBatchAndCommit(uncancellable: Bool) async throws { + let transport = URLSessionCaptureHTTP.ephemeral(timeout: 10) + let http: any CaptureHTTP = uncancellable ? UncancellableHTTP(transport) : transport + var policy = CaptureUploaderTests.Rig.fast + policy.maxConcurrentPuts = 2 + let rig = try CaptureUploaderTests.Rig(http: http, policy: policy) + defer { + rig.server.state.withLock { $0.held = [] } + rig.server.release("PUT upload") + rig.cleanUp() + } + rig.server.state.withLock { $0.held = ["PUT upload"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 3)) + for _ in 0..<500 where rig.server.state.withLock({ ($0.parked["PUT upload"]?.count ?? 0) < 2 }) { + try await Task.sleep(for: .milliseconds(10)) + } + try #require(rig.server.state.withLock { ($0.parked["PUT upload"]?.count ?? 0) == 2 }) + #expect(rig.uploader.withdrawConsent() == .marked) + await rig.uploader.kick() + rig.server.state.withLock { $0.held = [] } + rig.server.release("PUT upload") + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(rig.server.requests("PUT upload").count == 2) + #expect(rig.server.requests("POST captures/files:commit").isEmpty) + #expect(await rig.uploader.snapshot.end == .abandoned(CaptureUploader.withdrawnReason)) + } + + @Test func withdrawalBeforeConcurrentKicksAdmitsNoRequests() async throws { + let rig = try CaptureUploaderTests.Rig() + defer { rig.cleanUp() } + #expect(rig.uploader.withdrawConsent() == .marked) + await withTaskGroup(of: Void.self) { group in + for _ in 0..<20 { group.addTask { await rig.uploader.kick() } } + } + try #require(await CaptureUploaderTests.settles(rig.uploader)) + #expect(rig.server.state.withLock { $0.log.isEmpty }) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultSummaryTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultSummaryTests.swift new file mode 100644 index 000000000..e8c83cb24 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultSummaryTests.swift @@ -0,0 +1,142 @@ +import Foundation +import HouseScanKit +import simd +import Testing + +/// What the integration build shows of a capture-API result, from self-authored responses, and +/// the coordinator binding the result it fetched to its own scan. +@Suite struct CaptureResultSummaryTests { + static let world = CaptureResult.Association(sessionID: "session-A", captureID: "cap_A", runID: "run_A", epoch: "e1") + + static func response(_ json: String) throws -> CaptureResult.Response { + try JSONDecoder().decode(CaptureResult.Response.self, from: Data(json.utf8)) + } + + static func summary(_ json: String, association: CaptureResult.Association = world, analysis: CaptureResult.Analysis = .unverified) throws -> CaptureResultSummary { + CaptureResultSummary(.init(response: try response(json), association: association, analysis: analysis), world: world, meterAnchor: matrix_identity_float4x4) + } + + static let prompt = #"{"id":"vn1","kind":"wall_band","criteria":["C3"],"why":"unseen","promptId":"p1","prompt":{"title":"Step back and show the wall","body":"Include the ground under the meter."}}"# + + @Test func processingHasNoOutcomeAndNoMessage() throws { + let s = try Self.summary(#"{"runId":"run_A","status":"processing","viewsNeeded":[],"memberActions":[]}"#) + #expect(s.state == .working(.processing)) + #expect(s.message == nil && s.prompts.isEmpty && s.criteria == nil) + #expect(s.arUnavailable == .noOutcome(.processing)) + } + + /// A capture that failed or expired has no outcome, and it is over: it is not shown as + /// still working. + @Test(arguments: ["failed", "expired"]) func aCaptureThatEndedWithoutAnOutcomeIsNotWorking(status: String) throws { + let s = try Self.summary(#"{"runId":"run_A","status":"\#(status)","viewsNeeded":[],"memberActions":[]}"#) + #expect(s.state == .ended(CaptureResult.Status(rawValue: status))) + #expect(s.message == nil && s.prompts.isEmpty) + } + + @Test func manualReviewKeepsTheServersWordsAndNoInventedChecks() throws { + let s = try Self.summary(#""" + {"runId":"run_A","status":"manual_review","viewsNeeded":[],"memberActions":[], + "outcome":{"kind":"manual_review","profile":"C","message":"RECEIVER TEST: a person will look at this.","viewsNeeded":[],"reasons":[]}} + """#) + #expect(s.state == .decided(.manualReview)) + #expect(s.message == "RECEIVER TEST: a person will look at this.") + #expect(s.criteria == nil) + #expect(s.arUnavailable == .notEligible(.manualReview)) + } + + @Test func aMissingViewShowsItsPromptVerbatim() throws { + let s = try Self.summary(""" + {"runId":"run_A","status":"needs_views","viewsNeeded":[\(Self.prompt)],"memberActions":[], + "outcome":{"kind":"needs_more_photos","profile":"C","message":"One more view, please.","viewsNeeded":[\(Self.prompt)],"reasons":[]}} + """) + #expect(s.state == .decided(.needsMorePhotos)) + #expect(s.prompts.map(\.title) == ["Step back and show the wall"]) + #expect(s.prompts.map(\.body) == ["Include the ground under the meter."]) + } + + /// An eligible-shaped outcome from an unverified server stays without AR, and without the + /// criteria it didn't send. + @Test func eligibleWithoutCriteriaStaysUnverifiedAndOffAR() throws { + let s = try Self.summary(#""" + {"runId":"run_A","status":"complete","viewsNeeded":[],"memberActions":[], + "outcome":{"kind":"eligible","profile":"C","message":"A spot fits.","viewsNeeded":[],"reasons":[],"arkitEpoch":"e1", + "recommendedPlacement":{"wallId":"w1","startSM":1.2,"confidence":0.8, + "boxArkitWorld":{"pose":[1,0,0,0,0,1,0,0,0,0,1,0,0.5,0,-2,1],"size":[0.79,1.0,0.56]}}}} + """#) + #expect(s.state == .decided(.eligible)) + #expect(s.criteria == nil) + #expect(!s.analysisVerified) + #expect(s.arUnavailable == .analysisUnverified) + } + + @Test func anUnknownStatusStaysReadable() throws { + let s = try Self.summary(#"{"runId":"run_A","status":"paused_for_audit","viewsNeeded":[],"memberActions":[]}"#) + #expect(s.state == .working(.unknown("paused_for_audit"))) + #expect(s.arUnavailable == .noOutcome(.unknown("paused_for_audit"))) + } + + /// Even a verified result doesn't place a box for another session, capture or run. + @Test(arguments: [ + CaptureResult.Association(sessionID: "session-B", captureID: "cap_A", runID: "run_A", epoch: "e1"), + CaptureResult.Association(sessionID: "session-A", captureID: "cap_B", runID: "run_A", epoch: "e1"), + CaptureResult.Association(sessionID: "session-A", captureID: "cap_A", runID: "run_B", epoch: "e1"), + ]) + func aResultForAnotherScanPlacesNothing(_ association: CaptureResult.Association) throws { + let s = try Self.summary(#""" + {"runId":"run_A","status":"complete","viewsNeeded":[],"memberActions":[], + "outcome":{"kind":"eligible","profile":"C","message":"A spot fits.","viewsNeeded":[],"reasons":[],"arkitEpoch":"e1", + "recommendedPlacement":{"wallId":"w1","startSM":1.2,"confidence":0.8, + "boxArkitWorld":{"pose":[1,0,0,0,0,1,0,0,0,0,1,0,0.5,0,-2,1],"size":[0.79,1.0,0.56]}}}} + """#, association: association, analysis: .verified) + guard case .mismatch? = s.arUnavailable else { Issue.record("placed a box for another scan: \(String(describing: s.arUnavailable))"); return } + } +} + +/// The coordinator fetches the result through the real uploader and binds it to its own session. +@Suite(.serialized) @MainActor struct CaptureResultBindingTests { + @Test func theFetchedResultIsBoundToThisScanAndStaysUnverified() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "result-binding-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.resultMessage = "RECEIVER TEST: distinctive message 7f3" } + let coordinator = CaptureSessionCoordinator(environment: NativeCaptureFixture.environment( + endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: root.appending(path: "Captures"))) + var results: [CaptureResult.Record] = [] + coordinator.onResult = { results.append($0) } + try await NativeCaptureFixture(folder: root.appending(path: "store")).run(coordinator) + for _ in 0..<100 where results.isEmpty { try await Task.sleep(for: .milliseconds(20)) } + + let session = try #require(coordinator.session) + let record = try #require(results.first) + #expect(results.count == 1) + let state = await session.uploader!.snapshot + #expect(record.association == .init(sessionID: session.localID, captureID: state.captureID!, runID: state.finalized!.runID, epoch: "e1")) + #expect(record.response.outcome?.message == "RECEIVER TEST: distinctive message 7f3") + #expect(record.analysis == .unverified) + #expect(session.result == record) + } + + /// A result arriving for a session that has ended (a new scan started) is never published. + @Test func aResultForAnEndedSessionIsDropped() async throws { + let root = FileManager.default.temporaryDirectory.appending(path: "result-stale-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["GET captures/result"] } + let fixture = NativeCaptureFixture(folder: root.appending(path: "store")) + let coordinator = CaptureSessionCoordinator(environment: NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: root.appending(path: "Captures"))) + var results: [CaptureResult.Record] = [] + coordinator.onResult = { results.append($0) } + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 3)) + coordinator.captureEnded(acceptedCloseUpAt: nil) + for _ in 0..<500 where server.requests("GET captures/result").isEmpty { try await Task.sleep(for: .milliseconds(10)) } + let old = try #require(coordinator.session) + coordinator.newWorld("start over", recording: fixture.recording, newScan: true) + server.release("GET captures/result") + try await Task.sleep(for: .milliseconds(500)) + #expect(results.isEmpty) + #expect(old.result == nil) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultTests.swift new file mode 100644 index 000000000..ed3e9919a --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureResultTests.swift @@ -0,0 +1,320 @@ +import Foundation +import HouseScanKit +import Testing +import simd + +// Every body here is self-authored to the shape of the server's device result; none is a server +// example or a real capture. +@Suite struct CaptureResultTests { + typealias Association = CaptureResult.Association + + static let scan = Association(sessionID: "session-a", captureID: "cap-a", runID: "run-a", epoch: "e1") + + static let eligibleJSON = """ + {"runId": "run-a", "status": "complete", "viewsNeeded": [], "memberActions": [], + "outcome": {"kind": "eligible", "profile": "C", "message": "A battery fits on this wall.", + "viewsNeeded": [], "reasons": [], "arkitEpoch": "e1", + "frameToArkit": [1,0,0,0, 0,1,0,0, 0,0,1,0, 0,0,0,1], + "recommendedPlacement": {"wallId": "w1", "startSM": 1.1, "confidence": 0.7, "profile": "C", + "footprintM": {"w": 0.8, "h": 1.0, "d": 0.5}, + "boxSceneM": {"centerM": [1.5, 0.5, 0.3], "sizeM": [0.8, 1.0, 0.5], "yawDeg": 0}, + "boxArkitWorld": {"pose": [1,0,0,0, 0,1,0,0, 0,0,1,0, 1.5,0.5,0.3,1], "size": [0.8, 1.0, 0.5]}}, + "cableRoutePolylineArkit": null}, + "verdictUrl": null, "previewUrl": null, "reviewUrl": "/v1/captures/cap-a/review"} + """ + + static func decode(_ json: String) throws -> CaptureResult.Response { + try JSONDecoder().decode(CaptureResult.Response.self, from: Data(json.utf8)) + } + + static func eligible() throws -> CaptureResult.Response { try decode(eligibleJSON) } + + static func verified(_ response: CaptureResult.Response) -> CaptureResult.Record { + CaptureResult.Record(response: response, association: scan, analysis: .verified) + } + + // MARK: Decoding + + @Test func processingHasNoOutcome() throws { + let response = try Self.decode(""" + {"runId": null, "status": "processing", "viewsNeeded": [], "memberActions": [], "outcome": null} + """) + #expect(response.status == .processing) + #expect(response.runId == nil && response.outcome == nil && response.criteria == nil) + #expect(response.previewUrl == nil && response.reviewUrl == nil) + #expect(Self.verified(response).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + == .unavailable(.noOutcome(.processing))) + } + + @Test func eligibleWithoutCriteriaKeepsThemAbsent() throws { + let response = try Self.eligible() + #expect(response.outcome?.kind == .eligible) + #expect(response.criteria == nil) + #expect(response.reviewUrl == "/v1/captures/cap-a/review") + let box = try #require(response.outcome?.recommendedPlacement?.boxArkitWorld) + #expect(box.size == [0.8, 1.0, 0.5]) + guard case .box(let placed) = Self.verified(response).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) else { + Issue.record("expected a box") + return + } + #expect(placed.size == SIMD3(0.8, 1.0, 0.5)) + #expect(nearlyEqual(placed.anchorFromBox, translation(1.5, 0.5, 0.3))) + } + + @Test func eligibleWithCriteriaKeepsEachAsSent() throws { + let json = Self.eligibleJSON.replacingOccurrences( + of: "\"verdictUrl\": null", + with: """ + "criteria": [{"id": "c1", "outcome": "pass", "measuredFt": 4.2, "coverage": "observed"}, + {"id": "c2", "outcome": "unsure", "unsureCause": "unobserved"}, + {"id": "c3", "outcome": "waived"}], + "verdictUrl": null + """) + let criteria = try #require(try Self.decode(json).criteria) + #expect(criteria.map(\.outcome) == [.pass, .unsure, .unknown("waived")]) + #expect(criteria[0].measuredFt == 4.2 && criteria[0].coverage == "observed" && criteria[0].thresholdFt == nil) + #expect(criteria[1].unsureCause == "unobserved" && criteria[1].measuredFt == nil) + } + + @Test func manualReviewKeepsViewsAndPromptsVerbatim() throws { + let body = " Hold the phone at chest height,\nand include 3 ft left of the “meter”. " + let view = """ + {"id": "vn1", "kind": "past_end", "criteria": ["c2"], "why": "The left end was not seen.", + "promptId": "p-left-end", "prompt": {"title": "Show the left end", "body": \(try jsonString(body))}, + "side": "left", "wallId": "w1", "futureKey": {"nested": true}, + "referenceCrop": {"image": "k00002", "boxPx": [10, 20, 300, 400]}} + """ + let response = try Self.decode(""" + {"runId": "run-a", "status": "manual_review", "viewsNeeded": [\(view)], + "memberActions": ["retake_meter_closeups"], + "outcome": {"kind": "manual_review", "profile": "C", "message": "An installer will look at this scan.", + "viewsNeeded": [\(view)], "reasons": [], "arkitEpoch": "e1"}, + "criteria": [{"id": "c2", "outcome": "unsure"}], + "verdictUrl": null, "previewUrl": null, "reviewUrl": "/v1/captures/cap-a/review"} + """) + let outcome = try #require(response.outcome) + #expect(outcome.kind == .manualReview && outcome.reasons.isEmpty) + #expect(outcome.message == "An installer will look at this scan.") + #expect(response.viewsNeeded == outcome.viewsNeeded) + let needed = try #require(outcome.viewsNeeded.first) + #expect(needed.prompt.title == "Show the left end") + #expect(needed.prompt.body == body) + #expect(needed.kind == "past_end" && needed.wallId == "w1" && needed.band == nil) + #expect(needed.referenceCrop?.boxPx == [10, 20, 300, 400]) + #expect(response.memberActions == ["retake_meter_closeups"]) + #expect(Self.verified(response).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + == .unavailable(.notEligible(.manualReview))) + } + + @Test func unknownStatusAndKindDecodeAndWithholdAR() throws { + let unknownKind = try Self.decode(Self.eligibleJSON + .replacingOccurrences(of: "\"status\": \"complete\"", with: "\"status\": \"archived\"") + .replacingOccurrences(of: "\"kind\": \"eligible\"", with: "\"kind\": \"deferred\"")) + #expect(unknownKind.status == .unknown("archived")) + #expect(unknownKind.outcome?.kind == .unknown("deferred")) + #expect(unknownKind.outcome?.message == "A battery fits on this wall.") + #expect(Self.verified(unknownKind).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + == .unavailable(.notEligible(.unknown("deferred")))) + + var unknownStatus = try Self.eligible() + unknownStatus.status = .unknown("archived") + #expect(Self.verified(unknownStatus).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + == .unavailable(.statusNotComplete(.unknown("archived")))) + } + + @Test func missingStatusFailsToDecode() { + #expect(throws: DecodingError.self) { + try Self.decode(#"{"runId": "run-a", "viewsNeeded": [], "memberActions": [], "outcome": null}"#) + } + } + + // MARK: Gating + + @Test func eligibleResultIsUnverifiedByDefault() throws { + let record = CaptureResult.Record(response: try Self.eligible(), association: Self.scan) + #expect(record.analysis == .unverified) + #expect(record.placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) == .unavailable(.analysisUnverified)) + } + + enum Mismatch: CaseIterable, Sendable { + case worldSession, worldCapture, worldRun, worldEpoch, responseRun, responseRunMissing, outcomeEpoch + + var field: CaptureResult.IdentityField { + switch self { + case .worldSession: .session + case .worldCapture: .capture + case .worldRun, .responseRun, .responseRunMissing: .run + case .worldEpoch, .outcomeEpoch: .epoch + } + } + } + + @Test(arguments: Mismatch.allCases) + func everyIdentityMismatchWithholdsAR(_ mismatch: Mismatch) throws { + var response = try Self.eligible() + var world = Self.scan + switch mismatch { + case .worldSession: world.sessionID = "session-b" + case .worldCapture: world.captureID = "cap-b" + case .worldRun: world.runID = "run-b" + case .worldEpoch: world.epoch = "e2" + case .responseRun: response.runId = "run-b" + case .responseRunMissing: response.runId = nil + case .outcomeEpoch: response.outcome?.arkitEpoch = "e2" + } + let record = Self.verified(response) + #expect(record.placement(in: world, meterAnchor: matrix_identity_float4x4) == .unavailable(.mismatch(mismatch.field))) + #expect(record.response.outcome?.message == "A battery fits on this wall.") + } + + @Test func epochReusedByANewSessionIsNotAMatch() throws { + let newSession = Association(sessionID: "session-b", captureID: "cap-a", runID: "run-a", epoch: "e1") + #expect(newSession.epoch == Self.scan.epoch) + #expect(Self.verified(try Self.eligible()).placement(in: newSession, meterAnchor: matrix_identity_float4x4) + == .unavailable(.mismatch(.session))) + } + + enum BadBox: CaseIterable, Sendable { + case shortPose, twoSizes, nanPose, infiniteSize, zeroSize, scaledPose, shearedPose, mirroredPose, badBottomRow + case noBox, noEpoch, noPlacement + case floatOverflowSize, floatUnderflowSize, floatOverflowTranslation + + var expected: CaptureResult.Unavailable { + switch self { + case .shortPose: .malformedBox(.poseCount(15)) + case .twoSizes: .malformedBox(.sizeCount(2)) + case .nanPose, .infiniteSize, .floatOverflowSize, .floatOverflowTranslation: .malformedBox(.nonFinite) + case .zeroSize, .floatUnderflowSize: .malformedBox(.nonPositiveSize) + case .scaledPose, .shearedPose, .mirroredPose, .badBottomRow: .malformedBox(.notRigid) + case .noBox, .noEpoch: .notInARKitWorld + case .noPlacement: .noPlacement + } + } + + func apply(to outcome: inout CaptureResult.Outcome) { + switch self { + case .shortPose: outcome.recommendedPlacement?.boxArkitWorld?.pose.removeLast() + case .twoSizes: outcome.recommendedPlacement?.boxArkitWorld?.size = [0.8, 1.0] + case .nanPose: outcome.recommendedPlacement?.boxArkitWorld?.pose[13] = .nan + case .infiniteSize: outcome.recommendedPlacement?.boxArkitWorld?.size[1] = .infinity + case .zeroSize: outcome.recommendedPlacement?.boxArkitWorld?.size[2] = 0 + case .floatOverflowSize: outcome.recommendedPlacement?.boxArkitWorld?.size[0] = 1e40 + case .floatUnderflowSize: outcome.recommendedPlacement?.boxArkitWorld?.size[0] = 1e-50 + case .floatOverflowTranslation: outcome.recommendedPlacement?.boxArkitWorld?.pose[12] = 1e40 + case .scaledPose: + for i in 0..<11 { outcome.recommendedPlacement?.boxArkitWorld?.pose[i] *= 1.25 } + case .shearedPose: outcome.recommendedPlacement?.boxArkitWorld?.pose[4] = 0.3 + case .mirroredPose: outcome.recommendedPlacement?.boxArkitWorld?.pose[0] = -1 + case .badBottomRow: outcome.recommendedPlacement?.boxArkitWorld?.pose[3] = 0.5 + case .noBox: outcome.recommendedPlacement?.boxArkitWorld = nil + case .noEpoch: outcome.arkitEpoch = nil + case .noPlacement: outcome.recommendedPlacement = nil + } + } + } + + @Test(arguments: BadBox.allCases) + func malformedGeometryWithholdsARAndKeepsTheResult(_ bad: BadBox) throws { + var response = try Self.eligible() + var outcome = try #require(response.outcome) + bad.apply(to: &outcome) + response.outcome = outcome + let record = Self.verified(response) + #expect(record.placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) == .unavailable(bad.expected)) + #expect(record.response.outcome?.kind == .eligible) + #expect(record.response.outcome?.message == "A battery fits on this wall.") + } + + @Test func nonFiniteMeterAnchorWithholdsAR() throws { + var anchor = matrix_identity_float4x4 + anchor.columns.3.x = .nan + #expect(Self.verified(try Self.eligible()).placement(in: Self.scan, meterAnchor: anchor) == .unavailable(.invalidMeterAnchor)) + } + + // MARK: Transform + + @Test func boxIsExpressedInATranslatedRotatedMeterAnchor() throws { + let worldFromAnchor = pose(rotationY(degrees: 90), SIMD3(2, 0, -1)) + let anchorFromBox = pose(rotationY(degrees: 30), SIMD3(0.5, 0.4, 0.3)) + var response = try Self.eligible() + response.outcome?.recommendedPlacement?.boxArkitWorld?.pose = columnMajor(worldFromAnchor * anchorFromBox) + + let placement = Self.verified(response).placement(in: Self.scan, meterAnchor: float(worldFromAnchor)) + guard case .box(let placed) = placement else { + Issue.record("expected a box, got \(placement)") + return + } + #expect(nearlyEqual(placed.anchorFromBox, float(anchorFromBox))) + #expect(nearlyEqual(SIMD3(placed.anchorFromBox.columns.3.x, placed.anchorFromBox.columns.3.y, placed.anchorFromBox.columns.3.z), + SIMD3(0.5, 0.4, 0.3), 1e-5)) + #expect(placed.size == SIMD3(0.8, 1.0, 0.5)) + } + + /// The server builds the box's world pose from the scene box and the scene-to-ARKit similarity: + /// the similarity's scale is divided out of the rotation and multiplied into the size. The app + /// must take that size as it is, not scale it again by `frameToArkit`. + @Test func sizeCarriesTheServersScaleOnce() throws { + let scale = 1.25 + let frameToArkit = pose(rotationY(degrees: 40) * scale, SIMD3(1, 0, 2)) + let centerM = SIMD3(1.5, 0.5, 0.3), sizeM = SIMD3(0.8, 1.0, 0.56), yawDeg = 10.0 + + let rotation = rotation3(frameToArkit) * (1 / scale) * rotationY(degrees: yawDeg) + let center = frameToArkit * SIMD4(centerM, 1) + let serverPose = pose(rotation, SIMD3(center.x, center.y, center.z)) + let serverSize = sizeM * scale + + var response = try Self.eligible() + response.outcome?.recommendedPlacement?.boxArkitWorld?.pose = columnMajor(serverPose) + response.outcome?.recommendedPlacement?.boxArkitWorld?.size = [serverSize.x, serverSize.y, serverSize.z] + + let placement = Self.verified(response).placement(in: Self.scan, meterAnchor: matrix_identity_float4x4) + guard case .box(let placed) = placement else { + Issue.record("expected a box, got \(placement)") + return + } + #expect(nearlyEqual(placed.size, SIMD3(1.0, 1.25, 0.7), 1e-6)) + #expect(nearlyEqual(placed.anchorFromBox, float(serverPose))) + for column in [placed.anchorFromBox.columns.0, placed.anchorFromBox.columns.1, placed.anchorFromBox.columns.2] { + #expect(abs(simd_length(SIMD3(column.x, column.y, column.z)) - 1) < 1e-5) + } + } +} + +private func jsonString(_ text: String) throws -> String { + String(decoding: try JSONEncoder().encode(text), as: UTF8.self) +} + +private func rotationY(degrees: Double) -> simd_double3x3 { + let a = degrees * .pi / 180 + return simd_double3x3(SIMD3(cos(a), 0, -sin(a)), SIMD3(0, 1, 0), SIMD3(sin(a), 0, cos(a))) +} + +private func rotation3(_ m: simd_double4x4) -> simd_double3x3 { + simd_double3x3( + SIMD3(m.columns.0.x, m.columns.0.y, m.columns.0.z), + SIMD3(m.columns.1.x, m.columns.1.y, m.columns.1.z), + SIMD3(m.columns.2.x, m.columns.2.y, m.columns.2.z)) +} + +private func pose(_ r: simd_double3x3, _ t: SIMD3) -> simd_double4x4 { + simd_double4x4(SIMD4(r.columns.0, 0), SIMD4(r.columns.1, 0), SIMD4(r.columns.2, 0), SIMD4(t, 1)) +} + +private func translation(_ x: Float, _ y: Float, _ z: Float) -> simd_float4x4 { + var m = matrix_identity_float4x4 + m.columns.3 = SIMD4(x, y, z, 1) + return m +} + +private func columnMajor(_ m: simd_double4x4) -> [Double] { + [m.columns.0, m.columns.1, m.columns.2, m.columns.3].flatMap { [$0.x, $0.y, $0.z, $0.w] } +} + +private func float(_ m: simd_double4x4) -> simd_float4x4 { + simd_float4x4(SIMD4(m.columns.0), SIMD4(m.columns.1), SIMD4(m.columns.2), SIMD4(m.columns.3)) +} + +private func nearlyEqual(_ a: simd_float4x4, _ b: simd_float4x4, _ tolerance: Float = 1e-5) -> Bool { + [(a.columns.0, b.columns.0), (a.columns.1, b.columns.1), (a.columns.2, b.columns.2), (a.columns.3, b.columns.3)] + .allSatisfy { simd_reduce_max(simd_abs($0 - $1)) <= tolerance } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift new file mode 100644 index 000000000..1e1bfb85a --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureSessionCoordinatorTests.swift @@ -0,0 +1,711 @@ +import Foundation +import HouseScanKit +import simd +import Synchronization +import Testing + +/// A scan shaped like the app's live capture, for the coordinator: 1920 x 1440 sensor JPEGs, the +/// camera 2 m from a wall at 1.5 m height walking +x, the recorder's raw rows (a 60 Hz trajectory +/// with each frame's intrinsics beside it, and Core Motion's accelerometer and gyro at about +/// 100 Hz on their own jittered clocks), the meter tap's frame and pixel, and the meter close-up +/// from its own frame. Pattern images, no house; the packet says `source.kind: synthetic`. +struct NativeCaptureFixture: Sendable { + static let width = 1920, height = 1440 + static let k = SIMD4(1445, 1445, 960, 720) + static let meter = SIMD3(0.4, 1.2, -2) + let start = 5000.0 + let end = 5012.0 + let folder: URL + + func pose(_ t: Double) -> simd_float4x4 { + var m = matrix_identity_float4x4 + m.columns.3 = SIMD4(Float(t - start) * 0.1, 1.5, 0, 1) + return m + } + + /// Where `world` lands in the sensor image of the frame at `t`. + func pixel(of world: SIMD3, at t: Double) -> SIMD2 { + let p = world - SIMD3(pose(t).columns.3.x, pose(t).columns.3.y, pose(t).columns.3.z) + let depth = -p.z + return SIMD2(Double(Self.k.z + Self.k.x * p.x / depth), Double(Self.k.w - Self.k.y * p.y / depth)) + } + + var recording: RecordingSource { + let fixture = self + return RecordingSource(start: { (fixture.start, Date(timeIntervalSince1970: 1_790_000_000)) }, rows: { fixture.rows }) + } + + var rows: RecorderRows { + var trajectory: [[Double]] = [], intrinsics: [[Double]] = [] + for i in 0...Int((end - start) * 60) { + let t = start + Double(i) / 60 + let m = pose(t) + let columns: [SIMD4] = [m.columns.0, m.columns.1, m.columns.2, m.columns.3] + let values: [Double] = columns.flatMap { (c: SIMD4) -> [Double] in [Double(c.x), Double(c.y), Double(c.z), Double(c.w)] } + trajectory.append([t, 0, 0] + values) + intrinsics.append([t, 1445, 1445, 960, 720, 1920, 1440]) + } + // Two sensors on their own clocks: neither shares a timestamp with the other. + var accel: [[Double]] = [], gyro: [[Double]] = [] + for i in 0.. URL { + let url = folder.appending(path: "\(name).jpg") + if !FileManager.default.fileExists(atPath: url.path) { + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + try makeJPEG(width: Self.width, height: Self.height, at: url) + } + return url + } + + func photo(at t: Double, purpose: String? = nil) throws -> KeptPhoto { + KeptPhoto( + t: t, cameraToWorld: pose(t), cameraIntrinsics: Self.k, cameraImageSize: SIMD2(1920, 1440), width: Self.width, height: Self.height, + tracking: .normal, exposure: .init(duration: 0.004, offset: 0, iso: 64, fNumber: 1.6), jpeg: try jpeg(named: "frame-\(t)"), purpose: purpose) + } + + /// The raycast ran on the frame at `t`; the frame the app records is two frames newer, as + /// ARKit's current frame can be, and the tap is re-aimed at the hit in that frame. + func tap(at t: Double) throws -> (TapObservation, Packet04.TapHit) { + let data = try Data(contentsOf: try jpeg(named: "tap-\(t)")) + let recorded = t + 2.0 / 60 + let camera = pose(recorded).columns.3 + let stale = TapObservation( + t: recorded, cameraToWorld: pose(recorded), intrinsics: Self.k, width: Self.width, height: Self.height, tracking: .normal, + pixel: pixel(of: Self.meter, at: t), jpeg: { data }) + let tap = try #require(stale.pointing(at: Self.meter)) + let hit = Packet04.TapHit( + position: [Self.meter.x, Self.meter.y, Self.meter.z].map(Double.init), target: "estimatedPlane", alignment: "vertical", + distance: Double(simd_distance(Self.meter, SIMD3(camera.x, camera.y, camera.z)))) + return (tap, hit) + } + + static func environment( + endpoint: URL, http: any CaptureHTTP, captures: URL, sends: Bool = true, eventsWait: Int = 0, log: @escaping @Sendable (String) -> Void = { _ in } + ) -> CaptureSessionCoordinator.Environment { + var policy = CaptureUploader.Policy() + policy.eventsWait = eventsWait + let device = CaptureAPI.Device(model: "iPhone15,4", systemVersion: "26.0", appVersion: "0.1.0 (1)") + return .init( + endpoint: endpoint, sends: sends, http: http, capturesFolder: captures, + sessionInfo: { packetID, video in + Packet04SessionInfo( + packetID: packetID, sessionID: packetID, source: .synthetic, appVersion: device.appVersion, deviceModel: device.model, + systemVersion: device.systemVersion, lidarAvailable: false, sceneDepthEnabled: false, meshReconstructionSupported: nil, + sceneReconstruction: nil, planeDetection: ["horizontal", "vertical"], videoWidth: Int(video.x), videoHeight: Int(video.y), + framesPerSecond: nil, timeZone: "America/Chicago") + }, + device: device, tier: .arkit, policy: policy, log: log) + } + + /// The whole scan through `coordinator`: the tap, the close-up, keyframes (one of them the + /// close-up's own frame again), then the capture ends. + @MainActor + func run(_ coordinator: CaptureSessionCoordinator, consent: Bool? = true, beforeEnd: @MainActor () async -> Void = {}) async throws { + coordinator.begin(recording: recording) + // The integration build asks before the scan starts. + if let consent { coordinator.answerConsent(consent) } + let (tap, hit) = try self.tap(at: start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.kept(try photo(at: start + 2.5, purpose: "meter_close")) + for i in 0..<5 { coordinator.kept(try photo(at: start + 3 + Double(i) * 1.5)) } + // The app keeps the close-up's frame twice when it is also a walk frame. + coordinator.kept(try photo(at: start + 2.5)) + await coordinator.settle() + await beforeEnd() + coordinator.captureEnded(acceptedCloseUpAt: start + 2.5) + coordinator.captureEnded(acceptedCloseUpAt: start + 2.5) + await coordinator.settle() + } +} + +@Suite(.serialized) @MainActor struct CaptureSessionCoordinatorTests { + let root = FileManager.default.temporaryDirectory.appending(path: "coordinator-\(UUID().uuidString)") + var fixture: NativeCaptureFixture { NativeCaptureFixture(folder: root.appending(path: "store")) } + var captures: URL { root.appending(path: "Captures") } + + @Test func nativeScanExportsAValidPacketAndUploadsItDuringTheScan() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures)) + let committedBeforeEnd = Mutex(0) + try await fixture.run(coordinator) { + let uploader = coordinator.session!.uploader! + committedBeforeEnd.withLock { $0 = 0 } + let count = await uploader.snapshot.committedCount + committedBeforeEnd.withLock { $0 = count } + } + let session = try #require(coordinator.session) + let state = await session.uploader!.snapshot + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: try #require(state.packet)) + + #expect(Packet04Check.problems(packet, folder: session.folder).isEmpty) + // Tap frame, close-up frame and five walk frames; the repeated close-up frame is not a new keyframe. + #expect(packet.keyframes.count == 7) + #expect(packet.keyframes.map(\.reason) == ["tap", "still", "auto", "auto", "auto", "auto", "auto"]) + #expect(packet.stills?.map(\.purpose) == ["meter_close"]) + #expect(packet.scaleReference.meterCloseUp.still == "meter_close") + let tap = try #require(packet.taps?.first) + #expect(tap.label == "meter" && tap.keyframe == "k00001" && tap.epoch == "e1") + // The ray through the tap pixel reaches the meter the raycast hit. + let toMeter = simd_normalize(SIMD3(tap.hit!.position[0] - tap.rayOrigin[0], tap.hit!.position[1] - tap.rayOrigin[1], tap.hit!.position[2] - tap.rayOrigin[2])) + #expect(simd_distance(toMeter, SIMD3(tap.rayDirection[0], tap.rayDirection[1], tap.rayDirection[2])) < 1e-5) + // Raw world poses, straight from the frames. + #expect(packet.keyframes[2].pose[12...14].map { $0 } == [0.3, 1.5, 0]) + // Both sensors' own times travel beside the proposed pairing. + #expect(packet.ext?["imuRawPairingStatus"]?.hasPrefix("proposal") == true) + #expect(Set(packet.files.filter { $0.role == .stream }.map(\.path)) + == ["streams/arkit_poses.csv.gz", "streams/imu_raw.csv.gz", "streams/accelerometer_raw.csv.gz", "streams/gyroscope_raw.csv.gz"]) + + #expect(committedBeforeEnd.withLock { $0 } == packet.files.count - 4, "every image was received before the scan ended") + #expect(state.end == .finished(status: "manual_review")) + #expect(server.requests("POST captures").count == 1) + #expect(server.requests("POST captures/finalize").count == 1) + + if let dir = ProcessInfo.processInfo.environment["HOUSESCAN_EXPORT_EVIDENCE_DIR"] { + let target = URL(fileURLWithPath: dir).appending(path: "native-export-local") + try? FileManager.default.removeItem(at: target) + try FileManager.default.createDirectory(at: target.deletingLastPathComponent(), withIntermediateDirectories: true) + try FileManager.default.copyItem(at: session.folder, to: target) + } + } + + @Test func onlyTheIntegrationBuildRecordsAndOnlyAnAllowedBuildSends() { + let url = "https://capture.example.com/v1" + #expect(CaptureIntegrationMode.resolve(integrationBuild: "NO", endpoint: url, sendDeviceData: "YES") == .off("not the integration build")) + #expect(CaptureIntegrationMode.resolve(integrationBuild: nil, endpoint: url, sendDeviceData: "YES") == .off("not the integration build")) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "$(HOUSESCAN_INTEGRATION_BUILD)", endpoint: url, sendDeviceData: "YES") == .off("not the integration build")) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: "", sendDeviceData: "YES") == .off("no capture endpoint in this build")) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: url, sendDeviceData: "NO") == .recordOnly(URL(string: url)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: url, sendDeviceData: nil) == .recordOnly(URL(string: url)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: url, sendDeviceData: "YES") == .send(URL(string: url)!)) + // A replay goes only to a receiver on this machine, only when asked, whatever the device switch says. + let local = "http://127.0.0.1:8765/v1" + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: local, sendDeviceData: "NO", source: .replay(sendToLocalReceiver: true)) == .send(URL(string: local)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: local, sendDeviceData: "YES", source: .replay(sendToLocalReceiver: false)) == .recordOnly(URL(string: local)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "YES", endpoint: url, sendDeviceData: "YES", source: .replay(sendToLocalReceiver: true)) == .recordOnly(URL(string: url)!)) + #expect(CaptureIntegrationMode.resolve(integrationBuild: "NO", endpoint: local, sendDeviceData: "YES", source: .replay(sendToLocalReceiver: true)) == .off("not the integration build")) + } + + /// A build that may not send device data records the capture on the phone and never sends it, + /// even with a yes. + @Test func aRecordOnlyBuildKeepsTheCaptureOnThePhone() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures, sends: false)) + try await fixture.run(coordinator) + #expect(coordinator.needsConsent == false) + coordinator.answerConsent(true) + await coordinator.settle() + let session = try #require(coordinator.session) + #expect(session.uploader == nil) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: Data(contentsOf: session.folder.appending(path: "packet.json"))) + #expect(Packet04Check.problems(packet, folder: session.folder).isEmpty) + #expect(server.state.withLock { $0.log.isEmpty }) + } + + /// No answer or a no sends nothing; the next scan asks again. A yes before the scan sends + /// during it, and a world reset inside the scan keeps that yes for the new world's packet. + @Test func consentIsPerScanAndOnlyAYesSends() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let environment = NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures) + + let unanswered = CaptureSessionCoordinator(environment: environment) + try await fixture.run(unanswered, consent: nil) + #expect(unanswered.needsConsent) + #expect(unanswered.session?.uploader == nil) + + let declining = CaptureSessionCoordinator(environment: environment) + try await fixture.run(declining, consent: false) + #expect(declining.session?.uploader == nil) + #expect(server.state.withLock { $0.log.isEmpty }) + declining.newWorld("start over", recording: fixture.recording, newScan: true) + #expect(declining.consent == nil && declining.needsConsent) + + let agreeing = CaptureSessionCoordinator(environment: environment) + agreeing.begin(recording: fixture.recording) + agreeing.answerConsent(true) + let firstWorld = try #require(agreeing.session) + agreeing.newWorld("world reset", recording: fixture.recording, newScan: false) + #expect(agreeing.consent == true) + let secondWorld = try #require(agreeing.session) + #expect(secondWorld.uploader != nil && secondWorld.packetID != firstWorld.packetID) + agreeing.newWorld("start over", recording: fixture.recording, newScan: true) + #expect(agreeing.consent == nil) + #expect(agreeing.session?.uploader == nil) + } + + /// After a relaunch a frozen capture resumes only with the yes it recorded and only toward the + /// endpoint it was created on. + @Test func resumeNeedsTheCapturesYesAndTheSameEndpoint() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let original = try LoopbackCaptureAPI() + let other = try LoopbackCaptureAPI() + // A capture frozen on the phone for `original` whose upload never started. + let folder = captures.appending(path: "frozen") + let capture = try SyntheticCapture04(folder: folder) + let images = try await capture.sealImages() + let (streams, packet) = try await capture.finish() + var saved = CaptureUploadState( + attemptID: "old-process", packetID: capture.producer.info.packetID, + createBody: try JSONEncoder().encode(CaptureAPI.CreateRequest(packetId: capture.producer.info.packetID, tier: .arkit, device: .init(model: "m", systemVersion: "s", appVersion: "a")))) + saved.destination = original.base.absoluteString + // First without a recorded yes: never resumed. + for (i, file) in (images + streams).enumerated() { saved.files[file.path] = .init(sealed: file, sequence: i) } + saved.packet = packet + try saved.save(to: CaptureUploader.stateURL(in: folder)) + let http = URLSessionCaptureHTTP.ephemeral(timeout: 10) + + #expect(CaptureSessionCoordinator(environment: NativeCaptureFixture.environment(endpoint: original.base, http: http, captures: captures)) + .resumeSealedCaptures().isEmpty) + saved.consent = .init(grantedAt: Date(), destination: original.base.absoluteString) + try saved.save(to: CaptureUploader.stateURL(in: folder)) + #expect(CaptureSessionCoordinator(environment: NativeCaptureFixture.environment(endpoint: other.base, http: http, captures: captures)) + .resumeSealedCaptures().isEmpty) + #expect(CaptureSessionCoordinator(environment: NativeCaptureFixture.environment(endpoint: original.base, http: http, captures: captures, sends: false)) + .resumeSealedCaptures().isEmpty) + try await Task.sleep(for: .milliseconds(200)) + #expect(original.state.withLock { $0.log.isEmpty } && other.state.withLock { $0.log.isEmpty }) + + let resumed = CaptureSessionCoordinator(environment: NativeCaptureFixture.environment(endpoint: original.base, http: http, captures: captures)) + .resumeSealedCaptures() + #expect(resumed.count == 1) + for uploader in resumed { await uploader.kick(); await uploader.settled() } + #expect(await resumed.first?.snapshot.end == .finished(status: "manual_review")) + #expect(other.state.withLock { $0.log.isEmpty }) + } + + /// A refused close-up and its accepted retake, written over the same file as the app does: + /// both stay sealed with their own pixels, and the scale reference names the accepted one, + /// with its pixels and pose. A skipped close-up names none. + @Test func onlyTheAcceptedCloseUpIsTheScaleReference() async throws { + defer { try? FileManager.default.removeItem(at: root) } + for accepted in [true, false] { + let coordinator = CaptureSessionCoordinator(environment: NativeCaptureFixture.environment( + endpoint: URL(string: "https://capture.invalid/v1")!, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures, sends: false)) + coordinator.begin(recording: fixture.recording) + let source = fixture.folder.appending(path: "meter_close.jpg") + try FileManager.default.createDirectory(at: fixture.folder, withIntermediateDirectories: true) + try? FileManager.default.removeItem(at: source) + try makeJPEG(width: NativeCaptureFixture.width, height: NativeCaptureFixture.height, at: source, shift: 7) + let refusedBytes = try Data(contentsOf: source) + // Its depth too lives in files the retake overwrites: 4 x 3 meters and confidence. + let depthFile = fixture.folder.appending(path: "meter_close.f32"), confidenceFile = fixture.folder.appending(path: "meter_close.conf.u8") + func writeDepth(_ meters: Float, _ confidence: UInt8) throws { + try PacketFiles.depthData(meters: [Float](repeating: meters, count: 12)).write(to: depthFile) + try Data(repeating: confidence, count: 12).write(to: confidenceFile) + } + try writeDepth(1.5, 2) + func closeUp(at t: Double) -> KeptPhoto { + KeptPhoto( + t: t, cameraToWorld: fixture.pose(t), cameraIntrinsics: NativeCaptureFixture.k, cameraImageSize: SIMD2(1920, 1440), + width: NativeCaptureFixture.width, height: NativeCaptureFixture.height, tracking: .normal, exposure: nil, jpeg: source, purpose: "meter_close", + depth: { + guard let map = try? Data(contentsOf: depthFile), let confidence = try? Data(contentsOf: confidenceFile) else { return nil } + return Packet04Depth(meters: PacketFiles.floats(littleEndian: map), confidence: [UInt8](confidence), width: 4, height: 3) + }) + } + coordinator.kept(closeUp(at: fixture.start + 2)) + // The retake overwrites the same files before the first shot's sealing has run. + try FileManager.default.removeItem(at: source) + try makeJPEG(width: NativeCaptureFixture.width, height: NativeCaptureFixture.height, at: source, shift: 101) + try writeDepth(0.9, 1) + let retakeBytes = try Data(contentsOf: source) + coordinator.kept(closeUp(at: fixture.start + 4)) + coordinator.captureEnded(acceptedCloseUpAt: accepted ? fixture.start + 4 : nil) + await coordinator.settle() + + let session = try #require(coordinator.session) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: Data(contentsOf: session.folder.appending(path: "packet.json"))) + #expect(packet.stills?.map(\.id) == ["meter_close", "meter_close-2"]) + #expect(try Data(contentsOf: session.folder.appending(path: "stills/meter_close.jpg")) == refusedBytes) + #expect(try Data(contentsOf: session.folder.appending(path: "stills/meter_close-2.jpg")) == retakeBytes) + // Each shot keeps its own depth and confidence. + #expect(try Data(contentsOf: session.folder.appending(path: "keyframes/k00001.depth.f32")) == PacketFiles.depthData(meters: [Float](repeating: 1.5, count: 12))) + #expect(try Data(contentsOf: session.folder.appending(path: "keyframes/k00001.confidence.u8")) == Data(repeating: 2, count: 12)) + #expect(try Data(contentsOf: session.folder.appending(path: "keyframes/k00002.depth.f32")) == PacketFiles.depthData(meters: [Float](repeating: 0.9, count: 12))) + #expect(Packet04Check.problems(packet, folder: session.folder).isEmpty) + if accepted { + #expect(packet.scaleReference.meterCloseUp.still == "meter_close-2") + let frame = try #require(packet.keyframes.first { $0.id == packet.stills?.last?.keyframe }) + #expect(frame.timestamp == fixture.start + 4 && frame.pose[12] == 0.4) + } else { + #expect(packet.scaleReference.meterCloseUp == .notCaptured) + } + } + } + + /// A world reset while the first world's photos are still going up: that packet is abandoned, + /// the new world gets its own packet id, and nothing kept after the reset joins the old one. + @Test func aWorldResetStartsANewPacketAndDropsTheOld() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/files"] } + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures)) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + let first = try #require(coordinator.session) + for _ in 0..<500 where server.requests("POST captures/files").isEmpty { try await Task.sleep(for: .milliseconds(10)) } + + coordinator.newWorld("world reset", recording: fixture.recording, newScan: false) + coordinator.kept(try fixture.photo(at: fixture.start + 4)) + server.release("POST captures/files") + server.state.withLock { $0.held = [] } + await coordinator.settle() + await first.uploader!.settled() + + let second = try #require(coordinator.session) + #expect(second.packetID != first.packetID) + let old = await first.uploader!.snapshot + #expect(old.end == .abandoned("world reset")) + #expect(old.files.values.allSatisfy { $0.phase == .queued }) + #expect(await second.uploader!.snapshot.files.keys.sorted() == ["keyframes/k00001.jpg"]) + #expect(await second.uploader!.snapshot.committedCount == 1) + let creates = server.requests("POST captures").compactMap { try? JSONDecoder().decode(CaptureAPI.CreateRequest.self, from: $0.body).packetId } + #expect(Set(creates) == [first.packetID, second.packetID]) + } + + /// A reset that ends the session before its kept photos are sealed still deletes their staged + /// copies, so no copy of a home photo outlives the packet it was kept for. + @Test func aResetDeletesPhotosStagedForTheEndedSession() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures, sends: false)) + let staging = captures.appending(path: "staging") + func staged() -> [String] { (try? FileManager.default.contentsOfDirectory(atPath: staging.path)) ?? [] } + + coordinator.begin(recording: fixture.recording) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + coordinator.kept(try fixture.photo(at: fixture.start + 3)) + #expect(staged().count == 2) + // No suspension between keeping and the reset, so the queued work finds the session ended. + coordinator.newWorld("start over", recording: fixture.recording, newScan: true) + coordinator.kept(try fixture.photo(at: fixture.start + 4)) + await coordinator.settle() + for _ in 0..<500 where !staged().isEmpty { try await Task.sleep(for: .milliseconds(10)) } + #expect(staged().isEmpty) + } + + /// A no after a yes stops sending: the upload is abandoned, nothing kept later goes up, the + /// packet is never finalized, and a second yes in the same scan does not start it again. + @Test func aNoAfterAYesStopsSending() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures)) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + await coordinator.settle() + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + await uploader.settled() + let putsBefore = server.requests("PUT upload").count + #expect(putsBefore == 1) + + coordinator.answerConsent(false) + coordinator.kept(try fixture.photo(at: fixture.start + 4)) + coordinator.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.captureEnded(acceptedCloseUpAt: nil) + await coordinator.settle() + await uploader.settled() + + #expect(await uploader.snapshot.end == .abandoned("consent withdrawn")) + #expect(coordinator.session === session && session.uploader == nil) + #expect(server.requests("PUT upload").count == putsBefore) + #expect(server.requests("POST captures").count == 1) + #expect(server.requests("POST captures/finalize").isEmpty) + } + + /// A withdrawal is saved before `answerConsent(false)` returns: a relaunch straight after it + /// does not resume the sealed capture, even though the uploader has not saved its end yet. + @Test func aWithdrawnCaptureIsNotResumedAfterARelaunch() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/finalize"] } + let environment = NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + for i in 0..<3 { coordinator.kept(try fixture.photo(at: fixture.start + 3 + Double(i) * 1.5)) } + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + let stateURL = CaptureUploader.stateURL(in: session.folder) + for _ in 0..<500 where (try? CaptureUploadState.load(from: stateURL))?.packet == nil { try await Task.sleep(for: .milliseconds(10)) } + #expect(try CaptureUploadState.load(from: stateURL).packet != nil) + + coordinator.answerConsent(false) + // The same main-actor turn: the abandon task has not run yet. + #expect(CaptureSessionCoordinator(environment: environment).resumeSealedCaptures().isEmpty) + #expect(try CaptureUploader.resume(folder: session.folder, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) == nil) + + server.release("POST captures/finalize") + server.state.withLock { $0.held = [] } + // The withdrawal cancelled the loop; the abandon task ends the upload in its own time. + for _ in 0..<500 where await uploader.snapshot.end == nil { try await Task.sleep(for: .milliseconds(10)) } + #expect(await uploader.snapshot.end == .abandoned("consent withdrawn")) + } + + /// A relaunch scan in the live process must not create a second sender for the sealed + /// folder. On the old code B survives A's withdrawal and sends the remaining files. + @Test func withdrawalCoversTheFolderDuringResumeDiscovery() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/finalize"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + for i in 0..<3 { coordinator.kept(try fixture.photo(at: fixture.start + 3 + Double(i) * 1.5)) } + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + let uploader = try #require(coordinator.session?.uploader) + try await waitUntilParked(server, "POST captures/finalize") + + let duplicates = CaptureSessionCoordinator(environment: environment).resumeSealedCaptures() + for duplicate in duplicates { await duplicate.kick() } + if !duplicates.isEmpty { + for _ in 0..<500 where server.requests("POST captures/finalize").count < 2 { + try await Task.sleep(for: .milliseconds(10)) + } + try #require(server.requests("POST captures/finalize").count == 2) + } + #expect(duplicates.isEmpty) + coordinator.answerConsent(false) + let puts = server.requests("PUT upload").count + let commits = server.requests("POST captures/files:commit").count + server.state.withLock { $0.held = [] } + server.release("POST captures/finalize") + try #require(await CaptureUploaderTests.settles(uploader)) + for duplicate in duplicates { try #require(await CaptureUploaderTests.settles(duplicate)) } + #expect(server.requests("PUT upload").count == puts) + #expect(server.requests("POST captures/files:commit").count == commits) + #expect(server.requests("GET captures/result").isEmpty) + #expect(CaptureSessionCoordinator(environment: environment).resumeSealedCaptures().isEmpty) + } + + @Test func droppingAnIdleCoordinatorReleasesFolderOwnership() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let environment = NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures) + var coordinator: CaptureSessionCoordinator? = CaptureSessionCoordinator(environment: environment) + coordinator?.begin(recording: fixture.recording) + coordinator?.answerConsent(true) + coordinator?.kept(try fixture.photo(at: fixture.start + 2)) + await coordinator?.settle() + let folder = try #require(coordinator?.session?.folder) + weak let oldSession = coordinator?.session + weak let oldUploader = coordinator?.session?.uploader + coordinator = nil + for _ in 0..<500 where oldSession != nil || oldUploader != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(oldSession == nil && oldUploader == nil) + let resumed = try #require(try CaptureUploader.resume(folder: folder, base: server.base, http: environment.http)) + await resumed.kick() + try #require(await CaptureUploaderTests.settles(resumed)) + #expect(server.requests("POST captures").count == 1) + } + + @Test func anUnsavedWithdrawalIsReturnedToTheCaller() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/files"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + try await waitUntilParked(server, "POST captures/files") + let stateURL = CaptureUploader.stateURL(in: session.folder) + let before = try Data(contentsOf: stateURL) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: session.folder.path) + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: session.folder.path) } + + let result = coordinator.answerConsent(false) + guard case .failure(.notRecorded(let detail)) = result else { + Issue.record("The caller must receive the persistence failure") + server.state.withLock { $0.held = [] } + server.release("POST captures/files") + return + } + #expect(detail.contains("marker:") && detail.contains("state removal:")) + #expect(try Data(contentsOf: stateURL) == before) + // Repeated answers must not turn the failed withdrawal into a reported success. + if case .success = coordinator.answerConsent(false) { Issue.record("Repeated no lost the failure") } + if case .success = coordinator.answerConsent(true) { Issue.record("Later yes lost the failure") } + #expect(session.uploader == nil) + #expect(try CaptureUploader.resume(folder: session.folder, base: server.base, http: environment.http) == nil) + #expect(CaptureSessionCoordinator(environment: environment).resumeSealedCaptures().isEmpty) + server.state.withLock { $0.held = [] } + server.release("POST captures/files") + try #require(await CaptureUploaderTests.settles(uploader)) + #expect(server.requests("PUT upload").isEmpty) + #expect(server.requests("POST captures/files:commit").isEmpty) + // All writes still fail, so the app cannot promise the persisted yes was removed. + #expect(try Data(contentsOf: stateURL) == before) + } + + /// Waits, suspending, until the loopback has parked an answer for `route`. + func waitUntilParked(_ server: LoopbackCaptureAPI, _ route: String) async throws { + for _ in 0..<1000 where server.state.withLock({ $0.parked[route]?.isEmpty ?? true }) { try await Task.sleep(for: .milliseconds(10)) } + #expect(server.state.withLock { !($0.parked[route]?.isEmpty ?? true) }) + } + + /// Blocks the main actor until `stateURL` holds something other than `before`, so nothing + /// queued on the main actor (the abandon task a withdrawal schedules) can run meanwhile. + func blockUntilSaved(_ stateURL: URL, differentFrom before: Data?) { + let deadline = Date().addingTimeInterval(5) + while Date() < deadline, (try? Data(contentsOf: stateURL)) == before { usleep(10_000) } + } + + /// A reply that lands after the homeowner withdrew, but before the uploader is abandoned, + /// can't save a capture a relaunch would resume, whether or not the withdrawal marker could + /// be written. The transport lets the reply arrive even though the request's task was + /// cancelled, as a background session does. The main actor is held from the withdrawal to the + /// checks, so the abandon task the withdrawal schedules can't run first. + @Test(arguments: [false, true]) func aReplyAfterWithdrawalCannotSaveAResumableCapture(markerBlocked: Bool) async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/files"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + let stateURL = CaptureUploader.stateURL(in: session.folder) + try await waitUntilParked(server, "POST captures/files") + if markerBlocked { + // A directory where the marker goes makes writing it fail. + try FileManager.default.createDirectory( + at: CaptureUploadState.withdrawnURL(in: session.folder).appending(path: "blocked"), withIntermediateDirectories: true) + } + + coordinator.answerConsent(false) + // No suspension from here to the checks. + let before = try? Data(contentsOf: stateURL) + #expect((before == nil) == markerBlocked) + server.release("POST captures/files") + // The uploader saves the register reply on its own executor. + blockUntilSaved(stateURL, differentFrom: before) + let saved = try CaptureUploadState.load(from: stateURL) + #expect(saved.end == .abandoned("consent withdrawn")) + if markerBlocked { + // The blocking directory sits at the marker's path and would stop a resume by itself; + // without it only the saved end can. + try FileManager.default.removeItem(at: CaptureUploadState.withdrawnURL(in: session.folder)) + } else { + #expect(try CaptureUploader.resume(folder: session.folder, base: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10)) == nil) + } + #expect(CaptureSessionCoordinator(environment: environment).resumeSealedCaptures().isEmpty) + #expect(server.requests("PUT upload").isEmpty) + + server.state.withLock { $0.held = [] } + await uploader.settled() + #expect(server.requests("PUT upload").isEmpty) + } + + /// A result that arrives after the withdrawal can't save the upload as finished: the + /// withdrawal outranks the end a late reply sets. + @Test func aResultAfterWithdrawalSavesTheUploadAsAbandoned() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["GET captures/result"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10)), captures: captures) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + let (tap, hit) = try fixture.tap(at: fixture.start + 1) + coordinator.meterTapped(tap, hit: hit) + coordinator.kept(try fixture.photo(at: fixture.start + 2.5, purpose: "meter_close")) + for i in 0..<3 { coordinator.kept(try fixture.photo(at: fixture.start + 3 + Double(i) * 1.5)) } + coordinator.captureEnded(acceptedCloseUpAt: fixture.start + 2.5) + let session = try #require(coordinator.session) + let uploader = try #require(session.uploader) + let stateURL = CaptureUploader.stateURL(in: session.folder) + try await waitUntilParked(server, "GET captures/result") + + coordinator.answerConsent(false) + // No suspension from here to the check. + let before = try? Data(contentsOf: stateURL) + server.release("GET captures/result") + blockUntilSaved(stateURL, differentFrom: before) + #expect(try CaptureUploadState.load(from: stateURL).end == .abandoned("consent withdrawn")) + + server.state.withLock { $0.held = [] } + await uploader.settled() + #expect(await uploader.snapshot.end == .abandoned("consent withdrawn")) + } + + /// Withdrawing cancels the request the uploader is waiting on before the abandon task runs: + /// a held register request is given up at once instead of waiting for its answer, and no + /// photo is uploaded. + @Test func withdrawalCancelsTheRequestInFlight() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let logs = Mutex<[String]>([]) + let server = try LoopbackCaptureAPI() + server.state.withLock { $0.held = ["POST captures/files"] } + let environment = NativeCaptureFixture.environment( + endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures, + log: { line in logs.withLock { $0.append(line) } }) + let coordinator = CaptureSessionCoordinator(environment: environment) + coordinator.begin(recording: fixture.recording) + coordinator.answerConsent(true) + coordinator.kept(try fixture.photo(at: fixture.start + 2)) + let uploader = try #require(coordinator.session?.uploader) + try await waitUntilParked(server, "POST captures/files") + + coordinator.answerConsent(false) + // The main actor stays busy, so only the withdrawal itself can end the wait. + let deadline = Date().addingTimeInterval(5) + func gaveUp() -> Bool { logs.withLock { $0.contains { $0.contains("capture-upload retry step=register") } } } + while Date() < deadline, !gaveUp() { usleep(10_000) } + #expect(gaveUp()) + + server.release("POST captures/files") + server.state.withLock { $0.held = [] } + await uploader.settled() + #expect(server.requests("PUT upload").isEmpty) + } + + /// Photos kept after the scan was sent don't change the frozen packet or start more uploads. + @Test func photosAfterTheScanWasSentAreNotAdded() async throws { + defer { try? FileManager.default.removeItem(at: root) } + let server = try LoopbackCaptureAPI() + let coordinator = CaptureSessionCoordinator( + environment: NativeCaptureFixture.environment(endpoint: server.base, http: URLSessionCaptureHTTP.ephemeral(timeout: 10), captures: captures)) + try await fixture.run(coordinator) + let before = await coordinator.session!.uploader!.snapshot + coordinator.kept(try fixture.photo(at: fixture.end - 0.5)) + await coordinator.settle() + let after = await coordinator.session!.uploader!.snapshot + #expect(after.packet == before.packet) + #expect(after.files.count == before.files.count) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift new file mode 100644 index 000000000..664d5564b --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CaptureUploaderTests.swift @@ -0,0 +1,602 @@ +import Foundation +import HouseScanKit +import Synchronization +import Testing + +/// The uploader against `LoopbackCaptureAPI` over real HTTP, through the same `URLSession` +/// transport the app uses. Retries wait 10 ms instead of seconds. +@Suite(.serialized) struct CaptureUploaderTests { + struct Rig { + let server: LoopbackCaptureAPI + let root: URL + let capture: SyntheticCapture04 + private var ownedUploader: CaptureUploader? + var uploader: CaptureUploader { + guard let ownedUploader else { preconditionFailure("The test released this uploader to simulate process exit") } + return ownedUploader + } + + mutating func releaseUploader() { ownedUploader = nil } + let http: any CaptureHTTP + + init( + packetID: String = UUID().uuidString, appVersion: String = "test (1)", server: LoopbackCaptureAPI? = nil, + http: any CaptureHTTP = URLSessionCaptureHTTP.ephemeral(timeout: 10), policy: CaptureUploader.Policy = Rig.fast, + sleep: @escaping @Sendable (Double) async throws -> Void = { _ in try await Task.sleep(for: .milliseconds(10)) }, + now: @escaping @Sendable () -> Date = { Date() } + ) throws { + self.http = http + self.server = try server ?? LoopbackCaptureAPI() + root = FileManager.default.temporaryDirectory.appending(path: "uploader-\(UUID().uuidString)") + capture = try SyntheticCapture04(folder: root.appending(path: "packet"), packetID: packetID) + ownedUploader = try CaptureUploader.start( + folder: capture.folder, base: self.server.base, http: http, + create: .init(packetId: packetID, tier: .arkit, device: .init(model: "iPhone15,4", systemVersion: "26.0", appVersion: appVersion)), + consentedAt: Date(), policy: policy, now: now, sleep: sleep) + } + + static var fast: CaptureUploader.Policy { + var policy = CaptureUploader.Policy() + policy.eventsWait = 0 + return policy + } + + func finishAndSeal() async throws { + let (streams, packet) = try await capture.finish() + await uploader.seal(packet: packet, files: streams) + } + + func cleanUp() { try? FileManager.default.removeItem(at: root) } + } + + /// Photos go up during the capture; the frozen packet finalizes early, the streams follow, and + /// the capture ends with a result for its own run. + @Test func joinedPathCommitsPhotosBeforeTheCaptureEnds() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + let images = try await rig.capture.sealImages() + await rig.uploader.add(images) + await rig.uploader.settled() + + let during = await rig.uploader.snapshot + #expect(during.committedCount == images.count) + #expect(during.packet == nil) + #expect(rig.server.requests("POST captures/finalize").isEmpty) + + try await rig.finishAndSeal() + await rig.uploader.settled() + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "manual_review")) + #expect(done.finalized?.status == "awaiting_files") + #expect(done.finalized?.missing.sorted() == [ + "streams/accelerometer_raw.csv.gz", "streams/arkit_poses.csv.gz", "streams/gyroscope_raw.csv.gz", "streams/imu_raw.csv.gz", + ]) + #expect(done.committedCount == images.count + 4) + #expect(done.result != nil) + let marks = done.marks + #expect(try #require(marks["firstCommit"]) < #require(marks["sealed"])) + #expect(try #require(marks["sealed"]) <= #require(marks["lastCommit"])) + + let puts = rig.server.requests("PUT upload") + #expect(puts.count == images.count + 4) + #expect(puts.allSatisfy { $0.headers["authorization"] == nil && $0.headers["content-md5"] != nil }) + #expect(rig.server.requests("POST captures").count == 1) + } + + /// A create and a finalize whose answers are lost are sent again as the same bytes, and the + /// server keeps one capture. + @Test func lostAnswersAreReplayedWithTheFrozenBodies() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.dropNext = ["POST captures", "POST captures/finalize"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 2)) + await rig.uploader.settled() + try await rig.finishAndSeal() + await rig.uploader.settled() + + let creates = rig.server.requests("POST captures") + let finals = rig.server.requests("POST captures/finalize") + #expect(creates.count == 2) + #expect(Set(creates.map(\.body)).count == 1) + #expect(finals.count == 2) + #expect(Set(finals.map(\.body)).count == 1) + #expect(rig.server.state.withLock { $0.captures.count } == 1) + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + } + + /// A relaunch resumes the saved capture: same packet and capture ids, no second create, and the + /// old process's attempt id is replaced. + @Test func relaunchResumesTheSameCapture() async throws { + var rig = try Rig() + defer { rig.cleanUp() } + await rig.uploader.add(try await rig.capture.sealImages(count: 2)) + await rig.uploader.settled() + let before = await rig.uploader.snapshot + + // The old process releases every uploader before the new one resumes its saved state. + weak let released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + let resumed = try #require(try CaptureUploader.resume( + folder: rig.capture.folder, base: rig.server.base, http: rig.http, policy: Rig.fast, sleep: { _ in })) + let after = await resumed.snapshot + #expect(after.packetID == before.packetID) + #expect(after.captureID == before.captureID) + #expect(after.createBody == before.createBody) + #expect(after.attemptID != before.attemptID) + + let (streams, packet) = try await rig.capture.finish() + await resumed.seal(packet: packet, files: streams) + await resumed.settled() + #expect(await resumed.snapshot.end == .finished(status: "manual_review")) + #expect(rig.server.requests("POST captures").count == 1) + } + + /// The same packet id with a different create body is the server's idempotency conflict; the + /// uploader stops instead of retrying it. + @Test func aChangedCreateBodyIsRefusedNotRetried() async throws { + let packetID = UUID().uuidString + let first = try Rig(packetID: packetID) + defer { first.cleanUp() } + await first.uploader.kick() + await first.uploader.settled() + #expect(await first.uploader.snapshot.captureID != nil) + + let second = try Rig(packetID: packetID, appVersion: "test (2)", server: first.server) + defer { second.cleanUp() } + await second.uploader.kick() + await second.uploader.settled() + #expect(await second.uploader.snapshot.end == .failed(step: "create", codes: ["idempotency_conflict"], status: 409)) + #expect(first.server.requests("POST captures").count == 2) + } + + /// A storage URL answered 403 (expired) is registered again and the file sent to the new URL. + @Test func anExpiredURLIsFetchedAgain() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.expireNextPuts = 1 } + let images = try await rig.capture.sealImages(count: 1) + await rig.uploader.add(images) + await rig.uploader.settled() + let snapshot = await rig.uploader.snapshot + #expect(snapshot.committedCount == images.count) + #expect(rig.server.requests("PUT upload").count == images.count + 1) + let registered = rig.server.requests("POST captures/files").flatMap { request -> [String] in + let body = try? JSONDecoder().decode(CaptureAPI.RegisterRequest.self, from: request.body) + return body?.files.map(\.path) ?? [] + } + #expect(registered.count == images.count + 1) + } + + /// Only paths the commit names as committed advance; notFound and mismatch go up again. + @Test func mixedCommitAnswersAdvanceOnlyCommittedFiles() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + let images = try await rig.capture.sealImages(count: 2) + // Priority order: the still, then k00001, then k00002. + let paths = ["stills/meter_close.jpg", "keyframes/k00001.jpg", "keyframes/k00002.jpg"] + #expect(Set(images.map(\.path)) == Set(paths)) + rig.server.state.withLock { $0.commitOverride = (committed: [paths[0]], notFound: [paths[1]], mismatch: [paths[2]]) } + let seen = Mutex<[Int]>([]) + await rig.uploader.observe { status in seen.withLock { $0.append(status.committed) } } + await rig.uploader.add(images) + await rig.uploader.settled() + + #expect(await rig.uploader.snapshot.committedCount == 3) + // After the first commit exactly one file counted as received. + #expect(seen.withLock { $0.first { $0 > 0 } } == 1) + let puts = rig.server.requests("PUT upload") + #expect(puts.count == 5) + } + + /// A world reset while a register answer is in flight: the late answer changes nothing, and the + /// next capture is a new packet that never reuses the old one. + @Test func resetDuringAPendingAnswerDropsTheLateReply() async throws { + // A background session's transfer outlives the task that asked for it, so the reply + // still arrives after the reset; only the attempt check keeps it out. + let rig = try Rig(http: UncancellableHTTP(URLSessionCaptureHTTP.ephemeral(timeout: 10))) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.held = ["POST captures/files"] } + await rig.uploader.add(try await rig.capture.sealImages(count: 2)) + for _ in 0..<500 where rig.server.requests("POST captures/files").isEmpty { try await Task.sleep(for: .milliseconds(10)) } + #expect(rig.server.requests("POST captures/files").count == 1) + + await rig.uploader.abandon("world reset") + rig.server.release("POST captures/files") + try await Task.sleep(for: .milliseconds(300)) + await rig.uploader.settled() + + let after = await rig.uploader.snapshot + #expect(after.end == .abandoned("world reset")) + #expect(after.files.values.allSatisfy { $0.phase == .queued }) + #expect(rig.server.requests("PUT upload").isEmpty) + // Nothing more is sent for the abandoned capture. + let late = try await rig.capture.producer.sealKeyframe( + jpeg: try SyntheticCapture04.jpeg(in: rig.root), observation: rig.capture.observation(SyntheticCapture04.start + 1.9), reason: "motion") + await rig.uploader.add(late) + await rig.uploader.settled() + #expect(rig.server.requests("POST captures/files").count == 1) + + let next = try Rig(server: rig.server) + defer { next.cleanUp() } + await next.uploader.add(try await next.capture.sealImages(count: 1)) + await next.uploader.settled() + #expect(await next.uploader.snapshot.captureID != after.captureID) + #expect(await next.uploader.snapshot.committedCount == 2) + } + + /// A finalize the server accepted and then lost (`failed`, next `retry_finalize`) is sent + /// again as the same bytes, and the run then completes. + @Test func aLostFinalizeIsSentAgainOnRetryFinalize() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.loseNextFinalize = true } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + let finals = rig.server.requests("POST captures/finalize") + #expect(finals.count == 2) + #expect(Set(finals.map(\.body)).count == 1) + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + #expect(await rig.uploader.snapshot.finalizeRetries == 1) + } + + /// The run is done but its answer is not readable yet (404 `result_not_ready`): the uploader + /// reads again after a wait instead of failing the capture. + @Test func aResultNotReadableYetIsReadAgain() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.notReadyResults = 2 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "manual_review")) + #expect(try Self.outcomeKind(done.result) == .manualReview) + #expect(rig.server.requests("GET captures/result").count == 3) + } + + /// The API answers a result read with the capture's status and a null outcome until the run + /// has written its answer. A terminal status with that body is not the answer. + @Test func aResultWithoutItsOutcomeIsReadAgain() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.outcomelessResults = 2 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "manual_review")) + #expect(try Self.outcomeKind(done.result) == .manualReview) + #expect(rig.server.requests("GET captures/result").count == 3) + } + + /// An answer that never becomes readable ends the upload visibly after the policy's reads, + /// rather than waiting forever. + @Test func aResultThatNeverArrivesEndsTheUpload() async throws { + var policy = Rig.fast + policy.maxNotReadyResults = 3 + let rig = try Rig(policy: policy) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.outcomelessResults = 100 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .failed(step: "result", codes: ["result_not_ready"], status: 200)) + #expect(done.result == nil) + #expect(rig.server.requests("GET captures/result").count == 4) + } + + /// A 200 whose body is not a result is refused by name instead of being kept as the answer. + @Test func anUnreadableResultStopsTheUpload() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.unreadableResults = 1 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .failed(step: "result", codes: ["result_unreadable"], status: 200)) + #expect(done.result == nil) + #expect(rig.server.requests("GET captures/result").count == 1) + } + + /// An outcome missing fields the app needs is refused by name instead of finishing the upload + /// with an answer nothing can show. + @Test func aMalformedOutcomeStopsTheUpload() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.malformedOutcomeResults = 1 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .failed(step: "result", codes: ["result_unreadable"], status: 200)) + #expect(done.result == nil) + #expect(rig.server.requests("GET captures/result").count == 1) + } + + /// Whether `uploader` goes idle within `seconds`. A loop that never goes idle fails the test + /// here instead of hanging the run; the process ending stops it. + static func settles(_ uploader: CaptureUploader, within seconds: Int = 20) async -> Bool { + let done = Mutex(false) + Task.detached { await uploader.settled(); done.withLock { $0 = true } } + let deadline = ContinuousClock.now + .seconds(seconds) + while ContinuousClock.now < deadline { + if done.withLock({ $0 }) { return true } + try? await Task.sleep(for: .milliseconds(20)) + } + return false + } + + /// A storage URL that is not https (or http to this machine) is refused when it is + /// registered, before any bytes go to it; one that doesn't parse is refused the same way + /// instead of leaving the file registered and the loop spinning. + @Test(arguments: [("http://storage.example.com/upload/x", "upload_url_insecure"), ("", "upload_url_invalid")]) + func aStorageURLThatIsNotSafeIsRefusedBeforeAnyPut(url: String, code: String) async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.uploadURLOverride = url } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "register", codes: [code], status: 200)) + #expect(rig.server.requests("POST captures/files").count == 1) + #expect(rig.server.requests("PUT upload").isEmpty) + } + + /// Storage that answers a PUT with a redirect doesn't get the photo sent on: the real + /// `URLSession` transport refuses every redirect, so the photo reaches only the URL that passed + /// the check, and the 307 comes back as a storage refusal. The redirect goes to a second local + /// server, which must see nothing. + @Test func aStorageRedirectIsNotFollowed() async throws { + let elsewhere = try LoopbackCaptureAPI() + var policy = Rig.fast + policy.maxRefusedPuts = 2 + let rig = try Rig(policy: policy) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.redirects["PUT upload"] = "http://127.0.0.1:\(elsewhere.port)/upload/elsewhere" } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "put", codes: ["storage_refused"], status: 307)) + #expect(!rig.server.requests("PUT upload").isEmpty) + #expect(elsewhere.state.withLock { $0.log.isEmpty }) + } + + /// The API's own requests aren't redirected either: a 307 on create comes back as the reply, + /// which the uploader refuses by its status, and the second server sees nothing. + @Test func anAPIRedirectIsNotFollowed() async throws { + let elsewhere = try LoopbackCaptureAPI() + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.redirects["POST captures"] = "\(elsewhere.base.absoluteString)/captures" } + await rig.uploader.kick() + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "create", codes: [], status: 307)) + #expect(rig.server.requests("POST captures").count == 1) + #expect(elsewhere.state.withLock { $0.log.isEmpty }) + } + + /// Commits that never acknowledge a file back off between tries and stop after the policy's + /// limit, instead of sending the same photo again and again. + @Test func aCommitThatNeverAcknowledgesBacksOffAndStops() async throws { + var policy = Rig.fast + policy.maxUnacknowledged = 3 + let sleeps = Mutex<[Double]>([]) + let rig = try Rig(policy: policy, sleep: { seconds in sleeps.withLock { $0.append(seconds) } }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.commitNeverAcknowledges = true } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "commit", codes: ["not_acknowledged"], status: 200)) + // The images are the close-up still and its keyframe. + #expect(rig.server.requests("POST captures/files:commit").count == 4) + #expect(sleeps.withLock { $0.count } == 3) + } + + /// Register answers that leave a file out back off and stop the same way. + @Test func aRegisterThatOmitsFilesBacksOffAndStops() async throws { + var policy = Rig.fast + policy.maxUnacknowledged = 3 + let sleeps = Mutex<[Double]>([]) + let rig = try Rig(policy: policy, sleep: { seconds in sleeps.withLock { $0.append(seconds) } }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.registerOmitsFiles = true } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .failed(step: "register", codes: ["not_acknowledged"], status: 200)) + #expect(rig.server.requests("POST captures/files").count == 4) + #expect(rig.server.requests("PUT upload").isEmpty) + #expect(sleeps.withLock { $0.count } == 3) + } + + /// The limit on reads that find no answer is saved with the upload, so relaunching doesn't + /// start it again: across two processes the reads still stop at `maxNotReadyResults + 1`. + @Test func theNotReadyLimitSurvivesARelaunch() async throws { + var policy = Rig.fast + policy.maxNotReadyResults = 3 + let waits = Mutex(0) + // The first process quits during its second wait. + var rig = try Rig(policy: policy, sleep: { _ in + let n = waits.withLock { $0 += 1; return $0 } + if n >= 2 { throw CancellationError() } + }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.outcomelessResults = 100 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + try #require(await Self.settles(rig.uploader)) + #expect(rig.server.requests("GET captures/result").count == 2) + #expect(await rig.uploader.snapshot.end == nil) + + // The old process releases every uploader before the new one resumes its saved state. + weak let released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + let resumed = try #require(try CaptureUploader.resume( + folder: rig.capture.folder, base: rig.server.base, http: rig.http, policy: policy, sleep: { _ in })) + await resumed.kick() + try #require(await Self.settles(resumed)) + #expect(await resumed.snapshot.end == .failed(step: "result", codes: ["result_not_ready"], status: 200)) + #expect(rig.server.requests("GET captures/result").count == 4) + } + + /// A server that answers an events poll at once with nothing new is not polled again at once: + /// the uploader waits out `minimumPollInterval` first. The clock moves only when the uploader + /// sleeps, so every empty answer arrives at once and each wait is the whole interval. The + /// terminal answer is fetched without a wait. + @Test func emptyEventsAnsweredAtOnceArePaced() async throws { + let clock = Mutex(Date(timeIntervalSince1970: 1_000_000)) + let sleeps = Mutex<[Double]>([]) + let rig = try Rig( + sleep: { seconds in sleeps.withLock { $0.append(seconds) }; clock.withLock { $0 += seconds } }, + now: { clock.withLock { $0 } }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.processingPolls = 3 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + let interval = CaptureUploader.Policy().minimumPollInterval + #expect(sleeps.withLock { $0 } == [interval, interval, interval]) + #expect(rig.server.requests("GET captures/events").count == 4) + } + + /// An empty answer the server held for the whole interval needs no further wait. + @Test func emptyEventsHeldPastTheIntervalAreNotPaced() async throws { + // Every reading of the clock is 3 s after the last, longer than the 2 s interval. + let clock = Mutex(Date(timeIntervalSince1970: 1_000_000)) + let sleeps = Mutex<[Double]>([]) + let rig = try Rig( + sleep: { seconds in sleeps.withLock { $0.append(seconds) } }, + now: { clock.withLock { $0 += 3; return $0 } }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.processingPolls = 3 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + try #require(await Self.settles(rig.uploader)) + + #expect(await rig.uploader.snapshot.end == .finished(status: "manual_review")) + #expect(sleeps.withLock { $0 }.isEmpty) + #expect(rig.server.requests("GET captures/events").count == 4) + } + + /// A capture the server lost and the uploader opened again is a new run, so it gets a fresh + /// not-ready limit: the old run's reads don't end it. + @Test func aRecreatedCaptureStartsANewNotReadyLimit() async throws { + var policy = Rig.fast + policy.maxNotReadyResults = 3 + let waits = Mutex(0) + // The first process quits during its third wait, after three not-ready reads. + var rig = try Rig(policy: policy, sleep: { _ in + let n = waits.withLock { $0 += 1; return $0 } + if n >= 3 { throw CancellationError() } + }) + defer { rig.cleanUp() } + rig.server.state.withLock { $0.outcomelessResults = 100 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + try #require(await Self.settles(rig.uploader)) + #expect(try CaptureUploadState.load(from: CaptureUploader.stateURL(in: rig.capture.folder)).notReadyReads == 3) + + // The server forgets every capture (a redeploy with in-memory state). The next run has + // one not-ready read before its answer. + rig.server.state.withLock { $0.captures = [:]; $0.byPacket = [:]; $0.outcomelessResults = 1 } + // The old process releases every uploader before the new one resumes its saved state. + weak let released = rig.uploader + rig.releaseUploader() + for _ in 0..<500 where released != nil { try await Task.sleep(for: .milliseconds(10)) } + try #require(released == nil) + let resumed = try #require(try CaptureUploader.resume( + folder: rig.capture.folder, base: rig.server.base, http: rig.http, policy: policy, sleep: { _ in })) + await resumed.kick() + try #require(await Self.settles(resumed)) + #expect(await resumed.snapshot.end == .finished(status: "manual_review")) + #expect(rig.server.requests("POST captures").count == 2) + } + + /// `failed` ends a capture without an answer, so its outcomeless result is final at once. + @Test func aFailedCaptureEndsWithoutWaitingForAnOutcome() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.endStatus = "failed" } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + try await rig.finishAndSeal() + await rig.uploader.settled() + + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "failed")) + #expect(try Self.outcomeKind(done.result) == nil) + #expect(rig.server.requests("GET captures/result").count == 1) + } + + static func outcomeKind(_ body: Data?) throws -> CaptureResult.OutcomeKind? { + try JSONDecoder().decode(CaptureResult.Response.self, from: #require(body)).outcome?.kind + } + + /// Storage that keeps refusing the sealed bytes' digest stops the upload after one retry + /// instead of cycling through register and PUT. + @Test func aDigestRefusedTwiceStopsTheUpload() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + rig.server.state.withLock { $0.badDigestPuts = 100 } + await rig.uploader.add(try await rig.capture.sealImages(count: 1)) + await rig.uploader.settled() + #expect(await rig.uploader.snapshot.end == .failed(step: "put", codes: ["bad_digest"], status: 400)) + #expect(rig.server.requests("PUT upload").count <= 4) + } + + /// A file listed in the frozen packet that reached `seal` before its own `add` is still sent. + @Test func sealQueuesListedFilesItHasNotSeen() async throws { + let rig = try Rig() + defer { rig.cleanUp() } + let images = try await rig.capture.sealImages(count: 2) + await rig.uploader.add(Array(images.prefix(1))) + await rig.uploader.settled() + let (streams, packet) = try await rig.capture.finish() + await rig.uploader.seal(packet: packet, files: images + streams) + await rig.uploader.add(Array(images.suffix(1))) + await rig.uploader.settled() + let done = await rig.uploader.snapshot + #expect(done.end == .finished(status: "manual_review")) + #expect(done.committedCount == images.count + streams.count) + } + + @Test func noEndpointOrNoConsentMeansNoUpload() { + #expect(CaptureUploadGate.decide(endpoint: nil, consented: true) == .off("no capture endpoint in this build")) + #expect(CaptureUploadGate.decide(endpoint: "", consented: true) == .off("no capture endpoint in this build")) + #expect(CaptureUploadGate.decide(endpoint: "$(HOUSESCAN_CAPTURE_API_URL)", consented: true) == .off("no capture endpoint in this build")) + #expect(CaptureUploadGate.decide(endpoint: "http://example.com/v1", consented: true) == .off("capture endpoint must be https")) + #expect(CaptureUploadGate.decide(endpoint: "https://user:secret@example.com/v1", consented: true) == .off("capture endpoint is not a URL")) + #expect(CaptureUploadGate.decide(endpoint: "https://example.com/v1", consented: false) == .off("the homeowner has not agreed to send the capture")) + #expect(CaptureUploadGate.decide(endpoint: "https://example.com/v1", consented: true) == .on(URL(string: "https://example.com/v1")!)) + } +} + +/// Runs each request in a detached task, so cancelling the caller doesn't cancel the request: +/// the way a background URLSession behaves. +struct UncancellableHTTP: CaptureHTTP { + let inner: any CaptureHTTP + init(_ inner: any CaptureHTTP) { self.inner = inner } + + func send(_ request: URLRequest) async throws -> HTTPReply { + try await Task.detached { try await inner.send(request) }.value + } + + func upload(_ request: URLRequest, file: URL) async throws -> HTTPReply { + try await Task.detached { try await inner.upload(request, file: file) }.value + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift new file mode 100644 index 000000000..c10744ee1 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/LoopbackCaptureAPI.swift @@ -0,0 +1,391 @@ +import CryptoKit +import Foundation +import Network +import Synchronization +import HouseScanKit + +/// A local stand-in for the capture API over real HTTP on 127.0.0.1, so the tests drive the +/// actual `URLSession` transport. It implements the documented behaviour the uploader relies on: +/// idempotent create and finalize bound to the exact body, register with signed PUT URLs, +/// Content-MD5 checked on the PUT, commit as the only acknowledgement, early finalize with +/// `awaiting_files`, and events leading to a result. Faults are injected per test. +/// +/// Written for these tests from the public description of the API; it is not the server's code. +final class LoopbackCaptureAPI: Sendable { + struct Request: Sendable { + var method: String + var path: String + var query: [String: String] + var headers: [String: String] + var body: Data + } + + struct Registration { + var sha256: String + var bytes: Int + var md5: String + var contentType: String + } + + struct Capture { + var id: String + var packetID: String + var createBody: Data + var registered: [String: Registration] = [:] + var stored: [String: Data] = [:] + var committed: Set = [] + var packetSHA: String? + var listed: [String] = [] + var runID: String? + var events: [(type: String, data: [String: String])] = [] + var status = "uploading" + /// Accepted a finalize, then lost it to a storage failure: no run until it is sent again. + var finalizeLost = false + } + + struct State { + var captures: [String: Capture] = [:] + var byPacket: [String: String] = [:] + var tokens: [String: (capture: String, path: String)] = [:] + var log: [Request] = [] + var nextID = 1 + /// Close the connection without answering the next request whose route matches. + var dropNext: Set = [] + /// Answer the next N storage PUTs 403, as an expired signature. + var expireNextPuts = 0 + /// Answer the next N storage PUTs 400 BadDigest. + var badDigestPuts = 0 + /// Accept the next finalize, then report it lost with `failed` / `retry_finalize`. + var loseNextFinalize = false + /// The homeowner message the result carries. + var resultMessage = "The loopback receiver's test result." + /// Answer the next N result reads 404 `result_not_ready`: the run is done but its answer + /// is not readable yet. + var notReadyResults = 0 + /// Answer the next N result reads 200 with the capture's status and run but a null + /// outcome, the body the API serves before the run has written its answer. + var outcomelessResults = 0 + /// Answer the next N result reads 200 with a body that is not JSON. + var unreadableResults = 0 + /// Answer the next N result reads 200 with an outcome that has a kind but none of the + /// other fields an outcome must carry. + var malformedOutcomeResults = 0 + /// The status a capture ends in once every listed file is committed. Only `manual_review` + /// ends with an outcome; `failed` ends with a `failed` event and none. + var endStatus = "manual_review" + /// Every register answer gives this URL for its PUTs instead of the loopback's own. + var uploadURLOverride: String? + /// Register answers leave out every file, as if the server never saw the request's list. + var registerOmitsFiles = false + /// Commits never acknowledge a file: every path comes back in `notFound`. + var commitNeverAcknowledges = false + /// Once every listed file is committed, the next N event polls answer at once with status + /// `processing` and no events, as a server that doesn't hold the poll would; then the run ends. + var processingPolls = 0 + /// Routes ("PUT upload", "POST captures", ...) answered `307` with the given `Location`, as + /// a server sending the request somewhere else would. + var redirects: [String: String] = [:] + /// Answer the next commit with this split instead of the truth. + var commitOverride: ((committed: [String], notFound: [String], mismatch: [String]))? + /// Routes whose answers wait until `release` is called. + var held: Set = [] + var parked: [String: [@Sendable () -> Void]] = [:] + } + + let state = Mutex(State()) + private let listener: NWListener + private let queue = DispatchQueue(label: "loopback-capture-api") + var port: UInt16 { listener.port?.rawValue ?? 0 } + + init() throws { + let parameters = NWParameters.tcp + parameters.acceptLocalOnly = true + parameters.requiredLocalEndpoint = .hostPort(host: "127.0.0.1", port: .any) + listener = try NWListener(using: parameters) + let ready = DispatchSemaphore(value: 0) + listener.stateUpdateHandler = { if case .ready = $0 { ready.signal() } } + listener.newConnectionHandler = { [queue] connection in + connection.start(queue: queue) + Self.receive(connection, buffer: Data()) { request in self.handle(request, connection) } + } + listener.start(queue: queue) + guard ready.wait(timeout: .now() + 5) == .success, listener.port != nil else { throw URLError(.cannotConnectToHost) } + } + + deinit { listener.cancel() } + + var base: URL { URL(string: "http://127.0.0.1:\(port)/v1")! } + + func requests(_ route: String) -> [Request] { state.withLock { $0.log.filter { Self.route($0) == route } } } + + func release(_ route: String) { + let waiting = state.withLock { s -> [@Sendable () -> Void] in + s.held.remove(route) + return s.parked.removeValue(forKey: route) ?? [] + } + waiting.forEach { $0() } + } + + /// "POST /v1/captures/{id}/files" style names, ids replaced. + static func route(_ r: Request) -> String { + let parts = r.path.split(separator: "/").map(String.init) + if parts.first == "upload" { return "\(r.method) upload" } + guard parts.count >= 2 else { return "\(r.method) \(r.path)" } + if parts.count == 2 { return "\(r.method) captures" } + return "\(r.method) captures/\(parts[2...].joined(separator: "/"))" + } + + // MARK: HTTP over the connection + + private static func receive(_ connection: NWConnection, buffer: Data, handler: @escaping @Sendable (Request) -> Void) { + connection.receive(minimumIncompleteLength: 1, maximumLength: 1 << 20) { data, _, done, error in + var buffer = buffer + if let data { buffer.append(data) } + if let request = parse(buffer) { return handler(request) } + if done || error != nil { return connection.cancel() } + receive(connection, buffer: buffer, handler: handler) + } + } + + private static func parse(_ data: Data) -> Request? { + guard let end = data.range(of: Data("\r\n\r\n".utf8)) else { return nil } + let head = String(decoding: data[..= 2 else { return nil } + var headers: [String: String] = [:] + for field in head.dropFirst() { + guard let colon = field.firstIndex(of: ":") else { continue } + headers[field[..= length else { return nil } + let target = URLComponents(string: String(line[1])) + var query: [String: String] = [:] + for item in target?.queryItems ?? [] { query[item.name] = item.value } + var path = target?.path ?? "" + if path.hasPrefix("/v1/") { path.removeFirst(4) } else if path.hasPrefix("/") { path.removeFirst() } + return Request(method: String(line[0]), path: path, query: query, headers: headers, body: Data(body.prefix(length))) + } + + private func handle(_ request: Request, _ connection: NWConnection) { + let route = Self.route(request) + let (drop, hold, redirect) = state.withLock { s -> (Bool, Bool, String?) in + s.log.append(request) + return (s.dropNext.remove(route) != nil, s.held.contains(route), s.redirects[route]) + } + let (status, body) = redirect == nil ? answer(request) : (307, Data()) + let location = redirect.map { "Location: \($0)\r\n" } ?? "" + let send: @Sendable () -> Void = { + if drop { return connection.cancel() } + let head = "HTTP/1.1 \(status) X\r\n\(location)Content-Type: application/json\r\nContent-Length: \(body.count)\r\nConnection: close\r\n\r\n" + connection.send(content: Data(head.utf8) + body, completion: .contentProcessed { _ in connection.cancel() }) + } + if hold { + state.withLock { $0.parked[route, default: []].append(send) } + } else { + send() + } + } + + // MARK: API behaviour + + private func json(_ object: Any) -> Data { try! JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]) } + private func error(_ status: Int, _ code: String) -> (Int, Data) { (status, json(["errors": [["code": code, "pointer": "/", "message": code]]])) } + + private func answer(_ r: Request) -> (Int, Data) { + state.withLock { s in + let parts = r.path.split(separator: "/").map(String.init) + if parts.first == "upload" { return put(r, token: parts.last ?? "", &s) } + if r.headers["authorization"] != nil { return error(400, "unexpected_authorization") } + if r.method == "POST", parts == ["captures"] { return create(r, &s) } + guard parts.count >= 2, var capture = s.captures[parts[1]] else { return error(404, "capture_not_found") } + defer { s.captures[capture.id] = capture } + switch (r.method, parts.dropFirst(2).joined(separator: "/")) { + case ("POST", "files"): return register(r, &capture, &s) + case ("POST", "files:commit"): return commit(r, &capture, &s) + case ("POST", "finalize"): return finalize(r, &capture, &s) + case ("GET", "events"): return events(r, &capture, &s) + case ("GET", "result"): + if s.notReadyResults > 0 { + s.notReadyResults -= 1 + return error(404, "result_not_ready") + } + if s.unreadableResults > 0 { + s.unreadableResults -= 1 + return (200, Data("not a result".utf8)) + } + if s.malformedOutcomeResults > 0 { + s.malformedOutcomeResults -= 1 + return (200, json(["runId": capture.runID ?? "", "status": capture.status, "viewsNeeded": [], "memberActions": [], + "outcome": ["kind": "manual_review"]])) + } + let withheld = s.outcomelessResults > 0 + if withheld { s.outcomelessResults -= 1 } + let outcome: Any = capture.status == "manual_review" && !withheld + ? ["kind": "manual_review", "profile": "C", "message": s.resultMessage, "viewsNeeded": [], "reasons": []] as [String: Any] + : NSNull() + return (200, json(["runId": capture.runID ?? "", "status": capture.status, "viewsNeeded": [], "memberActions": [], "outcome": outcome])) + default: return error(404, "route") + } + } + } + + private func create(_ r: Request, _ s: inout State) -> (Int, Data) { + guard let body = try? JSONSerialization.jsonObject(with: r.body) as? [String: Any], let packetID = body["packetId"] as? String else { + return error(422, "schema") + } + guard let key = r.headers["idempotency-key"] else { return error(422, "idempotency_key_missing") } + guard key == packetID else { return error(422, "idempotency_key_mismatch") } + guard body["formatVersion"] as? String == "0.4" else { return error(422, "format_version_unsupported") } + let status: Int + let capture: Capture + if let existing = s.byPacket[packetID].flatMap({ s.captures[$0] }) { + guard existing.createBody == r.body else { return error(409, "idempotency_conflict") } + (status, capture) = (200, existing) + } else { + capture = Capture(id: "cap_\(s.nextID)", packetID: packetID, createBody: r.body) + s.nextID += 1 + s.captures[capture.id] = capture + s.byPacket[packetID] = capture.id + status = 201 + } + return (status, json([ + "captureId": capture.id, "status": "uploading", "eventsUrl": "/v1/captures/\(capture.id)/events", + "finalizeBy": "2099-01-01T00:00:00Z", "upload": ["maxSinglePutBytes": 16_777_216, "maxBatch": 50, "urlTtlS": 3600, "maxFiles": 2000, "partBytes": 8_388_608], + ])) + } + + private func register(_ r: Request, _ c: inout Capture, _ s: inout State) -> (Int, Data) { + guard let body = try? JSONSerialization.jsonObject(with: r.body) as? [String: Any], let files = body["files"] as? [[String: Any]], + (1...50).contains(files.count) else { return error(422, "batch_too_large") } + var out: [[String: Any]] = [] + for file in files { + guard let path = file["path"] as? String, Packet04.isRelPath(path), let sha = file["sha256"] as? String, sha.count == 64, + let bytes = file["bytes"] as? Int, let md5 = file["md5"] as? String, Data(base64Encoded: md5)?.count == 16, + let role = file["role"] as? String, Packet04.Role(rawValue: role) != nil, let type = file["contentType"] as? String + else { return error(422, "schema") } + let meta = file["meta"] as? [String: Any] + let image = role == "keyframe" || role == "still" + if image != (meta != nil) || (meta.map { $0.count != 1 } ?? false) { return error(422, "meta_mismatch") } + if let previous = c.registered[path], previous.sha256 != sha { return error(409, "sha256_conflict") } + if c.packetSHA != nil, !c.listed.contains(path) { return error(409, "not_in_manifest") } + c.registered[path] = Registration(sha256: sha, bytes: bytes, md5: md5, contentType: type) + if c.committed.contains(path) { + out.append(["path": path, "state": "committed"]) + continue + } + if s.registerOmitsFiles { continue } + let token = UUID().uuidString + s.tokens[token] = (c.id, path) + out.append(["path": path, "state": "pending", "upload": [ + "method": "PUT", "url": s.uploadURLOverride ?? "http://127.0.0.1:\(port)/upload/\(token)", + "headers": ["Content-Type": type, "Content-MD5": md5], "expiresAt": "2099-01-01T00:00:00Z", + ]]) + } + return (200, json(["files": out])) + } + + private func put(_ r: Request, token: String, _ s: inout State) -> (Int, Data) { + guard r.method == "PUT", let (captureID, path) = s.tokens[token], var c = s.captures[captureID], let reg = c.registered[path] else { + return (403, Data("token_invalid".utf8)) + } + if r.headers["authorization"] != nil { return (400, Data("authorization sent to storage".utf8)) } + if s.badDigestPuts > 0 { + s.badDigestPuts -= 1 + return (400, Data("BadDigest".utf8)) + } + if s.expireNextPuts > 0 { + s.expireNextPuts -= 1 + s.tokens[token] = nil + return (403, Data("AccessDeniedRequest has expired".utf8)) + } + guard r.headers["content-md5"] == reg.md5, Data(Insecure.MD5.hash(data: r.body)).base64EncodedString() == reg.md5, + r.headers["content-type"] == reg.contentType + else { return (400, Data("BadDigest".utf8)) } + c.stored[path] = r.body + s.captures[captureID] = c + return (200, Data()) + } + + private func commit(_ r: Request, _ c: inout Capture, _ s: inout State) -> (Int, Data) { + guard let body = try? JSONSerialization.jsonObject(with: r.body) as? [String: Any], let files = body["files"] as? [[String: Any]] else { + return error(422, "schema") + } + var committed: [String] = [], notFound: [String] = [], mismatch: [String] = [] + for file in files { + guard let path = file["path"] as? String, let sha = file["sha256"] as? String else { return error(422, "schema") } + guard let data = c.stored[path] else { notFound.append(path); continue } + if PacketFiles.sha256(data) != sha || c.registered[path]?.sha256 != sha { mismatch.append(path); continue } + committed.append(path) + } + if let forced = s.commitOverride { + s.commitOverride = nil + (committed, notFound, mismatch) = forced + } + if s.commitNeverAcknowledges { + notFound += committed + mismatch + committed = [] + mismatch = [] + } + c.committed.formUnion(committed) + if !committed.isEmpty { c.events.append(("files_committed", ["committed": String(c.committed.count)])) } + startIfComplete(&c, endStatus: s.endStatus) + return (200, json(["committed": committed, "notFound": notFound, "mismatch": mismatch])) + } + + private func finalize(_ r: Request, _ c: inout Capture, _ s: inout State) -> (Int, Data) { + guard r.headers["idempotency-key"] == c.packetID else { return error(422, "idempotency_key_mismatch") } + let sha = PacketFiles.sha256(r.body) + if let declared = r.headers["x-packet-sha256"], declared != sha { return error(422, "packet_sha_mismatch") } + if let previous = c.packetSHA, previous != sha { return error(409, "packet_sha_conflict") } + guard let packet = try? JSONDecoder().decode(Packet04.Packet.self, from: r.body) else { return error(422, "schema") } + guard packet.packetId == c.packetID else { return error(422, "packet_id_mismatch") } + let problems = Packet04Check.problems(packet, folder: nil) + if !problems.isEmpty { return error(422, "schema") } + for entry in packet.files { + if let reg = c.registered[entry.path], reg.sha256 != entry.sha256 || reg.bytes != entry.bytes { return error(422, "sha256_mismatch") } + } + if c.packetSHA == nil { + c.packetSHA = sha + c.listed = packet.files.map(\.path) + c.runID = "run_" + sha.prefix(16) + c.status = "awaiting_files" + if s.loseNextFinalize { + s.loseNextFinalize = false + c.finalizeLost = true + c.events.append(("failed", ["code": "storage_unavailable", "next": "retry_finalize"])) + } + startIfComplete(&c, endStatus: s.endStatus) + } else if c.finalizeLost { + c.finalizeLost = false + startIfComplete(&c, endStatus: s.endStatus) + } + let missing = c.listed.filter { !c.committed.contains($0) } + return (202, json(["status": missing.isEmpty ? "processing" : "awaiting_files", "missing": missing, "runId": c.runID!, "etaS": 1])) + } + + private func startIfComplete(_ c: inout Capture, endStatus: String) { + guard c.packetSHA != nil, !c.finalizeLost, c.status == "awaiting_files", c.listed.allSatisfy(c.committed.contains) else { return } + c.status = endStatus + c.events.append(("stage", ["stage": "ingest", "status": "ok", "runId": c.runID!])) + if endStatus == "manual_review" { + c.events.append(("verdict_ready", ["kind": "manual_review", "runId": c.runID!])) + } else { + c.events.append(("failed", ["code": "loopback_run_failed", "runId": c.runID!])) + } + } + + private func events(_ r: Request, _ c: inout Capture, _ s: inout State) -> (Int, Data) { + if c.status == "manual_review" || c.status == "failed", s.processingPolls > 0, c.runID != nil { + s.processingPolls -= 1 + let after = Int(r.query["after"] ?? "0") ?? 0 + return (200, json(["status": "processing", "next": after, "events": [] as [Any]])) + } + let after = Int(r.query["after"] ?? "0") ?? 0 + let list = c.events.enumerated().filter { $0.offset + 1 > after }.map { index, event in + ["seq": index + 1, "type": event.type, "at": "2026-09-27T00:00:00Z", "data": event.data] as [String: Any] + } + return (200, json(["status": c.status, "next": c.events.count, "events": list])) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/Packet04Tests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/Packet04Tests.swift new file mode 100644 index 000000000..909bb67b2 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/Packet04Tests.swift @@ -0,0 +1,277 @@ +import Foundation +import HouseScanKit +import simd +import Testing + +/// A synthetic ARKit capture for the 0.4 tests: a camera walking along +x at 1.5 m height, +/// 96 x 72 JPEGs with intrinsics that fit them, 60 Hz poses and 100 Hz IMU with the gyro 1 ms +/// behind the accelerometer. `source` is synthetic, so no packet from it is evidence. +struct SyntheticCapture04 { + static let start = 1000.0 + let folder: URL + let producer: Packet04Producer + + init(folder: URL, packetID: String = UUID().uuidString) throws { + self.folder = folder + producer = try Packet04Producer(folder: folder, info: Self.info(packetID: packetID), startedAtUptime: Self.start, startedAt: Date(timeIntervalSince1970: 1_790_000_000)) + } + + static func info(packetID: String) -> Packet04SessionInfo { + Packet04SessionInfo( + packetID: packetID, sessionID: "S-1", source: .synthetic, appVersion: "test (1)", deviceModel: "iPhone15,4", systemVersion: "26.0", + lidarAvailable: false, sceneDepthEnabled: false, meshReconstructionSupported: false, sceneReconstruction: "none", + planeDetection: ["horizontal", "vertical"], videoWidth: 1920, videoHeight: 1440, framesPerSecond: 60, timeZone: "America/Chicago") + } + + static func pose(_ t: Double) -> simd_float4x4 { + var m = matrix_identity_float4x4 + m.columns.3 = SIMD4(Float(t - start) * 0.5, 1.5, -2, 1) + return m + } + + static let intrinsics = SIMD4(80, 80, 48, 36) + + func observation(_ t: Double) -> Packet04Observation { + Packet04Observation(t: t, cameraToWorld: Self.pose(t), intrinsics: Self.intrinsics, width: 96, height: 72, tracking: .normal) + } + + static func jpeg(in folder: URL) throws -> Data { + let url = folder.appending(path: "source-\(UUID().uuidString).jpg") + try makeJPEG(width: 96, height: 72, at: url) + defer { try? FileManager.default.removeItem(at: url) } + return try Data(contentsOf: url) + } + + /// Seals `count` keyframes and the head-on meter still on the first one; returns every file. + func sealImages(count: Int = 3) async throws -> [SealedFile] { + let scratch = folder.deletingLastPathComponent() + var files: [SealedFile] = [] + for i in 0.. [Packet04Streams.PoseRow] { + stride(from: start, through: end, by: 1.0 / 60).map { .init(t: $0, tracking: .normal, cameraToWorld: pose($0), intrinsics: SIMD4(1440, 1440, 960, 720)) } + } + + static func imu(until end: Double = start + 2) -> (accel: [Packet04Streams.MotionRow], gyro: [Packet04Streams.MotionRow]) { + let times = Array(stride(from: start, through: end, by: 0.01)) + return (times.map { .init(t: $0, value: SIMD3(0, -1, 0)) }, times.map { .init(t: $0 + 0.001, value: SIMD3(0.01, 0, 0)) }) + } + + func finish() async throws -> (streams: [SealedFile], packet: Data) { + let imu = Self.imu() + return try await producer.finish(poses: Self.poses(), accelerometer: imu.accel, gyroscope: imu.gyro, endedAtUptime: Self.start + 2, acceptedCloseUpAt: Self.start + 0.2) + } +} + +func temporaryPacketFolder(_ name: String) -> URL { + FileManager.default.temporaryDirectory.appending(path: "\(name)-\(UUID().uuidString)/packet") +} + +@Suite struct Packet04Tests { + @Test func finishedPacketPassesItsChecksAndListsEverySealedFile() async throws { + let folder = temporaryPacketFolder("p04-normal") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + let images = try await capture.sealImages() + let (streams, data) = try await capture.finish() + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: data) + + #expect(Packet04Check.problems(packet, folder: folder).isEmpty) + #expect(Set(packet.files.map(\.path)) == Set((images + streams).map(\.path))) + #expect(packet.formatVersion == "0.4") + #expect(packet.session.tier == .arkit) + #expect(packet.epochs == [.init(id: "e1", startTime: SyntheticCapture04.start, reason: "sessionStart")]) + #expect(packet.keyframes.allSatisfy { $0.epoch == "e1" }) + #expect(packet.scaleReference.meterCloseUp == .init(captured: true, still: "meter_close", arkitDistanceM: nil, side: nil)) + #expect(packet.scaleReference.ext.obliqueCloseUp == .notCaptured) + #expect(packet.tracking == [.init(time: SyntheticCapture04.start, state: "normal", epoch: "e1")]) + // The exact bytes finish returned are what is on disk. + #expect(try Data(contentsOf: folder.appending(path: "packet.json")) == data) + } + + @Test func keyframeKeepsTheRawWorldPoseAndItsOwnIntrinsics() async throws { + let folder = temporaryPacketFolder("p04-pose") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + let t = SyntheticCapture04.start + 0.7 + let files = try await capture.producer.sealKeyframe(jpeg: try SyntheticCapture04.jpeg(in: folder.deletingLastPathComponent()), observation: capture.observation(t), reason: "motion") + guard case .keyframe(let record)? = files.first?.meta else { Issue.record("no keyframe record"); return } + #expect(record.pose == [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0.35, 1.5, -2, 1]) + #expect(record.intrinsics == [80, 80, 48, 36]) + #expect(record.timestamp == t) + #expect(files.first?.path == "keyframes/k00001.jpg") + #expect(files.first?.role == .keyframe) + } + + @Test func sealedDigestsMatchTheBytesOnDisk() async throws { + let folder = temporaryPacketFolder("p04-digest") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + for file in try await capture.sealImages() { + let data = try Data(contentsOf: folder.appending(path: file.path)) + #expect(file.bytes == data.count) + #expect(file.sha256 == PacketFiles.sha256(data)) + #expect(Data(base64Encoded: file.md5)?.count == 16) + } + } + + @Test func refusesARotatedImageAndIntrinsicsOfAnotherSize() async throws { + let folder = temporaryPacketFolder("p04-refuse") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + var portrait = capture.observation(SyntheticCapture04.start + 1) + (portrait.width, portrait.height) = (72, 96) + await #expect(throws: Packet04Error.self) { + _ = try await capture.producer.sealKeyframe(jpeg: try SyntheticCapture04.jpeg(in: folder.deletingLastPathComponent()), observation: portrait, reason: "motion") + } + var wrong = capture.observation(SyntheticCapture04.start + 1) + wrong.intrinsics = SIMD4(1440, 1440, 960, 720) + await #expect(throws: Packet04Error.self) { + _ = try await capture.producer.sealKeyframe(jpeg: try SyntheticCapture04.jpeg(in: folder.deletingLastPathComponent()), observation: wrong, reason: "motion") + } + } + + @Test func nothingIsSealedAfterFinishAndFinishReturnsTheSameBytes() async throws { + let folder = temporaryPacketFolder("p04-frozen") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + _ = try await capture.sealImages() + let first = try await capture.finish() + let second = try await capture.finish() + #expect(first.packet == second.packet) + await #expect(throws: Packet04Error.sealedAfterFinish("keyframe")) { + _ = try await capture.producer.sealKeyframe(jpeg: try SyntheticCapture04.jpeg(in: folder.deletingLastPathComponent()), observation: capture.observation(SyntheticCapture04.start + 1.9), reason: "motion") + } + } + + @Test func tapRayReplaysFromItsKeyframe() async throws { + let folder = temporaryPacketFolder("p04-tap") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + _ = try await capture.sealImages(count: 1) + try await capture.producer.addTap(id: "tap1", label: "meter", keyframe: "k00001", pixel: SIMD2(48, 36), time: SyntheticCapture04.start + 0.2, hit: nil) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: try await capture.finish().packet) + let tap = try #require(packet.taps?.first) + // The principal point looks straight down -z from the camera. + #expect(tap.rayOrigin == [0.1, 1.5, -2]) + #expect(tap.rayDirection == [0, 0, -1]) + + var broken = packet + broken.taps?[0].rayDirection = [0, 0.01, -1] + #expect(Packet04Check.problems(broken, folder: nil).contains { $0.hasPrefix("tap tap1 ray replays with error") }) + } + + /// A tap ray must be two 3-vectors of finite numbers. Joined and compared pairwise, a missing + /// or misshaped vector, or a NaN, could otherwise pass the replay check unseen. + @Test(arguments: [ + ([0.1, 1.5, -2], []), + ([0.1, 1.5], [-2, 0, 0, -1]), + ([0.1, 1.5, -2], [0, 0, -1, 0]), + ([0.1, 1.5, -2], [0, Double.nan, -1]), + ([0.1, .infinity, -2], [0, 0, -1]), + ] as [([Double], [Double])]) + func aTapRayThatIsNotTwoFinite3VectorsIsRefused(origin: [Double], direction: [Double]) async throws { + let folder = temporaryPacketFolder("p04-tap-shape") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + _ = try await capture.sealImages(count: 1) + try await capture.producer.addTap(id: "tap1", label: "meter", keyframe: "k00001", pixel: SIMD2(48, 36), time: SyntheticCapture04.start + 0.2, hit: nil) + let packet = try JSONDecoder().decode(Packet04.Packet.self, from: try await capture.finish().packet) + #expect(!Packet04Check.problems(packet, folder: nil).contains { $0.hasPrefix("tap tap1") }) + + var broken = packet + broken.taps?[0].rayOrigin = origin + broken.taps?[0].rayDirection = direction + #expect(Packet04Check.problems(broken, folder: nil).contains { $0.hasPrefix("tap tap1") }) + } + + @Test func checksCatchAMalformedPacket() async throws { + let folder = temporaryPacketFolder("p04-malformed") + defer { try? FileManager.default.removeItem(at: folder.deletingLastPathComponent()) } + let capture = try SyntheticCapture04(folder: folder) + _ = try await capture.sealImages() + let good = try JSONDecoder().decode(Packet04.Packet.self, from: try await capture.finish().packet) + + var unlisted = good + unlisted.files.removeAll { $0.path == "keyframes/k00002.jpg" } + #expect(Packet04Check.problems(unlisted, folder: folder) == ["keyframe k00002 image keyframes/k00002.jpg is not in files"]) + + var badPath = good + badPath.files[0].path = "../escape.jpg" + #expect(Packet04Check.problems(badPath, folder: nil).contains("file path ../escape.jpg is not a RelPath")) + + var badStill = good + badStill.stills?[0].keyframe = "k00009" + #expect(Packet04Check.problems(badStill, folder: nil).contains("still meter_close names unknown keyframe k00009")) + + try Data("changed".utf8).write(to: folder.appending(path: "keyframes/k00003.jpg")) + #expect(Packet04Check.problems(good, folder: folder).contains { $0.hasPrefix("file keyframes/k00003.jpg is 7 bytes") }) + } + + @Test func imuRowsPairOnlyCloseGyroSamplesAndReportTheMeasuredRate() throws { + let accel = (0..<100).map { Packet04Streams.MotionRow(t: 10 + Double($0) * 0.01, value: SIMD3(0, -1, 0)) } + // A gyro sample 1 ms after each accelerometer sample, except the 50th's, so the nearest to + // that accelerometer sample is 9 ms away: it is left out, not given a far reading. + var gyro = accel.map { Packet04Streams.MotionRow(t: $0.t + 0.001, value: SIMD3(0.5, 0, 0)) } + gyro.remove(at: 50) + let out = try Packet04Streams.imuRaw(accelerometer: accel, gyroscope: gyro) + let lines = String(decoding: out.csv, as: UTF8.self).split(separator: "\n") + #expect(lines.first == "t,ax,ay,az,gx,gy,gz") + #expect(lines.count == 1 + 99) + #expect(!lines.contains { $0.hasPrefix("10.5,") }) + #expect(abs(out.rateHz - 100) < 0.01) + #expect(out.pairingNote["imuRawRowsKept"] == "99 of 100") + } + + @Test func poseRowsJoinEachFrameToItsOwnIntrinsics() { + func trajectory(_ t: Double, x: Double) -> [Double] { [t, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, x, 1.5, -2, 1] } + let rows = Packet04Streams.poseRows( + trajectory: [trajectory(1, x: 0.1), trajectory(2, x: 0.2), trajectory(3, x: 0.3)], + // Frame 2 wrote no intrinsics row: it is left out, not given frame 1's or frame 3's. + intrinsics: [[1, 1440, 1441, 960, 720, 1920, 1440], [3, 1500, 1501, 961, 721, 1920, 1440]], + tracking: { state, _ in state == 0 ? .normal : .notAvailable }) + #expect(rows.map(\.t) == [1, 3]) + #expect(rows[1].intrinsics == SIMD4(1500, 1501, 961, 721)) + #expect(rows[1].cameraToWorld.columns.3 == SIMD4(0.3, 1.5, -2, 1)) + #expect(rows[0].tracking == .normal) + } + + @Test func aSlowIMUIsRefusedNotRelabelled() { + let accel = (0..<20).map { Packet04Streams.MotionRow(t: Double($0) * 0.05, value: SIMD3(0, -1, 0)) } + #expect(throws: Packet04Error.streamTooSlow(stream: "imuRaw", measuredHz: 1 / 0.05, minimumHz: 50)) { + _ = try Packet04Streams.imuRaw(accelerometer: accel, gyroscope: accel) + } + } + + @Test func gzipStreamsDecompressWithTheSystemGzip() throws { + let text = Data((0..<2000).map { "\($0),0.5,-1\n" }.joined().utf8) + let url = FileManager.default.temporaryDirectory.appending(path: "p04-\(UUID().uuidString).csv.gz") + defer { try? FileManager.default.removeItem(at: url) } + try Gzip.compress(text).write(to: url) + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/gzip") + process.arguments = ["-dc", url.path] + let pipe = Pipe() + process.standardOutput = pipe + try process.run() + let out = pipe.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + #expect(process.terminationStatus == 0) + #expect(out == text) + } + + @Test func idAndPathPatternsMatchTheServer() { + #expect(Packet04.isStorageID(UUID().uuidString)) + #expect(!Packet04.isStorageID("short")) + #expect(!Packet04.isStorageID("has.dot-in-it")) + #expect(Packet04.isSegmentID("k00001") && Packet04.isSegmentID("meter_close")) + #expect(!Packet04.isSegmentID("-lead") && !Packet04.isSegmentID("a.b")) + #expect(Packet04.isRelPath("keyframes/k00001.depth.f32") && Packet04.isRelPath("streams/imu_raw.csv.gz")) + #expect(!Packet04.isRelPath("a/../b") && !Packet04.isRelPath("a//b") && !Packet04.isRelPath("foo..bar") && !Packet04.isRelPath(".hidden")) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift index 0904fda17..120801e37 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/PacketWriterTests.swift @@ -19,7 +19,7 @@ enum PacketSchema { /// A JPEG of `width` x `height` with a colour pattern, written by ImageIO. `orientation` sets the /// EXIF orientation tag. -func makeJPEG(width: Int, height: Int, at url: URL, orientation: Int? = nil) throws { +func makeJPEG(width: Int, height: Int, at url: URL, orientation: Int? = nil, shift: Int = 0) throws { let space = CGColorSpaceCreateDeviceRGB() let context = try #require(CGContext( data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 4 * width, space: space, @@ -28,7 +28,7 @@ func makeJPEG(width: Int, height: Int, at url: URL, orientation: Int? = nil) thr for y in 0..