ios: live 3D map as the coverage model where depth exists

ios/.gitignore

@@ -9,6 +9,6 @@ xcuserdata/
 DerivedData/
 build/
 
-# SwiftPM build output of the capture package.
-HouseScanKit/.build/
+# SwiftPM build output, of the capture package or of a swift command run from here.
+.build/
 HouseScanKit/.swiftpm/

ios/HouseScan/Contract/ScanContract.swift

@@ -618,6 +618,10 @@ struct ResultPresentation: Equatable, Sendable {
     /// True when no server answered and the result is the offline sample used by tests and
     /// demos. The UI must say so on screen.
     var isSample: Bool
+    /// The wall the answer's s runs along: the one scene.json described. The result screens draw
+    /// along it rather than `ScanViewState.wall`, which is the walk's and can differ from it under
+    /// `-coverage map3d`. Nil in the UI demo, which uses the state's wall.
+    var wall: WallGeometry? = nil
 }
 
 // MARK: - State and intents
@@ -639,6 +643,9 @@ final class ScanViewState {
 
     var wall: WallGeometry?
     var coverage: CoverageStrip = .empty
+    /// The 3D map's unseen space and where to look next, for `Map3DOverlay`. Nil under
+    /// `-coverage legacy` and until the first snapshot after the meter is placed.
+    var map3D: Map3DFog?
     var closeUp: CloseUpState = .aiming(hold: 0, problem: nil)
     /// Close-up attempts that ended without a usable photo. "Can't get a clear shot" appears
     /// from the second one on, never earlier.

ios/HouseScan/Runtime/Autopilot.swift

@@ -30,7 +30,7 @@ final class Autopilot {
         guard await waitUntil(timeout: 60, { self.engine.replay != nil || self.engine.state.failure != nil }), let replay = engine.replay else {
             return fail("replay did not load")
         }
-        async let prepared: Void = replay.prepareHeldBack()
+        async let prepared: Void = replay.prepareHeldBack(map3D: engine.map3D != nil)
         await pause(hold)
         await engine.waitForGate(.onboarding)
         engine.finishOnboarding()
@@ -52,6 +52,7 @@ final class Autopilot {
         await pause(0.5)
         guard await waitUntil(timeout: 120, { !replay.isPlaying }) else { return fail("walk replay did not finish") }
         await pause(1.0)
+        await settleMap3D()
 
         await markFeatures(replay)
         if engine.options.autopilotCantGetThere {
@@ -70,6 +71,8 @@ final class Autopilot {
         // Mulch, so the replay's scene carries a ground patch over the ground its walk saw.
         // Answered after the UI test has finished with the screen, so its audit reads a still tree.
         engine.answerGround(.type(.mulch))
+        // The tilt-up frames went into the map too; the gap check reads what it made of them.
+        await settleMap3D()
         engine.confirmFeatures()
         await pause(0.3)
         if engine.state.phase == .gapRequest {
@@ -285,12 +288,17 @@ final class Autopilot {
         }
     }
 
-    /// With `-autopilotGate`, leaves the scene.json of the scan the result answers in the gate
-    /// folder, for the UI test to check its ends and what it reports as seen.
+    /// With `-autopilotGate`, leaves the scene.json of the scan the result answers (the one the
+    /// last upload sent) in the gate folder, for the UI test to check its ends and what it
+    /// reports as seen.
     private func writeSceneForTest() {
         guard let gate = engine.options.autopilotGate else { return }
+        guard let scene = engine.uploadedScene else {
+            log("no uploaded scene.json to write to the gate folder")
+            return
+        }
         do {
-            try engine.sceneJSON().write(to: gate.appending(path: "scene.json"))
+            try scene.write(to: gate.appending(path: "scene.json"))
             log("wrote scene.json to the gate folder")
         } catch {
             log("could not write scene.json to the gate folder: \(error)")
@@ -399,6 +407,7 @@ final class Autopilot {
         _ = await waitUntil(timeout: 60) { !replay.isPlaying || self.engine.state.gap?.id != request.id }
         // Time for the last frame's keyframe to be stored and counted.
         await pause(hold)
+        await settleMap3D()
         if engine.state.phase == .gapRequest, let gap = engine.state.gap, gap.id == request.id, !gap.isSatisfied {
             log("the replay does not settle request \(request.id) (\(Int(gap.progress * 100))% of it seen); skipping it")
             await engine.waitForGate(.gapRequest)
@@ -428,6 +437,17 @@ final class Autopilot {
 
     // MARK: Helpers
 
+    /// Under `-coverage map3d`, waits until the 3D map has integrated every frame played and its
+    /// last change has reached the coverage map, for at most a minute. Without it the steps after
+    /// the walk act on coverage the map has not caught up with: a debug build takes about 7 s per
+    /// snapshot (`Map3DSession`), and the replay's walk lasts about 6 s at 3x.
+    private func settleMap3D() async {
+        guard let map3D = engine.map3D else { return }
+        if await !waitUntil(timeout: 60, { !map3D.isCatchingUp }) {
+            log("the 3D map was still catching up after a minute; going on")
+        }
+    }
+
     /// Shows the replay frame that best shows `point`, then taps where it appears on screen.
     @discardableResult
     private func tap(_ point: SIMD3<Float>, replay: ReplayPlayer, action: (CGPoint) -> Void) async -> Bool {

ios/HouseScan/Runtime/DepthEstimator.swift

@@ -0,0 +1,227 @@
+import ARKit
+import CoreImage
+import CoreImage.CIFilterBuiltins
+import CoreML
+import Foundation
+import HouseScanKit
+import Synchronization
+
+/// Where the depth model is: compiled into the app bundle (Xcode compiles a bundled
+/// `.mlpackage` to `.mlmodelc`), else compiled in Application Support/Models. Nil when neither
+/// exists; the app then runs without estimated depth.
+enum DepthModelLocator {
+    /// Apple's Core ML Depth Anything V2 Small, Float16 (huggingface.co/apple/coreml-depth-anything-v2-small,
+    /// revision cfef6f6f2a70783dedc0bfae40cecbc2052285d3, Apache-2.0).
+    static let name = "DepthAnythingV2SmallF16"
+
+    static func url(bundle: Bundle = .main) -> URL? {
+        if let bundled = bundle.url(forResource: name, withExtension: "mlmodelc") { return bundled }
+        guard let support = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first else { return nil }
+        let cached = support.appending(path: "Models/\(name).mlmodelc", directoryHint: .isDirectory)
+        return FileManager.default.fileExists(atPath: cached.path) ? cached : nil
+    }
+}
+
+/// What the depth model needs from one ARKit frame, copied on the AR delegate queue: the camera
+/// image stretched to the model's input, and the frame's own geometry for the scale fit.
+struct DepthEstimatorInput: Sendable {
+    /// The `SourceFrame` id of the frame, so the engine can ask for it once the frame is kept.
+    let frameID: String
+    /// `DepthEstimator.inputWidth` x `inputHeight` BGRA, rows from the top of the unrotated
+    /// landscape sensor image, as the intrinsics describe it.
+    let bgra: [UInt8]
+    let camera: CameraFrame
+    /// ARKit's `rawFeaturePoints`, world meters.
+    let points: [SIMD3<Float>]
+    let planes: [PlaneObservation]
+}
+
+/// Estimated depth for a phone without LiDAR: runs the depth model on each kept keyframe, fits
+/// its scale to that frame's feature points and detected planes (`MonocularDepth`), and hands
+/// the `.estimated` depth frame to `onFrame` (the 3D map).
+///
+/// The delegate queue only converts a keyframe candidate (`input(from:id:context:)`) and offers
+/// it (`offer`); the model runs on this type's own queue when the engine keeps the frame
+/// (`estimate(frameID:)`), never on the delegate queue or the main actor. Candidates wait a few
+/// at a time, since a frame is kept only after its JPEG is encoded and stored.
+final class DepthEstimator: Sendable {
+    static let inputWidth = 518
+    static let inputHeight = 392
+    /// Candidates held for a keep decision. The encode queue delivers a candidate within about
+    /// one JPEG encode and the engine keeps it on arrival; four covers a second of candidates at
+    /// the walk's fastest encode rate (one per 0.3 s). A guess, not measured.
+    private static let waiting = 4
+    /// Kept frames waiting for the model, beyond the one it runs on; older ones are dropped. Each
+    /// holds the 518 x 392 image (812 KB) and the frame's geometry, and waiting longer only makes
+    /// its estimate staler. Two keeps the model busy without a backlog. A guess, not measured.
+    private static let queued = 2
+
+    private let model: ModelBox
+    private let queue = DispatchQueue(label: "dev.housescanning.housescan.depth-model", qos: .utility)
+    private let candidates = Mutex<[DepthEstimatorInput]>([])
+    /// Kept frames for the model with the generation they were kept in, and whether the model's
+    /// queue is running through them.
+    private let work = Mutex((pending: PendingWork<(input: DepthEstimatorInput, generation: Int)>(limit: DepthEstimator.queued), running: false))
+    private let onFrame: @Sendable (DepthFrame, Int) -> Void
+
+    /// Core ML documents a loaded model's predictions as safe to call from any thread; this
+    /// type calls it from its one serial queue only.
+    private struct ModelBox: @unchecked Sendable {
+        let model: MLModel
+    }
+
+    private init(model: MLModel, onFrame: @escaping @Sendable (DepthFrame, Int) -> Void) {
+        self.model = ModelBox(model: model)
+        self.onFrame = onFrame
+    }
+
+    /// Loads the model found by `DepthModelLocator`, or returns nil when there is none or it
+    /// fails to load. The first load on a device specializes the model for the Neural Engine
+    /// and can take seconds; later loads read Core ML's cache.
+    /// `onFrame` gets each estimate with the generation its frame was kept in (`estimate`).
+    static func load(onFrame: @escaping @Sendable (DepthFrame, Int) -> Void) async -> DepthEstimator? {
+        guard let url = DepthModelLocator.url() else {
+            RuntimeLog.engine.info("depth model: none in the bundle or Application Support; estimated depth off")
+            return nil
+        }
+        let configuration = MLModelConfiguration()
+        configuration.computeUnits = .all
+        do {
+            let started = ProcessInfo.processInfo.systemUptime
+            let model = try await MLModel.load(contentsOf: url, configuration: configuration)
+            RuntimeLog.engine.info("depth model loaded in \(ProcessInfo.processInfo.systemUptime - started) s")
+            return DepthEstimator(model: model, onFrame: onFrame)
+        } catch {
+            RuntimeLog.engine.error("depth model failed to load: \(error.localizedDescription, privacy: .public)")
+            return nil
+        }
+    }
+
+    // MARK: AR delegate queue
+
+    /// The model's input for a frame: its camera image scaled to 518 x 392 (Lanczos, stretched
+    /// from the sensor's 4:3, as Apple's evaluation of the model stretched its images) and its
+    /// feature points and planes. Nil when the render fails.
+    static func input(from frame: ARFrame, id: String, context: CIContext) -> DepthEstimatorInput? {
+        let image = CIImage(cvPixelBuffer: frame.capturedImage)
+        let size = image.extent.size
+        guard size.width > 0, size.height > 0 else { return nil }
+        let scale = CIFilter.lanczosScaleTransform()
+        scale.inputImage = image
+        scale.scale = Float(CGFloat(inputHeight) / size.height)
+        scale.aspectRatio = Float((CGFloat(inputWidth) / size.width) / (CGFloat(inputHeight) / size.height))
+        guard let scaled = scale.outputImage, let space = CGColorSpace(name: CGColorSpace.sRGB) else { return nil }
+        var bgra = [UInt8](repeating: 0, count: inputWidth * inputHeight * 4)
+        // Rows come out top first, as the pixel buffer's are.
+        bgra.withUnsafeMutableBytes { bytes in
+            context.render(
+                scaled, toBitmap: bytes.baseAddress!, rowBytes: inputWidth * 4,
+                bounds: CGRect(x: 0, y: 0, width: inputWidth, height: inputHeight), format: .BGRA8, colorSpace: space)
+        }
+        let feature = Map3DFeed.featureFrame(frame)
+        return DepthEstimatorInput(frameID: id, bgra: bgra, camera: feature.camera, points: feature.points, planes: Map3DFeed.planes(frame))
+    }
+
+    /// Keeps a candidate until the engine keeps or passes over its frame; the oldest goes first.
+    func offer(_ input: DepthEstimatorInput) {
+        candidates.withLock { waiting in
+            waiting.append(input)
+            if waiting.count > Self.waiting { waiting.removeFirst(waiting.count - Self.waiting) }
+        }
+    }
+
+    // MARK: Any actor
+
+    /// Estimates depth for a kept frame offered earlier; does nothing when it was not offered
+    /// (a replay, a LiDAR phone) or has already left the candidates. `generation` (the 3D map's,
+    /// `Map3DSession.generation`) goes back with the estimate, so one from before a reset is
+    /// dropped. At most `queued` frames wait for the model, the oldest dropped first.
+    func estimate(frameID: String, generation: Int) {
+        let input = candidates.withLock { waiting -> DepthEstimatorInput? in
+            guard let index = waiting.firstIndex(where: { $0.frameID == frameID }) else { return nil }
+            return waiting.remove(at: index)
+        }
+        guard let input else { return }
+        let start = work.withLock { work -> Bool in
+            work.pending.add((input, generation))
+            guard !work.running else { return false }
+            work.running = true
+            return true
+        }
+        if start { queue.async { [self] in runPending() } }
+    }
+
+    /// Drops the frames waiting for the model and the candidates: after a reset they belong to a
+    /// world frame the map no longer has.
+    func reset() {
+        candidates.withLock { $0.removeAll() }
+        work.withLock { $0.pending.removeAll() }
+    }
+
+    // MARK: Model queue
+
+    /// Runs the model on waiting frames, newest last, until none wait.
+    private func runPending() {
+        while let (input, generation) = work.withLock({ work -> (DepthEstimatorInput, Int)? in
+            guard let next = work.pending.take() else {
+                work.running = false
+                return nil
+            }
+            return (next.input, next.generation)
+        }) {
+            estimateNow(input, generation: generation)
+        }
+    }
+
+    private func estimateNow(_ input: DepthEstimatorInput, generation: Int) {
+        let started = ProcessInfo.processInfo.systemUptime
+        guard let prediction = predict(input.bgra) else { return }
+        let predicted = ProcessInfo.processInfo.systemUptime
+        let anchors = DepthAnchor.anchors(points: input.points, camera: input.camera) + DepthAnchor.anchors(planes: input.planes, confirmedBy: input.points, camera: input.camera)
+        guard let estimate = MonocularDepth.estimate(prediction, anchors: anchors, photo: input.camera) else {
+            RuntimeLog.engine.info("depth \(input.frameID, privacy: .public): no scale fit from \(anchors.count) anchors; frame skipped")
+            return
+        }
+        let fitted = ProcessInfo.processInfo.systemUptime
+        RuntimeLog.engine.info("depth \(input.frameID, privacy: .public): model \(Int((predicted - started) * 1000)) ms, fit \(Int((fitted - predicted) * 1000)) ms, \(estimate.fit.inlierCount)/\(anchors.count) anchors, relative sigma \(estimate.fit.relativeSigma)")
+        onFrame(estimate.frame, generation)
+    }
+
+    private func predict(_ bgra: [UInt8]) -> RelativeInverseDepth? {
+        var buffer: CVPixelBuffer?
+        guard CVPixelBufferCreate(nil, Self.inputWidth, Self.inputHeight, kCVPixelFormatType_32BGRA, nil, &buffer) == kCVReturnSuccess, let buffer else { return nil }
+        CVPixelBufferLockBaseAddress(buffer, [])
+        if let base = CVPixelBufferGetBaseAddress(buffer) {
+            let rowBytes = CVPixelBufferGetBytesPerRow(buffer)
+            bgra.withUnsafeBytes { source in
+                for row in 0..<Self.inputHeight {
+                    base.advanced(by: row * rowBytes).copyMemory(from: source.baseAddress!.advanced(by: row * Self.inputWidth * 4), byteCount: Self.inputWidth * 4)
+                }
+            }
+        }
+        CVPixelBufferUnlockBaseAddress(buffer, [])
+        do {
+            let output = try model.model.prediction(from: MLDictionaryFeatureProvider(dictionary: ["image": MLFeatureValue(pixelBuffer: buffer)]))
+            guard let depth = output.featureValue(for: "depth")?.imageBufferValue,
+                  CVPixelBufferGetPixelFormatType(depth) == kCVPixelFormatType_OneComponent16Half,
+                  CVPixelBufferGetWidth(depth) == Self.inputWidth, CVPixelBufferGetHeight(depth) == Self.inputHeight
+            else {
+                RuntimeLog.engine.error("depth model: output is not a 518 x 392 Float16 image")
+                return nil
+            }
+            CVPixelBufferLockBaseAddress(depth, .readOnly)
+            defer { CVPixelBufferUnlockBaseAddress(depth, .readOnly) }
+            guard let base = CVPixelBufferGetBaseAddress(depth) else { return nil }
+            let rowBytes = CVPixelBufferGetBytesPerRow(depth)
+            var values = [Float](repeating: 0, count: Self.inputWidth * Self.inputHeight)
+            for row in 0..<Self.inputHeight {
+                let line = base.advanced(by: row * rowBytes).assumingMemoryBound(to: Float16.self)
+                for column in 0..<Self.inputWidth { values[row * Self.inputWidth + column] = Float(line[column]) }
+            }
+            return RelativeInverseDepth(width: Self.inputWidth, height: Self.inputHeight, values: values)
+        } catch {
+            RuntimeLog.engine.error("depth model prediction failed: \(error.localizedDescription, privacy: .public)")
+            return nil
+        }
+    }
+}

ios/HouseScan/Runtime/LaunchOptions.swift

@@ -18,13 +18,29 @@ import OSLog
 ///   also writes the scan's scene.json there, for the test to check.
 /// - `-autopilotCantGetThere`: the autopilot ends the walk with "Can't get there" instead of
 ///   marking the ends (`Autopilot.endWalkByCantGetThere`).
+/// - `-coverage map3d|legacy`: where coverage comes from (default `map3d`). `map3d` is the 3D
+///   occupancy map (`Map3DSession`): scene.json's coverage and walls, the fog overlay, and on a
+///   phone with depth the strip's and planners' covered cells. `legacy` is the camera-sighting
+///   `CoverageMap` alone. Any other value stops the app: a mistyped flag must not run the other model.
+/// - `-estimatedDepth on|off`: on a phone without LiDAR under `map3d`, run the depth model on
+///   kept keyframes into the 3D map, for its fog of war and next view. Coverage stays with the
+///   camera coverage map: estimated depth certifies none (default `off`:
+///   `ScanEngine.estimatedDepthByDefault`). Without the model it stays off. Any other value
+///   stops the app.
 struct LaunchOptions: Equatable {
+    enum CoverageModel: String {
+        case map3d
+        case legacy
+    }
+
     var replayFolder: URL?
     var autopilot = false
     var serverURL: URL?
     var sampleResult = false
     var autopilotHold: Double = 1.2
     var autopilotGate: URL?
+    var coverage: CoverageModel = .map3d
+    var estimatedDepth = ScanEngine.estimatedDepthByDefault
     var autopilotCantGetThere = false
 
     init(
@@ -44,6 +60,14 @@ struct LaunchOptions: Equatable {
         sampleResult = arguments.contains("-sampleResult")
         if let gate = value(after: "-autopilotGate") { autopilotGate = URL(fileURLWithPath: gate, isDirectory: true) }
         if let hold = value(after: "-autopilotHold").flatMap(Double.init), hold > 0 { autopilotHold = hold }
+        if let model = value(after: "-coverage") {
+            guard let parsed = CoverageModel(rawValue: model) else { preconditionFailure("-coverage takes map3d or legacy, not \(model)") }
+            coverage = parsed
+        }
+        if let setting = value(after: "-estimatedDepth") {
+            guard setting == "on" || setting == "off" else { preconditionFailure("-estimatedDepth takes on or off, not \(setting)") }
+            estimatedDepth = setting == "on"
+        }
     }
 
     /// An http(s) URL with a host, or nil. An empty build setting leaves the plist value empty,