ios: AR result is hidden by people and objects in front of it (#41)

ios/HouseScan/Contract/ScanContract.swift

@@ -660,6 +660,9 @@ final class ScanViewState {
     var groundAnswer: GroundAnswer?
     var upload: UploadState = .idle
     var result: ResultPresentation?
+    /// True while the AR result is drawn into the live camera, where people and objects in front
+    /// of it hide it. The AR screen then draws no overlay of its own.
+    var resultInCamera = false
 
     /// True when frames come from a recorded session instead of the camera.
     var isReplay = false

ios/HouseScan/Runtime/LiveCapture.swift

@@ -49,6 +49,12 @@ final class LiveCapture {
     /// True when the mesh can carry ARKit's per-face classification (wall, floor, door, ...),
     /// which the packet's mesh records.
     static var supportsClassifiedMesh: Bool { ARWorldTrackingConfiguration.supportsSceneReconstruction(.meshWithClassification) }
+    /// True when this phone can hide virtual content behind a person standing in front of it.
+    static var supportsPeopleOcclusion: Bool { ARWorldTrackingConfiguration.supportsFrameSemantics(.personSegmentationWithDepth) }
+
+    /// The AR result in the camera view, while one is shown (`showResult`).
+    private var result: AnchorEntity?
+    private var occludesPeople = false
 
     init(onFrame: @escaping @MainActor @Sendable (SourceFrame) -> Void, onEvent: @escaping @MainActor @Sendable (LiveEvent) -> Void) {
         arView = ARView(frame: .zero, cameraMode: .ar, automaticallyConfigureSession: false)
@@ -58,7 +64,9 @@ final class LiveCapture {
         arView.renderOptions.insert(.disableMotionBlur)
     }
 
-    private static func configuration() -> ARWorldTrackingConfiguration {
+    /// People occlusion runs only while the AR result is on screen: the capture has no virtual
+    /// content for it to hide.
+    private static func configuration(occludingPeople: Bool = false) -> ARWorldTrackingConfiguration {
         let configuration = ARWorldTrackingConfiguration()
         // .gravity, not .gravityAndHeading: compass heading drifts near a house's metal and wiring.
         configuration.worldAlignment = .gravity
@@ -69,6 +77,7 @@ final class LiveCapture {
             configuration.sceneReconstruction = .mesh
         }
         if supportsDepth { configuration.frameSemantics.insert(.sceneDepth) }
+        if occludingPeople, supportsPeopleOcclusion { configuration.frameSemantics.insert(.personSegmentationWithDepth) }
         return configuration
     }
 
@@ -100,7 +109,7 @@ final class LiveCapture {
 
     func start() {
         RuntimeLog.engine.info("LiDAR: scene depth \(Self.supportsDepth ? "on" : "not available", privacy: .public), mesh \(Self.supportsMesh ? "on" : "not available", privacy: .public)")
-        arView.session.run(Self.configuration())
+        arView.session.run(Self.configuration(occludingPeople: occludesPeople))
     }
 
     func pause() {
@@ -110,10 +119,56 @@ final class LiveCapture {
     /// Starts world tracking over with a fresh map, after relocalization failed. The old mesh goes
     /// with the old anchors.
     func restart() {
-        arView.session.run(Self.configuration(), options: [.resetTracking, .removeExistingAnchors])
+        arView.session.run(Self.configuration(occludingPeople: occludesPeople), options: [.resetTracking, .removeExistingAnchors])
         delegate.shared.withLock { $0.meterAnchorID = nil }
     }
 
+    /// Shows `model` in the camera view, hung on the meter's anchor so it follows ARKit's
+    /// corrections to it. The model is in world axes with the meter at its origin. People and,
+    /// on a phone with LiDAR, the scene mesh hide it where they stand in front of it. False when
+    /// the meter has no anchor in the session; nothing is shown then.
+    func showResult(_ model: Entity) -> Bool {
+        removeResult()
+        guard let id = delegate.shared.withLock({ $0.meterAnchorID }),
+              let anchor = arView.session.currentFrame?.anchors.first(where: { $0.identifier == id }) else { return false }
+        let holder = AnchorEntity(.anchor(identifier: id))
+        model.orientation = simd_quatf(anchor.transform).inverse
+        holder.addChild(model)
+        arView.scene.addAnchor(holder)
+        result = holder
+        setOcclusion(true)
+        return true
+    }
+
+    func hideResult() {
+        removeResult()
+        setOcclusion(false)
+    }
+
+    /// Hides the shown result without taking it out, while the anchor's pose can't be trusted.
+    func setResultVisible(_ visible: Bool) {
+        result?.isEnabled = visible
+    }
+
+    private func removeResult() {
+        result?.removeFromParent()
+        result = nil
+    }
+
+    private func setOcclusion(_ on: Bool) {
+        if Self.supportsMesh {
+            if on {
+                arView.environment.sceneUnderstanding.options.insert(.occlusion)
+            } else {
+                arView.environment.sceneUnderstanding.options.remove(.occlusion)
+            }
+        }
+        guard Self.supportsPeopleOcclusion, occludesPeople != on else { return }
+        occludesPeople = on
+        // Without reset options the session keeps its tracking, anchors and mesh.
+        arView.session.run(Self.configuration(occludingPeople: on))
+    }
+
     /// ARKit's mesh in world meters, with one `ARMeshClassification` raw value per triangle (0,
     /// none, when the session runs without classification).
     struct MeshSnapshot: Sendable {

ios/HouseScan/Runtime/ScanEngine.swift

@@ -203,6 +203,7 @@ final class ScanEngine {
         if state.phase == .wallWalk || state.phase == .gapRequest {
             breakWalkedPath(because: "the walk paused (\(state.phase.rawValue) -> \(phase.rawValue))")
         }
+        if state.phase == .resultAR { hideResultInCamera() }
         state.phase = phase
         RuntimeLog.state.info("STATE=\(phase.rawValue, privacy: .public)")
         switch phase {
@@ -249,6 +250,7 @@ final class ScanEngine {
         case .resultAR:
             live?.setMode(.idle)
             if let replay, let index = bestFrameForResult() { replay.show(index: index) }
+            showResultInCamera(rising: true)
         case .onboarding, .unsupported:
             break
         }
@@ -330,6 +332,7 @@ final class ScanEngine {
             RuntimeLog.capture.info("tracking \(Self.name(self.state.tracking), privacy: .public) -> \(Self.name(frame.tracking), privacy: .public)")
             if state.tracking == .normal { breakWalkedPath(because: "tracking left normal") }
             state.tracking = frame.tracking
+            live?.setResultVisible(frame.tracking == .normal)
         }
         if !frame.groundPlanes.isEmpty, frame.groundPlanes != groundPlanes {
             groundPlanes = frame.groundPlanes
@@ -1141,6 +1144,24 @@ final class ScanEngine {
                 return WallGeometry.Segment(span: piece.span, along: piece.along, outward: piece.outward, anchor: piece.anchor, anchorS: piece.anchorS)
             }
         )
+        if state.phase == .resultAR { showResultInCamera(rising: false) }
+    }
+
+    /// "See it on your wall" on the live camera: the result goes into the AR scene on the meter's
+    /// anchor, where people and objects in front of it hide it. A replay, or a meter without an
+    /// anchor, leaves it to the screen's overlay.
+    private func showResultInCamera(rising: Bool) {
+        guard let live, let wall = state.wall, let result = state.result else { return }
+        let model = ResultARModel.build(wall: wall, result: result)
+        state.resultInCamera = live.showResult(model)
+        guard state.resultInCamera else { return }
+        live.setResultVisible(state.tracking == .normal)
+        if rising { ResultARModel.rise(model) }
+    }
+
+    private func hideResultInCamera() {
+        live?.hideResult()
+        state.resultInCamera = false
     }
 
     func publishCoverage() {

ios/HouseScan/UI/Result/ResultScene3D.swift

@@ -311,6 +311,96 @@ struct ResultScene3D: View {
     }
 }
 
+/// The result for "See it on your wall" (`LiveCapture.showResult`): the battery, the cable run and
+/// the clearance zones, the same pieces as `BatteryOverlay` draws over a replay. Built from
+/// `WallGeometry` in world axes with the meter at the origin, so it turns a corner where the wall does.
+@MainActor
+enum ResultARModel {
+    /// The LiDAR mesh sits a centimeter or two off the real wall and ground, and hides whatever is
+    /// behind it: anything flush with either would be cut into.
+    static let meshClearance: Float = 0.03
+
+    private static let unitName = "result-ar-battery"
+
+    static func build(wall: WallGeometry, result: ResultPresentation) -> Entity {
+        let root = Entity()
+        func local(_ s: Float, _ height: Float, _ out: Float) -> SIMD3<Float> {
+            wall.world(s: s, height: height, out: out) - wall.meter
+        }
+        for (index, zone) in result.clearances.enumerated() {
+            var material = UnlitMaterial(color: SceneColor.outcome(zone.outcome))
+            material.blending = .transparent(opacity: .init(floatLiteral: 0.35))
+            let width = zone.span.upperBound - zone.span.lowerBound
+            let middle = zone.span.lowerBound + width / 2
+            // Stacked zones sit a few millimeters apart so overlapping ones don't flicker.
+            let entity = ModelEntity(mesh: .generatePlane(width: width, depth: zone.depth), materials: [material])
+            entity.position = local(middle, meshClearance + Float(index) * 0.003, zone.depth / 2)
+            entity.orientation = facing(wall, atS: middle)
+            root.addChild(entity)
+        }
+        let cable = result.cableRoute.map { local($0.x, $0.y, meshClearance) }
+        let material = SimpleMaterial(color: SceneColor.signal, roughness: 0.85, isMetallic: false)
+        let radius: Float = 0.015
+        for (from, to) in zip(cable, cable.dropFirst()) {
+            let length = simd_distance(from, to)
+            guard length > 0.001 else { continue }
+            let segment = ModelEntity(mesh: .generateCylinder(height: length, radius: radius), materials: [material])
+            segment.position = (from + to) / 2
+            segment.orientation = simd_quatf(from: SIMD3(0, 1, 0), to: (to - from) / length)
+            root.addChild(segment)
+        }
+        for point in cable.dropFirst().dropLast() {
+            let joint = ModelEntity(mesh: .generateSphere(radius: radius), materials: [material])
+            joint.position = point
+            root.addChild(joint)
+        }
+        if let spot = result.spot {
+            let width = max(spot.span.upperBound - spot.span.lowerBound, 0.1)
+            let middle = (spot.span.lowerBound + spot.span.upperBound) / 2
+            let back = max(spot.offsetFromWall, meshClearance)
+            // Origin at the middle of the footprint on the ground, so `rise` grows it upward.
+            let unit = Entity()
+            unit.name = unitName
+            unit.position = local(middle, 0, back + spot.depth / 2)
+            unit.orientation = facing(wall, atS: middle)
+            var paint = PhysicallyBasedMaterial()
+            paint.baseColor = .init(tint: SceneColor.battery)
+            paint.roughness = 0.35
+            let corner = min(0.03, min(width, spot.height, spot.depth) / 4)
+            let body = ModelEntity(
+                mesh: .generateBox(width: width, height: spot.height, depth: spot.depth, cornerRadius: corner),
+                materials: [paint]
+            )
+            body.position = SIMD3(0, spot.height / 2, 0)
+            body.components.set(GroundingShadowComponent(castsShadow: true))
+            unit.addChild(body)
+            let bar = ModelEntity(
+                mesh: .generateBox(width: 0.04, height: spot.height * 0.7, depth: 0.006),
+                materials: [UnlitMaterial(color: SceneColor.signal)]
+            )
+            bar.position = SIMD3(0, spot.height / 2, spot.depth / 2 + 0.003)
+            unit.addChild(bar)
+            root.addChild(unit)
+        }
+        return root
+    }
+
+    /// Lifts the battery out of the ground. Call once the model is in the scene.
+    static func rise(_ model: Entity) {
+        guard let unit = model.findEntity(named: unitName) else { return }
+        let settled = unit.transform
+        unit.scale = SIMD3(1, 0.02, 1)
+        unit.move(to: settled, relativeTo: unit.parent, duration: UIAccessibility.isReduceMotionEnabled ? 0.2 : 0.7, timingFunction: .easeOut)
+    }
+
+    /// x along the wall, y up, z out from the wall toward the homeowner.
+    private static func facing(_ wall: WallGeometry, atS s: Float) -> simd_quatf {
+        let up = SIMD3<Float>(0, 1, 0)
+        let outward = simd_normalize(wall.outward(atS: s))
+        return simd_quatf(simd_float3x3(columns: (simd_normalize(simd_cross(up, outward)), up, outward)))
+    }
+}
+
 /// Fixed scene colors. They match the asset-catalog palette in light mode; a model of a house
 /// in daylight shouldn't change color with the phone's dark mode.
 private enum SceneColor {

ios/HouseScan/UI/Screens/ResultARScreen.swift

@@ -2,8 +2,9 @@ import simd
 import SwiftUI
 
 /// "See it on your wall": the battery drawn onto the live camera at the chosen spot, with the
-/// cable run from the meter and the clearance footprint tinted by outcome. Everything is
-/// projected from the meter-anchored wall frame, so it stays put as the homeowner moves.
+/// cable run from the meter and the clearance footprint tinted by outcome. On the live camera the
+/// engine draws it into the AR scene (`state.resultInCamera`); over a replay this screen projects
+/// it from the meter-anchored wall frame. Either way it stays put as the homeowner moves.
 struct ResultARScreen: View {
     let state: ScanViewState
     let actions: any ScanActions
@@ -13,7 +14,7 @@ struct ResultARScreen: View {
 
     var body: some View {
         ZStack {
-            if state.tracking == .normal, let projection = state.projection, let wall = state.wall, let result = state.result {
+            if !state.resultInCamera, state.tracking == .normal, let projection = state.projection, let wall = state.wall, let result = state.result {
                 BatteryOverlay(projection: projection, wall: wall, result: result, rise: appeared ? 1 : 0)
                     .ignoresSafeArea()
                     .accessibilityHidden(true)