From 2d33ceabfacc05a9db6498e068eed5c0a23b6a46 Mon Sep 17 00:00:00 2001 From: AidenJohnston Date: Sat, 26 Sep 2026 17:57:03 -0500 Subject: [PATCH] ios: draw the AR result in the scene so people and objects hide it (#41) The AR result was a SwiftUI Canvas over the camera, so nothing could occlude it. On the live camera it is now RealityKit content on the meter's anchor, with scene-mesh occlusion on LiDAR phones and people occlusion where supported, both on only while the AR screen is up. Replays keep the Canvas overlay. Co-authored-by: Cursor --- ios/HouseScan/Contract/ScanContract.swift | 3 + ios/HouseScan/Runtime/LiveCapture.swift | 61 ++++++++++++- ios/HouseScan/Runtime/ScanEngine.swift | 21 +++++ ios/HouseScan/UI/Result/ResultScene3D.swift | 90 +++++++++++++++++++ ios/HouseScan/UI/Screens/ResultARScreen.swift | 7 +- 5 files changed, 176 insertions(+), 6 deletions(-) diff --git a/ios/HouseScan/Contract/ScanContract.swift b/ios/HouseScan/Contract/ScanContract.swift index 71b42801e..26bec32f0 100644 --- a/ios/HouseScan/Contract/ScanContract.swift +++ b/ios/HouseScan/Contract/ScanContract.swift @@ -660,6 +660,9 @@ final class ScanViewState { var groundAnswer: GroundAnswer? var upload: UploadState = .idle var result: ResultPresentation? + /// True while the AR result is drawn into the live camera, where people and objects in front + /// of it hide it. The AR screen then draws no overlay of its own. + var resultInCamera = false /// True when frames come from a recorded session instead of the camera. var isReplay = false diff --git a/ios/HouseScan/Runtime/LiveCapture.swift b/ios/HouseScan/Runtime/LiveCapture.swift index 3ad22ccca..079b3d74b 100644 --- a/ios/HouseScan/Runtime/LiveCapture.swift +++ b/ios/HouseScan/Runtime/LiveCapture.swift @@ -49,6 +49,12 @@ final class LiveCapture { /// True when the mesh can carry ARKit's per-face classification (wall, floor, door, ...), /// which the packet's mesh records. static var supportsClassifiedMesh: Bool { ARWorldTrackingConfiguration.supportsSceneReconstruction(.meshWithClassification) } + /// True when this phone can hide virtual content behind a person standing in front of it. + static var supportsPeopleOcclusion: Bool { ARWorldTrackingConfiguration.supportsFrameSemantics(.personSegmentationWithDepth) } + + /// The AR result in the camera view, while one is shown (`showResult`). + private var result: AnchorEntity? + private var occludesPeople = false init(onFrame: @escaping @MainActor @Sendable (SourceFrame) -> Void, onEvent: @escaping @MainActor @Sendable (LiveEvent) -> Void) { arView = ARView(frame: .zero, cameraMode: .ar, automaticallyConfigureSession: false) @@ -58,7 +64,9 @@ final class LiveCapture { arView.renderOptions.insert(.disableMotionBlur) } - private static func configuration() -> ARWorldTrackingConfiguration { + /// People occlusion runs only while the AR result is on screen: the capture has no virtual + /// content for it to hide. + private static func configuration(occludingPeople: Bool = false) -> ARWorldTrackingConfiguration { let configuration = ARWorldTrackingConfiguration() // .gravity, not .gravityAndHeading: compass heading drifts near a house's metal and wiring. configuration.worldAlignment = .gravity @@ -69,6 +77,7 @@ final class LiveCapture { configuration.sceneReconstruction = .mesh } if supportsDepth { configuration.frameSemantics.insert(.sceneDepth) } + if occludingPeople, supportsPeopleOcclusion { configuration.frameSemantics.insert(.personSegmentationWithDepth) } return configuration } @@ -100,7 +109,7 @@ final class LiveCapture { func start() { RuntimeLog.engine.info("LiDAR: scene depth \(Self.supportsDepth ? "on" : "not available", privacy: .public), mesh \(Self.supportsMesh ? "on" : "not available", privacy: .public)") - arView.session.run(Self.configuration()) + arView.session.run(Self.configuration(occludingPeople: occludesPeople)) } func pause() { @@ -110,10 +119,56 @@ final class LiveCapture { /// Starts world tracking over with a fresh map, after relocalization failed. The old mesh goes /// with the old anchors. func restart() { - arView.session.run(Self.configuration(), options: [.resetTracking, .removeExistingAnchors]) + arView.session.run(Self.configuration(occludingPeople: occludesPeople), options: [.resetTracking, .removeExistingAnchors]) delegate.shared.withLock { $0.meterAnchorID = nil } } + /// Shows `model` in the camera view, hung on the meter's anchor so it follows ARKit's + /// corrections to it. The model is in world axes with the meter at its origin. People and, + /// on a phone with LiDAR, the scene mesh hide it where they stand in front of it. False when + /// the meter has no anchor in the session; nothing is shown then. + func showResult(_ model: Entity) -> Bool { + removeResult() + guard let id = delegate.shared.withLock({ $0.meterAnchorID }), + let anchor = arView.session.currentFrame?.anchors.first(where: { $0.identifier == id }) else { return false } + let holder = AnchorEntity(.anchor(identifier: id)) + model.orientation = simd_quatf(anchor.transform).inverse + holder.addChild(model) + arView.scene.addAnchor(holder) + result = holder + setOcclusion(true) + return true + } + + func hideResult() { + removeResult() + setOcclusion(false) + } + + /// Hides the shown result without taking it out, while the anchor's pose can't be trusted. + func setResultVisible(_ visible: Bool) { + result?.isEnabled = visible + } + + private func removeResult() { + result?.removeFromParent() + result = nil + } + + private func setOcclusion(_ on: Bool) { + if Self.supportsMesh { + if on { + arView.environment.sceneUnderstanding.options.insert(.occlusion) + } else { + arView.environment.sceneUnderstanding.options.remove(.occlusion) + } + } + guard Self.supportsPeopleOcclusion, occludesPeople != on else { return } + occludesPeople = on + // Without reset options the session keeps its tracking, anchors and mesh. + arView.session.run(Self.configuration(occludingPeople: on)) + } + /// ARKit's mesh in world meters, with one `ARMeshClassification` raw value per triangle (0, /// none, when the session runs without classification). struct MeshSnapshot: Sendable { diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index 270e4cfc5..5c822f53c 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -203,6 +203,7 @@ final class ScanEngine { if state.phase == .wallWalk || state.phase == .gapRequest { breakWalkedPath(because: "the walk paused (\(state.phase.rawValue) -> \(phase.rawValue))") } + if state.phase == .resultAR { hideResultInCamera() } state.phase = phase RuntimeLog.state.info("STATE=\(phase.rawValue, privacy: .public)") switch phase { @@ -249,6 +250,7 @@ final class ScanEngine { case .resultAR: live?.setMode(.idle) if let replay, let index = bestFrameForResult() { replay.show(index: index) } + showResultInCamera(rising: true) case .onboarding, .unsupported: break } @@ -330,6 +332,7 @@ final class ScanEngine { RuntimeLog.capture.info("tracking \(Self.name(self.state.tracking), privacy: .public) -> \(Self.name(frame.tracking), privacy: .public)") if state.tracking == .normal { breakWalkedPath(because: "tracking left normal") } state.tracking = frame.tracking + live?.setResultVisible(frame.tracking == .normal) } if !frame.groundPlanes.isEmpty, frame.groundPlanes != groundPlanes { groundPlanes = frame.groundPlanes @@ -1141,6 +1144,24 @@ final class ScanEngine { return WallGeometry.Segment(span: piece.span, along: piece.along, outward: piece.outward, anchor: piece.anchor, anchorS: piece.anchorS) } ) + if state.phase == .resultAR { showResultInCamera(rising: false) } + } + + /// "See it on your wall" on the live camera: the result goes into the AR scene on the meter's + /// anchor, where people and objects in front of it hide it. A replay, or a meter without an + /// anchor, leaves it to the screen's overlay. + private func showResultInCamera(rising: Bool) { + guard let live, let wall = state.wall, let result = state.result else { return } + let model = ResultARModel.build(wall: wall, result: result) + state.resultInCamera = live.showResult(model) + guard state.resultInCamera else { return } + live.setResultVisible(state.tracking == .normal) + if rising { ResultARModel.rise(model) } + } + + private func hideResultInCamera() { + live?.hideResult() + state.resultInCamera = false } func publishCoverage() { diff --git a/ios/HouseScan/UI/Result/ResultScene3D.swift b/ios/HouseScan/UI/Result/ResultScene3D.swift index 0da2383c6..ea46edae9 100644 --- a/ios/HouseScan/UI/Result/ResultScene3D.swift +++ b/ios/HouseScan/UI/Result/ResultScene3D.swift @@ -311,6 +311,96 @@ struct ResultScene3D: View { } } +/// The result for "See it on your wall" (`LiveCapture.showResult`): the battery, the cable run and +/// the clearance zones, the same pieces as `BatteryOverlay` draws over a replay. Built from +/// `WallGeometry` in world axes with the meter at the origin, so it turns a corner where the wall does. +@MainActor +enum ResultARModel { + /// The LiDAR mesh sits a centimeter or two off the real wall and ground, and hides whatever is + /// behind it: anything flush with either would be cut into. + static let meshClearance: Float = 0.03 + + private static let unitName = "result-ar-battery" + + static func build(wall: WallGeometry, result: ResultPresentation) -> Entity { + let root = Entity() + func local(_ s: Float, _ height: Float, _ out: Float) -> SIMD3 { + wall.world(s: s, height: height, out: out) - wall.meter + } + for (index, zone) in result.clearances.enumerated() { + var material = UnlitMaterial(color: SceneColor.outcome(zone.outcome)) + material.blending = .transparent(opacity: .init(floatLiteral: 0.35)) + let width = zone.span.upperBound - zone.span.lowerBound + let middle = zone.span.lowerBound + width / 2 + // Stacked zones sit a few millimeters apart so overlapping ones don't flicker. + let entity = ModelEntity(mesh: .generatePlane(width: width, depth: zone.depth), materials: [material]) + entity.position = local(middle, meshClearance + Float(index) * 0.003, zone.depth / 2) + entity.orientation = facing(wall, atS: middle) + root.addChild(entity) + } + let cable = result.cableRoute.map { local($0.x, $0.y, meshClearance) } + let material = SimpleMaterial(color: SceneColor.signal, roughness: 0.85, isMetallic: false) + let radius: Float = 0.015 + for (from, to) in zip(cable, cable.dropFirst()) { + let length = simd_distance(from, to) + guard length > 0.001 else { continue } + let segment = ModelEntity(mesh: .generateCylinder(height: length, radius: radius), materials: [material]) + segment.position = (from + to) / 2 + segment.orientation = simd_quatf(from: SIMD3(0, 1, 0), to: (to - from) / length) + root.addChild(segment) + } + for point in cable.dropFirst().dropLast() { + let joint = ModelEntity(mesh: .generateSphere(radius: radius), materials: [material]) + joint.position = point + root.addChild(joint) + } + if let spot = result.spot { + let width = max(spot.span.upperBound - spot.span.lowerBound, 0.1) + let middle = (spot.span.lowerBound + spot.span.upperBound) / 2 + let back = max(spot.offsetFromWall, meshClearance) + // Origin at the middle of the footprint on the ground, so `rise` grows it upward. + let unit = Entity() + unit.name = unitName + unit.position = local(middle, 0, back + spot.depth / 2) + unit.orientation = facing(wall, atS: middle) + var paint = PhysicallyBasedMaterial() + paint.baseColor = .init(tint: SceneColor.battery) + paint.roughness = 0.35 + let corner = min(0.03, min(width, spot.height, spot.depth) / 4) + let body = ModelEntity( + mesh: .generateBox(width: width, height: spot.height, depth: spot.depth, cornerRadius: corner), + materials: [paint] + ) + body.position = SIMD3(0, spot.height / 2, 0) + body.components.set(GroundingShadowComponent(castsShadow: true)) + unit.addChild(body) + let bar = ModelEntity( + mesh: .generateBox(width: 0.04, height: spot.height * 0.7, depth: 0.006), + materials: [UnlitMaterial(color: SceneColor.signal)] + ) + bar.position = SIMD3(0, spot.height / 2, spot.depth / 2 + 0.003) + unit.addChild(bar) + root.addChild(unit) + } + return root + } + + /// Lifts the battery out of the ground. Call once the model is in the scene. + static func rise(_ model: Entity) { + guard let unit = model.findEntity(named: unitName) else { return } + let settled = unit.transform + unit.scale = SIMD3(1, 0.02, 1) + unit.move(to: settled, relativeTo: unit.parent, duration: UIAccessibility.isReduceMotionEnabled ? 0.2 : 0.7, timingFunction: .easeOut) + } + + /// x along the wall, y up, z out from the wall toward the homeowner. + private static func facing(_ wall: WallGeometry, atS s: Float) -> simd_quatf { + let up = SIMD3(0, 1, 0) + let outward = simd_normalize(wall.outward(atS: s)) + return simd_quatf(simd_float3x3(columns: (simd_normalize(simd_cross(up, outward)), up, outward))) + } +} + /// Fixed scene colors. They match the asset-catalog palette in light mode; a model of a house /// in daylight shouldn't change color with the phone's dark mode. private enum SceneColor { diff --git a/ios/HouseScan/UI/Screens/ResultARScreen.swift b/ios/HouseScan/UI/Screens/ResultARScreen.swift index b9d16b5de..871da39cf 100644 --- a/ios/HouseScan/UI/Screens/ResultARScreen.swift +++ b/ios/HouseScan/UI/Screens/ResultARScreen.swift @@ -2,8 +2,9 @@ import simd import SwiftUI /// "See it on your wall": the battery drawn onto the live camera at the chosen spot, with the -/// cable run from the meter and the clearance footprint tinted by outcome. Everything is -/// projected from the meter-anchored wall frame, so it stays put as the homeowner moves. +/// cable run from the meter and the clearance footprint tinted by outcome. On the live camera the +/// engine draws it into the AR scene (`state.resultInCamera`); over a replay this screen projects +/// it from the meter-anchored wall frame. Either way it stays put as the homeowner moves. struct ResultARScreen: View { let state: ScanViewState let actions: any ScanActions @@ -13,7 +14,7 @@ struct ResultARScreen: View { var body: some View { ZStack { - if state.tracking == .normal, let projection = state.projection, let wall = state.wall, let result = state.result { + if !state.resultInCamera, state.tracking == .normal, let projection = state.projection, let wall = state.wall, let result = state.result { BatteryOverlay(projection: projection, wall: wall, result: result, rise: appeared ? 1 : 0) .ignoresSafeArea() .accessibilityHidden(true)