diff --git a/ios/HouseScan/Contract/ScanContract.swift b/ios/HouseScan/Contract/ScanContract.swift index 00f3e3819..795734ed9 100644 --- a/ios/HouseScan/Contract/ScanContract.swift +++ b/ios/HouseScan/Contract/ScanContract.swift @@ -794,8 +794,9 @@ final class ScanViewState { /// The spot check of the result's spot: the question during `.spotConfirm`, and the answer /// that stands for the spot on the result. var spotCheck: SpotCheck? - /// True while the AR result is drawn into the live camera, where people and objects in front - /// of it hide it. The AR screen then draws no overlay of its own. + /// True while the engine sees the AR scene drawing the result in the live camera + /// (`ResultOverlayPolicy`). The AR screen then draws no overlay of its own; otherwise it + /// draws `BatteryOverlay`. var resultInCamera = false /// False once the camera failed after the scan was sent: the answer stays, but "See it on /// your wall" is neither shown nor offered until a new scan starts the camera again. diff --git a/ios/HouseScan/Runtime/LiveCapture.swift b/ios/HouseScan/Runtime/LiveCapture.swift index 5c073dc1e..82c8966b8 100644 --- a/ios/HouseScan/Runtime/LiveCapture.swift +++ b/ios/HouseScan/Runtime/LiveCapture.swift @@ -56,7 +56,11 @@ final class LiveCapture { /// The AR result in the camera view, while one is shown (`showResult`). private var result: AnchorEntity? - private var occludesPeople = false + /// An empty entity in the result at the point it has to put on screen (the battery's + /// middle), for `resultIsDrawn`. + private var resultFocus: Entity? + /// The last `resultIsDrawn` findings logged, so they are logged when they change. + private var lastDrawnReport: String? init(onFrame: @escaping @MainActor @Sendable (SourceFrame) -> Void, onEvent: @escaping @MainActor @Sendable (LiveEvent) -> Void) { arView = ARView(frame: .zero, cameraMode: .ar, automaticallyConfigureSession: false) @@ -66,8 +70,9 @@ final class LiveCapture { arView.renderOptions.insert(.disableMotionBlur) } - /// People occlusion runs only while the AR result is on screen: the capture has no virtual - /// content for it to hide. + /// People occlusion is off: no caller asks for it. Turning it on while the AR result showed + /// re-ran the session under the result's freshly added anchor entity, and build 4.1 drew no + /// result on the phone it was tested on (#67). Whether and how to bring it back is #41. private static func configuration(occludingPeople: Bool = false) -> ARWorldTrackingConfiguration { let configuration = ARWorldTrackingConfiguration() // .gravity, not .gravityAndHeading: compass heading drifts near a house's metal and wiring. @@ -111,7 +116,7 @@ final class LiveCapture { func start() { RuntimeLog.engine.info("LiDAR: scene depth \(Self.supportsDepth ? "on" : "not available", privacy: .public), mesh \(Self.supportsMesh ? "on" : "not available", privacy: .public)") - arView.session.run(Self.configuration(occludingPeople: occludesPeople)) + arView.session.run(Self.configuration()) } func pause() { @@ -121,7 +126,7 @@ final class LiveCapture { /// Starts world tracking over with a fresh map, after relocalization failed. The old mesh goes /// with the old anchors. func restart() { - arView.session.run(Self.configuration(occludingPeople: occludesPeople), options: [.resetTracking, .removeExistingAnchors]) + arView.session.run(Self.configuration(), options: [.resetTracking, .removeExistingAnchors]) delegate.shared.withLock { $0.meterAnchorID = nil } } @@ -130,22 +135,63 @@ final class LiveCapture { /// when the anchor was at `pose` (`MeterAnchorTracking.pose`, the pose the engine's wall /// agrees with). It is attached in that pose's frame, so a turn ARKit has made to the anchor /// since carries the model with it. Cancelling the anchor's current turn instead drew the - /// wall as it stood before the turn. People and, on a phone with LiDAR, the scene mesh hide - /// it where they stand in front of it. False when the meter has no anchor in the session; - /// nothing is shown then. - func showResult(_ model: Entity, builtFor pose: simd_float4x4) -> Bool { + /// wall as it stood before the turn. On a phone with LiDAR the scene mesh hides it where + /// something stands in front of it. `focus` is the point the model has to put on screen, in + /// the model's coordinates (`resultIsDrawn`). + /// + /// True means the model was handed to RealityKit, not that anything is drawn: build 4.1 took + /// it for that and hid the screen's own drawing on a phone where nothing appeared (#67). + /// False when the meter has no anchor in the session; nothing is shown then. + func showResult(_ model: Entity, builtFor pose: simd_float4x4, focus: SIMD3) -> Bool { removeResult() guard let id = delegate.shared.withLock({ $0.meterAnchorID }), - arView.session.currentFrame?.anchors.contains(where: { $0.identifier == id }) == true else { return false } + arView.session.currentFrame?.anchors.contains(where: { $0.identifier == id }) == true else { + RuntimeLog.engine.info("AR result not shown: the meter's anchor is not in the session") + return false + } let holder = AnchorEntity(.anchor(identifier: id)) + // See-through until the engine hands the result to the AR scene (`setResultOwned`): a + // new model starts behind the screen's own drawing, and only one of them may show. + holder.components.set(OpacityComponent(opacity: 0)) model.orientation = simd_quatf(pose).inverse + let marker = Entity() + marker.position = focus + model.addChild(marker) holder.addChild(model) arView.scene.addAnchor(holder) result = holder + resultFocus = marker + lastDrawnReport = nil setOcclusion(true) return true } + /// Whether RealityKit holds the shown result now, and whether it can be drawing it: `held` + /// when its anchor entity is anchored to the meter's ARKit anchor and enabled, `drawn` when + /// the result's focus point also lands in front of the camera and inside the view. Both are + /// read the same whether the model is see-through or shown (`setResultOwned`). A check of + /// the chain, not of pixels: RealityKit says nothing about what it drew. `drawn` confirms the + /// model; `held` keeps it once confirmed (`ResultOverlayPolicy`). The findings are logged + /// when they change. + func resultIsDrawn() -> (drawn: Bool, held: Bool) { + guard let result, let resultFocus else { return (false, false) } + let id = delegate.shared.withLock { $0.meterAnchorID } + let inSession = id.map { id in arView.session.currentFrame?.anchors.contains(where: { $0.identifier == id }) == true } ?? false + let anchored = result.isAnchored + let enabled = result.isEnabledInHierarchy + let world = resultFocus.position(relativeTo: nil) + let inCamera = arView.cameraTransform.matrix.inverse * SIMD4(world, 1) + let point = inCamera.z < 0 ? arView.project(world) : nil + let onScreen = point.map { arView.bounds.contains($0) } ?? false + let report = "anchor in session \(inSession), anchored \(anchored), enabled \(enabled), on screen \(onScreen)" + if report != lastDrawnReport { + lastDrawnReport = report + let at = point.map { String(format: "(%.0f, %.0f) in %.0f x %.0f", Double($0.x), Double($0.y), Double(arView.bounds.width), Double(arView.bounds.height)) } ?? "none" + RuntimeLog.engine.info("AR result: \(report, privacy: .public), projected at \(at, privacy: .public)") + } + return (anchored && enabled && onScreen, anchored && enabled) + } + func hideResult() { removeResult() setOcclusion(false) @@ -156,23 +202,29 @@ final class LiveCapture { result?.isEnabled = visible } + /// Whether the AR scene owns the result (`ResultOverlayPolicy.usesRealityKit`): the model + /// shows only then, and is see-through otherwise, while the screen draws it. See-through + /// rather than disabled, so it stays anchored and enabled and `resultIsDrawn` can still + /// confirm it. The opacity applies to the model and everything under it. + func setResultOwned(_ owned: Bool) { + result?.components.set(OpacityComponent(opacity: owned ? 1 : 0)) + } + private func removeResult() { result?.removeFromParent() result = nil + resultFocus = nil } + /// The LiDAR mesh hides the result where something real stands in front of it. A RealityKit + /// setting only: the session is not re-run (see `configuration`). private func setOcclusion(_ on: Bool) { - if Self.supportsMesh { - if on { - arView.environment.sceneUnderstanding.options.insert(.occlusion) - } else { - arView.environment.sceneUnderstanding.options.remove(.occlusion) - } + guard Self.supportsMesh else { return } + if on { + arView.environment.sceneUnderstanding.options.insert(.occlusion) + } else { + arView.environment.sceneUnderstanding.options.remove(.occlusion) } - guard Self.supportsPeopleOcclusion, occludesPeople != on else { return } - occludesPeople = on - // Without reset options the session keeps its tracking, anchors and mesh. - arView.session.run(Self.configuration(occludingPeople: on)) } /// ARKit's mesh in world meters, with one `ARMeshClassification` raw value per triangle (0, diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index 438ad6ae4..700827637 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -125,6 +125,14 @@ final class ScanEngine { // Tracking recovery private var relocalizingSince: Double? + // AR result + /// Watches whether the AR scene draws the result (`watchResultInCamera`) while "See it on + /// your wall" is up on the live camera. + private var resultWatch: Task? + /// Which layer draws the result, and the wall the model in the AR scene was built for + /// (`showResultInCamera`, `watchResultInCamera`). + private var resultPolicy = ResultOverlayPolicy() + // Upload let resultClient: any ResultClient private var uploadTask: Task? @@ -358,6 +366,8 @@ final class ScanEngine { RuntimeLog.capture.info("tracking \(Self.name(self.state.tracking), privacy: .public) -> \(Self.name(frame.tracking), privacy: .public)") if state.tracking == .normal { breakWalkedPath(because: "tracking left normal") } state.tracking = frame.tracking + // The model stays see-through until the next look hands it the result + // (`setResultInCamera`), so enabling it here never shows it over the screen's drawing. live?.setResultVisible(frame.tracking == .normal) } if let planes = frame.groundPlanes { groundPlanes = planes } @@ -1204,20 +1214,85 @@ final class ScanEngine { refreshSpotPhoto() } - /// "See it on your wall" on the live camera: the result goes into the AR scene on the meter's - /// anchor, where people and objects in front of it hide it. A replay, or a meter without an - /// anchor, leaves it to the screen's overlay. + /// "See it on your wall" on the live camera. The screen draws the result over the camera + /// itself (`BatteryOverlay`) unless the AR scene is seen drawing it (`watchResultInCamera`). + /// Build 4.1 trusted the AR scene as soon as it had the model and showed no battery at all + /// (#67). Only a phone with LiDAR tries the AR scene, where the mesh can hide the result + /// behind things in front of it; without LiDAR the screen draws it, as build 3.1 did. A + /// replay, or a meter without an anchor, leaves it to the screen too. private func showResultInCamera(rising: Bool) { - guard let live, let wall = state.wall, let result = state.result, let pose = meterTracking?.pose, - state.spatialResultAvailable else { return } + guard LiveCapture.supportsMesh, let live, let wall = state.wall, let result = state.result, + let pose = meterTracking?.pose, state.spatialResultAvailable else { return } + if rising || resultWatch == nil { watchResultInCamera() } + // Each rebuild takes the model out of the scene and puts a new one in, so a wall that + // moved by a centimeter or two keeps the one it has. A new model starts on the screen's + // drawing and has to be seen drawn again (`ResultOverlayPolicy.needsModel`, review of + // #100); it goes in see-through (`LiveCapture.showResult`), so only one layer shows. + guard resultPolicy.needsModel(for: Self.modelShape(wall), rising: rising) else { return } + setResultInCamera(resultPolicy.usesRealityKit) let model = ResultARModel.build(wall: wall, result: result) - state.resultInCamera = live.showResult(model, builtFor: pose) - guard state.resultInCamera else { return } + // The battery's middle, or the meter without a spot, in the model's coordinates. + let focus = (result.spotCenter(on: wall) ?? wall.meter) - wall.meter + guard live.showResult(model, builtFor: pose, focus: focus) else { + resultPolicy.modelRemoved() + return + } live.setResultVisible(state.tracking == .normal) if rising { ResultARModel.rise(model) } } + /// What the AR result's model is built from, for `ResultOverlayPolicy.needsModel`: the + /// meter, the ground, the ends and each piece of wall. + private static func modelShape(_ wall: WallGeometry) -> ResultModelShape { + var shape = ResultModelShape( + points: [wall.meter], values: [wall.groundY, wall.leftEnd, wall.rightEnd], + directions: [wall.along, wall.outward] + ) + for piece in wall.cornerSegments { + shape.points.append(piece.anchor) + shape.values += [piece.span.lowerBound, piece.span.upperBound, piece.anchorS] + shape.directions += [piece.along, piece.outward] + } + return shape + } + + /// Looks at the AR scene about ten times a second while "See it on your wall" is up, and + /// lets the screen's drawing step aside once the scene is seen drawing the result, for as + /// long as the scene holds it (`ResultOverlayPolicy`). On its own clock, not in `ingest`: it + /// has to notice a scene that stopped holding it whether frames arrive or not. + private func watchResultInCamera() { + resultWatch?.cancel() + resultWatch = Task { [weak self] in + while !Task.isCancelled { + guard let self else { return } + guard self.state.phase == .resultAR, let live = self.live else { + // Nothing left to watch: the screen draws the result (review of #100). + self.setResultInCamera(false) + return + } + let look = live.resultIsDrawn() + self.setResultInCamera(self.resultPolicy.update(drawn: look.drawn, held: look.held, time: self.screenTime)) + try? await Task.sleep(for: .milliseconds(100)) + } + } + } + + /// Which layer draws the result: the AR scene (true) or the screen's overlay (false). The + /// AR scene's model shows only while it owns the result (`LiveCapture.setResultOwned`), and + /// both change here together, so the two layers are never both on screen: not while a new + /// model waits to be seen, and not while tracking comes back and `ingest` enables the model + /// before the next look (review of #100). + private func setResultInCamera(_ usesRealityKit: Bool) { + live?.setResultOwned(usesRealityKit) + guard state.resultInCamera != usesRealityKit else { return } + RuntimeLog.engine.info("AR result drawn by \(usesRealityKit ? "the AR scene" : "the screen overlay", privacy: .public)") + state.resultInCamera = usesRealityKit + } + private func hideResultInCamera() { + resultWatch?.cancel() + resultWatch = nil + resultPolicy = ResultOverlayPolicy() live?.hideResult() state.resultInCamera = false } diff --git a/ios/HouseScan/UI/Camera/WayfindingOverlay.swift b/ios/HouseScan/UI/Camera/WayfindingOverlay.swift index 43d954f4e..55a2c2461 100644 --- a/ios/HouseScan/UI/Camera/WayfindingOverlay.swift +++ b/ios/HouseScan/UI/Camera/WayfindingOverlay.swift @@ -90,8 +90,9 @@ struct WayfindingOverlay: View { } } -/// The aim ring, or a chevron at the screen edge pointing toward it. -private struct TargetMarker: View { +/// The aim ring, or a chevron at the screen edge pointing toward it. Also the chevron toward +/// the battery spot on "See it on your wall" (`ResultARScreen`). +struct TargetMarker: View { enum Placement: Equatable { case onScreen(CGPoint, radius: CGFloat) case offScreen(CGPoint, angle: Angle) diff --git a/ios/HouseScan/UI/Components/CameraChrome.swift b/ios/HouseScan/UI/Components/CameraChrome.swift index 78c03b8b2..5e9172e1c 100644 --- a/ios/HouseScan/UI/Components/CameraChrome.swift +++ b/ios/HouseScan/UI/Components/CameraChrome.swift @@ -15,6 +15,9 @@ struct CameraChrome: View { /// Taps on the open camera area, in the camera view's coordinates (full screen, which is /// the window's global space). Nil when the screen has nothing to tap. var onCameraTap: ((CGPoint) -> Void)? + /// Where the card and the bottom controls cover the camera, measured as laid out at the + /// current text size. Nil when the screen doesn't need it. + var cover: Binding? @ViewBuilder var bottom: Bottom var body: some View { @@ -52,8 +55,10 @@ struct CameraChrome: View { } .frame(minHeight: 36) InstructionCard(instruction: instruction, tone: tone, reply: reply, eyebrow: eyebrow) + .onGeometryChange(for: CGFloat.self) { $0.frame(in: .global).maxY } action: { value in if let cover { cover.wrappedValue.cardBottom = value } } Spacer(minLength: 0) bottom + .onGeometryChange(for: CGFloat.self) { $0.frame(in: .global).minY } action: { value in if let cover { cover.wrappedValue.controlsTop = value } } } .padding(.horizontal, Metrics.edge) .padding(.top, 4) @@ -61,6 +66,16 @@ struct CameraChrome: View { } } +/// Where `CameraChrome` covers the camera, in the window's coordinates: the instruction card's +/// bottom edge and the bottom controls' top edge. The camera between them is what the homeowner +/// can see. Measured, not assumed: at the largest text sizes the card grows far past its usual +/// 220 pt (review of #100). Nil until laid out. Measured with `onGeometryChange` rather than a +/// preference key, whose change handler can't set view state under Swift 6's checks. +struct ChromeCover: Equatable { + var cardBottom: CGFloat? + var controlsTop: CGFloat? +} + /// Reads the full-screen camera view size so buttons can send "the reticle" (nil point) /// with the right `viewSize`. struct CameraSizeReader: View { diff --git a/ios/HouseScan/UI/Preview/DemoEngine.swift b/ios/HouseScan/UI/Preview/DemoEngine.swift index 367e2a1cc..50e2b1601 100644 --- a/ios/HouseScan/UI/Preview/DemoEngine.swift +++ b/ios/HouseScan/UI/Preview/DemoEngine.swift @@ -18,6 +18,8 @@ final class DemoEngine: ScanActions { /// through the engine's own mapping, in place of the hand-made samples. private let resultFile: String? private let rejectUpload: Bool + /// `-uiDemoSpotOffScreen`: on "See it on your wall", the phone aims away from the spot. + private let spotOffScreen: Bool /// Which request the gap screen shows (`-uiDemoGap`); the phone's ground request by default. private let gapKind: String? /// The tilt-up step was answered or skipped. @@ -73,6 +75,7 @@ final class DemoEngine: ScanActions { resultFile = nil #endif rejectUpload = arguments.contains("-uiDemoRejected") + spotOffScreen = arguments.contains("-uiDemoSpotOffScreen") gapKind = value("-uiDemoGap") state.feed = DemoScene.image.map(CameraFeed.still) ?? .none state.isReplay = true @@ -234,6 +237,7 @@ final class DemoEngine: ScanActions { placeMeter() finishedWalkState() showResult() + if spotOffScreen { state.projection = DemoScene.awayFromSpotProjection } state.phase = .resultAR case .unsupported: state.failure = .arUnsupported diff --git a/ios/HouseScan/UI/Preview/DemoScene.swift b/ios/HouseScan/UI/Preview/DemoScene.swift index fcea399cc..00e37eb84 100644 --- a/ios/HouseScan/UI/Preview/DemoScene.swift +++ b/ios/HouseScan/UI/Preview/DemoScene.swift @@ -26,6 +26,9 @@ enum DemoScene { static let projection = camera(at: SIMD3(0.6, 1.5, 5.0), pitchDegrees: 18, focal: 1100) /// Standing 2.4 m back and aiming at the meter: finding it. static let meterProjection = camera(at: SIMD3(0.05, 1.5, 2.4), pitchDegrees: 0, focal: 1400) + /// 2.2 m left of the meter and 1.5 m back, aiming straight at the wall: the battery spot and + /// the meter are both well off to the right (`-uiDemoSpotOffScreen`). + static let awayFromSpotProjection = camera(at: SIMD3(-2.2, 1.5, 1.5), pitchDegrees: 0, focal: 1400) /// Close to the meter for its photo. static let closeUpProjection = camera(at: SIMD3(0, 1.5, 0.9), pitchDegrees: 0, focal: 1400) diff --git a/ios/HouseScan/UI/Preview/UIDemo.swift b/ios/HouseScan/UI/Preview/UIDemo.swift index b541f6422..c904cd8ad 100644 --- a/ios/HouseScan/UI/Preview/UIDemo.swift +++ b/ios/HouseScan/UI/Preview/UIDemo.swift @@ -16,6 +16,8 @@ import SwiftUI /// unsupported screen. /// - `-uiDemoPass`: the sample result is a pass with approved rules. /// - `-uiDemoOverlap`: the sample result's spot overlaps the meter's working space. +/// - `-uiDemoSpotOffScreen`: with `-uiDemoPhase resultAR`, the phone aims away from the spot, so +/// the edge chevron points to it. /// - `-uiDemoResultFile `: debug builds only. The result is the server answer in this JSON /// file, mapped as the engine maps one; the UI tests keep such files in `Fixtures/results/`. /// - `-uiDemoNoFeed`: no camera picture, to look at the chrome alone. diff --git a/ios/HouseScan/UI/Screens/ResultARScreen.swift b/ios/HouseScan/UI/Screens/ResultARScreen.swift index d0cd595e3..8929df6aa 100644 --- a/ios/HouseScan/UI/Screens/ResultARScreen.swift +++ b/ios/HouseScan/UI/Screens/ResultARScreen.swift @@ -2,31 +2,61 @@ import simd import SwiftUI /// "See it on your wall": the battery drawn onto the live camera at the chosen spot, with the -/// cable run from the meter and the clearance footprint tinted by outcome. On the live camera the -/// engine draws it into the AR scene (`state.resultInCamera`); over a replay this screen projects -/// it from the meter-anchored wall frame. Either way it stays put as the homeowner moves. +/// cable run from the meter and the clearance footprint tinted by outcome. This screen projects +/// it from the meter-anchored wall frame (`BatteryOverlay`) unless the engine has seen the AR +/// scene drawing it (`state.resultInCamera`). Either way it stays put as the homeowner moves. +/// While the spot can't be seen, a chevron at the edge points toward it and a caption above Done +/// says which way. struct ResultARScreen: View { let state: ScanViewState let actions: any ScanActions @State private var appeared = false + /// Where the card and the Done button cover the camera (`CameraChrome`), for the chevron. + @State private var cover = ChromeCover() + /// Which way the spot is while it can't be seen, in eighths of a turn from "right", + /// clockwise (`SpotDirection`); nil while it is in view. + @State private var spotOctant: Int? @Environment(\.accessibilityReduceMotion) private var reduceMotion var body: some View { ZStack { - if !state.resultInCamera, state.tracking == .normal, let projection = state.projection, let wall = state.wall, let result = state.result { - BatteryOverlay(projection: projection, wall: wall, result: result, rise: appeared ? 1 : 0) - .ignoresSafeArea() - .accessibilityHidden(true) + if state.tracking == .normal, let projection = state.projection, let wall = state.wall, let result = state.result { + if state.resultInCamera { + // The AR scene draws the result in the camera view; this only names it for + // VoiceOver and the UI tests, as the overlay below does. + Color.clear + .allowsHitTesting(false) + .accessibilityElement() + .accessibilityLabel(Self.overlayLabel(result)) + .accessibilityAddTraits(.isImage) + .accessibilityIdentifier("ar.overlay") + } else { + BatteryOverlay(projection: projection, wall: wall, result: result, rise: appeared ? 1 : 0) + .ignoresSafeArea() + .accessibilityElement() + .accessibilityLabel(Self.overlayLabel(result)) + .accessibilityAddTraits(.isImage) + .accessibilityIdentifier("ar.overlay") + } + if let spot = result.spotCenter(on: wall) { + SpotDirection(projection: projection, spot: spot, cover: cover, octant: $spotOctant) + } } CameraChrome( instruction: instruction, isReplay: state.isReplay, - isAutopilot: state.isAutopilot + isAutopilot: state.isAutopilot, + cover: $cover ) { - Button("Done") { actions.closeAR() } - .buttonStyle(.primary) - .accessibilityIdentifier("action.closeAR") + VStack(spacing: 10) { + if state.tracking == .normal, let spotOctant { + SpotDirectionCaption(octant: spotOctant) + } + Button("Done") { actions.closeAR() } + .buttonStyle(.primary) + .accessibilityIdentifier("action.closeAR") + } } } .onAppear { @@ -34,6 +64,10 @@ struct ResultARScreen: View { } } + private static func overlayLabel(_ result: ResultPresentation) -> String { + result.spot == nil ? "The cable run and clearances, drawn on your wall" : "The battery, drawn on your wall at its spot" + } + private var instruction: Instruction { if state.tracking != .normal { return Instruction(title: "Point at your meter", detail: "The battery comes back once your phone finds its place.") @@ -55,6 +89,117 @@ struct ResultARScreen: View { } } +extension ResultPresentation { + /// The middle of the battery on `wall`, in world meters; nil without a spot. The point + /// "See it on your wall" has to get on screen: the edge chevron points to it while it is off + /// screen, and the engine checks the AR scene puts it in view (`LiveCapture.resultIsDrawn`). + func spotCenter(on wall: WallGeometry) -> SIMD3? { + guard let spot else { return nil } + let middle = (spot.span.lowerBound + spot.span.upperBound) / 2 + return wall.world(s: middle, height: spot.height / 2, out: spot.offsetFromWall + spot.depth / 2) + } +} + +/// While the battery spot can't be seen, an edge chevron toward it in the clear part of the +/// camera, and which way it is (`octant`) for the caption above the Done button +/// (`SpotDirectionCaption`). The caption sits in the chrome's own stack rather than over the +/// camera: at the largest text sizes the card and the button leave no camera clear, and a +/// caption drawn there went under them (review of #100). The chevron shows only where it fits. +private struct SpotDirection: View { + var projection: CameraProjection + var spot: SIMD3 + /// Where the card and the Done button cover the camera, as `CameraChrome` measured them. + var cover: ChromeCover + /// Which way the spot is, in eighths of a turn clockwise from "right"; nil while in view. + @Binding var octant: Int? + + /// Half the chevron's disc (52 pt) and a gap. + private static let chevronReach: CGFloat = 34 + + var body: some View { + GeometryReader { proxy in + let size = proxy.size + let clear = clearArea(in: size) + let chevron = chevronPlacement(in: size, clear: clear) + ZStack { + Color.clear + if let chevron, clear.height >= 2 * Self.chevronReach { + TargetMarker(placement: .offScreen(chevron.point, angle: chevron.angle)) + .accessibilityHidden(true) + } + } + .frame(width: size.width, height: size.height) + .onChange(of: chevron.map { Self.octant($0.angle) }, initial: true) { _, new in octant = new } + } + .ignoresSafeArea() + .allowsHitTesting(false) + .onDisappear { octant = nil } + } + + /// `angle` (screen axes, y down) in eighths of a turn clockwise from "right", 0...7. + private static func octant(_ angle: Angle) -> Int { + let eighths = Int((angle.radians / (.pi / 4)).rounded()) + return ((eighths % 8) + 8) % 8 + } + + /// The part of the screen the chrome leaves clear, as `CameraChrome` measured it: between the + /// instruction card's bottom edge and the Done button's top edge. At the largest text sizes + /// the card reaches far further down than at the default size, so fixed insets hid the + /// chevron while the spot was under the card (review of #100). Before the first layout, + /// the default-size card (about 220 pt with a two-line detail) and button (about 100 pt up). + /// Empty when the chrome covers it all. + private func clearArea(in size: CGSize) -> CGRect { + let top = (cover.cardBottom ?? 252) + 8 + let bottom = (cover.controlsTop ?? size.height - 152) - 8 + return CGRect(x: 24, y: top, width: max(size.width - 48, 0), height: max(bottom - top, 0)) + } + + /// Where the chevron goes and which way it points, or nil while the spot is in view: in the + /// clear area. A spot under the card or the Done button can't be seen, so it gets the + /// chevron too. The chevron keeps to the clear area, toward the spot's side of it. + private func chevronPlacement(in size: CGSize, clear: CGRect) -> (point: CGPoint, angle: Angle)? { + if let point = projection.viewPoint(for: spot, in: size), clear.contains(point) { return nil } + guard let direction = projection.screenDirection(toward: spot) else { return nil } + let lane = clear.insetBy(dx: 16, dy: min(Self.chevronReach, clear.height / 2)) + let tx = direction.dx == 0 ? CGFloat.infinity : lane.width / 2 / abs(direction.dx) + let ty = direction.dy == 0 ? CGFloat.infinity : lane.height / 2 / abs(direction.dy) + let t = min(tx, ty) + let point = CGPoint(x: lane.midX + direction.dx * t, y: lane.midY + direction.dy * t) + return (point, .radians(atan2(direction.dy, direction.dx))) + } +} + +/// "Your battery spot is this way" with an arrow toward it, above the Done button while the +/// spot can't be seen (`SpotDirection`). VoiceOver hears which way. +private struct SpotDirectionCaption: View { + /// Eighths of a turn clockwise from "right", 0...7. + var octant: Int + + private static let words = ["to the right", "down and to the right", "down", "down and to the left", + "to the left", "up and to the left", "up", "up and to the right"] + + var body: some View { + HStack(spacing: 10) { + Image(systemName: "arrow.right") + .font(.body.weight(.heavy)) + .rotationEffect(.degrees(Double(octant) * 45)) + Text("Your battery spot is this way") + .font(.subheadline.weight(.semibold)) + .multilineTextAlignment(.leading) + .fixedSize(horizontal: false, vertical: true) + } + .foregroundStyle(Palette.chalk) + .padding(.horizontal, 16) + .padding(.vertical, 8) + // Solid, as the other camera captions: see-through black over a bright wall can fail + // the accessibility audit's contrast check. + .background(ScrimShape.rounded(20)) + .accessibilityElement(children: .ignore) + .accessibilityLabel("Your battery spot is off screen, \(Self.words[octant % 8]). Turn the phone that way.") + .accessibilityIdentifier("ar.spotDirection") + } +} + /// Canvas drawing of the battery box, cable and footprint. `rise` 0...1 lifts the box out of the /// ground for the entrance. struct BatteryOverlay: View, Animatable { diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultOverlayPolicy.swift b/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultOverlayPolicy.swift new file mode 100644 index 000000000..3aa329109 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Scene/ResultOverlayPolicy.swift @@ -0,0 +1,143 @@ +import Foundation +import simd + +/// Which layer draws the result on "See it on your wall": the AR scene (RealityKit) or the +/// screen's own drawing over the camera (the Canvas). Exactly one of them draws it at a time. +/// +/// The Canvas is the default. The AR scene takes over only once the app has seen it drawing the +/// result for `confirmation` seconds without a break. It keeps it while it holds the result +/// (anchored and enabled), whether or not the battery is in view, and gives it back at once when +/// it stops holding it. Build 4.1 hid the Canvas as soon as the model was handed to the AR scene; +/// on a phone where the scene then drew nothing, the homeowner saw no battery at all. +/// +/// A confirmation covers one model. The policy also decides when a model has to be built +/// (`needsModel`), and every new model starts back on the Canvas: a model rebuilt for a moved +/// wall is a new entity nobody has seen drawn yet. +/// +/// The engine shows only the owner: the Canvas while `usesRealityKit` is false, the AR scene's +/// model while it is true. The model is made see-through rather than disabled while the Canvas +/// owns the result, so the checks behind `drawn` and `held` still run on it +/// (`ScanEngine.setResultInCamera`). +public struct ResultOverlayPolicy: Sendable, Equatable { + /// How long the AR scene has to be seen drawing before the Canvas steps aside: long enough + /// to ride out a frame or two of an anchor settling. A guess, not measured on a phone. + public static let confirmation: Double = 0.5 + + /// True while the AR scene draws the result and the Canvas should not. + public private(set) var usesRealityKit = false + /// The wall the AR scene's model was built for; nil when there is none. + public private(set) var builtFor: ResultModelShape? + /// The AR scene has been seen drawing this model for `confirmation` seconds. + private var confirmed = false + /// When the current unbroken run of "drawn" began; nil while the AR scene isn't drawing. + private var drawnSince: Double? + + public init() {} + + /// Takes one look at the AR scene at `time` in seconds (any clock that only goes forward) and + /// returns `usesRealityKit`. + /// + /// - `drawn`: the AR scene would draw the result now, with the battery's middle in view. + /// Only this confirms the model. + /// - `held`: the AR scene holds the result, drawn or not: anchored and enabled, with the + /// battery perhaps out of view. Once the model is confirmed it keeps the result while + /// `held`, and gets it back as soon as it holds it again (tracking came back). Going back to + /// the Canvas whenever the battery left the view put a second, unoccluded cable and tint a + /// few centimetres off RealityKit's own, with the phone on the meter (review of #100). + /// `drawn` implies `held`. + @discardableResult + public mutating func update(drawn: Bool, held: Bool, time: Double) -> Bool { + let held = held || drawn + guard held else { + drawnSince = nil + usesRealityKit = false + return false + } + if confirmed { + usesRealityKit = true + return true + } + guard drawn else { + drawnSince = nil + return false + } + // A clock that ran back restarts the run rather than confirming it early. + if let since = drawnSince, since <= time { + if time - since >= Self.confirmation { + confirmed = true + usesRealityKit = true + } + } else { + drawnSince = time + } + return usesRealityKit + } + + /// Whether a model for a wall of `shape` has to go into the AR scene: there is none yet, + /// `rising` (the screen just opened), or the wall moved or turned past the tolerances since + /// the last one was built (`ResultModelShape.isClose`). A wall that moved by a centimeter or + /// two keeps its model, which hangs on the meter's anchor and follows its corrections anyway. + /// + /// When it has to, the result goes back to the Canvas at once, and the new model has to be + /// seen drawing for `confirmation` seconds before the Canvas steps aside again. Kept + /// confirmed, a rebuilt model that looked anchored and in view at the next look took over at + /// once, and if it drew nothing the homeowner saw no battery (Codex review of #100). The + /// caller then puts the model in, or calls `modelRemoved` if it couldn't. + public mutating func needsModel(for shape: ResultModelShape, rising: Bool) -> Bool { + if !rising, let builtFor, builtFor.isClose(to: shape) { return false } + builtFor = shape + backToTheCanvas() + return true + } + + /// There is no model in the AR scene any more: it couldn't go in, or it was taken out. + public mutating func modelRemoved() { + builtFor = nil + backToTheCanvas() + } + + private mutating func backToTheCanvas() { + confirmed = false + drawnSince = nil + usesRealityKit = false + } +} + +/// What the AR result's model is built from, reduced to what `ResultOverlayPolicy.needsModel` +/// compares: points (the meter, corner anchors), lengths (the ground's height, the ends, corner +/// spans) and directions (along the wall, out from it). The engine fills it from its wall in a +/// fixed order, so two shapes line up entry by entry. +public struct ResultModelShape: Sendable, Equatable { + public var points: [SIMD3] + public var values: [Float?] + public var directions: [SIMD3] + + public init(points: [SIMD3], values: [Float?], directions: [SIMD3]) { + self.points = points + self.values = values + self.directions = directions + } + + /// A wall change under these leaves the model as built: a few centimeters is within what the + /// ground and the meter's anchor are known to. Display choices, not measured. + public static let rebuildDistance: Float = 0.05 + public static let rebuildTurnDegrees: Float = 2 + + /// True when `other` is this shape moved by less than `rebuildDistance` and turned by less + /// than `rebuildTurnDegrees` in every entry, with the same number of each. + public func isClose(to other: ResultModelShape) -> Bool { + guard points.count == other.points.count, values.count == other.values.count, + directions.count == other.directions.count else { return false } + let turnCosine = cos(Self.rebuildTurnDegrees * .pi / 180) + let pointsClose = zip(points, other.points).allSatisfy { simd_distance($0, $1) < Self.rebuildDistance } + let valuesClose = zip(values, other.values).allSatisfy { pair in + guard let a = pair.0, let b = pair.1 else { return pair.0 == nil && pair.1 == nil } + // Equal first: a corner piece's span is infinite toward its open end. + return a == b || abs(a - b) < Self.rebuildDistance + } + let directionsClose = zip(directions, other.directions).allSatisfy { + simd_dot(simd_normalize($0), simd_normalize($1)) > turnCosine + } + return pointsClose && valuesClose && directionsClose + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/ResultOverlayPolicyTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/ResultOverlayPolicyTests.swift new file mode 100644 index 000000000..f2cac2c21 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/ResultOverlayPolicyTests.swift @@ -0,0 +1,213 @@ +import Foundation +import HouseScanKit +import Testing + +/// #67: "See it on your wall" keeps the Canvas drawing until the AR scene has been seen drawing +/// the result for half a second, and falls back to it the moment the AR scene stops holding it. +/// Review of #100: exactly one layer draws it, so the battery leaving the view doesn't bring the +/// Canvas back over RealityKit's copy, and a rebuilt model is confirmed afresh. +@Suite struct ResultOverlayPolicyTests { + /// One look at the AR scene: `drawn` (held, with the battery's middle in view), `held` + /// (anchored and enabled) and the time. + private typealias Look = (drawn: Bool, held: Bool, time: Double) + + /// Feeds `looks` to `policy` in order and returns each answer (true: the AR scene draws the + /// result, false: the Canvas does). + private static func answers(_ looks: [Look], to policy: inout ResultOverlayPolicy) -> [Bool] { + looks.map { policy.update(drawn: $0.drawn, held: $0.held, time: $0.time) } + } + + private static func answers(_ looks: [Look]) -> [Bool] { + var policy = ResultOverlayPolicy() + return answers(looks, to: &policy) + } + + /// Looks every 0.1 s from `start`, `count` times, as the engine does. + private static func polled(drawn: Bool, held: Bool, from start: Double, count: Int) -> [Look] { + (0.. = .zero, groundY: Float = -1.2, rightEnd: Float? = 3, + along: SIMD3 = SIMD3(1, 0, 0)) -> ResultModelShape { + ResultModelShape(points: [meter], values: [groundY, -2, rightEnd], directions: [along, SIMD3(0, 0, 1)]) + } + + /// The policy as the engine has it after "See it on your wall" opened and the AR scene was + /// seen drawing the first model. + private static func confirmedPolicy() -> ResultOverlayPolicy { + var policy = ResultOverlayPolicy() + let answer1 = policy.needsModel(for: shape(), rising: true) + #expect(answer1) + let answers = Self.answers(Self.confirming, to: &policy) + #expect(answers.last == true) + return policy + } + + @Test func theFirstModelIsBuiltAndStartsOnTheCanvas() { + var policy = ResultOverlayPolicy() + let answer2 = policy.needsModel(for: Self.shape(), rising: false) + #expect(answer2) + #expect(policy.builtFor == Self.shape()) + #expect(!policy.usesRealityKit) + } + + @Test func aWallThatMovedALittleKeepsItsConfirmedModel() { + var policy = Self.confirmedPolicy() + // The ground refined by 2 cm, the meter's anchor by 3 cm, the wall turned 1°. + let turned = SIMD3(cos(Float.pi / 180), 0, sin(Float.pi / 180)) + let moved = Self.shape(meter: SIMD3(0.03, 0, 0), groundY: -1.22, along: turned) + let answer3 = policy.needsModel(for: moved, rising: false) + #expect(!answer3) + #expect(policy.usesRealityKit) + let answer4 = policy.update(drawn: false, held: true, time: 0.8) + #expect(answer4) + } + + /// Codex and Sam on #100: a model rebuilt for a moved wall kept the old one's confirmation, + /// so a replacement that looked anchored and in view at the next look hid the Canvas at once. + @Test func aRebuiltModelGoesBackToTheCanvasAndIsConfirmedAgain() { + var policy = Self.confirmedPolicy() + // The ground moved 10 cm: a new model. + let answer5 = policy.needsModel(for: Self.shape(groundY: -1.3), rising: false) + #expect(answer5) + #expect(!policy.usesRealityKit) + + // The new model looks anchored and in view at the very next look. That alone isn't + // enough: it has to be seen for the half second again. + let replacement = Self.answers(Self.polled(drawn: true, held: true, from: 0.7, count: 6), to: &policy) + #expect(replacement == [false, false, false, false, false, true]) + } + + /// Sam on #100: a model rebuilt while the battery is out of view is held, not drawn, so it + /// stays behind the Canvas until the battery comes into view. The engine keeps it + /// see-through meanwhile, so only the Canvas shows. + @Test func aModelRebuiltOutOfViewStaysOnTheCanvasUntilSeen() { + var policy = Self.confirmedPolicy() + let answer6 = policy.needsModel(for: Self.shape(rightEnd: 3.5), rising: false) + #expect(answer6) + let heldOnly = Self.answers(Self.polled(drawn: false, held: true, from: 0.7, count: 30), to: &policy) + #expect(heldOnly.allSatisfy { !$0 }) + let inView = Self.answers(Self.polled(drawn: true, held: true, from: 3.7, count: 6), to: &policy) + #expect(inView == [false, false, false, false, false, true]) + } + + @Test func turningPastTwoDegreesOrAnotherPieceRebuilds() { + var policy = Self.confirmedPolicy() + let turned = SIMD3(cos(3 * Float.pi / 180), 0, sin(3 * Float.pi / 180)) + let answer7 = policy.needsModel(for: Self.shape(along: turned), rising: false) + #expect(answer7) + #expect(!policy.usesRealityKit) + + var cornered = Self.confirmedPolicy() + var withCorner = Self.shape() + withCorner.points.append(SIMD3(3, 0, 0)) + let answer8 = cornered.needsModel(for: withCorner, rising: false) + #expect(answer8) + } + + @Test func reopeningTheScreenAlwaysRebuilds() { + var policy = Self.confirmedPolicy() + let answer9 = policy.needsModel(for: Self.shape(), rising: true) + #expect(answer9) + #expect(!policy.usesRealityKit) + } + + @Test func aModelThatCouldNotGoInLeavesNone() { + var policy = Self.confirmedPolicy() + policy.modelRemoved() + #expect(policy.builtFor == nil) + #expect(!policy.usesRealityKit) + // The next wall update builds one, however little the wall moved. + let answer10 = policy.needsModel(for: Self.shape(), rising: false) + #expect(answer10) + } + + @Test func anOpenEndCountsAsTheSameOnlyIfBothAreOpen() { + #expect(Self.shape(rightEnd: nil).isClose(to: Self.shape(rightEnd: nil))) + #expect(!Self.shape(rightEnd: nil).isClose(to: Self.shape(rightEnd: 3))) + let open = ResultModelShape(points: [], values: [Float.infinity], directions: []) + #expect(open.isClose(to: open)) + } + + @Test func usesRealityKitMatchesTheLastAnswer() { + var policy = ResultOverlayPolicy() + _ = Self.answers(Self.polled(drawn: true, held: true, from: 0, count: 6), to: &policy) + let confirmed = policy.usesRealityKit + policy.update(drawn: false, held: true, time: 0.6) + let offScreen = policy.usesRealityKit + policy.update(drawn: false, held: false, time: 0.7) + let afterLoss = policy.usesRealityKit + #expect(confirmed) + #expect(offScreen) + #expect(!afterLoss) + } +} diff --git a/ios/HouseScanUITests/ScreenStatesUITests.swift b/ios/HouseScanUITests/ScreenStatesUITests.swift index eb9c1d546..7934e17c6 100644 --- a/ios/HouseScanUITests/ScreenStatesUITests.swift +++ b/ios/HouseScanUITests/ScreenStatesUITests.swift @@ -56,6 +56,7 @@ final class ScreenStatesUITests: XCTestCase { ("result-overlap", ["-uiDemoPhase", "result", "-uiDemoOverlap"], "result"), ("result-reject", ["-uiDemoPhase", "result", "-uiDemoResultFile", resultFile("reject-nearest")], "result"), ("resultAR", ["-uiDemoPhase", "resultAR"], "resultAR"), + ("resultAR-spotOffScreen", ["-uiDemoPhase", "resultAR", "-uiDemoSpotOffScreen"], "resultAR"), ("cameraDenied", ["-uiDemoFailure", "cameraDenied"], "unsupported"), ("arUnsupported", ["-uiDemoFailure", "arUnsupported"], "unsupported"), ("sessionFailed", ["-uiDemoFailure", "sessionFailed"], "unsupported"), @@ -74,6 +75,7 @@ final class ScreenStatesUITests: XCTestCase { "wallWalk-hidden", "wallWalk-seeBehind", "gapRequest-followUp", "uploading-followUp", "markFeatures-lostPlace", "spotConfirm", "spotConfirm-which", "spotConfirm-ground", "spotConfirm-answered", "spotConfirm-cantMark", "spotConfirm-unconfirmable", + "resultAR-spotOffScreen", ] /// Words a state must show: in the named element's label or value, or with no identifier, @@ -97,6 +99,11 @@ final class ScreenStatesUITests: XCTestCase { "result-review": ("result.headline", "One more look"), // A reject names the closest spot and the check it fails. "result-reject": ("result.nearest", "The closest spot"), + // #67: the screen draws the result itself unless the AR scene is seen drawing it, and + // the demo has no AR scene. + "resultAR": ("ar.overlay", "drawn on your wall"), + // Review of #100: the chevron toward a spot out of view, audited at AX5 too. + "resultAR-spotOffScreen": ("ar.spotDirection", "Your battery spot is off screen"), ] /// States where the scan is packaged, so "Share scan" must show.