diff --git a/ios/HouseScan/Contract/ScanContract.swift b/ios/HouseScan/Contract/ScanContract.swift index 4330f2fb8..51d5258df 100644 --- a/ios/HouseScan/Contract/ScanContract.swift +++ b/ios/HouseScan/Contract/ScanContract.swift @@ -272,6 +272,31 @@ struct CoverageStrip: Equatable, Sendable { } } +/// LiDAR phones: dots on the surfaces depth has measured, drawn over the camera by the live fog. +/// Guidance, never evidence: `coverage` alone decides what the scan has seen, and the fog lifts +/// only where it says so. Built from the same kept keyframes as coverage (`LiveDotsFeed`). +struct LiveDots: Sendable { + struct Dot: Sendable, Equatable { + /// Stable while the dot exists, so the renderer can tell a birth from a survivor. + var id: UInt64 + var position: SIMD3 + /// On a crease or a silhouette; drawn larger, with a stronger glow. + var isEdge: Bool + /// On something standing in front of the wall (a bin, a bush): drawn violet, the hidden + /// state's colour. + var onOccluder: Bool + /// 0.45 seen from one direction, up to 0.9 from four. + var opacity: Float + } + + /// At most `SurfaceDotConfig.maxDots` (5,000), those within 6 m of the latest keyframe. + var dots: [Dot] = [] + /// Increments on every change, so the renderer rebuilds its buffer only then. + var revision = 0 + + static let empty = LiveDots() +} + // MARK: - Guidance enum WallSide: String, Equatable, Sendable { @@ -698,6 +723,8 @@ final class ScanViewState { var wall: WallGeometry? var coverage: CoverageStrip = .empty + /// LiDAR phones: the live dots. Empty without depth. + var liveDots: LiveDots = .empty var closeUp: CloseUpState = .aiming(hold: 0, problem: nil) /// Close-up attempts that ended without a usable photo. "Can't get a clear shot" appears /// from the second one on, never earlier. diff --git a/ios/HouseScan/HouseScanApp.swift b/ios/HouseScan/HouseScanApp.swift index 013851423..a136ba964 100644 --- a/ios/HouseScan/HouseScanApp.swift +++ b/ios/HouseScan/HouseScanApp.swift @@ -4,11 +4,15 @@ import SwiftUI struct HouseScanApp: App { var body: some Scene { WindowGroup { - if ProcessInfo.processInfo.arguments.contains("-uiDemo") { - UIDemo.makeRoot() - } else { - AppRuntime.makeRoot() + Group { + if ProcessInfo.processInfo.arguments.contains("-uiDemo") { + UIDemo.makeRoot() + } else { + AppRuntime.makeRoot() + } } + // Compiles the live fog's shaders in the background long before the walk needs them. + .task { LiveFogSupport.shared.prepare() } } } } diff --git a/ios/HouseScan/Runtime/LiveDotsFeed.swift b/ios/HouseScan/Runtime/LiveDotsFeed.swift new file mode 100644 index 000000000..ea3a35f9d --- /dev/null +++ b/ios/HouseScan/Runtime/LiveDotsFeed.swift @@ -0,0 +1,81 @@ +import Foundation +import HouseScanKit +import os + +/// Runs the LiDAR dot field (`SurfaceDots`) off the main actor and publishes its dots for the +/// camera overlay (`ScanViewState.liveDots`). +/// +/// The engine hands it each kept keyframe that has depth, the same frames coverage observes, so +/// the dots and the fog move together. Fusing a 256 x 192 map takes a few milliseconds (3.4 ms +/// median on an M4 Pro in a Release build; not measured on a phone), so it runs on its own +/// serial queue. A keyframe arriving while two are still waiting is dropped: the next one covers +/// the same wall. An anchor correction is a matrix product (`SurfaceDots.apply`) and republishes +/// only when no keyframe is waiting to publish anyway, so neither kind of work can pile up. +final class LiveDotsFeed: @unchecked Sendable { + // `@unchecked Sendable`: everything below `queue` is read and written only on it; `pending` + // is a lock. + private let queue = DispatchQueue(label: "dev.housescanning.housescan.live-dots", qos: .utility) + private var field = SurfaceDots() + private var revision = 0 + private var last: (eye: SIMD3, wall: WallFrame)? + private let pending = OSAllocatedUnfairLock(initialState: 0) + private let publish: @MainActor @Sendable (LiveDots, _ generation: Int) -> Void + + /// Keyframes waiting on the queue past which new ones are dropped. + static let maxPending = 2 + + /// `publish` runs on the main actor with the scan generation the work was queued under, so + /// the engine can ignore dots from before a Start over. + init(publish: @escaping @MainActor @Sendable (LiveDots, _ generation: Int) -> Void) { + self.publish = publish + } + + func integrate(camera: CameraFrame, depth: DepthImage, wall: WallFrame, generation: Int) { + let accepted = pending.withLock { count -> Bool in + guard count < Self.maxPending else { return false } + count += 1 + return true + } + guard accepted else { + RuntimeLog.capture.info("live dots: dropped a keyframe, two still waiting") + return + } + queue.async { [self] in + field.integrate(camera: camera, depth: depth) + last = (camera.position, wall) + pending.withLock { $0 -= 1 } + send(generation) + } + } + + /// ARKit's correction to the meter's anchor, applied as coverage applies it. + func apply(_ correction: YawCorrection, wall: WallFrame, generation: Int) { + queue.async { [self] in + field.apply(correction) + if let eye = last?.eye { last = (correction.point(eye), wall) } + if pending.withLock({ $0 }) == 0 { send(generation) } + } + } + + func reset() { + queue.async { [self] in + field = SurfaceDots() + last = nil + } + } + + /// On `queue`. + private func send(_ generation: Int) { + guard let last else { return } + revision += 1 + let dots = LiveDots( + dots: field.dots(near: last.eye, wall: last.wall).map { + LiveDots.Dot(id: $0.id, position: $0.position, isEdge: $0.isEdge, onOccluder: $0.onOccluder, opacity: $0.opacity) + }, + revision: revision) + let publish = publish + DispatchQueue.main.async { + MainActor.assumeIsolated { publish(dots, generation) } + } + } +} diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index 609f468cd..add53477d 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -138,6 +138,12 @@ final class ScanEngine { /// The packet's sensor streams for the current world frame. private(set) var recorder: CaptureRecorder private let motion = MotionSource() + /// LiDAR phones: the dots over the camera, fused off the main actor from the keyframes + /// coverage observes. Guidance for the overlay only; nothing reads them back. + private lazy var liveDots = LiveDotsFeed { [weak self] dots, scan in + guard let self, scan == self.generation else { return } + self.state.liveDots = dots + } /// Every request the homeowner was shown, for the packet. var guidanceLog = GuidanceLog() /// The spot check (`ScanEngine+Confirm.swift`). @@ -394,6 +400,7 @@ final class ScanEngine { guard outcome.correction != nil || outcome.groundChanged else { return } coverage = map if let correction = outcome.correction { + liveDots.apply(correction, wall: map.wall, generation: generation) var features = state.features for index in features.indices { features[index].points = features[index].points.map(correction.point) } if features != state.features { state.features = features } @@ -669,7 +676,11 @@ final class ScanEngine { // The frame's own time lets the walked path join only poses kept close together in time. // With LiDAR depth, a cell counts only where depth confirms the camera saw it. // Corrected for anchor moves made while the photo was saving. - let delta = coverage?.observe(correctedCamera(frame), trackingNormal: frame.tracking == .normal, time: frame.timestamp, depth: frame.depth, segment: segment) + let camera = correctedCamera(frame) + let delta = coverage?.observe(camera, trackingNormal: frame.tracking == .normal, time: frame.timestamp, depth: frame.depth, segment: segment) + if let depth = frame.depth, frame.tracking == .normal, let wall = coverage?.wall { + liveDots.integrate(camera: camera, depth: depth, wall: wall, generation: scan) + } RuntimeLog.capture.info("stored \(frame.id, privacy: .public) as keyframe \(index)\(frame.depth == nil ? "" : " with depth", privacy: .public): \(delta?.newlySeen ?? 0) cells newly seen, \(delta?.newlyCovered ?? 0) newly covered, \(delta?.newlyHidden ?? 0) newly hidden") if overhead { recordOverhead(frame) } state.captureCount += 1 @@ -1584,6 +1595,8 @@ final class ScanEngine { uploadTask?.cancel() replay?.stop() coverage = nil + liveDots.reset() + state.liveDots = .empty meterAnchorID.map { live?.removeAnchor($0) } meterAnchorID = nil meterPlaneSource = .detectedPlane diff --git a/ios/HouseScan/UI/Camera/CameraOverlays.swift b/ios/HouseScan/UI/Camera/CameraOverlays.swift index da1916cee..929c02eaa 100644 --- a/ios/HouseScan/UI/Camera/CameraOverlays.swift +++ b/ios/HouseScan/UI/Camera/CameraOverlays.swift @@ -5,6 +5,9 @@ struct CameraOverlays: View { let state: ScanViewState var highlight: GapRequest? + @Environment(\.accessibilityReduceMotion) private var reduceMotion + @Environment(\.self) private var environment + var body: some View { ZStack { overlays @@ -14,11 +17,11 @@ struct CameraOverlays: View { @ViewBuilder private var overlays: some View { - // Hidden while tracking isn't normal: drawn from a pose the phone doesn't trust, the haze, - // path and pins would sit in the wrong place (checklist T3). They fade back on recovery. + // Hidden while tracking isn't normal: drawn from a pose the phone doesn't trust, the fog, + // path and marks would sit in the wrong place (checklist T3). They fade back on recovery. if state.tracking == .normal, let projection = state.projection, let wall = state.wall { ZStack { - FogOverlay(coverage: state.coverage, wall: wall, projection: projection, highlight: highlight) + fog(projection: projection, wall: wall) .ignoresSafeArea() WallMarksOverlay( projection: projection, @@ -34,4 +37,28 @@ struct CameraOverlays: View { } } } + + /// The live fog, with dots when the phone has depth. The frosted strip stands in while the + /// shaders compile at launch, and for good if Metal fails, so unseen wall never shows clear. + @ViewBuilder + private func fog(projection: CameraProjection, wall: WallGeometry) -> some View { + switch LiveFogSupport.shared.status { + case let .ready(gpu): + ZStack { + LiveFogView(gpu: gpu, input: LiveFogInput( + projection: projection, wall: wall, coverage: state.coverage, highlight: highlight, + dots: state.depthAvailable ? state.liveDots : .empty, reduceMotion: reduceMotion, + requestedColor: requestedColor)) + FogMarks(coverage: state.coverage, wall: wall, projection: projection, highlight: highlight) + } + .transition(.opacity) + case .preparing, .failed: + FogOverlay(coverage: state.coverage, wall: wall, projection: projection, highlight: highlight) + } + } + + private var requestedColor: SIMD3 { + let resolved = Palette.caution.resolve(in: environment) + return SIMD3(resolved.red, resolved.green, resolved.blue) + } } diff --git a/ios/HouseScan/UI/Camera/FogMarks.swift b/ios/HouseScan/UI/Camera/FogMarks.swift new file mode 100644 index 000000000..8b377feff --- /dev/null +++ b/ios/HouseScan/UI/Camera/FogMarks.swift @@ -0,0 +1,113 @@ +import SwiftUI +import simd + +/// The crisp lines over the live fog, which a soft mask can't draw: a dashed violet outline +/// round each stretch hidden behind something, a slate hatch where the homeowner said "I can't +/// get there", and an amber outline round the stretch a gap request asks for. The same marks +/// and dash the tape map uses, so the camera and the map read alike. +struct FogMarks: View { + var coverage: CoverageStrip + var wall: WallGeometry + var projection: CameraProjection + var highlight: GapRequest? + + var body: some View { + Canvas(rendersAsynchronously: false) { context, size in + let geometry = WallProjection(projection: projection, wall: wall, size: size) + let runs = runs() + // The map's dash, lengthened for the larger shapes on the camera. + let dash = StrokeStyle(lineWidth: 2, lineCap: .round, lineJoin: .round, dash: Palette.hiddenDash.map { $0 * 1.6 }) + for run in runs.hidden { + guard let outline = outline(run.band, run.edges, geometry) else { continue } + // A dark halo under the dash keeps it legible on a sunlit wall. + context.stroke(outline, with: .color(.black.opacity(0.35)), style: StrokeStyle(lineWidth: 4, lineJoin: .round)) + context.stroke(outline, with: .color(Palette.hidden), style: dash) + } + for run in runs.skipped { + guard let quad = outline(run.band, run.edges, geometry) else { continue } + context.fill(quad, with: .color(Palette.skipped.opacity(0.3))) + var hatched = context + hatched.clip(to: quad) + let bounds = quad.boundingRect + var lines = Path() + var x = bounds.minX - bounds.height + while x < bounds.maxX { + lines.move(to: CGPoint(x: x, y: bounds.maxY)) + lines.addLine(to: CGPoint(x: x + bounds.height, y: bounds.minY)) + x += 10 + } + hatched.stroke(lines, with: .color(.white.opacity(0.55)), lineWidth: 1.5) + } + for run in runs.requested { + guard let outline = outline(run.band, run.edges, geometry) else { continue } + context.stroke(outline, with: .color(.black.opacity(0.3)), style: StrokeStyle(lineWidth: 4, lineJoin: .round)) + context.stroke(outline, with: .color(Palette.caution), style: StrokeStyle(lineWidth: 2, lineJoin: .round)) + } + } + .allowsHitTesting(false) + .accessibilityHidden(true) + } + + private enum Kind { case hidden, skipped, requested } + + private struct Run { + var band: CoverageBand + /// The s of every cell edge along the run, clipped to the visible range. + var edges: [Float] + } + + /// Runs of hidden, of skipped, and of requested-but-not-covered cells in each band. Coverage + /// filling the middle of a request splits it, and each piece keeps its outline. + private func runs() -> (hidden: [Run], skipped: [Run], requested: [Run]) { + var hidden: [Run] = [], skipped: [Run] = [], requested: [Run] = [] + let visible = coverage.visibleRange + guard visible.upperBound > visible.lowerBound else { return ([], [], []) } + for band in CoverageBand.allCases { + let states = coverage.cells(band) + var current: (kind: Kind, run: Run)? + func close() { + if let current { + switch current.kind { + case .hidden: hidden.append(current.run) + case .skipped: skipped.append(current.run) + case .requested: requested.append(current.run) + } + } + current = nil + } + for index in states.indices { + let range = coverage.cellRange(index) + guard range.upperBound > visible.lowerBound, range.lowerBound < visible.upperBound else { close(); continue } + let s0 = max(range.lowerBound, visible.lowerBound), s1 = min(range.upperBound, visible.upperBound) + let state = states[index] + let isRequested = highlight.map { $0.band == band && $0.span.lowerBound < range.upperBound && range.lowerBound < $0.span.upperBound } ?? false + let kind: Kind? = if isRequested, state != .covered { .requested } + else if state == .hidden { .hidden } + else if state == .skipped { .skipped } + else { nil } + guard let kind else { close(); continue } + if var open = current, open.kind == kind { + open.run.edges.append(s1) + current = open + } else { + close() + current = (kind, Run(band: band, edges: [s0, s1])) + } + } + close() + } + return (hidden, skipped, requested) + } + + /// The outline of a run through every cell edge, so it follows the wall round a corner. Nil + /// when any point is behind the camera. + private func outline(_ band: CoverageBand, _ edges: [Float], _ geometry: WallProjection) -> Path? { + let corners: [SIMD3] = switch band { + case .wall: + edges.map { wall.world(s: $0, height: coverage.wallBandHeight) } + edges.reversed().map { wall.world(s: $0, height: 0) } + case .ground: + edges.map { wall.world(s: $0, height: 0, out: coverage.groundBandDepth) } + edges.reversed().map { wall.world(s: $0, height: 0) } + } + return geometry.polygon(corners) + } +} diff --git a/ios/HouseScan/UI/Camera/FogOverlay.swift b/ios/HouseScan/UI/Camera/FogOverlay.swift index 6e2ac6800..98db723e6 100644 --- a/ios/HouseScan/UI/Camera/FogOverlay.swift +++ b/ios/HouseScan/UI/Camera/FogOverlay.swift @@ -1,7 +1,9 @@ import SwiftUI import simd -/// The haze over the parts of the wall and ground the phone hasn't seen yet. +/// The haze over the parts of the wall and ground the phone hasn't seen yet: the frosted strip +/// that `LiveFogView` replaced. It still draws while the live fog's shaders compile at launch, +/// and for good if Metal fails, so unseen wall never shows clear. /// /// Unseen cells carry a soft frosted haze, seen cells a thinner one, covered cells none. When a /// cell's state changes (a new `coverage.revision`), its haze fades and drifts upward over diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift new file mode 100644 index 000000000..c74e2ae87 --- /dev/null +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift @@ -0,0 +1,347 @@ +import Metal +import os +import QuartzCore +import simd + +/// The compiled shaders and pipelines, made once per launch off the main thread +/// (`LiveFogSupport.prepare`). Immutable after init; Metal's devices, libraries and pipeline +/// states are safe to use from any thread. +final class LiveFogGPU: @unchecked Sendable { + enum SetupError: Error, CustomStringConvertible { + case noDevice + case missing(String) + + var description: String { + switch self { + case .noDevice: "no Metal device" + case let .missing(what): "Metal would not make the \(what)" + } + } + } + + static let drawableFormat = MTLPixelFormat.bgra8Unorm + /// Half floats: the blur sums eleven taps, and the fog's slow lift needs finer steps than 8 bits. + static let maskFormat = MTLPixelFormat.rgba16Float + static let depthFormat = MTLPixelFormat.depth32Float + + let device: MTLDevice + let queue: MTLCommandQueue + let mask: MTLRenderPipelineState + let pool: MTLRenderPipelineState + let blur: MTLRenderPipelineState + let fog: MTLRenderPipelineState + let dots: MTLRenderPipelineState + let depthTest: MTLDepthStencilState + + init() throws { + guard let device = MTLCreateSystemDefaultDevice() else { throw SetupError.noDevice } + guard let queue = device.makeCommandQueue() else { throw SetupError.missing("command queue") } + self.device = device + self.queue = queue + let library = try device.makeLibrary(source: LiveFogShaders.source, options: nil) + + func pipeline(_ vertex: String, _ fragment: String, format: MTLPixelFormat, depth: Bool = false, + blend: ((MTLRenderPipelineColorAttachmentDescriptor) -> Void)? = nil) throws -> MTLRenderPipelineState { + let descriptor = MTLRenderPipelineDescriptor() + guard let v = library.makeFunction(name: vertex), let f = library.makeFunction(name: fragment) else { + throw SetupError.missing("\(vertex) / \(fragment) functions") + } + descriptor.vertexFunction = v + descriptor.fragmentFunction = f + descriptor.colorAttachments[0].pixelFormat = format + if depth { descriptor.depthAttachmentPixelFormat = Self.depthFormat } + if let blend { + descriptor.colorAttachments[0].isBlendingEnabled = true + blend(descriptor.colorAttachments[0]) + } + return try device.makeRenderPipelineState(descriptor: descriptor) + } + mask = try pipeline("maskVertex", "maskFragment", format: Self.maskFormat, depth: true) + pool = try pipeline("fullVertex", "poolFragment", format: Self.maskFormat) + blur = try pipeline("fullVertex", "blurFragment", format: Self.maskFormat) + // The fog is the first thing drawn on a cleared target: premultiplied, written as is. + fog = try pipeline("fullVertex", "fogFragment", format: Self.drawableFormat) + // Additive light: colour adds, alpha stays, so a dot brightens the camera under it. + dots = try pipeline("dotVertex", "dotFragment", format: Self.drawableFormat) { a in + a.sourceRGBBlendFactor = .one + a.destinationRGBBlendFactor = .one + a.sourceAlphaBlendFactor = .zero + a.destinationAlphaBlendFactor = .one + } + let depthDescriptor = MTLDepthStencilDescriptor() + depthDescriptor.depthCompareFunction = .less + depthDescriptor.isDepthWriteEnabled = true + guard let depthTest = device.makeDepthStencilState(descriptor: depthDescriptor) else { throw SetupError.missing("depth state") } + self.depthTest = depthTest + } +} + +/// What one frame of the fog draws from, all of it read from `ScanViewState`. +struct LiveFogInput { + var projection: CameraProjection + var wall: WallGeometry + var coverage: CoverageStrip + var highlight: GapRequest? + var dots: LiveDots + var reduceMotion: Bool + /// The gap request's amber, sRGB. + var requestedColor: SIMD3 +} + +/// Draws the fog and dots into a transparent Metal layer over the camera (`LiveFogMTKView`), +/// every display frame while the walk is on screen. +/// +/// Per frame: the mask pass draws the coverage cells (`FogMaskGeometry`) into a texture 192 +/// texels wide with a depth test, a max-pool takes it to 64, two passes blur it, then the +/// drawable gets the fog and the dots. +/// Bounds: three frames in flight; the mask vertex ring holds 16,384 vertices a frame (a 60 m +/// strip of unmerged cells, two bands in two parts, needs about 9,600); the dot buffers hold `DotTimeline.capacity` dots +/// and are rewritten only when the field changes or a fading dot ends, at most once a frame. +/// Nothing grows with time: about 4.5 MB of buffers and 1 MB of textures, allocated once. +@MainActor +final class LiveFogRenderer { + private let gpu: LiveFogGPU + /// Seconds on the clock the animations run on. Replaceable so an offscreen render can step + /// time; the view leaves it on the display's media clock. + var clock: () -> Double = { CACurrentMediaTime() } + private var epoch: Double? + private let inFlight = DispatchSemaphore(value: LiveFogRenderer.framesInFlight) + private static let framesInFlight = 3 + private static let maskWidth = 64 + private static let maskVertexCapacity = 16_384 + private static let spriteCapacity = DotTimeline.capacity * 2 + + var input: LiveFogInput? + var queue: MTLCommandQueue { gpu.queue } + + private let animator = FogCellAnimator() + private let timeline = DotTimeline() + private var maskBuffers: [MTLBuffer] = [] + private var spriteBuffers: [MTLBuffer] = [] + private var frame = 0 + private var spriteBuffer = 0 + private var spriteCount = 0 + private var maskTextures: MaskTextures? + private var anchor: (point: SIMD2, scale: Float)? + + /// Called once if the mask textures can't be allocated, so the view can hand back to the + /// frosted strip instead of showing a clear layer. + var onFailure: (() -> Void)? + + /// Nil when Metal won't allocate the buffers. + init?(gpu: LiveFogGPU) { + self.gpu = gpu + for _ in 0...stride, options: .storageModeShared), + let sprites = gpu.device.makeBuffer(length: Self.spriteCapacity * MemoryLayout.stride, options: .storageModeShared) + else { return nil } + maskBuffers.append(masks) + spriteBuffers.append(sprites) + } + } + + /// Encodes one frame into `pass`, whose target is `pixels` in size for a view of `size` + /// points. Returns false, encoding nothing, when all three frames are still in flight: the + /// caller skips the frame rather than block the main thread on the GPU. `LiveFogMTKView` + /// calls this each display frame. + func encode(into commandBuffer: MTLCommandBuffer, pass: MTLRenderPassDescriptor, size: CGSize, pixels: SIMD2, pixelsPerPoint scale: Float) -> Bool { + guard let input, size.width > 1, size.height > 1 else { return false } + guard inFlight.wait(timeout: .now()) == .success else { return false } + guard let textures = textures(for: size) else { + inFlight.signal() + onFailure?() + onFailure = nil + return false + } + let semaphore = inFlight + commandBuffer.addCompletedHandler { _ in semaphore.signal() } + + let start = epoch ?? clock() + epoch = start + let now = clock() - start + let time = Float(now) + let slot = frame % Self.framesInFlight + frame &+= 1 + + var clip = Self.clipMatrix(input.projection, size: size) + let maskBuffer = maskBuffers[slot] + let maskCount = FogMaskGeometry.build( + coverage: input.coverage, wall: input.wall, highlight: input.highlight, animator: animator, + now: now, reduceMotion: input.reduceMotion, + into: maskBuffer.contents().bindMemory(to: FogMaskGeometry.Vertex.self, capacity: Self.maskVertexCapacity), + capacity: Self.maskVertexCapacity) + + let changed = timeline.update(input.dots, now: time) + let expired = timeline.expire(now: time) + if changed || expired { + spriteBuffer = (spriteBuffer + 1) % Self.framesInFlight + let buffer = spriteBuffers[spriteBuffer] + spriteCount = timeline.write(into: buffer.contents().bindMemory(to: DotTimeline.Sprite.self, capacity: Self.spriteCapacity), capacity: Self.spriteCapacity) + } + + // Mask: full fog wherever no cell reaches. + let maskPass = MTLRenderPassDescriptor() + maskPass.colorAttachments[0].texture = textures.fine + maskPass.colorAttachments[0].loadAction = .clear + maskPass.colorAttachments[0].clearColor = MTLClearColor(red: 1, green: 0, blue: 0, alpha: 0) + maskPass.colorAttachments[0].storeAction = .store + maskPass.depthAttachment.texture = textures.depth + maskPass.depthAttachment.loadAction = .clear + maskPass.depthAttachment.clearDepth = 1 + maskPass.depthAttachment.storeAction = .dontCare + if let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: maskPass) { + if maskCount > 0 { + encoder.setRenderPipelineState(gpu.mask) + encoder.setDepthStencilState(gpu.depthTest) + encoder.setVertexBuffer(maskBuffer, offset: 0, index: 0) + encoder.setVertexBytes(&clip, length: MemoryLayout.stride, index: 1) + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: maskCount) + } + encoder.endEncoding() + } + let poolPass = MTLRenderPassDescriptor() + poolPass.colorAttachments[0].texture = textures.pooled + poolPass.colorAttachments[0].loadAction = .dontCare + poolPass.colorAttachments[0].storeAction = .store + if let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: poolPass) { + encoder.setRenderPipelineState(gpu.pool) + encoder.setFragmentTexture(textures.fine, index: 0) + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3) + encoder.endEncoding() + } + for (source, target, direction) in [(textures.pooled, textures.horizontal, SIMD2(1, 0)), (textures.horizontal, textures.final, SIMD2(0, 1))] { + let blurPass = MTLRenderPassDescriptor() + blurPass.colorAttachments[0].texture = target + blurPass.colorAttachments[0].loadAction = .dontCare + blurPass.colorAttachments[0].storeAction = .store + guard let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: blurPass) else { continue } + var direction = direction + encoder.setRenderPipelineState(gpu.blur) + encoder.setFragmentTexture(source, index: 0) + encoder.setFragmentBytes(&direction, length: MemoryLayout>.stride, index: 0) + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3) + encoder.endEncoding() + } + + pass.colorAttachments[0].loadAction = .clear + pass.colorAttachments[0].clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: 0) + if let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: pass) { + let pin = anchor(input, size: size, pixelsPerPoint: scale) + var fog = FogUniforms( + viewSize: pixels, anchor: pin.point, anchorScale: pin.scale, time: time, + motion: input.reduceMotion ? 0 : 1, pad: 0, requested: SIMD4(input.requestedColor, 1)) + encoder.setRenderPipelineState(gpu.fog) + encoder.setFragmentBytes(&fog, length: MemoryLayout.stride, index: 0) + encoder.setFragmentTexture(textures.final, index: 0) + encoder.setFragmentTexture(textures.pooled, index: 1) + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3) + + if spriteCount > 0 { + var dots = DotUniforms( + clip: clip, cameraAndTime: SIMD4(input.projection.cameraPosition, time), + pointScale: scale, reduceMotion: input.reduceMotion ? 1 : 0, pad: .zero) + encoder.setRenderPipelineState(gpu.dots) + encoder.setVertexBuffer(spriteBuffers[spriteBuffer], offset: 0, index: 0) + encoder.setVertexBytes(&dots, length: MemoryLayout.stride, index: 1) + encoder.setVertexTexture(textures.final, index: 0) + encoder.drawPrimitives(type: .point, vertexStart: 0, vertexCount: spriteCount) + } + encoder.endEncoding() + } + return true + } + + /// The mask keeps the view's aspect at 64 texels across, about 6 points a texel on a phone; + /// the blur's 1.4-texel sigma, about 8 points, is the feather, and the noise-warped lookup + /// (`fogFragment`) rags the edges. + private struct MaskTextures { + /// Where the cells are drawn, three times the coarse size, with its depth buffer. + var fine: MTLTexture + var depth: MTLTexture + /// The fine mask max-pooled to the coarse size, then the two blur passes. + var pooled: MTLTexture + var horizontal: MTLTexture + var final: MTLTexture + } + + private func textures(for size: CGSize) -> MaskTextures? { + let width = Self.maskWidth + let height = max(1, Int((Double(width) * size.height / size.width).rounded())) + if let maskTextures, maskTextures.pooled.width == width, maskTextures.pooled.height == height { return maskTextures } + func texture(_ format: MTLPixelFormat, _ usage: MTLTextureUsage, scale: Int = 1) -> MTLTexture? { + let descriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: format, width: width * scale, height: height * scale, mipmapped: false) + descriptor.usage = usage + descriptor.storageMode = .private + return gpu.device.makeTexture(descriptor: descriptor) + } + guard let fine = texture(LiveFogGPU.maskFormat, [.renderTarget, .shaderRead], scale: 3), + let depth = texture(LiveFogGPU.depthFormat, [.renderTarget], scale: 3), + let pooled = texture(LiveFogGPU.maskFormat, [.renderTarget, .shaderRead]), + let horizontal = texture(LiveFogGPU.maskFormat, [.renderTarget, .shaderRead]), + let final = texture(LiveFogGPU.maskFormat, [.renderTarget, .shaderRead]) else { + RuntimeLog.engine.error("live fog: Metal would not allocate the mask textures; drawing the frosted strip instead") + return nil + } + maskTextures = MaskTextures(fine: fine, depth: depth, pooled: pooled, horizontal: horizontal, final: final) + return maskTextures + } + + /// Where the fog's noise is pinned: the meter on screen, one noise unit about 1.2 m at the + /// meter's distance, so the texture moves and scales with the wall. Kept from the last frame + /// that saw the meter in front of the camera. + private func anchor(_ input: LiveFogInput, size: CGSize, pixelsPerPoint: Float) -> (point: SIMD2, scale: Float) { + let width = Float(size.width) * pixelsPerPoint + let local = input.projection.cameraSpace(input.wall.meter) + if local.z < -0.1, let point = input.projection.viewPoint(for: input.wall.meter, in: size) { + let pointsPerMeter = input.projection.intrinsics.x * Float(input.projection.scale(in: size)) / -local.z + let scale = min(max(pointsPerMeter * pixelsPerPoint * 1.2, 0.25 * width), 2 * width) + anchor = (SIMD2(Float(point.x), Float(point.y)) * pixelsPerPoint, scale) + } + return anchor ?? (.zero, 0.45 * width) + } + + /// World to Metal clip space for a portrait view of `size` showing the landscape sensor image + /// turned 90 degrees clockwise and scaled to fill, the mapping `CameraProjection.viewPoint` + /// uses. Depth runs from 0 at 5 cm to 1 at 100 m, so the GPU clips anything behind the camera + /// and the mask's depth test keeps the nearest surface. + static func clipMatrix(_ projection: CameraProjection, size: CGSize, near: Float = 0.05, far: Float = 100) -> simd_float4x4 { + let k = projection.intrinsics + let imageHeight = projection.imageSize.y + let scale = Float(projection.scale(in: size)) + let width = Float(size.width), height = Float(size.height) + let offset = SIMD2( + (width - projection.imageSize.y * scale) / 2, + (height - projection.imageSize.x * scale) / 2) + // Rows act on camera-space (x, y, z, 1); w is the depth along -z. + let w = SIMD4(0, 0, -1, 0) + let portraitX = SIMD4(0, k.y, k.w - imageHeight, 0) + let portraitY = SIMD4(k.x, 0, -k.z, 0) + let screenX = offset.x * w + scale * portraitX + let screenY = offset.y * w + scale * portraitY + let clipX = (2 / width) * screenX - w + let clipY = w - (2 / height) * screenY + let clipZ = SIMD4(0, 0, -far / (far - near), -far * near / (far - near)) + let cameraToClip = simd_float4x4(rows: [clipX, clipY, clipZ, w]) + return cameraToClip * projection.cameraToWorld.inverse + } +} + +/// Matches `FogUniforms` in the shader. +struct FogUniforms { + var viewSize: SIMD2 + var anchor: SIMD2 + var anchorScale: Float + var time: Float + var motion: Float + var pad: Float + var requested: SIMD4 +} + +/// Matches `DotUniforms` in the shader. +struct DotUniforms { + var clip: simd_float4x4 + var cameraAndTime: SIMD4 + var pointScale: Float + var reduceMotion: Float + var pad: SIMD2 +} diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogScene.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogScene.swift new file mode 100644 index 000000000..b9c8e7bb5 --- /dev/null +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogScene.swift @@ -0,0 +1,320 @@ +import Foundation +import simd + +/// What the fog mask holds at one place: how thick the fog is, how much of it is the violet of a +/// hidden stretch, and how much is a gap request's amber. Channels of the mask texture, in order. +struct FogValue: Equatable { + var fog: Float + var hidden: Float + var requested: Float + + static let clear = FogValue(fog: 0, hidden: 0, requested: 0) + + /// The rule the fog follows, from the coverage map's own cell states: the fog lifts only + /// where coverage counts the cell, never where depth merely reached. + /// - unseen: full fog. + /// - seen from one position: thinned, not lifted, since one view is not evidence. + /// - covered: lifted. + /// - hidden (the camera looked, depth found something nearer): still fog, since the wall + /// behind is unseen, but thinner and violet, so the thing in front shows through with its + /// violet dots. + /// - skipped ("I can't get there"): lifted here; `FogMarks` hatches it, so it never reads as + /// seen. + /// - requested by a gap and not yet covered: amber instead of fog, so the camera shows where + /// to aim. + static func target(_ state: CellState, requested: Bool) -> FogValue { + if requested, state != .covered { return FogValue(fog: 0, hidden: 0, requested: 1) } + return fog(state) + } + + /// Above the wall band and past the ground band, where coverage measures nothing: fog while + /// the column below is unseen or seen once, clear once it is covered. A hidden stretch and a + /// gap request are about the band itself, so their violet and amber stop at its edge, and + /// the wall above a bin, which the camera did see past it, is not fogged. + static func beyond(_ state: CellState) -> FogValue { + switch state { + case .unseen, .seen: fog(state) + case .covered, .skipped, .hidden: .clear + } + } + + private static func fog(_ state: CellState) -> FogValue { + return switch state { + case .unseen: FogValue(fog: 1, hidden: 0, requested: 0) + // 0.42 and 0.4 from the taste check against the prototype's videos: at 0.55, seen once + // and unseen blurred into one grey; at 0.75 the hidden mist greyed out the bin in front. + case .seen: FogValue(fog: 0.42, hidden: 0, requested: 0) + case .hidden: FogValue(fog: 0.4, hidden: 1, requested: 0) + case .covered, .skipped: .clear + } + } + + func mixed(to other: FogValue, _ t: Float) -> FogValue { + FogValue(fog: fog + (other.fog - fog) * t, hidden: hidden + (other.hidden - hidden) * t, requested: requested + (other.requested - requested) * t) + } + + var total: Float { fog + hidden + requested } +} + +/// Eases each cell from its old fog to its new one when coverage changes it. Lifting is the +/// signature moment ("the phone saw that") and takes `Motion.fogLift` on the strong ease-out; +/// fog coming back (a cell turning hidden, a request raised) is quick, 150 ms, because it is the +/// system answering, not a reveal. Under Reduce Motion both are linear fades, 400 and 120 ms, as +/// in the prototype. Keyed by the cell's s, not its index in the strip, which shifts when the +/// strip grows to the left. +@MainActor +final class FogCellAnimator { + struct Key: Hashable { + var band: CoverageBand + /// The cell's left edge in whole cell widths from s = 0. + var cell: Int + /// The part above the wall band or past the ground band. + var beyond: Bool + } + + private struct Track { + var from: FogValue + var to: FogValue + var start: Double + var duration: Double + var linear: Bool + + func value(at now: Double) -> FogValue { + guard duration > 0 else { return to } + let p = Float(min(max((now - start) / duration, 0), 1)) + return from.mixed(to: to, linear ? p : Easing.strongEaseOut(p)) + } + + func isDone(at now: Double) -> Bool { now - start >= duration } + } + + private var tracks: [Key: Track] = [:] + private var seen: Set = [] + + /// The cell's value now. A cell seen for the first time starts at its target. + func value(_ key: Key, target: FogValue, now: Double, reduceMotion: Bool) -> FogValue { + seen.insert(key) + guard var track = tracks[key] else { + tracks[key] = Track(from: target, to: target, start: now, duration: 0, linear: false) + return target + } + if track.to != target { + let current = track.value(at: now) + let lifting = target.total < current.total + let duration = reduceMotion ? (lifting ? 0.4 : 0.12) : (lifting ? Motion.fogLift : 0.15) + track = Track(from: current, to: target, start: now, duration: duration, linear: reduceMotion) + tracks[key] = track + } + return track.value(at: now) + } + + /// True while the cell's fog is still easing toward its target. Call after `value`. + func isMoving(_ key: Key, now: Double) -> Bool { + tracks[key].map { !$0.isDone(at: now) } ?? false + } + + /// Forgets cells the last frame did not draw, so the table never outgrows the strip. + func endFrame() { + if tracks.count > seen.count { tracks = tracks.filter { seen.contains($0.key) } } + seen.removeAll(keepingCapacity: true) + } +} + +/// Builds the fog mask's geometry: each coverage cell as a wall column from the ground up +/// `columnHeight` and a stretch of ground out `groundReach`, carrying its `FogValue`. The column +/// and the stretch reach past the bands coverage measures (4.5 ft of wall, 4 ft of ground) on +/// purpose: fog there could never lift, and fog that walking can't clear stops meaning +/// "not seen yet". The wall above a covered column and the lawn in front of it are nothing the +/// scan decides on; past the band, a cell's part follows `FogValue.beyond`. Past a marked end the wall is not part of the scan, and is clear. Anything +/// no quad reaches (the wall past the strip, where the walk has not been) keeps the mask's clear +/// value, full fog. +enum FogMaskGeometry { + struct Vertex { + var position: SIMD4 + var value: SIMD4 + } + + static let columnHeight: Float = 8 + static let groundReach: Float = 8 + /// How far past a marked end the clear stretch runs. + static let pastEnd: Float = 30 + + /// Writes triangles into `vertices` (at most `capacity`) and returns how many it wrote. + /// Neighbouring cells with the same value on the same piece of wall merge into one quad. + @MainActor + static func build( + coverage: CoverageStrip, wall: WallGeometry, highlight: GapRequest?, animator: FogCellAnimator, + now: Double, reduceMotion: Bool, into vertices: UnsafeMutablePointer, capacity: Int + ) -> Int { + var count = 0 + let wallTop = coverage.wallBandHeight, groundEdge = coverage.groundBandDepth + /// A quad on the wall from height `near` to `far`, or on the ground from `near` to `far` + /// out; `beyond` picks the part past the band. + func quad(_ band: CoverageBand, beyond: Bool, _ s0: Float, _ s1: Float, _ value: FogValue, moving: Bool = false) { + guard count + 6 <= capacity else { return } + let (near, far) = switch (band, beyond) { + case (.wall, false): (Float(0), wallTop) + case (.wall, true): (wallTop, columnHeight) + case (.ground, false): (Float(0), groundEdge) + case (.ground, true): (groundEdge, groundReach) + } + func point(_ s: Float, _ t: Float) -> SIMD4 { + SIMD4(band == .wall ? wall.world(s: s, height: t) : wall.world(s: s, height: 0, out: t), 1) + } + let a = point(s0, near), b = point(s1, near), c = point(s1, far), d = point(s0, far) + let v = SIMD4(value.fog, value.hidden, value.requested, moving ? 1 : 0) + vertices[count] = Vertex(position: a, value: v) + vertices[count + 1] = Vertex(position: b, value: v) + vertices[count + 2] = Vertex(position: c, value: v) + vertices[count + 3] = Vertex(position: a, value: v) + vertices[count + 4] = Vertex(position: c, value: v) + vertices[count + 5] = Vertex(position: d, value: v) + count += 6 + } + + let visible = coverage.visibleRange + if visible.upperBound > visible.lowerBound { + for band in CoverageBand.allCases { + let states = coverage.cells(band) + for beyond in [false, true] { + var run: (s0: Float, s1: Float, value: FogValue, moving: Bool, along: SIMD3)? + for index in states.indices { + let range = coverage.cellRange(index) + guard range.upperBound > visible.lowerBound, range.lowerBound < visible.upperBound else { continue } + let s0 = max(range.lowerBound, visible.lowerBound), s1 = min(range.upperBound, visible.upperBound) + let requested = highlight.map { $0.band == band && $0.span.lowerBound < range.upperBound && range.lowerBound < $0.span.upperBound } ?? false + let key = FogCellAnimator.Key(band: band, cell: Int((range.lowerBound / coverage.cellWidth).rounded()), beyond: beyond) + let target = beyond ? FogValue.beyond(states[index]) : FogValue.target(states[index], requested: requested) + let value = animator.value(key, target: target, now: now, reduceMotion: reduceMotion) + let moving = animator.isMoving(key, now: now) + let along = wall.along(atS: (s0 + s1) / 2) + if var current = run, current.value == value, current.moving == moving, current.along == along, abs(current.s1 - s0) < 1e-4 { + current.s1 = s1 + run = current + } else { + if let current = run { quad(band, beyond: beyond, current.s0, current.s1, current.value, moving: current.moving) } + run = (s0, s1, value, moving, along) + } + } + if let current = run { quad(band, beyond: beyond, current.s0, current.s1, current.value, moving: current.moving) } + } + } + } + for band in CoverageBand.allCases { + for beyond in [false, true] { + if let left = wall.leftEnd { quad(band, beyond: beyond, left - pastEnd, left, .clear) } + if let right = wall.rightEnd { quad(band, beyond: beyond, right, right + pastEnd, .clear) } + } + } + animator.endFrame() + return count + } +} + +/// The dots' timing on the renderer's clock: when each was born, when its opacity last changed +/// and from what, when it turned edge, and, for a dot that left the field, when it died. The +/// shader evaluates the curves, so the buffer changes only when the field does or a death ends. +@MainActor +final class DotTimeline { + struct Sprite { + var a: SIMD4 + var b: SIMD4 + var c: SIMD4 + } + + private struct Track { + var birth: Float + var from: Float + var to: Float + var opacityTime: Float + var edgeSince: Float + var dot: LiveDots.Dot + + func opacity(at t: Float) -> Float { + from + (to - from) * Easing.strongEaseOut((t - opacityTime) / 0.25) + } + } + + /// Metal compiles with fast math, which assumes no infinities. + private static let never: Float = 1e6 + private static let deathFade: Float = 0.3 + + private var tracks: [UInt64: Track] = [:] + private var dying: [(track: Track, death: Float)] = [] + private var revision = -1 + /// Dots the renderer draws at once, living and dying: the field's own cap + /// (`SurfaceDotConfig.maxDots`, 5,000) and as many again fading out. + static let capacity = 10_000 + + /// Takes a new field. Returns true when the sprites changed. + func update(_ field: LiveDots, now: Float) -> Bool { + guard field.revision != revision else { return false } + revision = field.revision + var next: [UInt64: Track] = [:] + next.reserveCapacity(field.dots.count) + for dot in field.dots { + if var track = tracks[dot.id] { + if dot.opacity != track.to { + track.from = track.opacity(at: now) + track.to = dot.opacity + track.opacityTime = now + } + if dot.isEdge, track.edgeSince >= Self.never { track.edgeSince = now } + track.dot = dot + next[dot.id] = track + } else { + next[dot.id] = Track( + birth: now, from: dot.opacity, to: dot.opacity, opacityTime: -Self.never, + edgeSince: dot.isEdge ? -Self.never : Self.never, dot: dot) + } + } + for (id, track) in tracks where next[id] == nil { dying.append((track, now)) } + if dying.count > Self.capacity / 2 { dying.removeFirst(dying.count - Self.capacity / 2) } + tracks = next + return true + } + + /// Drops dots whose fade has finished. Returns true when any went. + func expire(now: Float) -> Bool { + let before = dying.count + dying.removeAll { now - $0.death > Self.deathFade } + return dying.count != before + } + + /// Two sprites per dot, its halo then its core, so the sharp cores sit on the glow. Returns + /// how many it wrote, at most `capacity`. + func write(into sprites: UnsafeMutablePointer, capacity: Int) -> Int { + var count = 0 + func add(_ track: Track, death: Float) { + guard count + 2 <= capacity else { return } + let d = track.dot + let b = SIMD4(track.birth, track.from, track.to, track.opacityTime) + let c = SIMD4(track.edgeSince, death, d.onOccluder ? 1 : 0, 0) + sprites[count] = Sprite(a: SIMD4(d.position, 1), b: b, c: c) + sprites[count + 1] = Sprite(a: SIMD4(d.position, 0), b: b, c: c) + count += 2 + } + for track in tracks.values { add(track, death: Self.never) } + for entry in dying { add(entry.track, death: entry.death) } + return count + } +} + +/// cubic-bezier(0.23, 1, 0.32, 1), the same curve the shader solves. +enum Easing { + static func strongEaseOut(_ progress: Float) -> Float { + if progress <= 0 { return 0 } + if progress >= 1 { return 1 } + let x1: Float = 0.23, x2: Float = 0.32 + var t = progress + for _ in 0..<8 { + let u = 1 - t + let x = 3 * u * u * t * x1 + 3 * u * t * t * x2 + t * t * t - progress + let dx = 3 * u * u * x1 + 6 * u * t * (x2 - x1) + 3 * t * t * (1 - x2) + if abs(dx) < 1e-6 { break } + t = min(max(t - x / dx, 0), 1) + } + let u = 1 - t + return 3 * u * u * t + 3 * u * t * t + t * t * t + } +} diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift new file mode 100644 index 000000000..3672486a4 --- /dev/null +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift @@ -0,0 +1,277 @@ +/// Metal source for the live fog and dots, compiled at runtime (`makeLibrary(source:)`) so the +/// build needs no Metal toolchain, as in the prototype (experiments/live-dots on t3/experience, +/// `Shaders.swift`). Struct layouts match the Swift structs in `LiveFogRenderer.swift`. +/// +/// Fog. Coverage decides where it lifts: `LiveFogScene` draws every wall column and ground +/// stretch of the coverage strip into a mask three times 64 texels wide (red: fog, green: hidden +/// behind something, blue: requested by a gap, alpha: still changing), clearing to full fog; a +/// max-pool takes it to 64 texels and a radius-3 Gaussian blur feathers it. The composite is the +/// prototype's: Chalk mixed toward a cool grey, textured by slow domain-warped fbm and breathing +/// slightly. Two changes: the noise is pinned to the meter +/// on screen and scaled by its distance, so the texture sits on the wall instead of on the lens +/// while the phone moves (the prototype cut between keyframes, where that never showed); and +/// part-lifted fog breaks up along the same noise instead of thinning evenly, so a lift reads as +/// mist burning off. +/// +/// Dots. The prototype's hologram scheme: additive point sprites, a halo under each core, births +/// that ease in over 350 ms from 60% size, opacity changes over 250 ms, deaths over 250 ms. A dot +/// under fog draws at 20%, so dots arrive where coverage lifts the fog; dots on an occluder +/// (violet) draw fully, since they are what the hidden state is about. +enum LiveFogShaders { + static let source = """ + #include + using namespace metal; + + // cubic-bezier(0.23, 1, 0.32, 1), solved by Newton. + static float strongEaseOut(float p) { + p = clamp(p, 0.0f, 1.0f); + if (p <= 0.0f || p >= 1.0f) { return p; } + const float x1 = 0.23f, x2 = 0.32f, y1 = 1.0f, y2 = 1.0f; + float t = p; + for (int i = 0; i < 8; i++) { + float u = 1.0f - t; + float x = 3.0f * u * u * t * x1 + 3.0f * u * t * t * x2 + t * t * t - p; + float dx = 3.0f * u * u * x1 + 6.0f * u * t * (x2 - x1) + 3.0f * t * t * (1.0f - x2); + if (fabs(dx) < 1e-6f) { break; } + t = clamp(t - x / dx, 0.0f, 1.0f); + } + float u = 1.0f - t; + return 3.0f * u * u * t * y1 + 3.0f * u * t * t * y2 + t * t * t; + } + + struct FullOut { float4 position [[position]]; }; + + vertex FullOut fullVertex(uint vid [[vertex_id]]) { + float2 p = float2((vid << 1) & 2, vid & 2); + FullOut out; + out.position = float4(p * 2.0f - 1.0f, 0.0f, 1.0f); + return out; + } + + // MARK: Mask + + struct MaskVertex { + float4 position; // world xyz + float4 value; // fog, hidden, requested, 1 while the cell's fog is changing + }; + + struct MaskOut { + float4 position [[position]]; + float4 value [[flat]]; + }; + + vertex MaskOut maskVertex(uint vid [[vertex_id]], + const device MaskVertex* vertices [[buffer(0)]], + constant float4x4& clip [[buffer(1)]]) { + MaskOut out; + out.position = clip * float4(vertices[vid].position.xyz, 1.0f); + out.value = vertices[vid].value; + return out; + } + + fragment float4 maskFragment(MaskOut in [[stage_in]]) { + return in.value; + } + + // Each texel takes the largest value of the 3 x 3 block under it in the mask drawn at three + // times the size, per channel: a cell as narrow as one fine texel (about 2 points) still + // reaches the coarse mask, where sampling alone could step over it. + fragment float4 poolFragment(FullOut in [[stage_in]], + texture2d fine [[texture(0)]]) { + uint2 base = uint2(in.position.xy) * 3u; + uint2 limit = uint2(fine.get_width() - 1, fine.get_height() - 1); + float4 m = float4(0.0f); + for (uint y = 0; y < 3; y++) { + for (uint x = 0; x < 3; x++) { + m = max(m, fine.read(min(base + uint2(x, y), limit))); + } + } + return m; + } + + // Separable Gaussian, radius 3 texels (sigma 1.4, about 8 points), all four channels. + fragment float4 blurFragment(FullOut in [[stage_in]], + texture2d source [[texture(0)]], + constant int2& direction [[buffer(0)]]) { + int2 p = int2(in.position.xy); + int2 limit = int2(source.get_width() - 1, source.get_height() - 1); + float4 sum = float4(0.0f); + float total = 0.0f; + for (int k = -3; k <= 3; k++) { + float w = exp(-float(k * k) / (2.0f * 1.4f * 1.4f)); + sum += w * source.read(uint2(clamp(p + k * direction, int2(0), limit))); + total += w; + } + return sum / total; + } + + // MARK: Fog + + struct FogUniforms { + float2 viewSize; // drawable pixels + float2 anchor; // pixels: the meter on screen, where the noise is pinned + float anchorScale; // pixels per noise unit (about 1.2 m at the meter) + float time; // seconds + float motion; // 0 under Reduce Motion: no drift, no breathing, no pulse + float pad; + float4 requested; // the gap request's amber, rgb + }; + + static float hash21(float2 p) { + p = fract(p * float2(123.34f, 456.21f)); + p += dot(p, p + 45.32f); + return fract(p.x * p.y); + } + + static float valueNoise(float2 p) { + float2 i = floor(p), f = fract(p); + float2 w = f * f * (3.0f - 2.0f * f); + float a = hash21(i), b = hash21(i + float2(1, 0)), c = hash21(i + float2(0, 1)), d = hash21(i + float2(1, 1)); + return mix(mix(a, b, w.x), mix(c, d, w.x), w.y); + } + + static float fbm(float2 p) { + float sum = 0.0f, amplitude = 0.5f; + for (int i = 0; i < 3; i++) { + sum += amplitude * valueNoise(p); + p = p * 2.03f + float2(17.1f, 9.2f); + amplitude *= 0.5f; + } + return sum / 0.875f; + } + + fragment float4 fogFragment(FullOut in [[stage_in]], + constant FogUniforms& u [[buffer(0)]], + texture2d mask [[texture(0)]], + texture2d pooled [[texture(1)]]) { + constexpr sampler linear(filter::linear, address::clamp_to_edge); + float2 screen = in.position.xy; + // Wrapped so the offsets stay small where Float precision is fine. + float2 uv = fmod((screen - u.anchor) / u.anchorScale, float2(256.0f)); + float2 q = uv + u.motion * u.time * 0.02f * float2(0.94f, 0.34f); + + // The mask is read through a fine noise offset, about 27 cm features at the meter and at + // most 2.5% of the screen's width either way, so the straight edges of cells come out + // ragged, as a fog bank's do (the noise-edged fog of war the prototype borrowed from + // Civilization VI). A coarser, larger offset (70 cm, 7%) smeared the edges into what + // read as a smudge on the lens in the first hosted screenshots. + float2 edgeNoise = float2(valueNoise(q * 4.5f + float2(7.3f, 2.1f)), valueNoise(q * 4.5f + float2(-4.1f, 8.6f))); + float2 look = screen + (edgeNoise - 0.5f) * 0.05f * u.viewSize.x; + // The blur and the noise shape the fog; neither may erase it. The unblurred pooled mask, + // read through a smaller offset, sets a floor: a cell coverage has not counted keeps 30% + // of its fog within a few points of where it is, however narrow. Read unwarped, the floor + // drew a straight step along every wide edge. + float2 near = screen + (edgeNoise - 0.5f) * 0.015f * u.viewSize.x; + float4 m = max(mask.sample(linear, look / u.viewSize), 0.3f * pooled.sample(linear, near / u.viewSize)); + float fog = saturate(m.r), hidden = saturate(m.g), requested = saturate(m.b); + if (fog < 0.002f && requested < 0.002f) { return float4(0.0f); } + + float2 warp = float2(fbm(q + float2(3.1f, 1.7f)), fbm(q + float2(-2.3f, 5.2f))); + float n = fbm(q + 0.8f * warp); + // The prototype's texture is barely there; deeper troughs read as dirt on the lens. + float noiseTerm = 0.88f + 0.12f * n; + float breathing = 1.0f + u.motion * 0.04f * sin(2.0f * M_PI_F * 0.08f * u.time); + + // Lifting fog breaks up along the noise, as mist burning off; fog that holds still (a cell + // seen once) stays an even, thinner veil, since held blotches read as a smudge. + float erosion = mix(0.4f, 1.6f, saturate(m.a)); + float shaped = saturate(fog + (n - 0.5f) * erosion * fog * (1.0f - fog)); + float a = saturate(0.66f * noiseTerm * shaped * breathing * mix(1.0f, 0.72f, hidden)); + + // Hidden is a tint of real violet, the tape map's colour, not a grey veil over the thing + // in front: half-mixed at 0.75 fog it read as a grey ghost over the bin. + float3 chalk = float3(0xF7, 0xF5, 0xEF) / 255.0f; + float3 cool = float3(0xD9, 0xE2, 0xEC) / 255.0f; + float3 violet = float3(0xA4, 0x8A, 0xFA) / 255.0f; + float3 color = mix(mix(chalk, cool, 0.2f), violet, 0.9f * hidden); + + float pulse = 1.0f - u.motion * 0.14f * (0.5f + 0.5f * sin(2.0f * M_PI_F * u.time / 1.8f)); + float r = saturate(0.28f * requested * noiseTerm * pulse); + + float3 rgb = color * a; + rgb = u.requested.rgb * r + rgb * (1.0f - r); + return float4(rgb, r + a * (1.0f - r)); + } + + // MARK: Dots + + struct Sprite { + float4 a; // world xyz, 1 for the halo sprite + float4 b; // birth time, from opacity, to opacity, opacity change time + float4 c; // edge since, death time, violet (0 or 1), unused + }; + + struct DotUniforms { + float4x4 clip; + float4 cameraAndTime; // camera xyz, seconds + float pointScale; // pixels per point + float reduceMotion; // 1 drops the birth scale + float2 pad; + }; + + struct DotOut { + float4 position [[position]]; + float size [[point_size]]; + float4 color; + float inner; // radius fraction where the soft edge starts + }; + + constant float3 hologram = float3(0xE6, 0xEC, 0xF4) / 255.0f; + constant float3 glow = float3(0x9C, 0xC8, 0xFF) / 255.0f; + constant float3 violetDot = float3(0xB4, 0x9C, 0xFF) / 255.0f; + + vertex DotOut dotVertex(uint vid [[vertex_id]], + const device Sprite* sprites [[buffer(0)]], + constant DotUniforms& u [[buffer(1)]], + texture2d mask [[texture(0)]]) { + constexpr sampler linear(filter::linear, address::clamp_to_edge); + Sprite s = sprites[vid]; + float t = u.cameraAndTime.w; + float birth = strongEaseOut((t - s.b.x) / 0.35f); + float evidence = mix(s.b.y, s.b.z, strongEaseOut((t - s.b.w) / 0.25f)); + float fade = 1.0f - strongEaseOut((t - s.c.y) / 0.25f); + float edge = strongEaseOut((t - s.c.x) / 0.25f); + bool halo = s.a.w > 0.5f; + bool violetOne = s.c.z > 0.5f; + + DotOut out; + out.position = u.clip * float4(s.a.xyz, 1.0f); + float fog = 1.0f; + if (out.position.w > 0.05f) { + float2 ndc = out.position.xy / out.position.w; + fog = mask.sample(linear, float2(0.5f + 0.5f * ndc.x, 0.5f - 0.5f * ndc.y), level(0)).r; + } + float veil = violetOne ? 1.0f : mix(1.0f, 0.2f, saturate(fog)); + float alpha = evidence * birth * fade * veil; + + // Flat 3.5 pt and edge 5 pt from 4 m out, growing to 5 and 7.5 pt at 1 m. The prototype's + // 2.5 to 4 pt over 1 to 2.5 m left dots at walking distance (1.5 to 3 m) as pinpricks. + float far = clamp((length(s.a.xyz - u.cameraAndTime.xyz) - 1.0f) / 3.0f, 0.0f, 1.0f); + float size = mix(mix(5.0f, 3.5f, far), mix(7.5f, 5.0f, far), edge); + float inner = 0.7f; + float3 color = hologram; + if (halo) { + // Edge 4x at 32%, flat 2.6x at 16%; a flat dot turning edge grows into it. The glow + // is what makes the dots read as light; the prototype's 8% flat halo didn't show. + size *= mix(2.6f, 4.0f, edge); + alpha *= mix(0.16f, 0.32f, edge); + inner = 0.0f; + color = glow; + } + if (violetOne) { color = violetDot; } + float scale = u.reduceMotion > 0.5f ? 1.0f : mix(0.6f, 1.0f, birth); + + if (alpha < 1.0f / 512.0f) { out.position = float4(0.0f, 0.0f, -1.0f, 1.0f); } + out.size = size * scale * u.pointScale; + // Alpha 0 with colour: added to what is under it, as light, when the layer composites. + out.color = float4(color * alpha, 0.0f); + out.inner = inner; + return out; + } + + fragment float4 dotFragment(DotOut in [[stage_in]], float2 pc [[point_coord]]) { + float r = length(pc * 2.0f - 1.0f); + return in.color * (1.0f - smoothstep(in.inner, 1.0f, r)); + } + """ +} diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift new file mode 100644 index 000000000..28b75abab --- /dev/null +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift @@ -0,0 +1,102 @@ +import MetalKit +import Observation +import SwiftUI + +/// Whether the live fog can draw. The shaders compile once per launch, off the main thread, as +/// soon as the app starts (`HouseScanApp`); until then, and if Metal fails, `CameraOverlays` +/// draws the frosted `FogOverlay` instead, so the camera never shows unseen wall as clear. +@MainActor +@Observable +final class LiveFogSupport { + enum Status { + case preparing + case ready(LiveFogGPU) + case failed + } + + static let shared = LiveFogSupport() + + private(set) var status: Status = .preparing + @ObservationIgnored private var started = false + + /// The renderer could not get its buffers or textures: back to the frosted strip for the + /// rest of the launch. + func fail(_ reason: String) { + RuntimeLog.engine.error("live fog unavailable, drawing the frosted strip instead: \(reason, privacy: .public)") + status = .failed + } + + func prepare() { + guard !started else { return } + started = true + Task.detached(priority: .userInitiated) { + let result: Result + do { result = .success(try LiveFogGPU()) } catch { result = .failure(error) } + await MainActor.run { + switch result { + case let .success(gpu): + RuntimeLog.engine.info("live fog ready") + LiveFogSupport.shared.status = .ready(gpu) + case let .failure(error): + // Logged as an error, not trapped: a trap in a Debug build would stop every UI + // test, while the frosted strip in their screenshots already shows the failure. + LiveFogSupport.shared.fail(String(describing: error)) + } + } + } + } +} + +/// The fog over what coverage has not counted and, on a LiDAR phone, the dots on what depth has +/// measured: a transparent Metal layer over the camera. See `LiveFogShaders` for the look and +/// `FogValue.target` for the rule. +struct LiveFogView: UIViewRepresentable { + let gpu: LiveFogGPU + let input: LiveFogInput + + func makeUIView(context: Context) -> LiveFogMTKView { + let view = LiveFogMTKView(frame: .zero, device: gpu.device) + view.colorPixelFormat = LiveFogGPU.drawableFormat + view.clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: 0) + view.isOpaque = false + view.backgroundColor = .clear + view.layer.isOpaque = false + view.framebufferOnly = true + view.autoResizeDrawable = true + // 60, not ProMotion's 120: the pose it draws from arrives at 30 Hz, and 60 keeps each new + // pose within one refresh of arriving. + view.preferredFramesPerSecond = 60 + view.isUserInteractionEnabled = false + view.accessibilityElementsHidden = true + guard let renderer = LiveFogRenderer(gpu: gpu) else { + // Not during this view update: the status change swaps this view out. + DispatchQueue.main.async { LiveFogSupport.shared.fail("Metal would not allocate the fog's buffers") } + return view + } + renderer.input = input + renderer.onFailure = { + DispatchQueue.main.async { LiveFogSupport.shared.fail("Metal would not allocate the fog's mask textures") } + } + view.renderer = renderer + return view + } + + func updateUIView(_ view: LiveFogMTKView, context: Context) { + view.renderer?.input = input + } +} + +/// An MTKView that hands each frame to its renderer. Subclassing keeps drawing on the main actor +/// without a delegate conformance. +final class LiveFogMTKView: MTKView { + var renderer: LiveFogRenderer? + + override func draw(_ rect: CGRect) { + guard let renderer, let pass = currentRenderPassDescriptor, let drawable = currentDrawable, + let commandBuffer = renderer.queue.makeCommandBuffer() else { return } + let pixels = SIMD2(Float(drawableSize.width), Float(drawableSize.height)) + guard renderer.encode(into: commandBuffer, pass: pass, size: bounds.size, pixels: pixels, pixelsPerPoint: Float(contentScaleFactor)) else { return } + commandBuffer.present(drawable) + commandBuffer.commit() + } +} diff --git a/ios/HouseScan/UI/Camera/WallMarksOverlay.swift b/ios/HouseScan/UI/Camera/WallMarksOverlay.swift index b9a4e4c69..b9f27a256 100644 --- a/ios/HouseScan/UI/Camera/WallMarksOverlay.swift +++ b/ios/HouseScan/UI/Camera/WallMarksOverlay.swift @@ -1,13 +1,30 @@ import SwiftUI +import simd /// What the homeowner has marked, pinned onto the camera image: the meter, the wall ends and -/// the marked features (openings as outlines, everything else as pins or lines). +/// the marked features. +/// +/// Things that stand on the wall get the prototype's corner brackets (after RoomPlan's +/// outlines) with a short caption: the meter in Signal blue, everything else in the dots' +/// Hologram white. A marked thing is settled, so its bracket is solid; the prototype's dashed, +/// unsettled bracket has no counterpart here. Driveways and fences stay dashed lines on the +/// ground, and the wall ends white posts. struct WallMarksOverlay: View { var projection: CameraProjection var wall: WallGeometry var features: [MarkedFeature] var wallBandHeight: Float + /// The dots' Hologram, #E6ECF4. + static let hologram = Color(red: 0xE6 / 255, green: 0xEC / 255, blue: 0xF4 / 255) + + /// Display sizes (width, height) for things marked with one tap, in meters: a bracket needs + /// a box, and one tap gives only the middle. Typical sizes, not measurements; the export uses + /// its own nominal width (`ScanEngine.project`). + static let meterBox = SIMD2(0.3, 0.4) + static let gasMeterBox = SIMD2(0.3, 0.4) + static let acUnitBox = SIMD2(0.6, 0.6) + var body: some View { Canvas { context, size in let geometry = WallProjection(projection: projection, wall: wall, size: size) @@ -15,21 +32,17 @@ struct WallMarksOverlay: View { for feature in features { drawFeature(feature, in: &context, geometry) } - drawMeter(in: &context, geometry) + let up = SIMD3(0, 1, 0) + let along = wall.along(atS: 0) + if let corners = box(center: wall.meter, along: along, up: up, size: Self.meterBox, geometry) { + drawBracket(corners, color: Palette.signal, caption: "Electric meter", in: &context) + } } .ignoresSafeArea() .allowsHitTesting(false) .accessibilityHidden(true) } - private func drawMeter(in context: inout GraphicsContext, _ geometry: WallProjection) { - guard let point = geometry.point(wall.meter) else { return } - let outer = CGRect(x: point.x - 13, y: point.y - 13, width: 26, height: 26) - context.fill(Path(ellipseIn: outer), with: .color(.white)) - context.fill(Path(ellipseIn: outer.insetBy(dx: 4, dy: 4)), with: .color(Palette.signal)) - context.stroke(Path(ellipseIn: outer.insetBy(dx: -5, dy: -5)), with: .color(.white.opacity(0.6)), lineWidth: 2) - } - private func drawEnds(in context: inout GraphicsContext, _ geometry: WallProjection) { for end in [wall.leftEnd, wall.rightEnd].compactMap(\.self) { guard let bottom = geometry.point(s: end, height: 0), let top = geometry.point(s: end, height: wallBandHeight) else { continue } @@ -44,30 +57,79 @@ struct WallMarksOverlay: View { } private func drawFeature(_ feature: MarkedFeature, in context: inout GraphicsContext, _ geometry: WallProjection) { - let color = Color.white + let caption = ScanCopy.name(feature.kind) + let up = SIMD3(0, 1, 0) switch feature.kind { case .door, .window: - if let bottom = feature.bottom, let top = feature.top, - let outline = geometry.wallQuad(s: feature.span, height: bottom...top, out: 0.01) { - context.fill(outline, with: .color(Palette.signal.opacity(0.18))) - context.stroke(outline, with: .color(color), style: StrokeStyle(lineWidth: 3, lineJoin: .round, dash: [9, 6])) + guard let bottom = feature.bottom, let top = feature.top else { return } + let corners = [ + wall.world(s: feature.span.lowerBound, height: top, out: 0.01), + wall.world(s: feature.span.upperBound, height: top, out: 0.01), + wall.world(s: feature.span.upperBound, height: bottom, out: 0.01), + wall.world(s: feature.span.lowerBound, height: bottom, out: 0.01), + ].compactMap(geometry.point) + if corners.count == 4 { drawBracket(corners, color: Self.hologram, caption: caption, in: &context) } + case .gasMeter, .acUnit: + guard let point = feature.points.first else { return } + let s = (feature.span.lowerBound + feature.span.upperBound) / 2 + let size = feature.kind == .gasMeter ? Self.gasMeterBox : Self.acUnitBox + if let corners = box(center: point, along: wall.along(atS: s), up: up, size: size, geometry) { + drawBracket(corners, color: Self.hologram, caption: caption, in: &context) } case .driveway, .fence: let points = feature.points.compactMap(geometry.point) - if points.count >= 2 { - var line = Path() - line.addLines(points) - context.stroke(line, with: .color(.black.opacity(0.3)), style: StrokeStyle(lineWidth: 8, lineCap: .round)) - context.stroke(line, with: .color(color), style: StrokeStyle(lineWidth: 4, lineCap: .round, dash: [10, 7])) + guard points.count >= 2 else { return } + var line = Path() + line.addLines(points) + context.stroke(line, with: .color(.black.opacity(0.3)), style: StrokeStyle(lineWidth: 8, lineCap: .round)) + context.stroke(line, with: .color(.white), style: StrokeStyle(lineWidth: 4, lineCap: .round, dash: [10, 7])) + for point in points { + let dot = CGRect(x: point.x - 6, y: point.y - 6, width: 12, height: 12) + context.fill(Path(ellipseIn: dot.insetBy(dx: -2, dy: -2)), with: .color(.black.opacity(0.25))) + context.fill(Path(ellipseIn: dot), with: .color(.white)) } - case .gasMeter, .acUnit: - break } - for point in feature.points.compactMap(geometry.point) { - let dot = CGRect(x: point.x - 8, y: point.y - 8, width: 16, height: 16) - context.fill(Path(ellipseIn: dot.insetBy(dx: -3, dy: -3)), with: .color(.black.opacity(0.25))) - context.fill(Path(ellipseIn: dot), with: .color(.white)) - context.fill(Path(ellipseIn: dot.insetBy(dx: 4, dy: 4)), with: .color(Palette.signal)) + } + + /// Four corners on screen, clockwise from top-left, of a box facing out of the wall around + /// `center`; nil when any corner is behind the camera. + private func box(center: SIMD3, along: SIMD3, up: SIMD3, size: SIMD2, _ geometry: WallProjection) -> [CGPoint]? { + let w = along * (size.x / 2), h = up * (size.y / 2) + let corners = [center - w + h, center + w + h, center + w - h, center - w - h].compactMap(geometry.point) + return corners.count == 4 ? corners : nil + } + + /// The prototype's bracket: four L-shaped corners 6 pt outside the box, each arm 18 pt or 40% + /// of its edge, with a 4 pt rounded bend, over a soft dark halo so it holds on a bright wall; + /// the caption sits 13 pt above the top edge. + private func drawBracket(_ box: [CGPoint], color: Color, caption: String, in context: inout GraphicsContext) { + let cx = box.map(\.x).reduce(0, +) / 4, cy = box.map(\.y).reduce(0, +) / 4 + let corners = box.map { p -> CGPoint in + let dx = p.x - cx, dy = p.y - cy + let length = max((dx * dx + dy * dy).squareRoot(), 0.001) + return CGPoint(x: p.x + dx / length * 6, y: p.y + dy / length * 6) } + var path = Path() + for i in corners.indices { + let corner = corners[i], previous = corners[(i + 3) % 4], next = corners[(i + 1) % 4] + func toward(_ other: CGPoint) -> CGPoint { + let dx = other.x - corner.x, dy = other.y - corner.y + let length = max((dx * dx + dy * dy).squareRoot(), 0.001) + let arm = min(18, 0.4 * length) + return CGPoint(x: corner.x + dx / length * arm, y: corner.y + dy / length * arm) + } + path.move(to: toward(previous)) + path.addArc(tangent1End: corner, tangent2End: toward(next), radius: 4) + path.addLine(to: toward(next)) + } + context.stroke(path, with: .color(.black.opacity(0.28)), style: StrokeStyle(lineWidth: 4.5, lineCap: .round, lineJoin: .round)) + context.stroke(path, with: .color(color), style: StrokeStyle(lineWidth: 2, lineCap: .round, lineJoin: .round)) + + let top = CGPoint(x: (corners[0].x + corners[1].x) / 2, y: min(corners[0].y, corners[1].y) - 13) + var layer = context + layer.addFilter(.shadow(color: .black.opacity(0.45), radius: 6, y: 1)) + layer.draw( + Text(caption).font(.system(size: 13, weight: .semibold, design: .rounded)).foregroundStyle(.white), + at: top, anchor: .center) } } diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Overlay/SurfaceDots.swift b/ios/HouseScanKit/Sources/HouseScanKit/Overlay/SurfaceDots.swift new file mode 100644 index 000000000..c21a004d1 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Overlay/SurfaceDots.swift @@ -0,0 +1,503 @@ +import simd + +// The live dots a LiDAR phone draws over the camera: the prototype's dot field +// (experiments/live-dots on t3/experience, `LiveDotsCore/DotField.swift`), bounded for a phone. +// +// Each kept keyframe's depth, every second pixel of it, with medium or high confidence and from +// 0.3 to 5 m, is unprojected and fused into 5 cm voxels. A voxel keeps its distinct view +// directions (a new one must differ from every stored one by more than 15 degrees, four at +// most), a normal from central differences on the depth map (averaged over keyframes), the most +// face-on view it has had, and a fixed jitter of up to 30% of the voxel in the plane of its +// first normal, so the field never reads as a grid. +// +// A voxel becomes an edge, and stays one, when +// (a) a face neighbour's normal differs by more than 35 degrees (the wall-to-ground corner, a +// bin's corners), or +// (b) it has surface along one side and seen-empty space along another (a bin's silhouette). +// Seen-empty means a confident depth reading passed the neighbour with 10 cm to spare, so +// the edge of what has been scanned so far is never an edge. +// The prototype's third rule, a colour step in the photo (painted outlines), is left out: it +// needs the camera image at depth resolution on every keyframe, and the things this scan cares +// about (meters, bins, AC units) stand off the wall, so (a) and (b) find them. +// +// Every edge voxel gets a dot. Flat voxels get one per 2 x 2 x 2 group, chosen once by lowest +// hash and kept, a quarter of the edge density on a wall. Opacity is 45% at one view and rises +// 15 points a view to 90% at four. An edge on a surface within 45 degrees of vertical stays at or +// below 60% until one view saw it within 30 degrees of face-on. A dot more than 25 cm in front of +// the wall and more than 10 cm above the ground is on an occluder, and draws violet. +// +// The dots are guidance, never evidence: coverage (`CoverageMap`) decides what the scan has +// seen, from the same keyframes, and nothing here feeds it. + +public typealias VoxelKey = SIMD3 + +public struct SurfaceDotConfig: Sendable, Equatable { + public var voxelSize: Float = 0.05 + /// ARKit confidence: 0 low, 1 medium, 2 high. + public var minimumConfidence: UInt8 = 1 + /// 5 m, not the prototype's 6: ARKit's confidence falls off past about 5 m outdoors, and + /// coverage does not count views from farther than `CoverageConfig.maxDistance` (6 m) anyway. + public var depthRange: ClosedRange = 0.3...5 + /// Every `sampleStride`-th depth pixel on both axes. At 2, a 256 x 192 map gives 12,288 + /// samples, and a pixel step at 5 m (about 5 cm) still reaches every 5 cm voxel it crosses. + public var sampleStride: Int = 2 + public var viewSeparationDegrees: Float = 15 + public var creaseDegrees: Float = 35 + /// A face neighbour lies along the surface when its direction is within 60 degrees of the + /// tangent plane. + public var alongSurfaceCosine: Float = 0.5 + public var freeSpaceMargin: Float = 0.1 + public var jitterFraction: Float = 0.3 + public var faceOnDegrees: Float = 30 + public var obliqueEdgeOpacityCap: Float = 0.6 + /// The oblique cap applies to edges whose normal is within 45 degrees of horizontal. + public var cappedNormalMaxY: Float = 0.7071 + public var occluderMinOut: Float = 0.25 + public var groundBand: Float = 0.1 + /// Voxels kept at once. Past it, the ones farthest from the camera go first, down to 75%. + /// A voxel costs about 100 bytes with its dictionary entry, so 60,000 is about 6 MB; a 20 m + /// walk with the wall and the ground in front of it is about 25,000 surface voxels. Not + /// measured on a phone. + public var maxVoxels: Int = 60_000 + /// Most dots handed to the renderer. The prototype drew 6,000 in 0.54 ms of an M4 Pro's GPU; + /// a phone GPU is several times slower and also draws the camera, so 5,000. Not measured on + /// a phone. + public var maxDots: Int = 5_000 + /// Dots farther than this from the latest camera are not handed out: off screen or too small + /// to matter, and they would crowd nearer ones out of `maxDots`. + public var dotRadius: Float = 6 + + public init() {} +} + +/// One dot as the renderer draws it, before timing. +public struct SurfaceDot: Sendable, Equatable { + /// Stable for the life of the voxel, so the renderer can tell a birth from a survivor. + public let id: UInt64 + public let position: SIMD3 + public let isEdge: Bool + /// Distinct view directions so far, up to four. + public let views: Int + /// Some view saw it within `faceOnDegrees` of face-on. Only edge dots are ever false. + public let faceOn: Bool + /// On something standing in front of the wall (a bin, a bush), drawn violet. + public let onOccluder: Bool + /// 0.45 at one view to 0.9 at four, capped at 0.6 for an edge never seen face-on. + public let opacity: Float + + public init(id: UInt64, position: SIMD3, isEdge: Bool, views: Int, faceOn: Bool, onOccluder: Bool, opacity: Float) { + self.id = id + self.position = position + self.isEdge = isEdge + self.views = views + self.faceOn = faceOn + self.onOccluder = onOccluder + self.opacity = opacity + } + + public static func opacity(views: Int, faceOn: Bool, cap: Float) -> Float { + guard views > 0 else { return 0 } + let level = min(0.45 + 0.15 * Float(views - 1), 0.9) + return faceOn ? level : min(level, cap) + } +} + +/// Up to four unit directions, stored inline so a voxel needs no heap allocation. +struct ViewSet: Sendable, Equatable { + private var d0 = SIMD3.zero, d1 = SIMD3.zero, d2 = SIMD3.zero, d3 = SIMD3.zero + private(set) var count: UInt8 = 0 + /// Opacity stops rising at four views. + static let capacity: UInt8 = 4 + + func direction(_ index: Int) -> SIMD3 { + switch index { + case 0: d0 + case 1: d1 + case 2: d2 + default: d3 + } + } + + /// Adds `unit` when it differs from every stored direction by more than the angle whose + /// cosine is `limit`. Returns whether it was added. + @discardableResult + mutating func insert(_ unit: SIMD3, limit: Float) -> Bool { + guard count < Self.capacity else { return false } + for index in 0..= limit { return false } + switch count { + case 0: d0 = unit + case 1: d1 = unit + case 2: d2 = unit + default: d3 = unit + } + count += 1 + return true + } +} + +struct Voxel: Sendable, Equatable { + /// Sum of per-keyframe unit normals. + var normalSum: SIMD3 + var views: ViewSet + /// Highest cosine between a view direction and the voxel's normal at that view. + var faceOnCosine: Float + /// Fixed at the first hit, in the plane of the first normal. + var jitter: SIMD3 + /// Sticky: an edge never turns back into a flat dot. + var isEdge: Bool + + var normal: SIMD3? { + let length = simd_length(normalSum) + return length > 1e-3 ? normalSum / length : nil + } +} + +/// The LiDAR dot field. A value type confined to one owner; `integrate` is the hot path and +/// touches only the voxels this keyframe saw and their face neighbours. +/// +/// Voxels live in the field's own frame, fixed at the first keyframe; `toWorld` carries them into +/// ARKit's world. An anchor correction changes only `toWorld`, so it is exact and costs nothing: +/// moving the voxels would snap them back to the 5 cm grid, and a run of 2 cm corrections would +/// leave the dots where they were while the wall moved. +public struct SurfaceDots: Sendable { + public let config: SurfaceDotConfig + /// Field frame to world. Identity until the first correction. + private var toWorld = matrix_identity_float4x4 + private var toField = matrix_identity_float4x4 + private(set) var voxels: [VoxelKey: Voxel] = [:] + /// Voxels a confident depth reading passed with room to spare. Only in-plane neighbours of + /// occupied voxels are tested, which is all the boundary rule needs. + private(set) var free: Set = [] + /// Each 2 x 2 x 2 group's flat voxel, keyed by the group. + private var representatives: [VoxelKey: VoxelKey] = [:] + + static let faceAxes: [VoxelKey] = [ + VoxelKey(1, 0, 0), VoxelKey(-1, 0, 0), VoxelKey(0, 1, 0), + VoxelKey(0, -1, 0), VoxelKey(0, 0, 1), VoxelKey(0, 0, -1), + ] + + public init(config: SurfaceDotConfig = SurfaceDotConfig()) { + self.config = config + } + + public var voxelCount: Int { voxels.count } + public var isEmpty: Bool { voxels.isEmpty } + + public func key(for p: SIMD3) -> VoxelKey { + VoxelKey((p / config.voxelSize).rounded(.toNearestOrAwayFromZero)) + } + + public func centre(of key: VoxelKey) -> SIMD3 { + SIMD3(key) * config.voxelSize + } + + // MARK: Keyframes + + /// Fuses one keyframe's depth. `depth` is ARKit's scene depth for `camera`, in the landscape + /// sensor orientation, with intrinsics for its own grid (`DepthImage.intrinsics(scaling:)`). + public mutating func integrate(camera world: CameraFrame, depth: DepthImage) { + let step = max(config.sampleStride, 1) + let camera = CameraFrame(cameraToWorld: toField * world.cameraToWorld, intrinsics: world.intrinsics, imageSize: world.imageSize) + let pose = camera.cameraToWorld + let eye = camera.position + + struct Accumulator { var normals = 0; var normalSum = SIMD3.zero } + var accumulators: [VoxelKey: Accumulator] = [:] + accumulators.reserveCapacity(depth.width * depth.height / (step * step * 4)) + for j in stride(from: step / 2, to: depth.height, by: step) { + for i in stride(from: step / 2, to: depth.width, by: step) { + guard let d = reading(depth, i, j) else { continue } + let local = Self.unproject(i, j, d, depth.intrinsics) + let world4 = pose * SIMD4(local, 1) + let world = SIMD3(world4.x, world4.y, world4.z) + let key = key(for: world) + var accumulator = accumulators[key] ?? Accumulator() + if let normal = cameraNormal(depth, i, j, d, step: step) { + let rotated = pose * SIMD4(normal, 0) + accumulator.normals += 1 + accumulator.normalSum += SIMD3(rotated.x, rotated.y, rotated.z) + } + accumulators[key] = accumulator + } + } + let limit = cos(config.viewSeparationDegrees * .pi / 180) + for (key, a) in accumulators { + observe(key, normal: a.normals > 0 ? a.normalSum / Float(a.normals) : nil, camera: eye, viewLimit: limit) + } + let touched = Set(accumulators.keys) + carveFreeSpace(around: touched, camera: camera, depth: depth) + classify(touched) + bound(around: eye) + } + + /// Records one keyframe's samples in a voxel. + mutating func observe(_ key: VoxelKey, normal: SIMD3?, camera: SIMD3, viewLimit: Float) { + let centre = centre(of: key) + var voxel = voxels[key] ?? Voxel( + normalSum: .zero, views: ViewSet(), faceOnCosine: -1, + jitter: Self.jitter(for: key, normal: normal ?? SIMD3(0, 0, 1), radius: config.jitterFraction * config.voxelSize), + isEdge: false) + if let normal, simd_length(normal) > 1e-3 { voxel.normalSum += simd_normalize(normal) } + let toCamera = camera - centre + let distance = simd_length(toCamera) + if distance > 1e-6 { + let unit = toCamera / distance + voxel.views.insert(unit, limit: viewLimit) + if let current = voxel.normal { voxel.faceOnCosine = max(voxel.faceOnCosine, simd_dot(unit, current)) } + } + voxels[key] = voxel + free.remove(key) + } + + /// Tests the empty in-plane neighbours of this keyframe's voxels against its depth: a + /// neighbour is seen-empty when a confident reading at its pixel is farther than it by + /// `freeSpaceMargin`. A neighbour outside the view, behind something, or under a low-confidence + /// reading stays unknown, which is why the edge of what has been scanned so far never counts + /// as a boundary. + mutating func carveFreeSpace(around touched: Set, camera: CameraFrame, depth: DepthImage) { + let toCamera = camera.worldToCamera + var found: [VoxelKey] = [] + for key in touched { + guard let normal = voxels[key]?.normal else { continue } + for axis in Self.faceAxes where abs(simd_dot(SIMD3(axis), normal)) < config.alongSurfaceCosine { + let neighbour = key &+ axis + guard voxels[neighbour] == nil, !free.contains(neighbour) else { continue } + let c4 = toCamera * SIMD4(centre(of: neighbour), 1) + let z = -c4.z + guard z >= config.depthRange.lowerBound else { continue } + let k = depth.intrinsics + let u = k.z + k.x * c4.x / z + let v = k.w - k.y * c4.y / z + guard u >= 0, v >= 0, u < Float(depth.width), v < Float(depth.height), + let measured = reading(depth, Int(u), Int(v), ignoringRange: true) else { continue } + if measured > z + config.freeSpaceMargin { found.append(neighbour) } + } + } + free.formUnion(found) + } + + func edgeReason(for key: VoxelKey) -> EdgeReason? { + guard let voxel = voxels[key] else { return nil } + let normal = voxel.normal + var surfaceNeighbour = false, emptyNeighbour = false + let creaseCosine = cos(config.creaseDegrees * .pi / 180) + for axis in Self.faceAxes { + let along = normal.map { abs(simd_dot(SIMD3(axis), $0)) < config.alongSurfaceCosine } ?? false + if let other = voxels[key &+ axis] { + if let normal, let otherNormal = other.normal, simd_dot(normal, otherNormal) < creaseCosine { + return .crease + } + if along { surfaceNeighbour = true } + } else if along, free.contains(key &+ axis) { + emptyNeighbour = true + } + } + return surfaceNeighbour && emptyNeighbour ? .boundary : nil + } + + enum EdgeReason: Equatable { + case crease + case boundary + } + + /// Marks new edges (sticky) among the touched voxels and their neighbours, then gives each + /// touched group without one a flat representative. + mutating func classify(_ touched: Set) { + var candidates = touched + for key in touched { + for axis in Self.faceAxes where voxels[key &+ axis] != nil { candidates.insert(key &+ axis) } + } + for key in candidates where voxels[key]?.isEdge == false && edgeReason(for: key) != nil { + voxels[key]?.isEdge = true + } + for key in touched { + let group = key &>> 1 + guard representatives[group] == nil else { continue } + var best: (key: VoxelKey, hash: UInt64)? + for dx in Int32(0)...1 { + for dy in Int32(0)...1 { + for dz in Int32(0)...1 { + let member = (group &<< 1) &+ VoxelKey(dx, dy, dz) + guard let voxel = voxels[member], !voxel.isEdge else { continue } + let hash = Self.hash(member, seed: 0xF1A7) + if best == nil || hash < best!.hash { best = (member, hash) } + } + } + } + if let best { representatives[group] = best.key } + } + } + + /// Keeps at most `maxVoxels`: past it, drops the voxels farthest from `eye` down to 75% of + /// the cap, with the free space and representatives that referred to them. + mutating func bound(around eye: SIMD3) { + if voxels.count > config.maxVoxels { + let keep = config.maxVoxels * 3 / 4 + let ranked = voxels.keys.map { ($0, simd_distance_squared(centre(of: $0), eye)) }.sorted { $0.1 < $1.1 } + let cutoff = ranked[keep].1 + voxels = voxels.filter { simd_distance_squared(centre(of: $0.key), eye) < cutoff } + representatives = representatives.filter { voxels[$0.value] != nil } + } + if free.count > config.maxVoxels { + let ranked = free.map { simd_distance_squared(centre(of: $0), eye) }.sorted() + let cutoff = ranked[config.maxVoxels * 3 / 4] + free = free.filter { simd_distance_squared(centre(of: $0), eye) < cutoff } + } + } + + /// ARKit's correction to the meter's anchor (`YawCorrection`): the dots move with the wall the + /// scan measured. Composes into `toWorld`; no voxel moves. + public mutating func apply(_ correction: YawCorrection) { + toWorld = correction.pose(toWorld) + toField = toWorld.inverse + } + + // MARK: Dots + + /// Every edge voxel and each group's flat representative within `dotRadius` of `eye`, at + /// most `maxDots`: past it, flat dots drop in hash order, then the farthest edges. `wall` + /// decides which dots stand on an occluder; without one, none do. Sorted by id. + public func dots(near worldEye: SIMD3, wall: WallFrame?) -> [SurfaceDot] { + let eye = Self.transform(toField, worldEye) + let radius2 = config.dotRadius * config.dotRadius + var edges: [(VoxelKey, Voxel, Float)] = [] + var flats: [(VoxelKey, Voxel)] = [] + for (key, voxel) in voxels where voxel.isEdge { + let d2 = simd_distance_squared(centre(of: key), eye) + if d2 <= radius2 { edges.append((key, voxel, d2)) } + } + for key in representatives.values { + guard let voxel = voxels[key], !voxel.isEdge, simd_distance_squared(centre(of: key), eye) <= radius2 else { continue } + flats.append((key, voxel)) + } + if edges.count > config.maxDots { + edges.sort { $0.2 < $1.2 } + edges.removeLast(edges.count - config.maxDots) + } + let room = config.maxDots - edges.count + if flats.count > room { + flats.sort { Self.hash($0.0, seed: 0xD075) < Self.hash($1.0, seed: 0xD075) } + flats.removeLast(flats.count - room) + } + var result = edges.map { dot($0.0, $0.1, wall: wall) } + flats.map { dot($0.0, $0.1, wall: wall) } + result.sort { $0.id < $1.id } + return result + } + + private func dot(_ key: VoxelKey, _ voxel: Voxel, wall: WallFrame?) -> SurfaceDot { + let centre = Self.transform(toWorld, centre(of: key)) + let jitter = Self.rotate(toWorld, voxel.jitter) + // Corrections turn only about the vertical, so a normal's y is the same in both frames. + let wallLike = voxel.normal.map { abs($0.y) < config.cappedNormalMaxY } ?? false + let faceOn = !voxel.isEdge || !wallLike || voxel.faceOnCosine >= cos(config.faceOnDegrees * .pi / 180) + let views = Int(voxel.views.count) + var onOccluder = false + if let wall { + let p = wall.wallPoint(centre) + onOccluder = p.out > config.occluderMinOut && p.height > config.groundBand + } + return SurfaceDot( + id: Self.id(for: key), position: centre + jitter, isEdge: voxel.isEdge, views: views, + faceOn: faceOn, onOccluder: onOccluder, + opacity: SurfaceDot.opacity(views: views, faceOn: faceOn, cap: config.obliqueEdgeOpacityCap)) + } + + // MARK: Depth helpers + + static func transform(_ m: simd_float4x4, _ p: SIMD3) -> SIMD3 { + let q = m * SIMD4(p, 1) + return SIMD3(q.x, q.y, q.z) + } + + static func rotate(_ m: simd_float4x4, _ v: SIMD3) -> SIMD3 { + let q = m * SIMD4(v, 0) + return SIMD3(q.x, q.y, q.z) + } + + /// Meters at a depth pixel, or nil without a confident reading in range. `ignoringRange` keeps + /// far readings, which the free-space test needs. + private func reading(_ depth: DepthImage, _ i: Int, _ j: Int, ignoringRange: Bool = false) -> Float? { + let index = j * depth.width + i + if let confidence = depth.confidence, confidence[index] < config.minimumConfidence { return nil } + let millimeters = depth.millimeters[index] + guard millimeters > 0 else { return nil } + let meters = Float(millimeters) / 1000 + return ignoringRange || config.depthRange.contains(meters) ? meters : nil + } + + /// Camera space: +x right and +y up in the sensor image, looking along -z; v grows down. + static func unproject(_ i: Int, _ j: Int, _ d: Float, _ k: SIMD4) -> SIMD3 { + let u = Float(i) + 0.5, v = Float(j) + 0.5 + return SIMD3((u - k.z) / k.x * d, (k.w - v) / k.y * d, -d) + } + + /// Camera-space unit normal from central differences `step` pixels apart, falling back to one + /// side where the other jumps by more than 10% of the depth (a silhouette), facing the + /// camera. Nil when neither side is usable on either axis. + private func cameraNormal(_ depth: DepthImage, _ i: Int, _ j: Int, _ d0: Float, step: Int) -> SIMD3? { + func point(_ i: Int, _ j: Int) -> SIMD3? { + guard i >= 0, j >= 0, i < depth.width, j < depth.height, let d = reading(depth, i, j), abs(d - d0) < 0.1 * d0 else { return nil } + return Self.unproject(i, j, d, depth.intrinsics) + } + let centre = Self.unproject(i, j, d0, depth.intrinsics) + func tangent(_ plus: SIMD3?, _ minus: SIMD3?) -> SIMD3? { + switch (plus, minus) { + case let (p?, m?): p - m + case let (p?, nil): p - centre + case let (nil, m?): centre - m + case (nil, nil): nil + } + } + guard let tx = tangent(point(i + step, j), point(i - step, j)), + let ty = tangent(point(i, j + step), point(i, j - step)) else { return nil } + let n = simd_cross(tx, ty) + let length = simd_length(n) + guard length > 1e-9 else { return nil } + let unit = n / length + return simd_dot(unit, -centre) < 0 ? -unit : unit + } + + // MARK: Hashing + + /// SplitMix64: Swift's `Hasher` is seeded per process, and the field must look the same twice. + static func mix(_ value: UInt64) -> UInt64 { + var z = value &+ 0x9E37_79B9_7F4A_7C15 + z = (z ^ (z >> 30)) &* 0xBF58_476D_1CE4_E5B9 + z = (z ^ (z >> 27)) &* 0x94D0_49BB_1331_11EB + return z ^ (z >> 31) + } + + static func hash(_ key: VoxelKey, seed: UInt64) -> UInt64 { + var h = mix(seed) + h = mix(h ^ UInt64(UInt32(bitPattern: key.x))) + h = mix(h ^ UInt64(UInt32(bitPattern: key.y))) + h = mix(h ^ UInt64(UInt32(bitPattern: key.z))) + return h + } + + static func unit(_ h: UInt64) -> Float { + Float(h >> 40) / Float(1 << 24) + } + + /// Uniform over a disc of `radius` perpendicular to `normal`, from a hash of the key. + static func jitter(for key: VoxelKey, normal: SIMD3, radius: Float) -> SIMD3 { + let length = simd_length(normal) + let n = length > 1e-6 ? normal / length : SIMD3(0, 0, 1) + let helper: SIMD3 = abs(n.y) < 0.9 ? SIMD3(0, 1, 0) : SIMD3(1, 0, 0) + let t = simd_normalize(simd_cross(helper, n)) + let b = simd_cross(n, t) + let h = hash(key, seed: 0x6A17) + let angle = unit(h) * 2 * .pi + let r = radius * unit(mix(h)).squareRoot() + return (t * cos(angle) + b * sin(angle)) * r + } + + /// Packs the key into 21 bits per axis (plus or minus 52 km at 5 cm), so ids never collide. + static func id(for key: VoxelKey) -> UInt64 { + let bias: Int64 = 1 << 20 + let x = UInt64(Int64(key.x) + bias) & 0x1F_FFFF + let y = UInt64(Int64(key.y) + bias) & 0x1F_FFFF + let z = UInt64(Int64(key.z) + bias) & 0x1F_FFFF + return x | (y << 21) | (z << 42) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/SurfaceDotsTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/SurfaceDotsTests.swift new file mode 100644 index 000000000..28e0a13c1 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/SurfaceDotsTests.swift @@ -0,0 +1,128 @@ +import HouseScanKit +import simd +import Testing + +// The LiDAR dot field on the standard wall (face z = 0, ground y = 0), with depth rendered from +// the scene mesh at 128 x 96 (`renderDepth`, fx = fy = 100): at 2 m a pixel is 2 cm, so every +// sampled pixel (stride 2) lands within one 5 cm voxel of the next. +@Suite struct SurfaceDotsTests { + /// A box standing in front of the wall: s in [-0.5, 0.5], 0.5 to 1.0 m out, 1.5 m tall. + static let box = (SIMD3(-0.5, 0, 0.5), SIMD3(0.5, 1.5, 1.0)) + + static func field(_ scene: TriangleMesh, cameras: [CameraFrame], config: SurfaceDotConfig = SurfaceDotConfig()) -> SurfaceDots { + var field = SurfaceDots(config: config) + for camera in cameras { field.integrate(camera: camera, depth: renderDepth(scene, from: camera)) } + return field + } + + /// Level at 1.5 m, 2 m from the wall: the view spans heights 0.22 to 2.78 m, all wall, so no + /// neighbour is ever seen empty and nothing creases. Every dot is flat, on the wall, one per + /// 2 x 2 x 2 group (four wall voxels share a group), at 45% for one view. + @Test func aPlainWallGivesSparseFlatDotsOnTheWall() { + let camera = makeCamera(at: SIMD3(0, 1.5, 2.0), forward: SIMD3(0, 0, -1), right: SIMD3(1, 0, 0)) + let field = Self.field(standardScene(), cameras: [camera]) + let dots = field.dots(near: camera.position, wall: standardWall()) + #expect(!dots.isEmpty) + #expect(dots.allSatisfy { !$0.isEdge }) + #expect(dots.allSatisfy { abs($0.position.z) < 0.02 }) + #expect(dots.allSatisfy { $0.opacity == 0.45 && !$0.onOccluder }) + #expect(dots.count * 3 <= field.voxelCount) + } + + /// Pitched down at the foot of the wall: the wall's and the ground's normals differ by 90 + /// degrees, so the voxels along the corner are edges, and only they are. + @Test func theWallToGroundCornerIsAnEdge() { + let camera = portraitCamera(at: SIMD3(0, 1.0, 2.0), forward: forwardFacingWall(pitchedDown: 30)) + let dots = Self.field(standardScene(), cameras: [camera]).dots(near: camera.position, wall: standardWall()) + let edges = dots.filter(\.isEdge) + #expect(edges.count > 20) + // The corner line is y = 0, z = 0; a corner voxel's centre is within one voxel diagonal, + // plus its jitter. + #expect(edges.allSatisfy { simd_length(SIMD2($0.position.y, $0.position.z)) < 0.1 }) + } + + /// The box's front stands 1 m out: its dots are on an occluder, the wall's are not, and its + /// silhouette against the wall behind is a boundary edge. + @Test func aBoxInFrontOfTheWallIsAnOccluderWithAnOutline() { + let camera = wallCamera(s: 0) + let dots = Self.field(standardScene(boxes: [Self.box]), cameras: [camera]).dots(near: camera.position, wall: standardWall()) + let onBox = dots.filter { $0.position.z > 0.9 && $0.position.y > 0.2 } + let onWall = dots.filter { abs($0.position.z) < 0.05 && $0.position.y > 0.2 } + #expect(!onBox.isEmpty && !onWall.isEmpty) + #expect(onBox.allSatisfy { $0.onOccluder }) + #expect(onWall.allSatisfy { !$0.onOccluder }) + let sides = onBox.filter { $0.isEdge && abs(abs($0.position.x) - 0.5) < 0.1 } + #expect(!sides.isEmpty) + } + + /// Opacity follows the distinct views: two positions 30 degrees apart as seen from the wall + /// give 60% where both saw it. + @Test func aSecondViewRaisesOpacity() { + let target = SIMD3(0, 1.0, 0) + let first = portraitCamera(at: SIMD3(0, 1.2, 2.0), lookingAt: target) + let second = portraitCamera(at: SIMD3(2.0 * tan(30 * .pi / 180), 1.2, 2.0), lookingAt: target) + let dots = Self.field(standardScene(), cameras: [first, second]).dots(near: second.position, wall: standardWall()) + #expect(dots.contains { $0.views == 2 && $0.opacity == 0.6 }) + #expect(dots.allSatisfy { $0.views <= 2 }) + } + + /// Low-confidence readings and readings past 5 m make no dots. + @Test func unconfidentOrFarDepthMakesNothing() { + let camera = wallCamera(s: 0) + var field = SurfaceDots() + field.integrate(camera: camera, depth: uniformDepth(2000, confidence: 0)) + field.integrate(camera: camera, depth: uniformDepth(5500)) + #expect(field.isEmpty) + } + + /// Past `maxVoxels` the field sheds the far voxels, so a long walk never grows it past the + /// cap; past `maxDots`, flat dots go before any edge. + @Test func dotsAndVoxelsStayUnderTheirCaps() { + var config = SurfaceDotConfig() + config.maxVoxels = 800 + var capped = config + capped.maxDots = 20 + var field = SurfaceDots(config: config) + var small = SurfaceDots(config: capped) + let scene = standardScene() + var last = portraitCamera(at: SIMD3(-4, 1.0, 2.0), forward: forwardFacingWall(pitchedDown: 30)) + for step in 0..<17 { + last = portraitCamera(at: SIMD3(-4 + Float(step) * 0.5, 1.0, 2.0), forward: forwardFacingWall(pitchedDown: 30)) + let depth = renderDepth(scene, from: last) + field.integrate(camera: last, depth: depth) + small.integrate(camera: last, depth: depth) + #expect(field.voxelCount <= config.maxVoxels) + } + let all = field.dots(near: last.position, wall: standardWall()) + let dots = small.dots(near: last.position, wall: standardWall()) + #expect(all.count > 20) + #expect(dots.count == 20) + #expect(dots.filter { $0.isEdge }.count == min(20, all.filter { $0.isEdge }.count)) + } + + /// An anchor correction moves the field with the wall, exactly, however small the steps: + /// ten 2.1 cm corrections, each under half a voxel, move every dot 21 cm, and a turn turns + /// them about the vertical. + @Test func correctionsMoveTheDotsExactly() { + let camera = wallCamera(s: 0) + var field = Self.field(standardScene(), cameras: [camera]) + let before = field.dots(near: camera.position, wall: nil) + for _ in 0..<10 { field.apply(YawCorrection(yaw: 0, translation: SIMD3(0.021, 0, 0))) } + let turn = YawCorrection(yaw: 0.1, translation: .zero) + field.apply(turn) + let after = field.dots(near: turn.point(camera.position + SIMD3(0.21, 0, 0)), wall: nil) + #expect(after.count == before.count) + for (a, b) in zip(before, after) { + #expect(a.id == b.id) + #expect(nearlyEqual(b.position, turn.point(a.position + SIMD3(0.21, 0, 0)), 1e-4)) + } + } + + /// The same keyframes give the same dots: jitter, thinning and order come from stable hashes. + @Test func theFieldIsDeterministic() { + let cameras = [wallCamera(s: 0), wallCamera(s: 0.5)] + let a = Self.field(standardScene(boxes: [Self.box]), cameras: cameras).dots(near: cameras[1].position, wall: standardWall()) + let b = Self.field(standardScene(boxes: [Self.box]), cameras: cameras).dots(near: cameras[1].position, wall: standardWall()) + #expect(a == b) + } +} diff --git a/ios/HouseScanUITests/FullFlowUITests.swift b/ios/HouseScanUITests/FullFlowUITests.swift index 6f3d2ec1d..ee8c02f80 100644 --- a/ios/HouseScanUITests/FullFlowUITests.swift +++ b/ios/HouseScanUITests/FullFlowUITests.swift @@ -58,6 +58,34 @@ final class FullFlowUITests: XCTestCase { XCTAssertTrue(showedHidden, "the wall map never reported hidden cells behind the bin") } + /// The LiDAR walk under the live fog, screenshotted as fast as the runner allows from the + /// moment the walk appears, for comparing against the prototype's keyframes (see + /// `LiveFogShaders`). No audit here: the audit's six-second wait would span the whole walk, + /// which plays in about 6 s at the autopilot's 3x. The run stops at the walk. + @MainActor + func testLiveFogWalkScreenshots() throws { + let app = XCUIApplication() + let gate = URL(fileURLWithPath: NSTemporaryDirectory()).appending(path: "housescan-gate-\(UUID().uuidString)", directoryHint: .isDirectory) + try FileManager.default.createDirectory(at: gate, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: gate) } + // The screens before the walk may leave as soon as the autopilot is done with them. + for phase in ["onboarding", "findMeter", "meterCloseUp"] { try Data().write(to: gate.appending(path: phase)) } + app.launchArguments = ["-replay", Self.lidarFixture, "-autopilot", "-autopilotHold", "1.5", "-autopilotGate", gate.path, "-sampleResult"] + app.launch() + XCTAssertTrue(app.descendants(matching: .any)["screen.wallWalk"].waitForExistence(timeout: 90), "screen.wallWalk never appeared") + let start = Date() + var index = 0 + while Date().timeIntervalSince(start) < 16 { + let shot = XCTAttachment(screenshot: app.screenshot()) + shot.name = String(format: "liveFog-%02d-%04.1fs", index, Date().timeIntervalSince(start)) + shot.lifetime = .keepAlways + add(shot) + index += 1 + Thread.sleep(forTimeInterval: 0.25) + } + app.terminate() + } + /// Device run 1's failure on the synthetic replay (`-autopilotCantGetThere`): "Can't get there" /// goes to the ground by the meter before it is seen, then ends each side where the walk went /// farthest, 5 m left and right of the meter (16 ft 5 in). The old rule put the ends at the diff --git a/ios/README.md b/ios/README.md index 90a5e9628..6c9082eaa 100644 --- a/ios/README.md +++ b/ios/README.md @@ -24,6 +24,8 @@ Native iPhone app for the AR capture walk. The homeowner marks the electric mete **Spot check.** Before the result, the app shows the kept photo that best sees the answer's spot, with the spot and its clearance area outlined, and asks whether anything stands in front of the wall or on the ground there (`STATE=spotConfirm`). Photos without depth and the walked path claim wall, ground and clear space they never saw; "It's clear" backs those claims for that spot, and "Something's there" takes them back over the area, so scene.json reports it unseen, and sends the scan again. The rules are in `Runtime/ScanEngine+Confirm.swift` and HouseScanKit's `Confirm/SpotConfirmation.swift`. +**Live fog.** During the walk and a gap request, a soft fog covers what the coverage map has not counted and lifts only where it counts a cell, the same rule the scan's decisions use (`FogValue.target` in `UI/Camera/LiveFog/LiveFogScene.swift`). On a LiDAR phone, dots settle on the surfaces the kept keyframes' depth measured (HouseScanKit's `Overlay/SurfaceDots.swift`, run by `Runtime/LiveDotsFeed.swift`); they are guidance and feed nothing. The look is the prototype's (experiments/live-dots on t3/experience). Phones without LiDAR get the same fog without dots. The shaders compile at launch, and the old frosted strip (`FogOverlay`) draws until they are ready, and for good if Metal fails. + Every screen change is logged as `STATE=` under subsystem `dev.housescanning.housescan`, category `state`. ## Layout