From 51184fee407077a4b2ee769b44c5e934b70c6f1a Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 03:33:58 -0500 Subject: [PATCH 1/5] Draw the prototype's fog and live dots over the walk The fog lifts only where the coverage map counts a cell, in wall columns and ground stretches, feathered and textured as in the prototype. LiDAR phones get dots fused from the depth of the keyframes coverage observes. Marks get the prototype's corner brackets. The frosted strip stays as the fallback while the shaders compile and if Metal fails. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/Contract/ScanContract.swift | 27 + ios/HouseScan/HouseScanApp.swift | 12 +- ios/HouseScan/Runtime/LiveDotsFeed.swift | 80 +++ ios/HouseScan/Runtime/ScanEngine.swift | 12 + ios/HouseScan/UI/Camera/CameraOverlays.swift | 33 +- ios/HouseScan/UI/Camera/FogMarks.swift | 112 ++++ ios/HouseScan/UI/Camera/FogOverlay.swift | 4 +- .../UI/Camera/LiveFog/LiveFogRenderer.swift | 316 +++++++++++ .../UI/Camera/LiveFog/LiveFogScene.swift | 281 ++++++++++ .../UI/Camera/LiveFog/LiveFogShaders.swift | 239 +++++++++ .../UI/Camera/LiveFog/LiveFogView.swift | 82 +++ .../UI/Camera/WallMarksOverlay.swift | 116 +++- .../HouseScanKit/Overlay/SurfaceDots.swift | 503 ++++++++++++++++++ .../HouseScanKitTests/SurfaceDotsTests.swift | 122 +++++ ios/HouseScanUITests/FullFlowUITests.swift | 28 + 15 files changed, 1932 insertions(+), 35 deletions(-) create mode 100644 ios/HouseScan/Runtime/LiveDotsFeed.swift create mode 100644 ios/HouseScan/UI/Camera/FogMarks.swift create mode 100644 ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift create mode 100644 ios/HouseScan/UI/Camera/LiveFog/LiveFogScene.swift create mode 100644 ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift create mode 100644 ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Overlay/SurfaceDots.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/SurfaceDotsTests.swift diff --git a/ios/HouseScan/Contract/ScanContract.swift b/ios/HouseScan/Contract/ScanContract.swift index 801525c49..b7d202cd4 100644 --- a/ios/HouseScan/Contract/ScanContract.swift +++ b/ios/HouseScan/Contract/ScanContract.swift @@ -272,6 +272,31 @@ struct CoverageStrip: Equatable, Sendable { } } +/// LiDAR phones: dots on the surfaces depth has measured, drawn over the camera by the live fog. +/// Guidance, never evidence: `coverage` alone decides what the scan has seen, and the fog lifts +/// only where it says so. Built from the same kept keyframes as coverage (`LiveDotsFeed`). +struct LiveDots: Sendable { + struct Dot: Sendable, Equatable { + /// Stable while the dot exists, so the renderer can tell a birth from a survivor. + var id: UInt64 + var position: SIMD3 + /// On a crease or a silhouette; drawn larger, with a stronger glow. + var isEdge: Bool + /// On something standing in front of the wall (a bin, a bush): drawn violet, the hidden + /// state's colour. + var onOccluder: Bool + /// 0.45 seen from one direction, up to 0.9 from four. + var opacity: Float + } + + /// At most `SurfaceDotConfig.maxDots` (5,000), those within 6 m of the latest keyframe. + var dots: [Dot] = [] + /// Increments on every change, so the renderer rebuilds its buffer only then. + var revision = 0 + + static let empty = LiveDots() +} + // MARK: - Guidance enum WallSide: String, Equatable, Sendable { @@ -685,6 +710,8 @@ final class ScanViewState { var wall: WallGeometry? var coverage: CoverageStrip = .empty + /// LiDAR phones: the live dots. Empty without depth. + var liveDots: LiveDots = .empty var closeUp: CloseUpState = .aiming(hold: 0, problem: nil) /// Close-up attempts that ended without a usable photo. "Can't get a clear shot" appears /// from the second one on, never earlier. diff --git a/ios/HouseScan/HouseScanApp.swift b/ios/HouseScan/HouseScanApp.swift index 013851423..a136ba964 100644 --- a/ios/HouseScan/HouseScanApp.swift +++ b/ios/HouseScan/HouseScanApp.swift @@ -4,11 +4,15 @@ import SwiftUI struct HouseScanApp: App { var body: some Scene { WindowGroup { - if ProcessInfo.processInfo.arguments.contains("-uiDemo") { - UIDemo.makeRoot() - } else { - AppRuntime.makeRoot() + Group { + if ProcessInfo.processInfo.arguments.contains("-uiDemo") { + UIDemo.makeRoot() + } else { + AppRuntime.makeRoot() + } } + // Compiles the live fog's shaders in the background long before the walk needs them. + .task { LiveFogSupport.shared.prepare() } } } } diff --git a/ios/HouseScan/Runtime/LiveDotsFeed.swift b/ios/HouseScan/Runtime/LiveDotsFeed.swift new file mode 100644 index 000000000..7f04bc98b --- /dev/null +++ b/ios/HouseScan/Runtime/LiveDotsFeed.swift @@ -0,0 +1,80 @@ +import Foundation +import HouseScanKit +import os + +/// Runs the LiDAR dot field (`SurfaceDots`) off the main actor and publishes its dots for the +/// camera overlay (`ScanViewState.liveDots`). +/// +/// The engine hands it each kept keyframe that has depth, the same frames coverage observes, so +/// the dots and the fog move together. Fusing a 256 x 192 map takes milliseconds, not a frame +/// budget, so it runs on its own serial queue. A keyframe arriving while two are still waiting +/// is dropped: the next one covers the same wall, and the queue can never grow. +final class LiveDotsFeed: @unchecked Sendable { + // `@unchecked Sendable`: everything below `queue` is read and written only on it; `pending` + // is a lock. + private let queue = DispatchQueue(label: "dev.housescanning.housescan.live-dots", qos: .utility) + private var field = SurfaceDots() + private var revision = 0 + private var last: (eye: SIMD3, wall: WallFrame)? + private let pending = OSAllocatedUnfairLock(initialState: 0) + private let publish: @MainActor @Sendable (LiveDots, _ generation: Int) -> Void + + /// Keyframes waiting on the queue past which new ones are dropped. + static let maxPending = 2 + + /// `publish` runs on the main actor with the scan generation the work was queued under, so + /// the engine can ignore dots from before a Start over. + init(publish: @escaping @MainActor @Sendable (LiveDots, _ generation: Int) -> Void) { + self.publish = publish + } + + func integrate(camera: CameraFrame, depth: DepthImage, wall: WallFrame, generation: Int) { + let accepted = pending.withLock { count -> Bool in + guard count < Self.maxPending else { return false } + count += 1 + return true + } + guard accepted else { + RuntimeLog.capture.info("live dots: dropped a keyframe, two still waiting") + return + } + queue.async { [self] in + field.integrate(camera: camera, depth: depth) + last = (camera.position, wall) + pending.withLock { $0 -= 1 } + send(generation) + } + } + + /// ARKit's correction to the meter's anchor, applied as coverage applies it. + func apply(_ correction: YawCorrection, wall: WallFrame, generation: Int) { + queue.async { [self] in + guard !field.isEmpty else { return } + field.apply(correction) + if let eye = last?.eye { last = (correction.point(eye), wall) } + send(generation) + } + } + + func reset() { + queue.async { [self] in + field = SurfaceDots() + last = nil + } + } + + /// On `queue`. + private func send(_ generation: Int) { + guard let last else { return } + revision += 1 + let dots = LiveDots( + dots: field.dots(near: last.eye, wall: last.wall).map { + LiveDots.Dot(id: $0.id, position: $0.position, isEdge: $0.isEdge, onOccluder: $0.onOccluder, opacity: $0.opacity) + }, + revision: revision) + let publish = publish + DispatchQueue.main.async { + MainActor.assumeIsolated { publish(dots, generation) } + } + } +} diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index de568c086..7acda2943 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -133,6 +133,12 @@ final class ScanEngine { /// The packet's sensor streams for the current world frame. private(set) var recorder: CaptureRecorder private let motion = MotionSource() + /// LiDAR phones: the dots over the camera, fused off the main actor from the keyframes + /// coverage observes. Guidance for the overlay only; nothing reads them back. + private lazy var liveDots = LiveDotsFeed { [weak self] dots, scan in + guard let self, scan == self.generation else { return } + self.state.liveDots = dots + } /// Every request the homeowner was shown, for the packet. var guidanceLog = GuidanceLog() /// The spot check (`ScanEngine+Confirm.swift`). @@ -409,6 +415,7 @@ final class ScanEngine { private func refreshMeterFromAnchor(_ frame: SourceFrame) { guard let anchor = frame.meterAnchor, coverage != nil, let correction = meterTracking?.update(to: anchor) else { return } coverage?.apply(correction) + if let wall = coverage?.wall { liveDots.apply(correction, wall: wall, generation: generation) } var features = state.features for index in features.indices { features[index].points = features[index].points.map(correction.point) } if features != state.features { state.features = features } @@ -653,6 +660,9 @@ final class ScanEngine { // The frame's own time lets the walked path join only poses kept close together in time. // With LiDAR depth, a cell counts only where depth confirms the camera saw it. let delta = coverage?.observe(frame.camera, trackingNormal: frame.tracking == .normal, time: frame.timestamp, depth: frame.depth, segment: segment) + if let depth = frame.depth, frame.tracking == .normal, let wall = coverage?.wall { + liveDots.integrate(camera: frame.camera, depth: depth, wall: wall, generation: scan) + } RuntimeLog.capture.info("stored \(frame.id, privacy: .public) as keyframe \(index)\(frame.depth == nil ? "" : " with depth", privacy: .public): \(delta?.newlySeen ?? 0) cells newly seen, \(delta?.newlyCovered ?? 0) newly covered, \(delta?.newlyHidden ?? 0) newly hidden") if overhead { recordOverhead(frame) } state.captureCount += 1 @@ -1545,6 +1555,8 @@ final class ScanEngine { uploadTask?.cancel() replay?.stop() coverage = nil + liveDots.reset() + state.liveDots = .empty meterAnchorID.map { live?.removeAnchor($0) } meterAnchorID = nil meterPlaneSource = .detectedPlane diff --git a/ios/HouseScan/UI/Camera/CameraOverlays.swift b/ios/HouseScan/UI/Camera/CameraOverlays.swift index da1916cee..929c02eaa 100644 --- a/ios/HouseScan/UI/Camera/CameraOverlays.swift +++ b/ios/HouseScan/UI/Camera/CameraOverlays.swift @@ -5,6 +5,9 @@ struct CameraOverlays: View { let state: ScanViewState var highlight: GapRequest? + @Environment(\.accessibilityReduceMotion) private var reduceMotion + @Environment(\.self) private var environment + var body: some View { ZStack { overlays @@ -14,11 +17,11 @@ struct CameraOverlays: View { @ViewBuilder private var overlays: some View { - // Hidden while tracking isn't normal: drawn from a pose the phone doesn't trust, the haze, - // path and pins would sit in the wrong place (checklist T3). They fade back on recovery. + // Hidden while tracking isn't normal: drawn from a pose the phone doesn't trust, the fog, + // path and marks would sit in the wrong place (checklist T3). They fade back on recovery. if state.tracking == .normal, let projection = state.projection, let wall = state.wall { ZStack { - FogOverlay(coverage: state.coverage, wall: wall, projection: projection, highlight: highlight) + fog(projection: projection, wall: wall) .ignoresSafeArea() WallMarksOverlay( projection: projection, @@ -34,4 +37,28 @@ struct CameraOverlays: View { } } } + + /// The live fog, with dots when the phone has depth. The frosted strip stands in while the + /// shaders compile at launch, and for good if Metal fails, so unseen wall never shows clear. + @ViewBuilder + private func fog(projection: CameraProjection, wall: WallGeometry) -> some View { + switch LiveFogSupport.shared.status { + case let .ready(gpu): + ZStack { + LiveFogView(gpu: gpu, input: LiveFogInput( + projection: projection, wall: wall, coverage: state.coverage, highlight: highlight, + dots: state.depthAvailable ? state.liveDots : .empty, reduceMotion: reduceMotion, + requestedColor: requestedColor)) + FogMarks(coverage: state.coverage, wall: wall, projection: projection, highlight: highlight) + } + .transition(.opacity) + case .preparing, .failed: + FogOverlay(coverage: state.coverage, wall: wall, projection: projection, highlight: highlight) + } + } + + private var requestedColor: SIMD3 { + let resolved = Palette.caution.resolve(in: environment) + return SIMD3(resolved.red, resolved.green, resolved.blue) + } } diff --git a/ios/HouseScan/UI/Camera/FogMarks.swift b/ios/HouseScan/UI/Camera/FogMarks.swift new file mode 100644 index 000000000..9f35af181 --- /dev/null +++ b/ios/HouseScan/UI/Camera/FogMarks.swift @@ -0,0 +1,112 @@ +import SwiftUI +import simd + +/// The crisp lines over the live fog, which a soft mask can't draw: a dashed violet outline +/// round each stretch hidden behind something, a slate hatch where the homeowner said "I can't +/// get there", and an amber outline round the stretch a gap request asks for. The same marks +/// and dash the tape map uses, so the camera and the map read alike. +struct FogMarks: View { + var coverage: CoverageStrip + var wall: WallGeometry + var projection: CameraProjection + var highlight: GapRequest? + + var body: some View { + Canvas(rendersAsynchronously: false) { context, size in + let geometry = WallProjection(projection: projection, wall: wall, size: size) + let runs = runs() + // The map's dash, lengthened for the larger shapes on the camera. + let dash = StrokeStyle(lineWidth: 2, lineCap: .round, lineJoin: .round, dash: Palette.hiddenDash.map { $0 * 1.6 }) + for run in runs.hidden { + guard let outline = outline(run.band, run.edges, geometry) else { continue } + // A dark halo under the dash keeps it legible on a sunlit wall. + context.stroke(outline, with: .color(.black.opacity(0.35)), style: StrokeStyle(lineWidth: 4, lineJoin: .round)) + context.stroke(outline, with: .color(Palette.hidden), style: dash) + } + for run in runs.skipped { + guard let quad = outline(run.band, run.edges, geometry) else { continue } + context.fill(quad, with: .color(Palette.skipped.opacity(0.3))) + var hatched = context + hatched.clip(to: quad) + let bounds = quad.boundingRect + var lines = Path() + var x = bounds.minX - bounds.height + while x < bounds.maxX { + lines.move(to: CGPoint(x: x, y: bounds.maxY)) + lines.addLine(to: CGPoint(x: x + bounds.height, y: bounds.minY)) + x += 10 + } + hatched.stroke(lines, with: .color(.white.opacity(0.55)), lineWidth: 1.5) + } + if let requested = runs.requested, let outline = outline(requested.band, requested.edges, geometry) { + context.stroke(outline, with: .color(.black.opacity(0.3)), style: StrokeStyle(lineWidth: 4, lineJoin: .round)) + context.stroke(outline, with: .color(Palette.caution), style: StrokeStyle(lineWidth: 2, lineJoin: .round)) + } + } + .allowsHitTesting(false) + .accessibilityHidden(true) + } + + private struct Run { + var band: CoverageBand + /// The s of every cell edge along the run, clipped to the visible range. + var edges: [Float] + } + + /// Runs of hidden and of skipped cells in each band, and the requested cells not yet covered. + private func runs() -> (hidden: [Run], skipped: [Run], requested: Run?) { + var hidden: [Run] = [], skipped: [Run] = [] + var requested: Run? + let visible = coverage.visibleRange + guard visible.upperBound > visible.lowerBound else { return ([], [], nil) } + for band in CoverageBand.allCases { + let states = coverage.cells(band) + var current: (state: CellState, run: Run)? + func close() { + if let current { + if current.state == .hidden { hidden.append(current.run) } else { skipped.append(current.run) } + } + current = nil + } + for index in states.indices { + let range = coverage.cellRange(index) + guard range.upperBound > visible.lowerBound, range.lowerBound < visible.upperBound else { close(); continue } + let s0 = max(range.lowerBound, visible.lowerBound), s1 = min(range.upperBound, visible.upperBound) + let state = states[index] + let isRequested = highlight.map { $0.band == band && $0.span.lowerBound < range.upperBound && range.lowerBound < $0.span.upperBound } ?? false + if isRequested, state != .covered { + close() + if var run = requested, run.band == band, let last = run.edges.last, abs(last - s0) < 1e-4 { + run.edges.append(s1) + requested = run + } else if requested == nil { + requested = Run(band: band, edges: [s0, s1]) + } + continue + } + guard state == .hidden || state == .skipped else { close(); continue } + if var open = current, open.state == state { + open.run.edges.append(s1) + current = open + } else { + close() + current = (state, Run(band: band, edges: [s0, s1])) + } + } + close() + } + return (hidden, skipped, requested) + } + + /// The outline of a run through every cell edge, so it follows the wall round a corner. Nil + /// when any point is behind the camera. + private func outline(_ band: CoverageBand, _ edges: [Float], _ geometry: WallProjection) -> Path? { + let corners: [SIMD3] = switch band { + case .wall: + edges.map { wall.world(s: $0, height: coverage.wallBandHeight) } + edges.reversed().map { wall.world(s: $0, height: 0) } + case .ground: + edges.map { wall.world(s: $0, height: 0, out: coverage.groundBandDepth) } + edges.reversed().map { wall.world(s: $0, height: 0) } + } + return geometry.polygon(corners) + } +} diff --git a/ios/HouseScan/UI/Camera/FogOverlay.swift b/ios/HouseScan/UI/Camera/FogOverlay.swift index 6e2ac6800..98db723e6 100644 --- a/ios/HouseScan/UI/Camera/FogOverlay.swift +++ b/ios/HouseScan/UI/Camera/FogOverlay.swift @@ -1,7 +1,9 @@ import SwiftUI import simd -/// The haze over the parts of the wall and ground the phone hasn't seen yet. +/// The haze over the parts of the wall and ground the phone hasn't seen yet: the frosted strip +/// that `LiveFogView` replaced. It still draws while the live fog's shaders compile at launch, +/// and for good if Metal fails, so unseen wall never shows clear. /// /// Unseen cells carry a soft frosted haze, seen cells a thinner one, covered cells none. When a /// cell's state changes (a new `coverage.revision`), its haze fades and drifts upward over diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift new file mode 100644 index 000000000..4d5b8f5c3 --- /dev/null +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift @@ -0,0 +1,316 @@ +import Metal +import MetalKit +import os +import QuartzCore +import simd + +/// The compiled shaders and pipelines, made once per launch off the main thread +/// (`LiveFogSupport.prepare`). Immutable after init; Metal's devices, libraries and pipeline +/// states are safe to use from any thread. +final class LiveFogGPU: @unchecked Sendable { + enum SetupError: Error, CustomStringConvertible { + case noDevice + case missing(String) + + var description: String { + switch self { + case .noDevice: "no Metal device" + case let .missing(what): "Metal would not make the \(what)" + } + } + } + + static let drawableFormat = MTLPixelFormat.bgra8Unorm + /// Half floats: the blur sums eleven taps, and the fog's slow lift needs finer steps than 8 bits. + static let maskFormat = MTLPixelFormat.rgba16Float + static let depthFormat = MTLPixelFormat.depth32Float + + let device: MTLDevice + let queue: MTLCommandQueue + let mask: MTLRenderPipelineState + let blur: MTLRenderPipelineState + let fog: MTLRenderPipelineState + let dots: MTLRenderPipelineState + let depthTest: MTLDepthStencilState + + init() throws { + guard let device = MTLCreateSystemDefaultDevice() else { throw SetupError.noDevice } + guard let queue = device.makeCommandQueue() else { throw SetupError.missing("command queue") } + self.device = device + self.queue = queue + let library = try device.makeLibrary(source: LiveFogShaders.source, options: nil) + + func pipeline(_ vertex: String, _ fragment: String, format: MTLPixelFormat, depth: Bool = false, + blend: ((MTLRenderPipelineColorAttachmentDescriptor) -> Void)? = nil) throws -> MTLRenderPipelineState { + let descriptor = MTLRenderPipelineDescriptor() + guard let v = library.makeFunction(name: vertex), let f = library.makeFunction(name: fragment) else { + throw SetupError.missing("\(vertex) / \(fragment) functions") + } + descriptor.vertexFunction = v + descriptor.fragmentFunction = f + descriptor.colorAttachments[0].pixelFormat = format + if depth { descriptor.depthAttachmentPixelFormat = Self.depthFormat } + if let blend { + descriptor.colorAttachments[0].isBlendingEnabled = true + blend(descriptor.colorAttachments[0]) + } + return try device.makeRenderPipelineState(descriptor: descriptor) + } + mask = try pipeline("maskVertex", "maskFragment", format: Self.maskFormat, depth: true) + blur = try pipeline("fullVertex", "blurFragment", format: Self.maskFormat) + // The fog is the first thing drawn on a cleared target: premultiplied, written as is. + fog = try pipeline("fullVertex", "fogFragment", format: Self.drawableFormat) + // Additive light: colour adds, alpha stays, so a dot brightens the camera under it. + dots = try pipeline("dotVertex", "dotFragment", format: Self.drawableFormat) { a in + a.sourceRGBBlendFactor = .one + a.destinationRGBBlendFactor = .one + a.sourceAlphaBlendFactor = .zero + a.destinationAlphaBlendFactor = .one + } + let depthDescriptor = MTLDepthStencilDescriptor() + depthDescriptor.depthCompareFunction = .less + depthDescriptor.isDepthWriteEnabled = true + guard let depthTest = device.makeDepthStencilState(descriptor: depthDescriptor) else { throw SetupError.missing("depth state") } + self.depthTest = depthTest + } +} + +/// What one frame of the fog draws from, all of it read from `ScanViewState`. +struct LiveFogInput { + var projection: CameraProjection + var wall: WallGeometry + var coverage: CoverageStrip + var highlight: GapRequest? + var dots: LiveDots + var reduceMotion: Bool + /// The gap request's amber, sRGB. + var requestedColor: SIMD3 +} + +/// Draws the fog and dots into a transparent MTKView over the camera, every display frame while +/// the walk is on screen. +/// +/// Per frame: the mask pass draws the coverage cells (`FogMaskGeometry`) into a 96-texel-wide +/// texture with a depth test, two passes blur it, then the drawable gets the fog and the dots. +/// Bounds: three frames in flight; the mask vertex ring holds 8,192 vertices a frame (a 60 m +/// strip of unmerged cells needs about 4,800); the dot buffers hold `DotTimeline.capacity` dots +/// and are rewritten only when the field changes or a fading dot ends, at most once a frame. +/// Nothing grows with time: about 4 MB of buffers and 0.2 MB of textures, allocated once. +@MainActor +final class LiveFogRenderer { + private let gpu: LiveFogGPU + private let epoch = CACurrentMediaTime() + private let inFlight = DispatchSemaphore(value: LiveFogRenderer.framesInFlight) + private static let framesInFlight = 3 + private static let maskWidth = 96 + private static let maskVertexCapacity = 8_192 + private static let spriteCapacity = DotTimeline.capacity * 2 + + var input: LiveFogInput? + + private let animator = FogCellAnimator() + private let timeline = DotTimeline() + private var maskBuffers: [MTLBuffer] = [] + private var spriteBuffers: [MTLBuffer] = [] + private var frame = 0 + private var spriteBuffer = 0 + private var spriteCount = 0 + private var maskTextures: (mask: MTLTexture, horizontal: MTLTexture, final: MTLTexture, depth: MTLTexture)? + private var anchor: (point: SIMD2, scale: Float)? + + init(gpu: LiveFogGPU) { + self.gpu = gpu + for _ in 0...stride, options: .storageModeShared) { + maskBuffers.append(buffer) + } + if let buffer = gpu.device.makeBuffer(length: Self.spriteCapacity * MemoryLayout.stride, options: .storageModeShared) { + spriteBuffers.append(buffer) + } + } + if maskBuffers.count < Self.framesInFlight || spriteBuffers.count < Self.framesInFlight { + RuntimeLog.engine.error("live fog: Metal would not allocate its buffers; nothing will draw") + } + } + + func draw(in view: MTKView) { + guard let input, maskBuffers.count == Self.framesInFlight, spriteBuffers.count == Self.framesInFlight else { return } + let size = view.bounds.size + guard size.width > 1, size.height > 1 else { return } + // Never block the main thread on the GPU: a frame that finds all three in flight is skipped. + guard inFlight.wait(timeout: .now()) == .success else { return } + guard let pass = view.currentRenderPassDescriptor, let drawable = view.currentDrawable, + let commandBuffer = gpu.queue.makeCommandBuffer(), let textures = textures(for: size) else { + inFlight.signal() + return + } + let semaphore = inFlight + commandBuffer.addCompletedHandler { _ in semaphore.signal() } + + let now = CACurrentMediaTime() - epoch + let time = Float(now) + let slot = frame % Self.framesInFlight + frame &+= 1 + + var clip = Self.clipMatrix(input.projection, size: size) + let maskBuffer = maskBuffers[slot] + let maskCount = FogMaskGeometry.build( + coverage: input.coverage, wall: input.wall, highlight: input.highlight, animator: animator, + now: now, reduceMotion: input.reduceMotion, + into: maskBuffer.contents().bindMemory(to: FogMaskGeometry.Vertex.self, capacity: Self.maskVertexCapacity), + capacity: Self.maskVertexCapacity) + + let changed = timeline.update(input.dots, now: time) + let expired = timeline.expire(now: time) + if changed || expired { + spriteBuffer = (spriteBuffer + 1) % Self.framesInFlight + let buffer = spriteBuffers[spriteBuffer] + spriteCount = timeline.write(into: buffer.contents().bindMemory(to: DotTimeline.Sprite.self, capacity: Self.spriteCapacity), capacity: Self.spriteCapacity) + } + + // Mask: full fog wherever no cell reaches. + let maskPass = MTLRenderPassDescriptor() + maskPass.colorAttachments[0].texture = textures.mask + maskPass.colorAttachments[0].loadAction = .clear + maskPass.colorAttachments[0].clearColor = MTLClearColor(red: 1, green: 0, blue: 0, alpha: 0) + maskPass.colorAttachments[0].storeAction = .store + maskPass.depthAttachment.texture = textures.depth + maskPass.depthAttachment.loadAction = .clear + maskPass.depthAttachment.clearDepth = 1 + maskPass.depthAttachment.storeAction = .dontCare + if let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: maskPass) { + if maskCount > 0 { + encoder.setRenderPipelineState(gpu.mask) + encoder.setDepthStencilState(gpu.depthTest) + encoder.setVertexBuffer(maskBuffer, offset: 0, index: 0) + encoder.setVertexBytes(&clip, length: MemoryLayout.stride, index: 1) + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: maskCount) + } + encoder.endEncoding() + } + for (source, target, direction) in [(textures.mask, textures.horizontal, SIMD2(1, 0)), (textures.horizontal, textures.final, SIMD2(0, 1))] { + let blurPass = MTLRenderPassDescriptor() + blurPass.colorAttachments[0].texture = target + blurPass.colorAttachments[0].loadAction = .dontCare + blurPass.colorAttachments[0].storeAction = .store + guard let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: blurPass) else { continue } + var direction = direction + encoder.setRenderPipelineState(gpu.blur) + encoder.setFragmentTexture(source, index: 0) + encoder.setFragmentBytes(&direction, length: MemoryLayout>.stride, index: 0) + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3) + encoder.endEncoding() + } + + pass.colorAttachments[0].loadAction = .clear + pass.colorAttachments[0].clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: 0) + if let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: pass) { + let scale = Float(view.contentScaleFactor) + let pixels = SIMD2(Float(view.drawableSize.width), Float(view.drawableSize.height)) + let pin = anchor(input, size: size, pixelsPerPoint: scale) + var fog = FogUniforms( + viewSize: pixels, anchor: pin.point, anchorScale: pin.scale, time: time, + motion: input.reduceMotion ? 0 : 1, pad: 0, requested: SIMD4(input.requestedColor, 1)) + encoder.setRenderPipelineState(gpu.fog) + encoder.setFragmentBytes(&fog, length: MemoryLayout.stride, index: 0) + encoder.setFragmentTexture(textures.final, index: 0) + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3) + + if spriteCount > 0 { + var dots = DotUniforms( + clip: clip, cameraAndTime: SIMD4(input.projection.cameraPosition, time), + pointScale: scale, reduceMotion: input.reduceMotion ? 1 : 0, pad: .zero) + encoder.setRenderPipelineState(gpu.dots) + encoder.setVertexBuffer(spriteBuffers[spriteBuffer], offset: 0, index: 0) + encoder.setVertexBytes(&dots, length: MemoryLayout.stride, index: 1) + encoder.setVertexTexture(textures.final, index: 0) + encoder.drawPrimitives(type: .point, vertexStart: 0, vertexCount: spriteCount) + } + encoder.endEncoding() + } + commandBuffer.present(drawable) + commandBuffer.commit() + } + + /// The mask keeps the view's aspect at 96 texels across, about 4 points a texel on a phone: + /// the blur's 10-point sigma is the feather. + private func textures(for size: CGSize) -> (mask: MTLTexture, horizontal: MTLTexture, final: MTLTexture, depth: MTLTexture)? { + let width = Self.maskWidth + let height = max(1, Int((Double(width) * size.height / size.width).rounded())) + if let maskTextures, maskTextures.mask.width == width, maskTextures.mask.height == height { return maskTextures } + func texture(_ format: MTLPixelFormat, _ usage: MTLTextureUsage) -> MTLTexture? { + let descriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: format, width: width, height: height, mipmapped: false) + descriptor.usage = usage + descriptor.storageMode = .private + return gpu.device.makeTexture(descriptor: descriptor) + } + guard let mask = texture(LiveFogGPU.maskFormat, [.renderTarget, .shaderRead]), + let horizontal = texture(LiveFogGPU.maskFormat, [.renderTarget, .shaderRead]), + let final = texture(LiveFogGPU.maskFormat, [.renderTarget, .shaderRead]), + let depth = texture(LiveFogGPU.depthFormat, [.renderTarget]) else { + RuntimeLog.engine.error("live fog: Metal would not allocate the mask textures") + return nil + } + maskTextures = (mask, horizontal, final, depth) + return maskTextures + } + + /// Where the fog's noise is pinned: the meter on screen, one noise unit about 1.2 m at the + /// meter's distance, so the texture moves and scales with the wall. Kept from the last frame + /// that saw the meter in front of the camera. + private func anchor(_ input: LiveFogInput, size: CGSize, pixelsPerPoint: Float) -> (point: SIMD2, scale: Float) { + let width = Float(size.width) * pixelsPerPoint + let local = input.projection.cameraSpace(input.wall.meter) + if local.z < -0.1, let point = input.projection.viewPoint(for: input.wall.meter, in: size) { + let pointsPerMeter = input.projection.intrinsics.x * Float(input.projection.scale(in: size)) / -local.z + let scale = min(max(pointsPerMeter * pixelsPerPoint * 1.2, 0.25 * width), 2 * width) + anchor = (SIMD2(Float(point.x), Float(point.y)) * pixelsPerPoint, scale) + } + return anchor ?? (.zero, 0.45 * width) + } + + /// World to Metal clip space for a portrait view of `size` showing the landscape sensor image + /// turned 90 degrees clockwise and scaled to fill, the mapping `CameraProjection.viewPoint` + /// uses. Depth runs from 0 at 5 cm to 1 at 100 m, so the GPU clips anything behind the camera + /// and the mask's depth test keeps the nearest surface. + static func clipMatrix(_ projection: CameraProjection, size: CGSize, near: Float = 0.05, far: Float = 100) -> simd_float4x4 { + let k = projection.intrinsics + let imageHeight = projection.imageSize.y + let scale = Float(projection.scale(in: size)) + let width = Float(size.width), height = Float(size.height) + let offset = SIMD2( + (width - projection.imageSize.y * scale) / 2, + (height - projection.imageSize.x * scale) / 2) + // Rows act on camera-space (x, y, z, 1); w is the depth along -z. + let w = SIMD4(0, 0, -1, 0) + let portraitX = SIMD4(0, k.y, k.w - imageHeight, 0) + let portraitY = SIMD4(k.x, 0, -k.z, 0) + let screenX = offset.x * w + scale * portraitX + let screenY = offset.y * w + scale * portraitY + let clipX = (2 / width) * screenX - w + let clipY = w - (2 / height) * screenY + let clipZ = SIMD4(0, 0, -far / (far - near), -far * near / (far - near)) + let cameraToClip = simd_float4x4(rows: [clipX, clipY, clipZ, w]) + return cameraToClip * projection.cameraToWorld.inverse + } +} + +/// Matches `FogUniforms` in the shader. +struct FogUniforms { + var viewSize: SIMD2 + var anchor: SIMD2 + var anchorScale: Float + var time: Float + var motion: Float + var pad: Float + var requested: SIMD4 +} + +/// Matches `DotUniforms` in the shader. +struct DotUniforms { + var clip: simd_float4x4 + var cameraAndTime: SIMD4 + var pointScale: Float + var reduceMotion: Float + var pad: SIMD2 +} diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogScene.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogScene.swift new file mode 100644 index 000000000..f1c9595a8 --- /dev/null +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogScene.swift @@ -0,0 +1,281 @@ +import Foundation +import simd + +/// What the fog mask holds at one place: how thick the fog is, how much of it is the violet of a +/// hidden stretch, and how much is a gap request's amber. Channels of the mask texture, in order. +struct FogValue: Equatable { + var fog: Float + var hidden: Float + var requested: Float + + static let clear = FogValue(fog: 0, hidden: 0, requested: 0) + + /// The rule the fog follows, from the coverage map's own cell states: the fog lifts only + /// where coverage counts the cell, never where depth merely reached. + /// - unseen: full fog. + /// - seen from one position: thinned, not lifted, since one view is not evidence. + /// - covered: lifted. + /// - hidden (the camera looked, depth found something nearer): still fog, since the wall + /// behind is unseen, but thinner and violet, so the thing in front shows through with its + /// violet dots. + /// - skipped ("I can't get there"): lifted here; `FogMarks` hatches it, so it never reads as + /// seen. + /// - requested by a gap and not yet covered: amber instead of fog, so the camera shows where + /// to aim. + static func target(_ state: CellState, requested: Bool) -> FogValue { + if requested, state != .covered { return FogValue(fog: 0, hidden: 0, requested: 1) } + return switch state { + case .unseen: FogValue(fog: 1, hidden: 0, requested: 0) + case .seen: FogValue(fog: 0.55, hidden: 0, requested: 0) + case .hidden: FogValue(fog: 0.75, hidden: 1, requested: 0) + case .covered, .skipped: .clear + } + } + + func mixed(to other: FogValue, _ t: Float) -> FogValue { + FogValue(fog: fog + (other.fog - fog) * t, hidden: hidden + (other.hidden - hidden) * t, requested: requested + (other.requested - requested) * t) + } + + var total: Float { fog + hidden + requested } +} + +/// Eases each cell from its old fog to its new one when coverage changes it. Lifting is the +/// signature moment ("the phone saw that") and takes `Motion.fogLift` on the strong ease-out; +/// fog coming back (a cell turning hidden, a request raised) is quick, 150 ms, because it is the +/// system answering, not a reveal. Under Reduce Motion both are linear fades, 400 and 120 ms, as +/// in the prototype. Keyed by the cell's s, not its index in the strip, which shifts when the +/// strip grows to the left. +@MainActor +final class FogCellAnimator { + struct Key: Hashable { + var band: CoverageBand + /// The cell's left edge in whole cell widths from s = 0. + var cell: Int + } + + private struct Track { + var from: FogValue + var to: FogValue + var start: Double + var duration: Double + var linear: Bool + + func value(at now: Double) -> FogValue { + guard duration > 0 else { return to } + let p = Float(min(max((now - start) / duration, 0), 1)) + return from.mixed(to: to, linear ? p : Easing.strongEaseOut(p)) + } + + func isDone(at now: Double) -> Bool { now - start >= duration } + } + + private var tracks: [Key: Track] = [:] + private var seen: Set = [] + + /// The cell's value now. A cell seen for the first time starts at its target. + func value(_ key: Key, target: FogValue, now: Double, reduceMotion: Bool) -> FogValue { + seen.insert(key) + guard var track = tracks[key] else { + tracks[key] = Track(from: target, to: target, start: now, duration: 0, linear: false) + return target + } + if track.to != target { + let current = track.value(at: now) + let lifting = target.total < current.total + let duration = reduceMotion ? (lifting ? 0.4 : 0.12) : (lifting ? Motion.fogLift : 0.15) + track = Track(from: current, to: target, start: now, duration: duration, linear: reduceMotion) + tracks[key] = track + } + return track.value(at: now) + } + + /// Forgets cells the last frame did not draw, so the table never outgrows the strip. + func endFrame() { + if tracks.count > seen.count { tracks = tracks.filter { seen.contains($0.key) } } + seen.removeAll(keepingCapacity: true) + } +} + +/// Builds the fog mask's geometry: each coverage cell as a wall column from the ground up +/// `columnHeight` and a stretch of ground out `groundReach`, carrying its `FogValue`. The column +/// and the stretch reach past the bands coverage measures (4.5 ft of wall, 4 ft of ground) on +/// purpose: fog there could never lift, and fog that walking can't clear stops meaning +/// "not seen yet". The wall above a covered column and the lawn in front of it are nothing the +/// scan decides on. Past a marked end the wall is not part of the scan, and is clear. Anything +/// no quad reaches (the wall past the strip, where the walk has not been) keeps the mask's clear +/// value, full fog. +enum FogMaskGeometry { + struct Vertex { + var position: SIMD4 + var value: SIMD4 + } + + static let columnHeight: Float = 8 + static let groundReach: Float = 8 + /// How far past a marked end the clear stretch runs. + static let pastEnd: Float = 30 + + /// Writes triangles into `vertices` (at most `capacity`) and returns how many it wrote. + /// Neighbouring cells with the same value on the same piece of wall merge into one quad. + @MainActor + static func build( + coverage: CoverageStrip, wall: WallGeometry, highlight: GapRequest?, animator: FogCellAnimator, + now: Double, reduceMotion: Bool, into vertices: UnsafeMutablePointer, capacity: Int + ) -> Int { + var count = 0 + func quad(_ band: CoverageBand, _ s0: Float, _ s1: Float, _ value: FogValue) { + guard count + 6 <= capacity else { return } + let far0 = band == .wall ? wall.world(s: s0, height: columnHeight) : wall.world(s: s0, height: 0, out: groundReach) + let far1 = band == .wall ? wall.world(s: s1, height: columnHeight) : wall.world(s: s1, height: 0, out: groundReach) + let a = SIMD4(wall.world(s: s0, height: 0), 1), b = SIMD4(wall.world(s: s1, height: 0), 1) + let c = SIMD4(far1, 1), d = SIMD4(far0, 1) + let v = SIMD4(value.fog, value.hidden, value.requested, 0) + vertices[count] = Vertex(position: a, value: v) + vertices[count + 1] = Vertex(position: b, value: v) + vertices[count + 2] = Vertex(position: c, value: v) + vertices[count + 3] = Vertex(position: a, value: v) + vertices[count + 4] = Vertex(position: c, value: v) + vertices[count + 5] = Vertex(position: d, value: v) + count += 6 + } + + let visible = coverage.visibleRange + if visible.upperBound > visible.lowerBound { + for band in CoverageBand.allCases { + let states = coverage.cells(band) + var run: (s0: Float, s1: Float, value: FogValue, along: SIMD3)? + for index in states.indices { + let range = coverage.cellRange(index) + guard range.upperBound > visible.lowerBound, range.lowerBound < visible.upperBound else { continue } + let s0 = max(range.lowerBound, visible.lowerBound), s1 = min(range.upperBound, visible.upperBound) + let requested = highlight.map { $0.band == band && $0.span.lowerBound < range.upperBound && range.lowerBound < $0.span.upperBound } ?? false + let key = FogCellAnimator.Key(band: band, cell: Int((range.lowerBound / coverage.cellWidth).rounded())) + let value = animator.value(key, target: FogValue.target(states[index], requested: requested), now: now, reduceMotion: reduceMotion) + let along = wall.along(atS: (s0 + s1) / 2) + if var current = run, current.value == value, current.along == along, abs(current.s1 - s0) < 1e-4 { + current.s1 = s1 + run = current + } else { + if let current = run { quad(band, current.s0, current.s1, current.value) } + run = (s0, s1, value, along) + } + } + if let current = run { quad(band, current.s0, current.s1, current.value) } + } + } + for band in CoverageBand.allCases { + if let left = wall.leftEnd { quad(band, left - pastEnd, left, .clear) } + if let right = wall.rightEnd { quad(band, right, right + pastEnd, .clear) } + } + animator.endFrame() + return count + } +} + +/// The dots' timing on the renderer's clock: when each was born, when its opacity last changed +/// and from what, when it turned edge, and, for a dot that left the field, when it died. The +/// shader evaluates the curves, so the buffer changes only when the field does or a death ends. +@MainActor +final class DotTimeline { + struct Sprite { + var a: SIMD4 + var b: SIMD4 + var c: SIMD4 + } + + private struct Track { + var birth: Float + var from: Float + var to: Float + var opacityTime: Float + var edgeSince: Float + var dot: LiveDots.Dot + + func opacity(at t: Float) -> Float { + from + (to - from) * Easing.strongEaseOut((t - opacityTime) / 0.25) + } + } + + /// Metal compiles with fast math, which assumes no infinities. + private static let never: Float = 1e6 + private static let deathFade: Float = 0.3 + + private var tracks: [UInt64: Track] = [:] + private var dying: [(track: Track, death: Float)] = [] + private var revision = -1 + /// Dots the renderer draws at once, living and dying: the field's own cap + /// (`SurfaceDotConfig.maxDots`, 5,000) and as many again fading out. + static let capacity = 10_000 + + /// Takes a new field. Returns true when the sprites changed. + func update(_ field: LiveDots, now: Float) -> Bool { + guard field.revision != revision else { return false } + revision = field.revision + var next: [UInt64: Track] = [:] + next.reserveCapacity(field.dots.count) + for dot in field.dots { + if var track = tracks[dot.id] { + if dot.opacity != track.to { + track.from = track.opacity(at: now) + track.to = dot.opacity + track.opacityTime = now + } + if dot.isEdge, track.edgeSince >= Self.never { track.edgeSince = now } + track.dot = dot + next[dot.id] = track + } else { + next[dot.id] = Track( + birth: now, from: dot.opacity, to: dot.opacity, opacityTime: -Self.never, + edgeSince: dot.isEdge ? -Self.never : Self.never, dot: dot) + } + } + for (id, track) in tracks where next[id] == nil { dying.append((track, now)) } + if dying.count > Self.capacity / 2 { dying.removeFirst(dying.count - Self.capacity / 2) } + tracks = next + return true + } + + /// Drops dots whose fade has finished. Returns true when any went. + func expire(now: Float) -> Bool { + let before = dying.count + dying.removeAll { now - $0.death > Self.deathFade } + return dying.count != before + } + + /// Two sprites per dot, its halo then its core, so the sharp cores sit on the glow. Returns + /// how many it wrote, at most `capacity`. + func write(into sprites: UnsafeMutablePointer, capacity: Int) -> Int { + var count = 0 + func add(_ track: Track, death: Float) { + guard count + 2 <= capacity else { return } + let d = track.dot + let b = SIMD4(track.birth, track.from, track.to, track.opacityTime) + let c = SIMD4(track.edgeSince, death, d.onOccluder ? 1 : 0, 0) + sprites[count] = Sprite(a: SIMD4(d.position, 1), b: b, c: c) + sprites[count + 1] = Sprite(a: SIMD4(d.position, 0), b: b, c: c) + count += 2 + } + for track in tracks.values { add(track, death: Self.never) } + for entry in dying { add(entry.track, death: entry.death) } + return count + } +} + +/// cubic-bezier(0.23, 1, 0.32, 1), the same curve the shader solves. +enum Easing { + static func strongEaseOut(_ progress: Float) -> Float { + if progress <= 0 { return 0 } + if progress >= 1 { return 1 } + let x1: Float = 0.23, x2: Float = 0.32 + var t = progress + for _ in 0..<8 { + let u = 1 - t + let x = 3 * u * u * t * x1 + 3 * u * t * t * x2 + t * t * t - progress + let dx = 3 * u * u * x1 + 6 * u * t * (x2 - x1) + 3 * t * t * (1 - x2) + if abs(dx) < 1e-6 { break } + t = min(max(t - x / dx, 0), 1) + } + let u = 1 - t + return 3 * u * u * t + 3 * u * t * t + t * t * t + } +} diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift new file mode 100644 index 000000000..de2234e5d --- /dev/null +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift @@ -0,0 +1,239 @@ +/// Metal source for the live fog and dots, compiled at runtime (`makeLibrary(source:)`) so the +/// build needs no Metal toolchain, as in the prototype (experiments/live-dots on t3/experience, +/// `Shaders.swift`). Struct layouts match the Swift structs in `LiveFogRenderer.swift`. +/// +/// Fog. Coverage decides where it lifts: `LiveFogScene` draws every wall column and ground +/// stretch of the coverage strip into a 96-texel-wide mask (red: fog, green: hidden behind +/// something, blue: requested by a gap), clearing to full fog, and a radius-5 Gaussian blur +/// feathers it. The composite is the prototype's: Chalk mixed toward a cool grey, textured by +/// slow domain-warped fbm and breathing slightly. Two changes: the noise is pinned to the meter +/// on screen and scaled by its distance, so the texture sits on the wall instead of on the lens +/// while the phone moves (the prototype cut between keyframes, where that never showed); and +/// part-lifted fog breaks up along the same noise instead of thinning evenly, so a lift reads as +/// mist burning off. +/// +/// Dots. The prototype's hologram scheme: additive point sprites, a halo under each core, births +/// that ease in over 350 ms from 60% size, opacity changes over 250 ms, deaths over 250 ms. A dot +/// under fog draws at 20%, so dots arrive where coverage lifts the fog; dots on an occluder +/// (violet) draw fully, since they are what the hidden state is about. +enum LiveFogShaders { + static let source = """ + #include + using namespace metal; + + // cubic-bezier(0.23, 1, 0.32, 1), solved by Newton. + static float strongEaseOut(float p) { + p = clamp(p, 0.0f, 1.0f); + if (p <= 0.0f || p >= 1.0f) { return p; } + const float x1 = 0.23f, x2 = 0.32f, y1 = 1.0f, y2 = 1.0f; + float t = p; + for (int i = 0; i < 8; i++) { + float u = 1.0f - t; + float x = 3.0f * u * u * t * x1 + 3.0f * u * t * t * x2 + t * t * t - p; + float dx = 3.0f * u * u * x1 + 6.0f * u * t * (x2 - x1) + 3.0f * t * t * (1.0f - x2); + if (fabs(dx) < 1e-6f) { break; } + t = clamp(t - x / dx, 0.0f, 1.0f); + } + float u = 1.0f - t; + return 3.0f * u * u * t * y1 + 3.0f * u * t * t * y2 + t * t * t; + } + + struct FullOut { float4 position [[position]]; }; + + vertex FullOut fullVertex(uint vid [[vertex_id]]) { + float2 p = float2((vid << 1) & 2, vid & 2); + FullOut out; + out.position = float4(p * 2.0f - 1.0f, 0.0f, 1.0f); + return out; + } + + // MARK: Mask + + struct MaskVertex { + float4 position; // world xyz + float4 value; // fog, hidden, requested, unused + }; + + struct MaskOut { + float4 position [[position]]; + float4 value [[flat]]; + }; + + vertex MaskOut maskVertex(uint vid [[vertex_id]], + const device MaskVertex* vertices [[buffer(0)]], + constant float4x4& clip [[buffer(1)]]) { + MaskOut out; + out.position = clip * float4(vertices[vid].position.xyz, 1.0f); + out.value = vertices[vid].value; + return out; + } + + fragment float4 maskFragment(MaskOut in [[stage_in]]) { + return in.value; + } + + // Separable Gaussian, radius 5 texels (sigma 2.5), all four channels. + fragment float4 blurFragment(FullOut in [[stage_in]], + texture2d source [[texture(0)]], + constant int2& direction [[buffer(0)]]) { + int2 p = int2(in.position.xy); + int2 limit = int2(source.get_width() - 1, source.get_height() - 1); + float4 sum = float4(0.0f); + float total = 0.0f; + for (int k = -5; k <= 5; k++) { + float w = exp(-float(k * k) / (2.0f * 2.5f * 2.5f)); + sum += w * source.read(uint2(clamp(p + k * direction, int2(0), limit))); + total += w; + } + return sum / total; + } + + // MARK: Fog + + struct FogUniforms { + float2 viewSize; // drawable pixels + float2 anchor; // pixels: the meter on screen, where the noise is pinned + float anchorScale; // pixels per noise unit (about 1.2 m at the meter) + float time; // seconds + float motion; // 0 under Reduce Motion: no drift, no breathing, no pulse + float pad; + float4 requested; // the gap request's amber, rgb + }; + + static float hash21(float2 p) { + p = fract(p * float2(123.34f, 456.21f)); + p += dot(p, p + 45.32f); + return fract(p.x * p.y); + } + + static float valueNoise(float2 p) { + float2 i = floor(p), f = fract(p); + float2 w = f * f * (3.0f - 2.0f * f); + float a = hash21(i), b = hash21(i + float2(1, 0)), c = hash21(i + float2(0, 1)), d = hash21(i + float2(1, 1)); + return mix(mix(a, b, w.x), mix(c, d, w.x), w.y); + } + + static float fbm(float2 p) { + float sum = 0.0f, amplitude = 0.5f; + for (int i = 0; i < 3; i++) { + sum += amplitude * valueNoise(p); + p = p * 2.03f + float2(17.1f, 9.2f); + amplitude *= 0.5f; + } + return sum / 0.875f; + } + + fragment float4 fogFragment(FullOut in [[stage_in]], + constant FogUniforms& u [[buffer(0)]], + texture2d mask [[texture(0)]]) { + constexpr sampler linear(filter::linear, address::clamp_to_edge); + float2 screen = in.position.xy; + float4 m = mask.sample(linear, screen / u.viewSize); + float fog = saturate(m.r), hidden = saturate(m.g), requested = saturate(m.b); + if (fog < 0.002f && requested < 0.002f) { return float4(0.0f); } + + // Wrapped so the offsets stay small where Float precision is fine. + float2 uv = fmod((screen - u.anchor) / u.anchorScale, float2(256.0f)); + float2 q = uv + u.motion * u.time * 0.02f * float2(0.94f, 0.34f); + float2 warp = float2(fbm(q + float2(3.1f, 1.7f)), fbm(q + float2(-2.3f, 5.2f))); + float n = fbm(q + 0.8f * warp); + float noiseTerm = 0.78f + 0.22f * n; + float breathing = 1.0f + u.motion * 0.04f * sin(2.0f * M_PI_F * 0.08f * u.time); + + // Part-lifted fog breaks up along the noise; full fog and clear stay as they are. + float shaped = saturate(fog + (n - 0.5f) * 1.6f * fog * (1.0f - fog)); + float a = saturate(0.6f * noiseTerm * shaped * breathing * mix(1.0f, 0.72f, hidden)); + + float3 chalk = float3(0xF7, 0xF5, 0xEF) / 255.0f; + float3 cool = float3(0xD9, 0xE2, 0xEC) / 255.0f; + float3 violet = float3(0xB4, 0x9C, 0xFF) / 255.0f; + float3 color = mix(mix(chalk, cool, 0.35f), violet, 0.5f * hidden); + + float pulse = 1.0f - u.motion * 0.14f * (0.5f + 0.5f * sin(2.0f * M_PI_F * u.time / 1.8f)); + float r = saturate(0.36f * requested * noiseTerm * pulse); + + float3 rgb = color * a; + rgb = u.requested.rgb * r + rgb * (1.0f - r); + return float4(rgb, r + a * (1.0f - r)); + } + + // MARK: Dots + + struct Sprite { + float4 a; // world xyz, 1 for the halo sprite + float4 b; // birth time, from opacity, to opacity, opacity change time + float4 c; // edge since, death time, violet (0 or 1), unused + }; + + struct DotUniforms { + float4x4 clip; + float4 cameraAndTime; // camera xyz, seconds + float pointScale; // pixels per point + float reduceMotion; // 1 drops the birth scale + float2 pad; + }; + + struct DotOut { + float4 position [[position]]; + float size [[point_size]]; + float4 color; + float inner; // radius fraction where the soft edge starts + }; + + constant float3 hologram = float3(0xE6, 0xEC, 0xF4) / 255.0f; + constant float3 glow = float3(0x9C, 0xC8, 0xFF) / 255.0f; + constant float3 violetDot = float3(0xB4, 0x9C, 0xFF) / 255.0f; + + vertex DotOut dotVertex(uint vid [[vertex_id]], + const device Sprite* sprites [[buffer(0)]], + constant DotUniforms& u [[buffer(1)]], + texture2d mask [[texture(0)]]) { + constexpr sampler linear(filter::linear, address::clamp_to_edge); + Sprite s = sprites[vid]; + float t = u.cameraAndTime.w; + float birth = strongEaseOut((t - s.b.x) / 0.35f); + float evidence = mix(s.b.y, s.b.z, strongEaseOut((t - s.b.w) / 0.25f)); + float fade = 1.0f - strongEaseOut((t - s.c.y) / 0.25f); + float edge = strongEaseOut((t - s.c.x) / 0.25f); + bool halo = s.a.w > 0.5f; + bool violetOne = s.c.z > 0.5f; + + DotOut out; + out.position = u.clip * float4(s.a.xyz, 1.0f); + float fog = 1.0f; + if (out.position.w > 0.05f) { + float2 ndc = out.position.xy / out.position.w; + fog = mask.sample(linear, float2(0.5f + 0.5f * ndc.x, 0.5f - 0.5f * ndc.y), level(0)).r; + } + float veil = violetOne ? 1.0f : mix(1.0f, 0.2f, saturate(fog)); + float alpha = evidence * birth * fade * veil; + + // Flat 2.5 pt and edge 3.5 pt from 2.5 m out, growing to 4 and 6 pt at 1 m. + float far = clamp((length(s.a.xyz - u.cameraAndTime.xyz) - 1.0f) / 1.5f, 0.0f, 1.0f); + float size = mix(mix(4.0f, 2.5f, far), mix(6.0f, 3.5f, far), edge); + float inner = 0.7f; + float3 color = hologram; + if (halo) { + // Edge 4x at 22%, flat 2x at 8%; a flat dot turning edge grows into it. + size *= mix(2.0f, 4.0f, edge); + alpha *= mix(0.08f, 0.22f, edge); + inner = 0.0f; + color = glow; + } + if (violetOne) { color = violetDot; } + float scale = u.reduceMotion > 0.5f ? 1.0f : mix(0.6f, 1.0f, birth); + + if (alpha < 1.0f / 512.0f) { out.position = float4(0.0f, 0.0f, -1.0f, 1.0f); } + out.size = size * scale * u.pointScale; + // Alpha 0 with colour: added to what is under it, as light, when the layer composites. + out.color = float4(color * alpha, 0.0f); + out.inner = inner; + return out; + } + + fragment float4 dotFragment(DotOut in [[stage_in]], float2 pc [[point_coord]]) { + float r = length(pc * 2.0f - 1.0f); + return in.color * (1.0f - smoothstep(in.inner, 1.0f, r)); + } + """ +} diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift new file mode 100644 index 000000000..8c58457df --- /dev/null +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift @@ -0,0 +1,82 @@ +import MetalKit +import Observation +import SwiftUI + +/// Whether the live fog can draw. The shaders compile once per launch, off the main thread, as +/// soon as the app starts (`HouseScanApp`); until then, and if Metal fails, `CameraOverlays` +/// draws the frosted `FogOverlay` instead, so the camera never shows unseen wall as clear. +@MainActor +@Observable +final class LiveFogSupport { + enum Status { + case preparing + case ready(LiveFogGPU) + case failed + } + + static let shared = LiveFogSupport() + + private(set) var status: Status = .preparing + @ObservationIgnored private var started = false + + func prepare() { + guard !started else { return } + started = true + Task.detached(priority: .userInitiated) { + let result: Result + do { result = .success(try LiveFogGPU()) } catch { result = .failure(error) } + await MainActor.run { + switch result { + case let .success(gpu): + LiveFogSupport.shared.status = .ready(gpu) + case let .failure(error): + // Logged as an error, not trapped: a trap in a Debug build would stop every UI + // test, while the frosted strip in their screenshots already shows the failure. + RuntimeLog.engine.error("live fog unavailable, drawing the frosted strip instead: \(String(describing: error), privacy: .public)") + LiveFogSupport.shared.status = .failed + } + } + } + } +} + +/// The fog over what coverage has not counted and, on a LiDAR phone, the dots on what depth has +/// measured: a transparent Metal layer over the camera. See `LiveFogShaders` for the look and +/// `FogValue.target` for the rule. +struct LiveFogView: UIViewRepresentable { + let gpu: LiveFogGPU + let input: LiveFogInput + + func makeUIView(context: Context) -> LiveFogMTKView { + let view = LiveFogMTKView(frame: .zero, device: gpu.device) + view.colorPixelFormat = LiveFogGPU.drawableFormat + view.clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: 0) + view.isOpaque = false + view.backgroundColor = .clear + view.layer.isOpaque = false + view.framebufferOnly = true + view.autoResizeDrawable = true + // 60, not ProMotion's 120: the pose it draws from arrives at 30 Hz, and 60 keeps each new + // pose within one refresh of arriving. + view.preferredFramesPerSecond = 60 + view.isUserInteractionEnabled = false + view.accessibilityElementsHidden = true + view.renderer = LiveFogRenderer(gpu: gpu) + view.renderer?.input = input + return view + } + + func updateUIView(_ view: LiveFogMTKView, context: Context) { + view.renderer?.input = input + } +} + +/// An MTKView that hands each frame to its renderer. Subclassing keeps drawing on the main actor +/// without a delegate conformance. +final class LiveFogMTKView: MTKView { + var renderer: LiveFogRenderer? + + override func draw(_ rect: CGRect) { + renderer?.draw(in: self) + } +} diff --git a/ios/HouseScan/UI/Camera/WallMarksOverlay.swift b/ios/HouseScan/UI/Camera/WallMarksOverlay.swift index b9a4e4c69..b9f27a256 100644 --- a/ios/HouseScan/UI/Camera/WallMarksOverlay.swift +++ b/ios/HouseScan/UI/Camera/WallMarksOverlay.swift @@ -1,13 +1,30 @@ import SwiftUI +import simd /// What the homeowner has marked, pinned onto the camera image: the meter, the wall ends and -/// the marked features (openings as outlines, everything else as pins or lines). +/// the marked features. +/// +/// Things that stand on the wall get the prototype's corner brackets (after RoomPlan's +/// outlines) with a short caption: the meter in Signal blue, everything else in the dots' +/// Hologram white. A marked thing is settled, so its bracket is solid; the prototype's dashed, +/// unsettled bracket has no counterpart here. Driveways and fences stay dashed lines on the +/// ground, and the wall ends white posts. struct WallMarksOverlay: View { var projection: CameraProjection var wall: WallGeometry var features: [MarkedFeature] var wallBandHeight: Float + /// The dots' Hologram, #E6ECF4. + static let hologram = Color(red: 0xE6 / 255, green: 0xEC / 255, blue: 0xF4 / 255) + + /// Display sizes (width, height) for things marked with one tap, in meters: a bracket needs + /// a box, and one tap gives only the middle. Typical sizes, not measurements; the export uses + /// its own nominal width (`ScanEngine.project`). + static let meterBox = SIMD2(0.3, 0.4) + static let gasMeterBox = SIMD2(0.3, 0.4) + static let acUnitBox = SIMD2(0.6, 0.6) + var body: some View { Canvas { context, size in let geometry = WallProjection(projection: projection, wall: wall, size: size) @@ -15,21 +32,17 @@ struct WallMarksOverlay: View { for feature in features { drawFeature(feature, in: &context, geometry) } - drawMeter(in: &context, geometry) + let up = SIMD3(0, 1, 0) + let along = wall.along(atS: 0) + if let corners = box(center: wall.meter, along: along, up: up, size: Self.meterBox, geometry) { + drawBracket(corners, color: Palette.signal, caption: "Electric meter", in: &context) + } } .ignoresSafeArea() .allowsHitTesting(false) .accessibilityHidden(true) } - private func drawMeter(in context: inout GraphicsContext, _ geometry: WallProjection) { - guard let point = geometry.point(wall.meter) else { return } - let outer = CGRect(x: point.x - 13, y: point.y - 13, width: 26, height: 26) - context.fill(Path(ellipseIn: outer), with: .color(.white)) - context.fill(Path(ellipseIn: outer.insetBy(dx: 4, dy: 4)), with: .color(Palette.signal)) - context.stroke(Path(ellipseIn: outer.insetBy(dx: -5, dy: -5)), with: .color(.white.opacity(0.6)), lineWidth: 2) - } - private func drawEnds(in context: inout GraphicsContext, _ geometry: WallProjection) { for end in [wall.leftEnd, wall.rightEnd].compactMap(\.self) { guard let bottom = geometry.point(s: end, height: 0), let top = geometry.point(s: end, height: wallBandHeight) else { continue } @@ -44,30 +57,79 @@ struct WallMarksOverlay: View { } private func drawFeature(_ feature: MarkedFeature, in context: inout GraphicsContext, _ geometry: WallProjection) { - let color = Color.white + let caption = ScanCopy.name(feature.kind) + let up = SIMD3(0, 1, 0) switch feature.kind { case .door, .window: - if let bottom = feature.bottom, let top = feature.top, - let outline = geometry.wallQuad(s: feature.span, height: bottom...top, out: 0.01) { - context.fill(outline, with: .color(Palette.signal.opacity(0.18))) - context.stroke(outline, with: .color(color), style: StrokeStyle(lineWidth: 3, lineJoin: .round, dash: [9, 6])) + guard let bottom = feature.bottom, let top = feature.top else { return } + let corners = [ + wall.world(s: feature.span.lowerBound, height: top, out: 0.01), + wall.world(s: feature.span.upperBound, height: top, out: 0.01), + wall.world(s: feature.span.upperBound, height: bottom, out: 0.01), + wall.world(s: feature.span.lowerBound, height: bottom, out: 0.01), + ].compactMap(geometry.point) + if corners.count == 4 { drawBracket(corners, color: Self.hologram, caption: caption, in: &context) } + case .gasMeter, .acUnit: + guard let point = feature.points.first else { return } + let s = (feature.span.lowerBound + feature.span.upperBound) / 2 + let size = feature.kind == .gasMeter ? Self.gasMeterBox : Self.acUnitBox + if let corners = box(center: point, along: wall.along(atS: s), up: up, size: size, geometry) { + drawBracket(corners, color: Self.hologram, caption: caption, in: &context) } case .driveway, .fence: let points = feature.points.compactMap(geometry.point) - if points.count >= 2 { - var line = Path() - line.addLines(points) - context.stroke(line, with: .color(.black.opacity(0.3)), style: StrokeStyle(lineWidth: 8, lineCap: .round)) - context.stroke(line, with: .color(color), style: StrokeStyle(lineWidth: 4, lineCap: .round, dash: [10, 7])) + guard points.count >= 2 else { return } + var line = Path() + line.addLines(points) + context.stroke(line, with: .color(.black.opacity(0.3)), style: StrokeStyle(lineWidth: 8, lineCap: .round)) + context.stroke(line, with: .color(.white), style: StrokeStyle(lineWidth: 4, lineCap: .round, dash: [10, 7])) + for point in points { + let dot = CGRect(x: point.x - 6, y: point.y - 6, width: 12, height: 12) + context.fill(Path(ellipseIn: dot.insetBy(dx: -2, dy: -2)), with: .color(.black.opacity(0.25))) + context.fill(Path(ellipseIn: dot), with: .color(.white)) } - case .gasMeter, .acUnit: - break } - for point in feature.points.compactMap(geometry.point) { - let dot = CGRect(x: point.x - 8, y: point.y - 8, width: 16, height: 16) - context.fill(Path(ellipseIn: dot.insetBy(dx: -3, dy: -3)), with: .color(.black.opacity(0.25))) - context.fill(Path(ellipseIn: dot), with: .color(.white)) - context.fill(Path(ellipseIn: dot.insetBy(dx: 4, dy: 4)), with: .color(Palette.signal)) + } + + /// Four corners on screen, clockwise from top-left, of a box facing out of the wall around + /// `center`; nil when any corner is behind the camera. + private func box(center: SIMD3, along: SIMD3, up: SIMD3, size: SIMD2, _ geometry: WallProjection) -> [CGPoint]? { + let w = along * (size.x / 2), h = up * (size.y / 2) + let corners = [center - w + h, center + w + h, center + w - h, center - w - h].compactMap(geometry.point) + return corners.count == 4 ? corners : nil + } + + /// The prototype's bracket: four L-shaped corners 6 pt outside the box, each arm 18 pt or 40% + /// of its edge, with a 4 pt rounded bend, over a soft dark halo so it holds on a bright wall; + /// the caption sits 13 pt above the top edge. + private func drawBracket(_ box: [CGPoint], color: Color, caption: String, in context: inout GraphicsContext) { + let cx = box.map(\.x).reduce(0, +) / 4, cy = box.map(\.y).reduce(0, +) / 4 + let corners = box.map { p -> CGPoint in + let dx = p.x - cx, dy = p.y - cy + let length = max((dx * dx + dy * dy).squareRoot(), 0.001) + return CGPoint(x: p.x + dx / length * 6, y: p.y + dy / length * 6) } + var path = Path() + for i in corners.indices { + let corner = corners[i], previous = corners[(i + 3) % 4], next = corners[(i + 1) % 4] + func toward(_ other: CGPoint) -> CGPoint { + let dx = other.x - corner.x, dy = other.y - corner.y + let length = max((dx * dx + dy * dy).squareRoot(), 0.001) + let arm = min(18, 0.4 * length) + return CGPoint(x: corner.x + dx / length * arm, y: corner.y + dy / length * arm) + } + path.move(to: toward(previous)) + path.addArc(tangent1End: corner, tangent2End: toward(next), radius: 4) + path.addLine(to: toward(next)) + } + context.stroke(path, with: .color(.black.opacity(0.28)), style: StrokeStyle(lineWidth: 4.5, lineCap: .round, lineJoin: .round)) + context.stroke(path, with: .color(color), style: StrokeStyle(lineWidth: 2, lineCap: .round, lineJoin: .round)) + + let top = CGPoint(x: (corners[0].x + corners[1].x) / 2, y: min(corners[0].y, corners[1].y) - 13) + var layer = context + layer.addFilter(.shadow(color: .black.opacity(0.45), radius: 6, y: 1)) + layer.draw( + Text(caption).font(.system(size: 13, weight: .semibold, design: .rounded)).foregroundStyle(.white), + at: top, anchor: .center) } } diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Overlay/SurfaceDots.swift b/ios/HouseScanKit/Sources/HouseScanKit/Overlay/SurfaceDots.swift new file mode 100644 index 000000000..3187fffda --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Overlay/SurfaceDots.swift @@ -0,0 +1,503 @@ +import simd + +// The live dots a LiDAR phone draws over the camera: the prototype's dot field +// (experiments/live-dots on t3/experience, `LiveDotsCore/DotField.swift`), bounded for a phone. +// +// Each kept keyframe's depth, every second pixel of it, with medium or high confidence and from +// 0.3 to 5 m, is unprojected and fused into 5 cm voxels. A voxel keeps its distinct view +// directions (a new one must differ from every stored one by more than 15 degrees, four at +// most), a normal from central differences on the depth map (averaged over keyframes), the most +// face-on view it has had, and a fixed jitter of up to 30% of the voxel in the plane of its +// first normal, so the field never reads as a grid. +// +// A voxel becomes an edge, and stays one, when +// (a) a face neighbour's normal differs by more than 35 degrees (the wall-to-ground corner, a +// bin's corners), or +// (b) it has surface along one side and seen-empty space along another (a bin's silhouette). +// Seen-empty means a confident depth reading passed the neighbour with 10 cm to spare, so +// the edge of what has been scanned so far is never an edge. +// The prototype's third rule, a colour step in the photo (painted outlines), is left out: it +// needs the camera image at depth resolution on every keyframe, and the things this scan cares +// about (meters, bins, AC units) stand off the wall, so (a) and (b) find them. +// +// Every edge voxel gets a dot. Flat voxels get one per 2 x 2 x 2 group, chosen once by lowest +// hash and kept, a quarter of the edge density on a wall. Opacity is 45% at one view and rises +// 15 points a view to 90% at four. An edge on a surface within 45 degrees of vertical stays at or +// below 60% until one view saw it within 30 degrees of face-on. A dot more than 25 cm in front of +// the wall and more than 10 cm above the ground is on an occluder, and draws violet. +// +// The dots are guidance, never evidence: coverage (`CoverageMap`) decides what the scan has +// seen, from the same keyframes, and nothing here feeds it. + +public typealias VoxelKey = SIMD3 + +public struct SurfaceDotConfig: Sendable, Equatable { + public var voxelSize: Float = 0.05 + /// ARKit confidence: 0 low, 1 medium, 2 high. + public var minimumConfidence: UInt8 = 1 + /// 5 m, not the prototype's 6: ARKit's confidence falls off past about 5 m outdoors, and + /// coverage does not count views from farther than `CoverageConfig.maxDistance` (6 m) anyway. + public var depthRange: ClosedRange = 0.3...5 + /// Every `sampleStride`-th depth pixel on both axes. At 2, a 256 x 192 map gives 12,288 + /// samples, and a pixel step at 5 m (about 5 cm) still reaches every 5 cm voxel it crosses. + public var sampleStride: Int = 2 + public var viewSeparationDegrees: Float = 15 + public var creaseDegrees: Float = 35 + /// A face neighbour lies along the surface when its direction is within 60 degrees of the + /// tangent plane. + public var alongSurfaceCosine: Float = 0.5 + public var freeSpaceMargin: Float = 0.1 + public var jitterFraction: Float = 0.3 + public var faceOnDegrees: Float = 30 + public var obliqueEdgeOpacityCap: Float = 0.6 + /// The oblique cap applies to edges whose normal is within 45 degrees of horizontal. + public var cappedNormalMaxY: Float = 0.7071 + public var occluderMinOut: Float = 0.25 + public var groundBand: Float = 0.1 + /// Voxels kept at once. Past it, the ones farthest from the camera go first, down to 75%. + /// A voxel costs about 100 bytes with its dictionary entry, so 60,000 is about 6 MB; a 20 m + /// walk with the wall and the ground in front of it is about 25,000 surface voxels. Not + /// measured on a phone. + public var maxVoxels: Int = 60_000 + /// Most dots handed to the renderer. The prototype drew 6,000 in 0.54 ms of an M4 Pro's GPU; + /// a phone GPU is several times slower and also draws the camera, so 5,000. Not measured on + /// a phone. + public var maxDots: Int = 5_000 + /// Dots farther than this from the latest camera are not handed out: off screen or too small + /// to matter, and they would crowd nearer ones out of `maxDots`. + public var dotRadius: Float = 6 + + public init() {} +} + +/// One dot as the renderer draws it, before timing. +public struct SurfaceDot: Sendable, Equatable { + /// Stable for the life of the voxel, so the renderer can tell a birth from a survivor. + public let id: UInt64 + public let position: SIMD3 + public let isEdge: Bool + /// Distinct view directions so far, up to four. + public let views: Int + /// Some view saw it within `faceOnDegrees` of face-on. Only edge dots are ever false. + public let faceOn: Bool + /// On something standing in front of the wall (a bin, a bush), drawn violet. + public let onOccluder: Bool + /// 0.45 at one view to 0.9 at four, capped at 0.6 for an edge never seen face-on. + public let opacity: Float + + public init(id: UInt64, position: SIMD3, isEdge: Bool, views: Int, faceOn: Bool, onOccluder: Bool, opacity: Float) { + self.id = id + self.position = position + self.isEdge = isEdge + self.views = views + self.faceOn = faceOn + self.onOccluder = onOccluder + self.opacity = opacity + } + + public static func opacity(views: Int, faceOn: Bool, cap: Float) -> Float { + guard views > 0 else { return 0 } + let level = min(0.45 + 0.15 * Float(views - 1), 0.9) + return faceOn ? level : min(level, cap) + } +} + +/// Up to four unit directions, stored inline so a voxel needs no heap allocation. +struct ViewSet: Sendable, Equatable { + private var d0 = SIMD3.zero, d1 = SIMD3.zero, d2 = SIMD3.zero, d3 = SIMD3.zero + private(set) var count: UInt8 = 0 + /// Opacity stops rising at four views. + static let capacity: UInt8 = 4 + + func direction(_ index: Int) -> SIMD3 { + switch index { + case 0: d0 + case 1: d1 + case 2: d2 + default: d3 + } + } + + /// Adds `unit` when it differs from every stored direction by more than the angle whose + /// cosine is `limit`. Returns whether it was added. + @discardableResult + mutating func insert(_ unit: SIMD3, limit: Float) -> Bool { + guard count < Self.capacity else { return false } + for index in 0..= limit { return false } + switch count { + case 0: d0 = unit + case 1: d1 = unit + case 2: d2 = unit + default: d3 = unit + } + count += 1 + return true + } + + func rotated(_ rotate: (SIMD3) -> SIMD3) -> ViewSet { + var copy = self + copy.d0 = rotate(d0) + copy.d1 = rotate(d1) + copy.d2 = rotate(d2) + copy.d3 = rotate(d3) + return copy + } +} + +struct Voxel: Sendable, Equatable { + /// Sum of per-keyframe unit normals. + var normalSum: SIMD3 + var views: ViewSet + /// Highest cosine between a view direction and the voxel's normal at that view. + var faceOnCosine: Float + /// Fixed at the first hit, in the plane of the first normal. + var jitter: SIMD3 + /// Sticky: an edge never turns back into a flat dot. + var isEdge: Bool + + var normal: SIMD3? { + let length = simd_length(normalSum) + return length > 1e-3 ? normalSum / length : nil + } +} + +/// The LiDAR dot field. A value type confined to one owner; `integrate` is the hot path and +/// touches only the voxels this keyframe saw and their face neighbours. +public struct SurfaceDots: Sendable { + public let config: SurfaceDotConfig + private(set) var voxels: [VoxelKey: Voxel] = [:] + /// Voxels a confident depth reading passed with room to spare. Only in-plane neighbours of + /// occupied voxels are tested, which is all the boundary rule needs. + private(set) var free: Set = [] + /// Each 2 x 2 x 2 group's flat voxel, keyed by the group. + private var representatives: [VoxelKey: VoxelKey] = [:] + + static let faceAxes: [VoxelKey] = [ + VoxelKey(1, 0, 0), VoxelKey(-1, 0, 0), VoxelKey(0, 1, 0), + VoxelKey(0, -1, 0), VoxelKey(0, 0, 1), VoxelKey(0, 0, -1), + ] + + public init(config: SurfaceDotConfig = SurfaceDotConfig()) { + self.config = config + } + + public var voxelCount: Int { voxels.count } + public var isEmpty: Bool { voxels.isEmpty } + + public func key(for p: SIMD3) -> VoxelKey { + VoxelKey((p / config.voxelSize).rounded(.toNearestOrAwayFromZero)) + } + + public func centre(of key: VoxelKey) -> SIMD3 { + SIMD3(key) * config.voxelSize + } + + // MARK: Keyframes + + /// Fuses one keyframe's depth. `depth` is ARKit's scene depth for `camera`, in the landscape + /// sensor orientation, with intrinsics for its own grid (`DepthImage.intrinsics(scaling:)`). + public mutating func integrate(camera: CameraFrame, depth: DepthImage) { + let step = max(config.sampleStride, 1) + let pose = camera.cameraToWorld + let eye = camera.position + + struct Accumulator { var normals = 0; var normalSum = SIMD3.zero } + var accumulators: [VoxelKey: Accumulator] = [:] + accumulators.reserveCapacity(depth.width * depth.height / (step * step * 4)) + for j in stride(from: step / 2, to: depth.height, by: step) { + for i in stride(from: step / 2, to: depth.width, by: step) { + guard let d = reading(depth, i, j) else { continue } + let local = Self.unproject(i, j, d, depth.intrinsics) + let world4 = pose * SIMD4(local, 1) + let world = SIMD3(world4.x, world4.y, world4.z) + let key = key(for: world) + var accumulator = accumulators[key] ?? Accumulator() + if let normal = cameraNormal(depth, i, j, d, step: step) { + let rotated = pose * SIMD4(normal, 0) + accumulator.normals += 1 + accumulator.normalSum += SIMD3(rotated.x, rotated.y, rotated.z) + } + accumulators[key] = accumulator + } + } + let limit = cos(config.viewSeparationDegrees * .pi / 180) + for (key, a) in accumulators { + observe(key, normal: a.normals > 0 ? a.normalSum / Float(a.normals) : nil, camera: eye, viewLimit: limit) + } + let touched = Set(accumulators.keys) + carveFreeSpace(around: touched, camera: camera, depth: depth) + classify(touched) + bound(around: eye) + } + + /// Records one keyframe's samples in a voxel. + mutating func observe(_ key: VoxelKey, normal: SIMD3?, camera: SIMD3, viewLimit: Float) { + let centre = centre(of: key) + var voxel = voxels[key] ?? Voxel( + normalSum: .zero, views: ViewSet(), faceOnCosine: -1, + jitter: Self.jitter(for: key, normal: normal ?? SIMD3(0, 0, 1), radius: config.jitterFraction * config.voxelSize), + isEdge: false) + if let normal, simd_length(normal) > 1e-3 { voxel.normalSum += simd_normalize(normal) } + let toCamera = camera - centre + let distance = simd_length(toCamera) + if distance > 1e-6 { + let unit = toCamera / distance + voxel.views.insert(unit, limit: viewLimit) + if let current = voxel.normal { voxel.faceOnCosine = max(voxel.faceOnCosine, simd_dot(unit, current)) } + } + voxels[key] = voxel + free.remove(key) + } + + /// Tests the empty in-plane neighbours of this keyframe's voxels against its depth: a + /// neighbour is seen-empty when a confident reading at its pixel is farther than it by + /// `freeSpaceMargin`. A neighbour outside the view, behind something, or under a low-confidence + /// reading stays unknown, which is why the edge of what has been scanned so far never counts + /// as a boundary. + mutating func carveFreeSpace(around touched: Set, camera: CameraFrame, depth: DepthImage) { + let toCamera = camera.worldToCamera + var found: [VoxelKey] = [] + for key in touched { + guard let normal = voxels[key]?.normal else { continue } + for axis in Self.faceAxes where abs(simd_dot(SIMD3(axis), normal)) < config.alongSurfaceCosine { + let neighbour = key &+ axis + guard voxels[neighbour] == nil, !free.contains(neighbour) else { continue } + let c4 = toCamera * SIMD4(centre(of: neighbour), 1) + let z = -c4.z + guard z >= config.depthRange.lowerBound else { continue } + let k = depth.intrinsics + let u = k.z + k.x * c4.x / z + let v = k.w - k.y * c4.y / z + guard u >= 0, v >= 0, u < Float(depth.width), v < Float(depth.height), + let measured = reading(depth, Int(u), Int(v), ignoringRange: true) else { continue } + if measured > z + config.freeSpaceMargin { found.append(neighbour) } + } + } + free.formUnion(found) + } + + func edgeReason(for key: VoxelKey) -> EdgeReason? { + guard let voxel = voxels[key] else { return nil } + let normal = voxel.normal + var surfaceNeighbour = false, emptyNeighbour = false + let creaseCosine = cos(config.creaseDegrees * .pi / 180) + for axis in Self.faceAxes { + let along = normal.map { abs(simd_dot(SIMD3(axis), $0)) < config.alongSurfaceCosine } ?? false + if let other = voxels[key &+ axis] { + if let normal, let otherNormal = other.normal, simd_dot(normal, otherNormal) < creaseCosine { + return .crease + } + if along { surfaceNeighbour = true } + } else if along, free.contains(key &+ axis) { + emptyNeighbour = true + } + } + return surfaceNeighbour && emptyNeighbour ? .boundary : nil + } + + enum EdgeReason: Equatable { + case crease + case boundary + } + + /// Marks new edges (sticky) among the touched voxels and their neighbours, then gives each + /// touched group without one a flat representative. + mutating func classify(_ touched: Set) { + var candidates = touched + for key in touched { + for axis in Self.faceAxes where voxels[key &+ axis] != nil { candidates.insert(key &+ axis) } + } + for key in candidates where voxels[key]?.isEdge == false && edgeReason(for: key) != nil { + voxels[key]?.isEdge = true + } + for key in touched { + let group = key &>> 1 + guard representatives[group] == nil else { continue } + var best: (key: VoxelKey, hash: UInt64)? + for dx in Int32(0)...1 { + for dy in Int32(0)...1 { + for dz in Int32(0)...1 { + let member = (group &<< 1) &+ VoxelKey(dx, dy, dz) + guard let voxel = voxels[member], !voxel.isEdge else { continue } + let hash = Self.hash(member, seed: 0xF1A7) + if best == nil || hash < best!.hash { best = (member, hash) } + } + } + } + if let best { representatives[group] = best.key } + } + } + + /// Keeps at most `maxVoxels`: past it, drops the voxels farthest from `eye` down to 75% of + /// the cap, with the free space and representatives that referred to them. + mutating func bound(around eye: SIMD3) { + if voxels.count > config.maxVoxels { + let keep = config.maxVoxels * 3 / 4 + let ranked = voxels.keys.map { ($0, simd_distance_squared(centre(of: $0), eye)) }.sorted { $0.1 < $1.1 } + let cutoff = ranked[keep].1 + voxels = voxels.filter { simd_distance_squared(centre(of: $0.key), eye) < cutoff } + representatives = representatives.filter { voxels[$0.value] != nil } + } + if free.count > config.maxVoxels { + let ranked = free.map { simd_distance_squared(centre(of: $0), eye) }.sorted() + let cutoff = ranked[config.maxVoxels * 3 / 4] + free = free.filter { simd_distance_squared(centre(of: $0), eye) < cutoff } + } + } + + /// ARKit's correction to the meter's anchor (`YawCorrection`), applied to every voxel, so the + /// dots move with the wall the scan measured. + public mutating func apply(_ correction: YawCorrection) { + var moved: [VoxelKey: Voxel] = [:] + moved.reserveCapacity(voxels.count) + for (key, voxel) in voxels { + var copy = voxel + copy.normalSum = correction.direction(voxel.normalSum) + copy.views = voxel.views.rotated(correction.direction) + copy.jitter = correction.direction(voxel.jitter) + let newKey = self.key(for: correction.point(centre(of: key))) + if let existing = moved[newKey], existing.views.count >= copy.views.count { continue } + moved[newKey] = copy + } + voxels = moved + free = Set(free.map { key(for: correction.point(centre(of: $0))) }.filter { moved[$0] == nil }) + representatives = [:] + classify(Set(voxels.keys)) + } + + // MARK: Dots + + /// Every edge voxel and each group's flat representative within `dotRadius` of `eye`, at + /// most `maxDots`: past it, flat dots drop in hash order, then the farthest edges. `wall` + /// decides which dots stand on an occluder; without one, none do. Sorted by id. + public func dots(near eye: SIMD3, wall: WallFrame?) -> [SurfaceDot] { + let radius2 = config.dotRadius * config.dotRadius + var edges: [(VoxelKey, Voxel, Float)] = [] + var flats: [(VoxelKey, Voxel)] = [] + for (key, voxel) in voxels where voxel.isEdge { + let d2 = simd_distance_squared(centre(of: key), eye) + if d2 <= radius2 { edges.append((key, voxel, d2)) } + } + for key in representatives.values { + guard let voxel = voxels[key], !voxel.isEdge, simd_distance_squared(centre(of: key), eye) <= radius2 else { continue } + flats.append((key, voxel)) + } + if edges.count > config.maxDots { + edges.sort { $0.2 < $1.2 } + edges.removeLast(edges.count - config.maxDots) + } + let room = config.maxDots - edges.count + if flats.count > room { + flats.sort { Self.hash($0.0, seed: 0xD075) < Self.hash($1.0, seed: 0xD075) } + flats.removeLast(flats.count - room) + } + var result = edges.map { dot($0.0, $0.1, wall: wall) } + flats.map { dot($0.0, $0.1, wall: wall) } + result.sort { $0.id < $1.id } + return result + } + + private func dot(_ key: VoxelKey, _ voxel: Voxel, wall: WallFrame?) -> SurfaceDot { + let centre = centre(of: key) + let wallLike = voxel.normal.map { abs($0.y) < config.cappedNormalMaxY } ?? false + let faceOn = !voxel.isEdge || !wallLike || voxel.faceOnCosine >= cos(config.faceOnDegrees * .pi / 180) + let views = Int(voxel.views.count) + var onOccluder = false + if let wall { + let p = wall.wallPoint(centre) + onOccluder = p.out > config.occluderMinOut && p.height > config.groundBand + } + return SurfaceDot( + id: Self.id(for: key), position: centre + voxel.jitter, isEdge: voxel.isEdge, views: views, + faceOn: faceOn, onOccluder: onOccluder, + opacity: SurfaceDot.opacity(views: views, faceOn: faceOn, cap: config.obliqueEdgeOpacityCap)) + } + + // MARK: Depth helpers + + /// Meters at a depth pixel, or nil without a confident reading in range. `ignoringRange` keeps + /// far readings, which the free-space test needs. + private func reading(_ depth: DepthImage, _ i: Int, _ j: Int, ignoringRange: Bool = false) -> Float? { + let index = j * depth.width + i + if let confidence = depth.confidence, confidence[index] < config.minimumConfidence { return nil } + let millimeters = depth.millimeters[index] + guard millimeters > 0 else { return nil } + let meters = Float(millimeters) / 1000 + return ignoringRange || config.depthRange.contains(meters) ? meters : nil + } + + /// Camera space: +x right and +y up in the sensor image, looking along -z; v grows down. + static func unproject(_ i: Int, _ j: Int, _ d: Float, _ k: SIMD4) -> SIMD3 { + let u = Float(i) + 0.5, v = Float(j) + 0.5 + return SIMD3((u - k.z) / k.x * d, (k.w - v) / k.y * d, -d) + } + + /// Camera-space unit normal from central differences `step` pixels apart, falling back to one + /// side where the other jumps by more than 10% of the depth (a silhouette), facing the + /// camera. Nil when neither side is usable on either axis. + private func cameraNormal(_ depth: DepthImage, _ i: Int, _ j: Int, _ d0: Float, step: Int) -> SIMD3? { + func point(_ i: Int, _ j: Int) -> SIMD3? { + guard i >= 0, j >= 0, i < depth.width, j < depth.height, let d = reading(depth, i, j), abs(d - d0) < 0.1 * d0 else { return nil } + return Self.unproject(i, j, d, depth.intrinsics) + } + let centre = Self.unproject(i, j, d0, depth.intrinsics) + func tangent(_ plus: SIMD3?, _ minus: SIMD3?) -> SIMD3? { + switch (plus, minus) { + case let (p?, m?): p - m + case let (p?, nil): p - centre + case let (nil, m?): centre - m + case (nil, nil): nil + } + } + guard let tx = tangent(point(i + step, j), point(i - step, j)), + let ty = tangent(point(i, j + step), point(i, j - step)) else { return nil } + let n = simd_cross(tx, ty) + let length = simd_length(n) + guard length > 1e-9 else { return nil } + let unit = n / length + return simd_dot(unit, -centre) < 0 ? -unit : unit + } + + // MARK: Hashing + + /// SplitMix64: Swift's `Hasher` is seeded per process, and the field must look the same twice. + static func mix(_ value: UInt64) -> UInt64 { + var z = value &+ 0x9E37_79B9_7F4A_7C15 + z = (z ^ (z >> 30)) &* 0xBF58_476D_1CE4_E5B9 + z = (z ^ (z >> 27)) &* 0x94D0_49BB_1331_11EB + return z ^ (z >> 31) + } + + static func hash(_ key: VoxelKey, seed: UInt64) -> UInt64 { + var h = mix(seed) + h = mix(h ^ UInt64(UInt32(bitPattern: key.x))) + h = mix(h ^ UInt64(UInt32(bitPattern: key.y))) + h = mix(h ^ UInt64(UInt32(bitPattern: key.z))) + return h + } + + static func unit(_ h: UInt64) -> Float { + Float(h >> 40) / Float(1 << 24) + } + + /// Uniform over a disc of `radius` perpendicular to `normal`, from a hash of the key. + static func jitter(for key: VoxelKey, normal: SIMD3, radius: Float) -> SIMD3 { + let length = simd_length(normal) + let n = length > 1e-6 ? normal / length : SIMD3(0, 0, 1) + let helper: SIMD3 = abs(n.y) < 0.9 ? SIMD3(0, 1, 0) : SIMD3(1, 0, 0) + let t = simd_normalize(simd_cross(helper, n)) + let b = simd_cross(n, t) + let h = hash(key, seed: 0x6A17) + let angle = unit(h) * 2 * .pi + let r = radius * unit(mix(h)).squareRoot() + return (t * cos(angle) + b * sin(angle)) * r + } + + /// Packs the key into 21 bits per axis (plus or minus 52 km at 5 cm), so ids never collide. + static func id(for key: VoxelKey) -> UInt64 { + let bias: Int64 = 1 << 20 + let x = UInt64(Int64(key.x) + bias) & 0x1F_FFFF + let y = UInt64(Int64(key.y) + bias) & 0x1F_FFFF + let z = UInt64(Int64(key.z) + bias) & 0x1F_FFFF + return x | (y << 21) | (z << 42) + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/SurfaceDotsTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/SurfaceDotsTests.swift new file mode 100644 index 000000000..14e20288a --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/SurfaceDotsTests.swift @@ -0,0 +1,122 @@ +import HouseScanKit +import simd +import Testing + +// The LiDAR dot field on the standard wall (face z = 0, ground y = 0), with depth rendered from +// the scene mesh at 128 x 96 (`renderDepth`, fx = fy = 100): at 2 m a pixel is 2 cm, so every +// sampled pixel (stride 2) lands within one 5 cm voxel of the next. +@Suite struct SurfaceDotsTests { + /// A box standing in front of the wall: s in [-0.5, 0.5], 0.5 to 1.0 m out, 1.5 m tall. + static let box = (SIMD3(-0.5, 0, 0.5), SIMD3(0.5, 1.5, 1.0)) + + static func field(_ scene: TriangleMesh, cameras: [CameraFrame], config: SurfaceDotConfig = SurfaceDotConfig()) -> SurfaceDots { + var field = SurfaceDots(config: config) + for camera in cameras { field.integrate(camera: camera, depth: renderDepth(scene, from: camera)) } + return field + } + + /// Level at 1.5 m, 2 m from the wall: the view spans heights 0.22 to 2.78 m, all wall, so no + /// neighbour is ever seen empty and nothing creases. Every dot is flat, on the wall, one per + /// 2 x 2 x 2 group (four wall voxels share a group), at 45% for one view. + @Test func aPlainWallGivesSparseFlatDotsOnTheWall() { + let camera = makeCamera(at: SIMD3(0, 1.5, 2.0), forward: SIMD3(0, 0, -1), right: SIMD3(1, 0, 0)) + let field = Self.field(standardScene(), cameras: [camera]) + let dots = field.dots(near: camera.position, wall: standardWall()) + #expect(!dots.isEmpty) + #expect(dots.allSatisfy { !$0.isEdge }) + #expect(dots.allSatisfy { abs($0.position.z) < 0.02 }) + #expect(dots.allSatisfy { $0.opacity == 0.45 && !$0.onOccluder }) + #expect(dots.count * 3 <= field.voxelCount) + } + + /// Pitched down at the foot of the wall: the wall's and the ground's normals differ by 90 + /// degrees, so the voxels along the corner are edges, and only they are. + @Test func theWallToGroundCornerIsAnEdge() { + let camera = portraitCamera(at: SIMD3(0, 1.0, 2.0), forward: forwardFacingWall(pitchedDown: 30)) + let dots = Self.field(standardScene(), cameras: [camera]).dots(near: camera.position, wall: standardWall()) + let edges = dots.filter(\.isEdge) + #expect(edges.count > 20) + // The corner line is y = 0, z = 0; a corner voxel's centre is within one voxel diagonal, + // plus its jitter. + #expect(edges.allSatisfy { simd_length(SIMD2($0.position.y, $0.position.z)) < 0.1 }) + } + + /// The box's front stands 1 m out: its dots are on an occluder, the wall's are not, and its + /// silhouette against the wall behind is a boundary edge. + @Test func aBoxInFrontOfTheWallIsAnOccluderWithAnOutline() { + let camera = wallCamera(s: 0) + let dots = Self.field(standardScene(boxes: [Self.box]), cameras: [camera]).dots(near: camera.position, wall: standardWall()) + let onBox = dots.filter { $0.position.z > 0.9 && $0.position.y > 0.2 } + let onWall = dots.filter { abs($0.position.z) < 0.05 && $0.position.y > 0.2 } + #expect(!onBox.isEmpty && !onWall.isEmpty) + #expect(onBox.allSatisfy { $0.onOccluder }) + #expect(onWall.allSatisfy { !$0.onOccluder }) + let sides = onBox.filter { $0.isEdge && abs(abs($0.position.x) - 0.5) < 0.1 } + #expect(!sides.isEmpty) + } + + /// Opacity follows the distinct views: two positions 30 degrees apart as seen from the wall + /// give 60% where both saw it. + @Test func aSecondViewRaisesOpacity() { + let target = SIMD3(0, 1.0, 0) + let first = portraitCamera(at: SIMD3(0, 1.2, 2.0), lookingAt: target) + let second = portraitCamera(at: SIMD3(2.0 * tan(30 * .pi / 180), 1.2, 2.0), lookingAt: target) + let dots = Self.field(standardScene(), cameras: [first, second]).dots(near: second.position, wall: standardWall()) + #expect(dots.contains { $0.views == 2 && $0.opacity == 0.6 }) + #expect(dots.allSatisfy { $0.views <= 2 }) + } + + /// Low-confidence readings and readings past 5 m make no dots. + @Test func unconfidentOrFarDepthMakesNothing() { + let camera = wallCamera(s: 0) + var field = SurfaceDots() + field.integrate(camera: camera, depth: uniformDepth(2000, confidence: 0)) + field.integrate(camera: camera, depth: uniformDepth(5500)) + #expect(field.isEmpty) + } + + /// Past `maxVoxels` the field sheds the far voxels, so a long walk never grows it past the + /// cap; past `maxDots`, flat dots go before any edge. + @Test func dotsAndVoxelsStayUnderTheirCaps() { + var config = SurfaceDotConfig() + config.maxVoxels = 800 + var capped = config + capped.maxDots = 20 + var field = SurfaceDots(config: config) + var small = SurfaceDots(config: capped) + let scene = standardScene() + var last = portraitCamera(at: SIMD3(-4, 1.0, 2.0), forward: forwardFacingWall(pitchedDown: 30)) + for step in 0..<17 { + last = portraitCamera(at: SIMD3(-4 + Float(step) * 0.5, 1.0, 2.0), forward: forwardFacingWall(pitchedDown: 30)) + let depth = renderDepth(scene, from: last) + field.integrate(camera: last, depth: depth) + small.integrate(camera: last, depth: depth) + #expect(field.voxelCount <= config.maxVoxels) + } + let all = field.dots(near: last.position, wall: standardWall()) + let dots = small.dots(near: last.position, wall: standardWall()) + #expect(all.count > 20) + #expect(dots.count == 20) + #expect(dots.filter { $0.isEdge }.count == min(20, all.filter { $0.isEdge }.count)) + } + + /// An anchor correction moves the field with the wall: a 0.5 m shift moves every dot 0.5 m. + @Test func aCorrectionMovesTheDots() { + let camera = wallCamera(s: 0) + var field = Self.field(standardScene(), cameras: [camera]) + let before = field.dots(near: camera.position, wall: nil) + field.apply(YawCorrection(yaw: 0, translation: SIMD3(0.5, 0, 0))) + let after = field.dots(near: camera.position + SIMD3(0.5, 0, 0), wall: nil) + func meanX(_ dots: [SurfaceDot]) -> Float { dots.map(\.position.x).reduce(0, +) / Float(dots.count) } + #expect(abs(after.count - before.count) <= before.count / 20) + #expect(nearlyEqual(meanX(after) - meanX(before), 0.5, 0.03)) + } + + /// The same keyframes give the same dots: jitter, thinning and order come from stable hashes. + @Test func theFieldIsDeterministic() { + let cameras = [wallCamera(s: 0), wallCamera(s: 0.5)] + let a = Self.field(standardScene(boxes: [Self.box]), cameras: cameras).dots(near: cameras[1].position, wall: standardWall()) + let b = Self.field(standardScene(boxes: [Self.box]), cameras: cameras).dots(near: cameras[1].position, wall: standardWall()) + #expect(a == b) + } +} diff --git a/ios/HouseScanUITests/FullFlowUITests.swift b/ios/HouseScanUITests/FullFlowUITests.swift index 6f3d2ec1d..ee8c02f80 100644 --- a/ios/HouseScanUITests/FullFlowUITests.swift +++ b/ios/HouseScanUITests/FullFlowUITests.swift @@ -58,6 +58,34 @@ final class FullFlowUITests: XCTestCase { XCTAssertTrue(showedHidden, "the wall map never reported hidden cells behind the bin") } + /// The LiDAR walk under the live fog, screenshotted as fast as the runner allows from the + /// moment the walk appears, for comparing against the prototype's keyframes (see + /// `LiveFogShaders`). No audit here: the audit's six-second wait would span the whole walk, + /// which plays in about 6 s at the autopilot's 3x. The run stops at the walk. + @MainActor + func testLiveFogWalkScreenshots() throws { + let app = XCUIApplication() + let gate = URL(fileURLWithPath: NSTemporaryDirectory()).appending(path: "housescan-gate-\(UUID().uuidString)", directoryHint: .isDirectory) + try FileManager.default.createDirectory(at: gate, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: gate) } + // The screens before the walk may leave as soon as the autopilot is done with them. + for phase in ["onboarding", "findMeter", "meterCloseUp"] { try Data().write(to: gate.appending(path: phase)) } + app.launchArguments = ["-replay", Self.lidarFixture, "-autopilot", "-autopilotHold", "1.5", "-autopilotGate", gate.path, "-sampleResult"] + app.launch() + XCTAssertTrue(app.descendants(matching: .any)["screen.wallWalk"].waitForExistence(timeout: 90), "screen.wallWalk never appeared") + let start = Date() + var index = 0 + while Date().timeIntervalSince(start) < 16 { + let shot = XCTAttachment(screenshot: app.screenshot()) + shot.name = String(format: "liveFog-%02d-%04.1fs", index, Date().timeIntervalSince(start)) + shot.lifetime = .keepAlways + add(shot) + index += 1 + Thread.sleep(forTimeInterval: 0.25) + } + app.terminate() + } + /// Device run 1's failure on the synthetic replay (`-autopilotCantGetThere`): "Can't get there" /// goes to the ground by the meter before it is seen, then ends each side where the walk went /// farthest, 5 m left and right of the meter (16 ft 5 in). The old rule put the ends at the From 94c2e2a71dbbd05d4eed4ecc73b95aede5dcabab Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 03:40:04 -0500 Subject: [PATCH 2/5] Keep hidden and requested tints in the band, and rag the fog's edges The offscreen preview on the LiDAR fixture showed lifted columns as straight strips and a hidden stretch's violet climbing the whole column. The mask is now read through a noise offset and blurred at 64 texels across, and only the coverage band carries the violet and amber. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../UI/Camera/LiveFog/LiveFogRenderer.swift | 56 +++++++------ .../UI/Camera/LiveFog/LiveFogScene.swift | 79 +++++++++++++------ .../UI/Camera/LiveFog/LiveFogShaders.swift | 18 +++-- .../UI/Camera/LiveFog/LiveFogView.swift | 7 +- 4 files changed, 103 insertions(+), 57 deletions(-) diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift index 4d5b8f5c3..2022157b8 100644 --- a/ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift @@ -1,5 +1,4 @@ import Metal -import MetalKit import os import QuartzCore import simd @@ -87,26 +86,30 @@ struct LiveFogInput { var requestedColor: SIMD3 } -/// Draws the fog and dots into a transparent MTKView over the camera, every display frame while -/// the walk is on screen. +/// Draws the fog and dots into a transparent Metal layer over the camera (`LiveFogMTKView`), +/// every display frame while the walk is on screen. /// -/// Per frame: the mask pass draws the coverage cells (`FogMaskGeometry`) into a 96-texel-wide +/// Per frame: the mask pass draws the coverage cells (`FogMaskGeometry`) into a 64-texel-wide /// texture with a depth test, two passes blur it, then the drawable gets the fog and the dots. -/// Bounds: three frames in flight; the mask vertex ring holds 8,192 vertices a frame (a 60 m -/// strip of unmerged cells needs about 4,800); the dot buffers hold `DotTimeline.capacity` dots +/// Bounds: three frames in flight; the mask vertex ring holds 16,384 vertices a frame (a 60 m +/// strip of unmerged cells, two bands in two parts, needs about 9,600); the dot buffers hold `DotTimeline.capacity` dots /// and are rewritten only when the field changes or a fading dot ends, at most once a frame. -/// Nothing grows with time: about 4 MB of buffers and 0.2 MB of textures, allocated once. +/// Nothing grows with time: about 4.5 MB of buffers and 0.1 MB of textures, allocated once. @MainActor final class LiveFogRenderer { private let gpu: LiveFogGPU - private let epoch = CACurrentMediaTime() + /// Seconds on the clock the animations run on. Replaceable so an offscreen render can step + /// time; the view leaves it on the display's media clock. + var clock: () -> Double = { CACurrentMediaTime() } + private var epoch: Double? private let inFlight = DispatchSemaphore(value: LiveFogRenderer.framesInFlight) private static let framesInFlight = 3 - private static let maskWidth = 96 - private static let maskVertexCapacity = 8_192 + private static let maskWidth = 64 + private static let maskVertexCapacity = 16_384 private static let spriteCapacity = DotTimeline.capacity * 2 var input: LiveFogInput? + var queue: MTLCommandQueue { gpu.queue } private let animator = FogCellAnimator() private let timeline = DotTimeline() @@ -133,21 +136,24 @@ final class LiveFogRenderer { } } - func draw(in view: MTKView) { - guard let input, maskBuffers.count == Self.framesInFlight, spriteBuffers.count == Self.framesInFlight else { return } - let size = view.bounds.size - guard size.width > 1, size.height > 1 else { return } - // Never block the main thread on the GPU: a frame that finds all three in flight is skipped. - guard inFlight.wait(timeout: .now()) == .success else { return } - guard let pass = view.currentRenderPassDescriptor, let drawable = view.currentDrawable, - let commandBuffer = gpu.queue.makeCommandBuffer(), let textures = textures(for: size) else { + /// Encodes one frame into `pass`, whose target is `pixels` in size for a view of `size` + /// points. Returns false, encoding nothing, when all three frames are still in flight: the + /// caller skips the frame rather than block the main thread on the GPU. `LiveFogMTKView` + /// calls this each display frame. + func encode(into commandBuffer: MTLCommandBuffer, pass: MTLRenderPassDescriptor, size: CGSize, pixels: SIMD2, pixelsPerPoint scale: Float) -> Bool { + guard let input, maskBuffers.count == Self.framesInFlight, spriteBuffers.count == Self.framesInFlight, + size.width > 1, size.height > 1 else { return false } + guard inFlight.wait(timeout: .now()) == .success else { return false } + guard let textures = textures(for: size) else { inFlight.signal() - return + return false } let semaphore = inFlight commandBuffer.addCompletedHandler { _ in semaphore.signal() } - let now = CACurrentMediaTime() - epoch + let start = epoch ?? clock() + epoch = start + let now = clock() - start let time = Float(now) let slot = frame % Self.framesInFlight frame &+= 1 @@ -205,8 +211,6 @@ final class LiveFogRenderer { pass.colorAttachments[0].loadAction = .clear pass.colorAttachments[0].clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: 0) if let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: pass) { - let scale = Float(view.contentScaleFactor) - let pixels = SIMD2(Float(view.drawableSize.width), Float(view.drawableSize.height)) let pin = anchor(input, size: size, pixelsPerPoint: scale) var fog = FogUniforms( viewSize: pixels, anchor: pin.point, anchorScale: pin.scale, time: time, @@ -228,12 +232,12 @@ final class LiveFogRenderer { } encoder.endEncoding() } - commandBuffer.present(drawable) - commandBuffer.commit() + return true } - /// The mask keeps the view's aspect at 96 texels across, about 4 points a texel on a phone: - /// the blur's 10-point sigma is the feather. + /// The mask keeps the view's aspect at 64 texels across, about 6 points a texel on a phone, + /// so the blur's sigma is about 15 points: the prototype's 96 texels left cell columns reading + /// as strips in the offscreen preview. The noise-warped lookup (`fogFragment`) ragged the rest. private func textures(for size: CGSize) -> (mask: MTLTexture, horizontal: MTLTexture, final: MTLTexture, depth: MTLTexture)? { let width = Self.maskWidth let height = max(1, Int((Double(width) * size.height / size.width).rounded())) diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogScene.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogScene.swift index f1c9595a8..9dd0863d6 100644 --- a/ios/HouseScan/UI/Camera/LiveFog/LiveFogScene.swift +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogScene.swift @@ -24,6 +24,21 @@ struct FogValue: Equatable { /// to aim. static func target(_ state: CellState, requested: Bool) -> FogValue { if requested, state != .covered { return FogValue(fog: 0, hidden: 0, requested: 1) } + return fog(state) + } + + /// Above the wall band and past the ground band, where coverage measures nothing: fog while + /// the column below is unseen or seen once, clear once it is covered. A hidden stretch and a + /// gap request are about the band itself, so their violet and amber stop at its edge, and + /// the wall above a bin, which the camera did see past it, is not fogged. + static func beyond(_ state: CellState) -> FogValue { + switch state { + case .unseen, .seen: fog(state) + case .covered, .skipped, .hidden: .clear + } + } + + private static func fog(_ state: CellState) -> FogValue { return switch state { case .unseen: FogValue(fog: 1, hidden: 0, requested: 0) case .seen: FogValue(fog: 0.55, hidden: 0, requested: 0) @@ -51,6 +66,8 @@ final class FogCellAnimator { var band: CoverageBand /// The cell's left edge in whole cell widths from s = 0. var cell: Int + /// The part above the wall band or past the ground band. + var beyond: Bool } private struct Track { @@ -101,7 +118,7 @@ final class FogCellAnimator { /// and the stretch reach past the bands coverage measures (4.5 ft of wall, 4 ft of ground) on /// purpose: fog there could never lift, and fog that walking can't clear stops meaning /// "not seen yet". The wall above a covered column and the lawn in front of it are nothing the -/// scan decides on. Past a marked end the wall is not part of the scan, and is clear. Anything +/// scan decides on; past the band, a cell's part follows `FogValue.beyond`. Past a marked end the wall is not part of the scan, and is clear. Anything /// no quad reaches (the wall past the strip, where the walk has not been) keeps the mask's clear /// value, full fog. enum FogMaskGeometry { @@ -123,12 +140,21 @@ enum FogMaskGeometry { now: Double, reduceMotion: Bool, into vertices: UnsafeMutablePointer, capacity: Int ) -> Int { var count = 0 - func quad(_ band: CoverageBand, _ s0: Float, _ s1: Float, _ value: FogValue) { + let wallTop = coverage.wallBandHeight, groundEdge = coverage.groundBandDepth + /// A quad on the wall from height `near` to `far`, or on the ground from `near` to `far` + /// out; `beyond` picks the part past the band. + func quad(_ band: CoverageBand, beyond: Bool, _ s0: Float, _ s1: Float, _ value: FogValue) { guard count + 6 <= capacity else { return } - let far0 = band == .wall ? wall.world(s: s0, height: columnHeight) : wall.world(s: s0, height: 0, out: groundReach) - let far1 = band == .wall ? wall.world(s: s1, height: columnHeight) : wall.world(s: s1, height: 0, out: groundReach) - let a = SIMD4(wall.world(s: s0, height: 0), 1), b = SIMD4(wall.world(s: s1, height: 0), 1) - let c = SIMD4(far1, 1), d = SIMD4(far0, 1) + let (near, far) = switch (band, beyond) { + case (.wall, false): (Float(0), wallTop) + case (.wall, true): (wallTop, columnHeight) + case (.ground, false): (Float(0), groundEdge) + case (.ground, true): (groundEdge, groundReach) + } + func point(_ s: Float, _ t: Float) -> SIMD4 { + SIMD4(band == .wall ? wall.world(s: s, height: t) : wall.world(s: s, height: 0, out: t), 1) + } + let a = point(s0, near), b = point(s1, near), c = point(s1, far), d = point(s0, far) let v = SIMD4(value.fog, value.hidden, value.requested, 0) vertices[count] = Vertex(position: a, value: v) vertices[count + 1] = Vertex(position: b, value: v) @@ -143,29 +169,34 @@ enum FogMaskGeometry { if visible.upperBound > visible.lowerBound { for band in CoverageBand.allCases { let states = coverage.cells(band) - var run: (s0: Float, s1: Float, value: FogValue, along: SIMD3)? - for index in states.indices { - let range = coverage.cellRange(index) - guard range.upperBound > visible.lowerBound, range.lowerBound < visible.upperBound else { continue } - let s0 = max(range.lowerBound, visible.lowerBound), s1 = min(range.upperBound, visible.upperBound) - let requested = highlight.map { $0.band == band && $0.span.lowerBound < range.upperBound && range.lowerBound < $0.span.upperBound } ?? false - let key = FogCellAnimator.Key(band: band, cell: Int((range.lowerBound / coverage.cellWidth).rounded())) - let value = animator.value(key, target: FogValue.target(states[index], requested: requested), now: now, reduceMotion: reduceMotion) - let along = wall.along(atS: (s0 + s1) / 2) - if var current = run, current.value == value, current.along == along, abs(current.s1 - s0) < 1e-4 { - current.s1 = s1 - run = current - } else { - if let current = run { quad(band, current.s0, current.s1, current.value) } - run = (s0, s1, value, along) + for beyond in [false, true] { + var run: (s0: Float, s1: Float, value: FogValue, along: SIMD3)? + for index in states.indices { + let range = coverage.cellRange(index) + guard range.upperBound > visible.lowerBound, range.lowerBound < visible.upperBound else { continue } + let s0 = max(range.lowerBound, visible.lowerBound), s1 = min(range.upperBound, visible.upperBound) + let requested = highlight.map { $0.band == band && $0.span.lowerBound < range.upperBound && range.lowerBound < $0.span.upperBound } ?? false + let key = FogCellAnimator.Key(band: band, cell: Int((range.lowerBound / coverage.cellWidth).rounded()), beyond: beyond) + let target = beyond ? FogValue.beyond(states[index]) : FogValue.target(states[index], requested: requested) + let value = animator.value(key, target: target, now: now, reduceMotion: reduceMotion) + let along = wall.along(atS: (s0 + s1) / 2) + if var current = run, current.value == value, current.along == along, abs(current.s1 - s0) < 1e-4 { + current.s1 = s1 + run = current + } else { + if let current = run { quad(band, beyond: beyond, current.s0, current.s1, current.value) } + run = (s0, s1, value, along) + } } + if let current = run { quad(band, beyond: beyond, current.s0, current.s1, current.value) } } - if let current = run { quad(band, current.s0, current.s1, current.value) } } } for band in CoverageBand.allCases { - if let left = wall.leftEnd { quad(band, left - pastEnd, left, .clear) } - if let right = wall.rightEnd { quad(band, right, right + pastEnd, .clear) } + for beyond in [false, true] { + if let left = wall.leftEnd { quad(band, beyond: beyond, left - pastEnd, left, .clear) } + if let right = wall.rightEnd { quad(band, beyond: beyond, right, right + pastEnd, .clear) } + } } animator.endFrame() return count diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift index de2234e5d..c7e7fe6ac 100644 --- a/ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift @@ -3,7 +3,7 @@ /// `Shaders.swift`). Struct layouts match the Swift structs in `LiveFogRenderer.swift`. /// /// Fog. Coverage decides where it lifts: `LiveFogScene` draws every wall column and ground -/// stretch of the coverage strip into a 96-texel-wide mask (red: fog, green: hidden behind +/// stretch of the coverage strip into a 64-texel-wide mask (red: fog, green: hidden behind /// something, blue: requested by a gap), clearing to full fog, and a radius-5 Gaussian blur /// feathers it. The composite is the prototype's: Chalk mixed toward a cool grey, textured by /// slow domain-warped fbm and breathing slightly. Two changes: the noise is pinned to the meter @@ -128,13 +128,19 @@ enum LiveFogShaders { texture2d mask [[texture(0)]]) { constexpr sampler linear(filter::linear, address::clamp_to_edge); float2 screen = in.position.xy; - float4 m = mask.sample(linear, screen / u.viewSize); - float fog = saturate(m.r), hidden = saturate(m.g), requested = saturate(m.b); - if (fog < 0.002f && requested < 0.002f) { return float4(0.0f); } - // Wrapped so the offsets stay small where Float precision is fine. float2 uv = fmod((screen - u.anchor) / u.anchorScale, float2(256.0f)); float2 q = uv + u.motion * u.time * 0.02f * float2(0.94f, 0.34f); + + // The mask is read through a noise offset of up to 7% of the screen's width, so the + // straight edges of cells and columns come out as ragged, drifting fog banks (the + // noise-edged fog of war the prototype borrowed from Civilization VI). + float2 edgeNoise = float2(valueNoise(q * 1.7f + float2(7.3f, 2.1f)), valueNoise(q * 1.7f + float2(-4.1f, 8.6f))); + float2 look = screen + (edgeNoise - 0.5f) * 0.14f * u.viewSize.x; + float4 m = mask.sample(linear, look / u.viewSize); + float fog = saturate(m.r), hidden = saturate(m.g), requested = saturate(m.b); + if (fog < 0.002f && requested < 0.002f) { return float4(0.0f); } + float2 warp = float2(fbm(q + float2(3.1f, 1.7f)), fbm(q + float2(-2.3f, 5.2f))); float n = fbm(q + 0.8f * warp); float noiseTerm = 0.78f + 0.22f * n; @@ -142,7 +148,7 @@ enum LiveFogShaders { // Part-lifted fog breaks up along the noise; full fog and clear stay as they are. float shaped = saturate(fog + (n - 0.5f) * 1.6f * fog * (1.0f - fog)); - float a = saturate(0.6f * noiseTerm * shaped * breathing * mix(1.0f, 0.72f, hidden)); + float a = saturate(0.66f * noiseTerm * shaped * breathing * mix(1.0f, 0.72f, hidden)); float3 chalk = float3(0xF7, 0xF5, 0xEF) / 255.0f; float3 cool = float3(0xD9, 0xE2, 0xEC) / 255.0f; diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift index 8c58457df..214a4bba8 100644 --- a/ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift @@ -77,6 +77,11 @@ final class LiveFogMTKView: MTKView { var renderer: LiveFogRenderer? override func draw(_ rect: CGRect) { - renderer?.draw(in: self) + guard let renderer, let pass = currentRenderPassDescriptor, let drawable = currentDrawable, + let commandBuffer = renderer.queue.makeCommandBuffer() else { return } + let pixels = SIMD2(Float(drawableSize.width), Float(drawableSize.height)) + guard renderer.encode(into: commandBuffer, pass: pass, size: bounds.size, pixels: pixels, pixelsPerPoint: Float(contentScaleFactor)) else { return } + commandBuffer.present(drawable) + commandBuffer.commit() } } From 48d024a56845fa97d188fe1cecfb6581dc57019c Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 03:41:52 -0500 Subject: [PATCH 3/5] Say in the iOS README where the live fog's rule and dots live Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ios/README.md b/ios/README.md index 90a5e9628..6c9082eaa 100644 --- a/ios/README.md +++ b/ios/README.md @@ -24,6 +24,8 @@ Native iPhone app for the AR capture walk. The homeowner marks the electric mete **Spot check.** Before the result, the app shows the kept photo that best sees the answer's spot, with the spot and its clearance area outlined, and asks whether anything stands in front of the wall or on the ground there (`STATE=spotConfirm`). Photos without depth and the walked path claim wall, ground and clear space they never saw; "It's clear" backs those claims for that spot, and "Something's there" takes them back over the area, so scene.json reports it unseen, and sends the scan again. The rules are in `Runtime/ScanEngine+Confirm.swift` and HouseScanKit's `Confirm/SpotConfirmation.swift`. +**Live fog.** During the walk and a gap request, a soft fog covers what the coverage map has not counted and lifts only where it counts a cell, the same rule the scan's decisions use (`FogValue.target` in `UI/Camera/LiveFog/LiveFogScene.swift`). On a LiDAR phone, dots settle on the surfaces the kept keyframes' depth measured (HouseScanKit's `Overlay/SurfaceDots.swift`, run by `Runtime/LiveDotsFeed.swift`); they are guidance and feed nothing. The look is the prototype's (experiments/live-dots on t3/experience). Phones without LiDAR get the same fog without dots. The shaders compile at launch, and the old frosted strip (`FogOverlay`) draws until they are ready, and for good if Metal fails. + Every screen change is logged as `STATE=` under subsystem `dev.housescanning.housescan`, category `state`. ## Layout From 9defff1977eae2ed5e84f65360a714412929fb85 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 03:51:24 -0500 Subject: [PATCH 4/5] Keep narrow unseen cells fogged, move dots exactly on corrections From review: a cell narrower than a mask texel could drop out of the fog, so the mask is drawn at three times the size, max-pooled, and sets a floor the blur and noise can't erase. The dot field now keeps its own frame and a cumulative transform, so anchor corrections no longer snap to the 5 cm grid. A renderer that can't allocate hands back to the frosted strip, every piece of a split gap request gets its outline, and corrections coalesce their republish. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/Runtime/LiveDotsFeed.swift | 11 +-- ios/HouseScan/UI/Camera/FogMarks.swift | 43 +++++----- .../UI/Camera/LiveFog/LiveFogRenderer.swift | 81 ++++++++++++------- .../UI/Camera/LiveFog/LiveFogShaders.swift | 32 ++++++-- .../UI/Camera/LiveFog/LiveFogView.swift | 22 ++++- .../HouseScanKit/Overlay/SurfaceDots.swift | 60 +++++++------- .../HouseScanKitTests/SurfaceDotsTests.swift | 20 +++-- 7 files changed, 170 insertions(+), 99 deletions(-) diff --git a/ios/HouseScan/Runtime/LiveDotsFeed.swift b/ios/HouseScan/Runtime/LiveDotsFeed.swift index 7f04bc98b..ea3a35f9d 100644 --- a/ios/HouseScan/Runtime/LiveDotsFeed.swift +++ b/ios/HouseScan/Runtime/LiveDotsFeed.swift @@ -6,9 +6,11 @@ import os /// camera overlay (`ScanViewState.liveDots`). /// /// The engine hands it each kept keyframe that has depth, the same frames coverage observes, so -/// the dots and the fog move together. Fusing a 256 x 192 map takes milliseconds, not a frame -/// budget, so it runs on its own serial queue. A keyframe arriving while two are still waiting -/// is dropped: the next one covers the same wall, and the queue can never grow. +/// the dots and the fog move together. Fusing a 256 x 192 map takes a few milliseconds (3.4 ms +/// median on an M4 Pro in a Release build; not measured on a phone), so it runs on its own +/// serial queue. A keyframe arriving while two are still waiting is dropped: the next one covers +/// the same wall. An anchor correction is a matrix product (`SurfaceDots.apply`) and republishes +/// only when no keyframe is waiting to publish anyway, so neither kind of work can pile up. final class LiveDotsFeed: @unchecked Sendable { // `@unchecked Sendable`: everything below `queue` is read and written only on it; `pending` // is a lock. @@ -49,10 +51,9 @@ final class LiveDotsFeed: @unchecked Sendable { /// ARKit's correction to the meter's anchor, applied as coverage applies it. func apply(_ correction: YawCorrection, wall: WallFrame, generation: Int) { queue.async { [self] in - guard !field.isEmpty else { return } field.apply(correction) if let eye = last?.eye { last = (correction.point(eye), wall) } - send(generation) + if pending.withLock({ $0 }) == 0 { send(generation) } } } diff --git a/ios/HouseScan/UI/Camera/FogMarks.swift b/ios/HouseScan/UI/Camera/FogMarks.swift index 9f35af181..8b377feff 100644 --- a/ios/HouseScan/UI/Camera/FogMarks.swift +++ b/ios/HouseScan/UI/Camera/FogMarks.swift @@ -38,7 +38,8 @@ struct FogMarks: View { } hatched.stroke(lines, with: .color(.white.opacity(0.55)), lineWidth: 1.5) } - if let requested = runs.requested, let outline = outline(requested.band, requested.edges, geometry) { + for run in runs.requested { + guard let outline = outline(run.band, run.edges, geometry) else { continue } context.stroke(outline, with: .color(.black.opacity(0.3)), style: StrokeStyle(lineWidth: 4, lineJoin: .round)) context.stroke(outline, with: .color(Palette.caution), style: StrokeStyle(lineWidth: 2, lineJoin: .round)) } @@ -47,24 +48,30 @@ struct FogMarks: View { .accessibilityHidden(true) } + private enum Kind { case hidden, skipped, requested } + private struct Run { var band: CoverageBand /// The s of every cell edge along the run, clipped to the visible range. var edges: [Float] } - /// Runs of hidden and of skipped cells in each band, and the requested cells not yet covered. - private func runs() -> (hidden: [Run], skipped: [Run], requested: Run?) { - var hidden: [Run] = [], skipped: [Run] = [] - var requested: Run? + /// Runs of hidden, of skipped, and of requested-but-not-covered cells in each band. Coverage + /// filling the middle of a request splits it, and each piece keeps its outline. + private func runs() -> (hidden: [Run], skipped: [Run], requested: [Run]) { + var hidden: [Run] = [], skipped: [Run] = [], requested: [Run] = [] let visible = coverage.visibleRange - guard visible.upperBound > visible.lowerBound else { return ([], [], nil) } + guard visible.upperBound > visible.lowerBound else { return ([], [], []) } for band in CoverageBand.allCases { let states = coverage.cells(band) - var current: (state: CellState, run: Run)? + var current: (kind: Kind, run: Run)? func close() { if let current { - if current.state == .hidden { hidden.append(current.run) } else { skipped.append(current.run) } + switch current.kind { + case .hidden: hidden.append(current.run) + case .skipped: skipped.append(current.run) + case .requested: requested.append(current.run) + } } current = nil } @@ -74,23 +81,17 @@ struct FogMarks: View { let s0 = max(range.lowerBound, visible.lowerBound), s1 = min(range.upperBound, visible.upperBound) let state = states[index] let isRequested = highlight.map { $0.band == band && $0.span.lowerBound < range.upperBound && range.lowerBound < $0.span.upperBound } ?? false - if isRequested, state != .covered { - close() - if var run = requested, run.band == band, let last = run.edges.last, abs(last - s0) < 1e-4 { - run.edges.append(s1) - requested = run - } else if requested == nil { - requested = Run(band: band, edges: [s0, s1]) - } - continue - } - guard state == .hidden || state == .skipped else { close(); continue } - if var open = current, open.state == state { + let kind: Kind? = if isRequested, state != .covered { .requested } + else if state == .hidden { .hidden } + else if state == .skipped { .skipped } + else { nil } + guard let kind else { close(); continue } + if var open = current, open.kind == kind { open.run.edges.append(s1) current = open } else { close() - current = (state, Run(band: band, edges: [s0, s1])) + current = (kind, Run(band: band, edges: [s0, s1])) } } close() diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift index 2022157b8..ba9e57917 100644 --- a/ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift @@ -27,6 +27,7 @@ final class LiveFogGPU: @unchecked Sendable { let device: MTLDevice let queue: MTLCommandQueue let mask: MTLRenderPipelineState + let pool: MTLRenderPipelineState let blur: MTLRenderPipelineState let fog: MTLRenderPipelineState let dots: MTLRenderPipelineState @@ -56,6 +57,7 @@ final class LiveFogGPU: @unchecked Sendable { return try device.makeRenderPipelineState(descriptor: descriptor) } mask = try pipeline("maskVertex", "maskFragment", format: Self.maskFormat, depth: true) + pool = try pipeline("fullVertex", "poolFragment", format: Self.maskFormat) blur = try pipeline("fullVertex", "blurFragment", format: Self.maskFormat) // The fog is the first thing drawn on a cleared target: premultiplied, written as is. fog = try pipeline("fullVertex", "fogFragment", format: Self.drawableFormat) @@ -89,12 +91,13 @@ struct LiveFogInput { /// Draws the fog and dots into a transparent Metal layer over the camera (`LiveFogMTKView`), /// every display frame while the walk is on screen. /// -/// Per frame: the mask pass draws the coverage cells (`FogMaskGeometry`) into a 64-texel-wide -/// texture with a depth test, two passes blur it, then the drawable gets the fog and the dots. +/// Per frame: the mask pass draws the coverage cells (`FogMaskGeometry`) into a texture 192 +/// texels wide with a depth test, a max-pool takes it to 64, two passes blur it, then the +/// drawable gets the fog and the dots. /// Bounds: three frames in flight; the mask vertex ring holds 16,384 vertices a frame (a 60 m /// strip of unmerged cells, two bands in two parts, needs about 9,600); the dot buffers hold `DotTimeline.capacity` dots /// and are rewritten only when the field changes or a fading dot ends, at most once a frame. -/// Nothing grows with time: about 4.5 MB of buffers and 0.1 MB of textures, allocated once. +/// Nothing grows with time: about 4.5 MB of buffers and 1 MB of textures, allocated once. @MainActor final class LiveFogRenderer { private let gpu: LiveFogGPU @@ -118,21 +121,22 @@ final class LiveFogRenderer { private var frame = 0 private var spriteBuffer = 0 private var spriteCount = 0 - private var maskTextures: (mask: MTLTexture, horizontal: MTLTexture, final: MTLTexture, depth: MTLTexture)? + private var maskTextures: MaskTextures? private var anchor: (point: SIMD2, scale: Float)? - init(gpu: LiveFogGPU) { + /// Called once if the mask textures can't be allocated, so the view can hand back to the + /// frosted strip instead of showing a clear layer. + var onFailure: (() -> Void)? + + /// Nil when Metal won't allocate the buffers. + init?(gpu: LiveFogGPU) { self.gpu = gpu for _ in 0...stride, options: .storageModeShared) { - maskBuffers.append(buffer) - } - if let buffer = gpu.device.makeBuffer(length: Self.spriteCapacity * MemoryLayout.stride, options: .storageModeShared) { - spriteBuffers.append(buffer) - } - } - if maskBuffers.count < Self.framesInFlight || spriteBuffers.count < Self.framesInFlight { - RuntimeLog.engine.error("live fog: Metal would not allocate its buffers; nothing will draw") + guard let masks = gpu.device.makeBuffer(length: Self.maskVertexCapacity * MemoryLayout.stride, options: .storageModeShared), + let sprites = gpu.device.makeBuffer(length: Self.spriteCapacity * MemoryLayout.stride, options: .storageModeShared) + else { return nil } + maskBuffers.append(masks) + spriteBuffers.append(sprites) } } @@ -141,11 +145,12 @@ final class LiveFogRenderer { /// caller skips the frame rather than block the main thread on the GPU. `LiveFogMTKView` /// calls this each display frame. func encode(into commandBuffer: MTLCommandBuffer, pass: MTLRenderPassDescriptor, size: CGSize, pixels: SIMD2, pixelsPerPoint scale: Float) -> Bool { - guard let input, maskBuffers.count == Self.framesInFlight, spriteBuffers.count == Self.framesInFlight, - size.width > 1, size.height > 1 else { return false } + guard let input, size.width > 1, size.height > 1 else { return false } guard inFlight.wait(timeout: .now()) == .success else { return false } guard let textures = textures(for: size) else { inFlight.signal() + onFailure?() + onFailure = nil return false } let semaphore = inFlight @@ -176,7 +181,7 @@ final class LiveFogRenderer { // Mask: full fog wherever no cell reaches. let maskPass = MTLRenderPassDescriptor() - maskPass.colorAttachments[0].texture = textures.mask + maskPass.colorAttachments[0].texture = textures.fine maskPass.colorAttachments[0].loadAction = .clear maskPass.colorAttachments[0].clearColor = MTLClearColor(red: 1, green: 0, blue: 0, alpha: 0) maskPass.colorAttachments[0].storeAction = .store @@ -194,7 +199,17 @@ final class LiveFogRenderer { } encoder.endEncoding() } - for (source, target, direction) in [(textures.mask, textures.horizontal, SIMD2(1, 0)), (textures.horizontal, textures.final, SIMD2(0, 1))] { + let poolPass = MTLRenderPassDescriptor() + poolPass.colorAttachments[0].texture = textures.pooled + poolPass.colorAttachments[0].loadAction = .dontCare + poolPass.colorAttachments[0].storeAction = .store + if let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: poolPass) { + encoder.setRenderPipelineState(gpu.pool) + encoder.setFragmentTexture(textures.fine, index: 0) + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3) + encoder.endEncoding() + } + for (source, target, direction) in [(textures.pooled, textures.horizontal, SIMD2(1, 0)), (textures.horizontal, textures.final, SIMD2(0, 1))] { let blurPass = MTLRenderPassDescriptor() blurPass.colorAttachments[0].texture = target blurPass.colorAttachments[0].loadAction = .dontCare @@ -218,6 +233,7 @@ final class LiveFogRenderer { encoder.setRenderPipelineState(gpu.fog) encoder.setFragmentBytes(&fog, length: MemoryLayout.stride, index: 0) encoder.setFragmentTexture(textures.final, index: 0) + encoder.setFragmentTexture(textures.pooled, index: 1) encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3) if spriteCount > 0 { @@ -238,24 +254,35 @@ final class LiveFogRenderer { /// The mask keeps the view's aspect at 64 texels across, about 6 points a texel on a phone, /// so the blur's sigma is about 15 points: the prototype's 96 texels left cell columns reading /// as strips in the offscreen preview. The noise-warped lookup (`fogFragment`) ragged the rest. - private func textures(for size: CGSize) -> (mask: MTLTexture, horizontal: MTLTexture, final: MTLTexture, depth: MTLTexture)? { + private struct MaskTextures { + /// Where the cells are drawn, three times the coarse size, with its depth buffer. + var fine: MTLTexture + var depth: MTLTexture + /// The fine mask max-pooled to the coarse size, then the two blur passes. + var pooled: MTLTexture + var horizontal: MTLTexture + var final: MTLTexture + } + + private func textures(for size: CGSize) -> MaskTextures? { let width = Self.maskWidth let height = max(1, Int((Double(width) * size.height / size.width).rounded())) - if let maskTextures, maskTextures.mask.width == width, maskTextures.mask.height == height { return maskTextures } - func texture(_ format: MTLPixelFormat, _ usage: MTLTextureUsage) -> MTLTexture? { - let descriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: format, width: width, height: height, mipmapped: false) + if let maskTextures, maskTextures.pooled.width == width, maskTextures.pooled.height == height { return maskTextures } + func texture(_ format: MTLPixelFormat, _ usage: MTLTextureUsage, scale: Int = 1) -> MTLTexture? { + let descriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: format, width: width * scale, height: height * scale, mipmapped: false) descriptor.usage = usage descriptor.storageMode = .private return gpu.device.makeTexture(descriptor: descriptor) } - guard let mask = texture(LiveFogGPU.maskFormat, [.renderTarget, .shaderRead]), + guard let fine = texture(LiveFogGPU.maskFormat, [.renderTarget, .shaderRead], scale: 3), + let depth = texture(LiveFogGPU.depthFormat, [.renderTarget], scale: 3), + let pooled = texture(LiveFogGPU.maskFormat, [.renderTarget, .shaderRead]), let horizontal = texture(LiveFogGPU.maskFormat, [.renderTarget, .shaderRead]), - let final = texture(LiveFogGPU.maskFormat, [.renderTarget, .shaderRead]), - let depth = texture(LiveFogGPU.depthFormat, [.renderTarget]) else { - RuntimeLog.engine.error("live fog: Metal would not allocate the mask textures") + let final = texture(LiveFogGPU.maskFormat, [.renderTarget, .shaderRead]) else { + RuntimeLog.engine.error("live fog: Metal would not allocate the mask textures; drawing the frosted strip instead") return nil } - maskTextures = (mask, horizontal, final, depth) + maskTextures = MaskTextures(fine: fine, depth: depth, pooled: pooled, horizontal: horizontal, final: final) return maskTextures } diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift index c7e7fe6ac..9341f0cf1 100644 --- a/ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift @@ -3,9 +3,9 @@ /// `Shaders.swift`). Struct layouts match the Swift structs in `LiveFogRenderer.swift`. /// /// Fog. Coverage decides where it lifts: `LiveFogScene` draws every wall column and ground -/// stretch of the coverage strip into a 64-texel-wide mask (red: fog, green: hidden behind -/// something, blue: requested by a gap), clearing to full fog, and a radius-5 Gaussian blur -/// feathers it. The composite is the prototype's: Chalk mixed toward a cool grey, textured by +/// stretch of the coverage strip into a mask three times 64 texels wide (red: fog, green: hidden +/// behind something, blue: requested by a gap), clearing to full fog; a max-pool takes it to 64 +/// texels and a radius-5 Gaussian blur feathers it. The composite is the prototype's: Chalk mixed toward a cool grey, textured by /// slow domain-warped fbm and breathing slightly. Two changes: the noise is pinned to the meter /// on screen and scaled by its distance, so the texture sits on the wall instead of on the lens /// while the phone moves (the prototype cut between keyframes, where that never showed); and @@ -72,6 +72,22 @@ enum LiveFogShaders { return in.value; } + // Each texel takes the largest value of the 3 x 3 block under it in the mask drawn at three + // times the size, per channel: a cell as narrow as one fine texel (about 2 points) still + // reaches the coarse mask, where sampling alone could step over it. + fragment float4 poolFragment(FullOut in [[stage_in]], + texture2d fine [[texture(0)]]) { + uint2 base = uint2(in.position.xy) * 3u; + uint2 limit = uint2(fine.get_width() - 1, fine.get_height() - 1); + float4 m = float4(0.0f); + for (uint y = 0; y < 3; y++) { + for (uint x = 0; x < 3; x++) { + m = max(m, fine.read(min(base + uint2(x, y), limit))); + } + } + return m; + } + // Separable Gaussian, radius 5 texels (sigma 2.5), all four channels. fragment float4 blurFragment(FullOut in [[stage_in]], texture2d source [[texture(0)]], @@ -125,7 +141,8 @@ enum LiveFogShaders { fragment float4 fogFragment(FullOut in [[stage_in]], constant FogUniforms& u [[buffer(0)]], - texture2d mask [[texture(0)]]) { + texture2d mask [[texture(0)]], + texture2d pooled [[texture(1)]]) { constexpr sampler linear(filter::linear, address::clamp_to_edge); float2 screen = in.position.xy; // Wrapped so the offsets stay small where Float precision is fine. @@ -137,7 +154,12 @@ enum LiveFogShaders { // noise-edged fog of war the prototype borrowed from Civilization VI). float2 edgeNoise = float2(valueNoise(q * 1.7f + float2(7.3f, 2.1f)), valueNoise(q * 1.7f + float2(-4.1f, 8.6f))); float2 look = screen + (edgeNoise - 0.5f) * 0.14f * u.viewSize.x; - float4 m = mask.sample(linear, look / u.viewSize); + // The blur and the noise shape the fog; neither may erase it. The unblurred pooled mask, + // read through a fifth of the offset, sets a floor: a cell coverage has not counted keeps + // 30% of its fog within 3% of the screen's width of where it is, however narrow. Read + // unwarped, the floor drew a straight step along every wide edge. + float2 near = screen + (edgeNoise - 0.5f) * 0.03f * u.viewSize.x; + float4 m = max(mask.sample(linear, look / u.viewSize), 0.3f * pooled.sample(linear, near / u.viewSize)); float fog = saturate(m.r), hidden = saturate(m.g), requested = saturate(m.b); if (fog < 0.002f && requested < 0.002f) { return float4(0.0f); } diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift index 214a4bba8..47a3542f6 100644 --- a/ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift @@ -19,6 +19,13 @@ final class LiveFogSupport { private(set) var status: Status = .preparing @ObservationIgnored private var started = false + /// The renderer could not get its buffers or textures: back to the frosted strip for the + /// rest of the launch. + func fail(_ reason: String) { + RuntimeLog.engine.error("live fog unavailable, drawing the frosted strip instead: \(reason, privacy: .public)") + status = .failed + } + func prepare() { guard !started else { return } started = true @@ -32,8 +39,7 @@ final class LiveFogSupport { case let .failure(error): // Logged as an error, not trapped: a trap in a Debug build would stop every UI // test, while the frosted strip in their screenshots already shows the failure. - RuntimeLog.engine.error("live fog unavailable, drawing the frosted strip instead: \(String(describing: error), privacy: .public)") - LiveFogSupport.shared.status = .failed + LiveFogSupport.shared.fail(String(describing: error)) } } } @@ -61,8 +67,16 @@ struct LiveFogView: UIViewRepresentable { view.preferredFramesPerSecond = 60 view.isUserInteractionEnabled = false view.accessibilityElementsHidden = true - view.renderer = LiveFogRenderer(gpu: gpu) - view.renderer?.input = input + guard let renderer = LiveFogRenderer(gpu: gpu) else { + // Not during this view update: the status change swaps this view out. + DispatchQueue.main.async { LiveFogSupport.shared.fail("Metal would not allocate the fog's buffers") } + return view + } + renderer.input = input + renderer.onFailure = { + DispatchQueue.main.async { LiveFogSupport.shared.fail("Metal would not allocate the fog's mask textures") } + } + view.renderer = renderer return view } diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Overlay/SurfaceDots.swift b/ios/HouseScanKit/Sources/HouseScanKit/Overlay/SurfaceDots.swift index 3187fffda..c21a004d1 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Overlay/SurfaceDots.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Overlay/SurfaceDots.swift @@ -133,15 +133,6 @@ struct ViewSet: Sendable, Equatable { count += 1 return true } - - func rotated(_ rotate: (SIMD3) -> SIMD3) -> ViewSet { - var copy = self - copy.d0 = rotate(d0) - copy.d1 = rotate(d1) - copy.d2 = rotate(d2) - copy.d3 = rotate(d3) - return copy - } } struct Voxel: Sendable, Equatable { @@ -163,8 +154,16 @@ struct Voxel: Sendable, Equatable { /// The LiDAR dot field. A value type confined to one owner; `integrate` is the hot path and /// touches only the voxels this keyframe saw and their face neighbours. +/// +/// Voxels live in the field's own frame, fixed at the first keyframe; `toWorld` carries them into +/// ARKit's world. An anchor correction changes only `toWorld`, so it is exact and costs nothing: +/// moving the voxels would snap them back to the 5 cm grid, and a run of 2 cm corrections would +/// leave the dots where they were while the wall moved. public struct SurfaceDots: Sendable { public let config: SurfaceDotConfig + /// Field frame to world. Identity until the first correction. + private var toWorld = matrix_identity_float4x4 + private var toField = matrix_identity_float4x4 private(set) var voxels: [VoxelKey: Voxel] = [:] /// Voxels a confident depth reading passed with room to spare. Only in-plane neighbours of /// occupied voxels are tested, which is all the boundary rule needs. @@ -196,8 +195,9 @@ public struct SurfaceDots: Sendable { /// Fuses one keyframe's depth. `depth` is ARKit's scene depth for `camera`, in the landscape /// sensor orientation, with intrinsics for its own grid (`DepthImage.intrinsics(scaling:)`). - public mutating func integrate(camera: CameraFrame, depth: DepthImage) { + public mutating func integrate(camera world: CameraFrame, depth: DepthImage) { let step = max(config.sampleStride, 1) + let camera = CameraFrame(cameraToWorld: toField * world.cameraToWorld, intrinsics: world.intrinsics, imageSize: world.imageSize) let pose = camera.cameraToWorld let eye = camera.position @@ -345,24 +345,11 @@ public struct SurfaceDots: Sendable { } } - /// ARKit's correction to the meter's anchor (`YawCorrection`), applied to every voxel, so the - /// dots move with the wall the scan measured. + /// ARKit's correction to the meter's anchor (`YawCorrection`): the dots move with the wall the + /// scan measured. Composes into `toWorld`; no voxel moves. public mutating func apply(_ correction: YawCorrection) { - var moved: [VoxelKey: Voxel] = [:] - moved.reserveCapacity(voxels.count) - for (key, voxel) in voxels { - var copy = voxel - copy.normalSum = correction.direction(voxel.normalSum) - copy.views = voxel.views.rotated(correction.direction) - copy.jitter = correction.direction(voxel.jitter) - let newKey = self.key(for: correction.point(centre(of: key))) - if let existing = moved[newKey], existing.views.count >= copy.views.count { continue } - moved[newKey] = copy - } - voxels = moved - free = Set(free.map { key(for: correction.point(centre(of: $0))) }.filter { moved[$0] == nil }) - representatives = [:] - classify(Set(voxels.keys)) + toWorld = correction.pose(toWorld) + toField = toWorld.inverse } // MARK: Dots @@ -370,7 +357,8 @@ public struct SurfaceDots: Sendable { /// Every edge voxel and each group's flat representative within `dotRadius` of `eye`, at /// most `maxDots`: past it, flat dots drop in hash order, then the farthest edges. `wall` /// decides which dots stand on an occluder; without one, none do. Sorted by id. - public func dots(near eye: SIMD3, wall: WallFrame?) -> [SurfaceDot] { + public func dots(near worldEye: SIMD3, wall: WallFrame?) -> [SurfaceDot] { + let eye = Self.transform(toField, worldEye) let radius2 = config.dotRadius * config.dotRadius var edges: [(VoxelKey, Voxel, Float)] = [] var flats: [(VoxelKey, Voxel)] = [] @@ -397,7 +385,9 @@ public struct SurfaceDots: Sendable { } private func dot(_ key: VoxelKey, _ voxel: Voxel, wall: WallFrame?) -> SurfaceDot { - let centre = centre(of: key) + let centre = Self.transform(toWorld, centre(of: key)) + let jitter = Self.rotate(toWorld, voxel.jitter) + // Corrections turn only about the vertical, so a normal's y is the same in both frames. let wallLike = voxel.normal.map { abs($0.y) < config.cappedNormalMaxY } ?? false let faceOn = !voxel.isEdge || !wallLike || voxel.faceOnCosine >= cos(config.faceOnDegrees * .pi / 180) let views = Int(voxel.views.count) @@ -407,13 +397,23 @@ public struct SurfaceDots: Sendable { onOccluder = p.out > config.occluderMinOut && p.height > config.groundBand } return SurfaceDot( - id: Self.id(for: key), position: centre + voxel.jitter, isEdge: voxel.isEdge, views: views, + id: Self.id(for: key), position: centre + jitter, isEdge: voxel.isEdge, views: views, faceOn: faceOn, onOccluder: onOccluder, opacity: SurfaceDot.opacity(views: views, faceOn: faceOn, cap: config.obliqueEdgeOpacityCap)) } // MARK: Depth helpers + static func transform(_ m: simd_float4x4, _ p: SIMD3) -> SIMD3 { + let q = m * SIMD4(p, 1) + return SIMD3(q.x, q.y, q.z) + } + + static func rotate(_ m: simd_float4x4, _ v: SIMD3) -> SIMD3 { + let q = m * SIMD4(v, 0) + return SIMD3(q.x, q.y, q.z) + } + /// Meters at a depth pixel, or nil without a confident reading in range. `ignoringRange` keeps /// far readings, which the free-space test needs. private func reading(_ depth: DepthImage, _ i: Int, _ j: Int, ignoringRange: Bool = false) -> Float? { diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/SurfaceDotsTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/SurfaceDotsTests.swift index 14e20288a..28e0a13c1 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/SurfaceDotsTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/SurfaceDotsTests.swift @@ -100,16 +100,22 @@ import Testing #expect(dots.filter { $0.isEdge }.count == min(20, all.filter { $0.isEdge }.count)) } - /// An anchor correction moves the field with the wall: a 0.5 m shift moves every dot 0.5 m. - @Test func aCorrectionMovesTheDots() { + /// An anchor correction moves the field with the wall, exactly, however small the steps: + /// ten 2.1 cm corrections, each under half a voxel, move every dot 21 cm, and a turn turns + /// them about the vertical. + @Test func correctionsMoveTheDotsExactly() { let camera = wallCamera(s: 0) var field = Self.field(standardScene(), cameras: [camera]) let before = field.dots(near: camera.position, wall: nil) - field.apply(YawCorrection(yaw: 0, translation: SIMD3(0.5, 0, 0))) - let after = field.dots(near: camera.position + SIMD3(0.5, 0, 0), wall: nil) - func meanX(_ dots: [SurfaceDot]) -> Float { dots.map(\.position.x).reduce(0, +) / Float(dots.count) } - #expect(abs(after.count - before.count) <= before.count / 20) - #expect(nearlyEqual(meanX(after) - meanX(before), 0.5, 0.03)) + for _ in 0..<10 { field.apply(YawCorrection(yaw: 0, translation: SIMD3(0.021, 0, 0))) } + let turn = YawCorrection(yaw: 0.1, translation: .zero) + field.apply(turn) + let after = field.dots(near: turn.point(camera.position + SIMD3(0.21, 0, 0)), wall: nil) + #expect(after.count == before.count) + for (a, b) in zip(before, after) { + #expect(a.id == b.id) + #expect(nearlyEqual(b.position, turn.point(a.position + SIMD3(0.21, 0, 0)), 1e-4)) + } } /// The same keyframes give the same dots: jitter, thinning and order come from stable hashes. From f57310cac5577fef61baadca86729c0b4b43b080 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Sun, 27 Sep 2026 04:43:54 -0500 Subject: [PATCH 5/5] Tune the fog, the hidden tint and the dots against the prototype videos A taste check of the hosted screenshots against the prototype found partial fog smearing, the hidden state greying out the bin, and dots too small to glow. The fog now has a tighter feather, finer ragged edges, erosion only while it lifts, a thinner seen-once veil and a whiter colour; hidden is a light violet tint; dots are larger with a visible halo. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../UI/Camera/LiveFog/LiveFogRenderer.swift | 6 +- .../UI/Camera/LiveFog/LiveFogScene.swift | 26 +++++--- .../UI/Camera/LiveFog/LiveFogShaders.swift | 66 +++++++++++-------- .../UI/Camera/LiveFog/LiveFogView.swift | 1 + 4 files changed, 59 insertions(+), 40 deletions(-) diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift index ba9e57917..c74e2ae87 100644 --- a/ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogRenderer.swift @@ -251,9 +251,9 @@ final class LiveFogRenderer { return true } - /// The mask keeps the view's aspect at 64 texels across, about 6 points a texel on a phone, - /// so the blur's sigma is about 15 points: the prototype's 96 texels left cell columns reading - /// as strips in the offscreen preview. The noise-warped lookup (`fogFragment`) ragged the rest. + /// The mask keeps the view's aspect at 64 texels across, about 6 points a texel on a phone; + /// the blur's 1.4-texel sigma, about 8 points, is the feather, and the noise-warped lookup + /// (`fogFragment`) rags the edges. private struct MaskTextures { /// Where the cells are drawn, three times the coarse size, with its depth buffer. var fine: MTLTexture diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogScene.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogScene.swift index 9dd0863d6..b9c8e7bb5 100644 --- a/ios/HouseScan/UI/Camera/LiveFog/LiveFogScene.swift +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogScene.swift @@ -41,8 +41,10 @@ struct FogValue: Equatable { private static func fog(_ state: CellState) -> FogValue { return switch state { case .unseen: FogValue(fog: 1, hidden: 0, requested: 0) - case .seen: FogValue(fog: 0.55, hidden: 0, requested: 0) - case .hidden: FogValue(fog: 0.75, hidden: 1, requested: 0) + // 0.42 and 0.4 from the taste check against the prototype's videos: at 0.55, seen once + // and unseen blurred into one grey; at 0.75 the hidden mist greyed out the bin in front. + case .seen: FogValue(fog: 0.42, hidden: 0, requested: 0) + case .hidden: FogValue(fog: 0.4, hidden: 1, requested: 0) case .covered, .skipped: .clear } } @@ -106,6 +108,11 @@ final class FogCellAnimator { return track.value(at: now) } + /// True while the cell's fog is still easing toward its target. Call after `value`. + func isMoving(_ key: Key, now: Double) -> Bool { + tracks[key].map { !$0.isDone(at: now) } ?? false + } + /// Forgets cells the last frame did not draw, so the table never outgrows the strip. func endFrame() { if tracks.count > seen.count { tracks = tracks.filter { seen.contains($0.key) } } @@ -143,7 +150,7 @@ enum FogMaskGeometry { let wallTop = coverage.wallBandHeight, groundEdge = coverage.groundBandDepth /// A quad on the wall from height `near` to `far`, or on the ground from `near` to `far` /// out; `beyond` picks the part past the band. - func quad(_ band: CoverageBand, beyond: Bool, _ s0: Float, _ s1: Float, _ value: FogValue) { + func quad(_ band: CoverageBand, beyond: Bool, _ s0: Float, _ s1: Float, _ value: FogValue, moving: Bool = false) { guard count + 6 <= capacity else { return } let (near, far) = switch (band, beyond) { case (.wall, false): (Float(0), wallTop) @@ -155,7 +162,7 @@ enum FogMaskGeometry { SIMD4(band == .wall ? wall.world(s: s, height: t) : wall.world(s: s, height: 0, out: t), 1) } let a = point(s0, near), b = point(s1, near), c = point(s1, far), d = point(s0, far) - let v = SIMD4(value.fog, value.hidden, value.requested, 0) + let v = SIMD4(value.fog, value.hidden, value.requested, moving ? 1 : 0) vertices[count] = Vertex(position: a, value: v) vertices[count + 1] = Vertex(position: b, value: v) vertices[count + 2] = Vertex(position: c, value: v) @@ -170,7 +177,7 @@ enum FogMaskGeometry { for band in CoverageBand.allCases { let states = coverage.cells(band) for beyond in [false, true] { - var run: (s0: Float, s1: Float, value: FogValue, along: SIMD3)? + var run: (s0: Float, s1: Float, value: FogValue, moving: Bool, along: SIMD3)? for index in states.indices { let range = coverage.cellRange(index) guard range.upperBound > visible.lowerBound, range.lowerBound < visible.upperBound else { continue } @@ -179,16 +186,17 @@ enum FogMaskGeometry { let key = FogCellAnimator.Key(band: band, cell: Int((range.lowerBound / coverage.cellWidth).rounded()), beyond: beyond) let target = beyond ? FogValue.beyond(states[index]) : FogValue.target(states[index], requested: requested) let value = animator.value(key, target: target, now: now, reduceMotion: reduceMotion) + let moving = animator.isMoving(key, now: now) let along = wall.along(atS: (s0 + s1) / 2) - if var current = run, current.value == value, current.along == along, abs(current.s1 - s0) < 1e-4 { + if var current = run, current.value == value, current.moving == moving, current.along == along, abs(current.s1 - s0) < 1e-4 { current.s1 = s1 run = current } else { - if let current = run { quad(band, beyond: beyond, current.s0, current.s1, current.value) } - run = (s0, s1, value, along) + if let current = run { quad(band, beyond: beyond, current.s0, current.s1, current.value, moving: current.moving) } + run = (s0, s1, value, moving, along) } } - if let current = run { quad(band, beyond: beyond, current.s0, current.s1, current.value) } + if let current = run { quad(band, beyond: beyond, current.s0, current.s1, current.value, moving: current.moving) } } } } diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift index 9341f0cf1..3672486a4 100644 --- a/ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogShaders.swift @@ -4,9 +4,10 @@ /// /// Fog. Coverage decides where it lifts: `LiveFogScene` draws every wall column and ground /// stretch of the coverage strip into a mask three times 64 texels wide (red: fog, green: hidden -/// behind something, blue: requested by a gap), clearing to full fog; a max-pool takes it to 64 -/// texels and a radius-5 Gaussian blur feathers it. The composite is the prototype's: Chalk mixed toward a cool grey, textured by -/// slow domain-warped fbm and breathing slightly. Two changes: the noise is pinned to the meter +/// behind something, blue: requested by a gap, alpha: still changing), clearing to full fog; a +/// max-pool takes it to 64 texels and a radius-3 Gaussian blur feathers it. The composite is the +/// prototype's: Chalk mixed toward a cool grey, textured by slow domain-warped fbm and breathing +/// slightly. Two changes: the noise is pinned to the meter /// on screen and scaled by its distance, so the texture sits on the wall instead of on the lens /// while the phone moves (the prototype cut between keyframes, where that never showed); and /// part-lifted fog breaks up along the same noise instead of thinning evenly, so a lift reads as @@ -51,7 +52,7 @@ enum LiveFogShaders { struct MaskVertex { float4 position; // world xyz - float4 value; // fog, hidden, requested, unused + float4 value; // fog, hidden, requested, 1 while the cell's fog is changing }; struct MaskOut { @@ -88,7 +89,7 @@ enum LiveFogShaders { return m; } - // Separable Gaussian, radius 5 texels (sigma 2.5), all four channels. + // Separable Gaussian, radius 3 texels (sigma 1.4, about 8 points), all four channels. fragment float4 blurFragment(FullOut in [[stage_in]], texture2d source [[texture(0)]], constant int2& direction [[buffer(0)]]) { @@ -96,8 +97,8 @@ enum LiveFogShaders { int2 limit = int2(source.get_width() - 1, source.get_height() - 1); float4 sum = float4(0.0f); float total = 0.0f; - for (int k = -5; k <= 5; k++) { - float w = exp(-float(k * k) / (2.0f * 2.5f * 2.5f)); + for (int k = -3; k <= 3; k++) { + float w = exp(-float(k * k) / (2.0f * 1.4f * 1.4f)); sum += w * source.read(uint2(clamp(p + k * direction, int2(0), limit))); total += w; } @@ -149,36 +150,43 @@ enum LiveFogShaders { float2 uv = fmod((screen - u.anchor) / u.anchorScale, float2(256.0f)); float2 q = uv + u.motion * u.time * 0.02f * float2(0.94f, 0.34f); - // The mask is read through a noise offset of up to 7% of the screen's width, so the - // straight edges of cells and columns come out as ragged, drifting fog banks (the - // noise-edged fog of war the prototype borrowed from Civilization VI). - float2 edgeNoise = float2(valueNoise(q * 1.7f + float2(7.3f, 2.1f)), valueNoise(q * 1.7f + float2(-4.1f, 8.6f))); - float2 look = screen + (edgeNoise - 0.5f) * 0.14f * u.viewSize.x; + // The mask is read through a fine noise offset, about 27 cm features at the meter and at + // most 2.5% of the screen's width either way, so the straight edges of cells come out + // ragged, as a fog bank's do (the noise-edged fog of war the prototype borrowed from + // Civilization VI). A coarser, larger offset (70 cm, 7%) smeared the edges into what + // read as a smudge on the lens in the first hosted screenshots. + float2 edgeNoise = float2(valueNoise(q * 4.5f + float2(7.3f, 2.1f)), valueNoise(q * 4.5f + float2(-4.1f, 8.6f))); + float2 look = screen + (edgeNoise - 0.5f) * 0.05f * u.viewSize.x; // The blur and the noise shape the fog; neither may erase it. The unblurred pooled mask, - // read through a fifth of the offset, sets a floor: a cell coverage has not counted keeps - // 30% of its fog within 3% of the screen's width of where it is, however narrow. Read - // unwarped, the floor drew a straight step along every wide edge. - float2 near = screen + (edgeNoise - 0.5f) * 0.03f * u.viewSize.x; + // read through a smaller offset, sets a floor: a cell coverage has not counted keeps 30% + // of its fog within a few points of where it is, however narrow. Read unwarped, the floor + // drew a straight step along every wide edge. + float2 near = screen + (edgeNoise - 0.5f) * 0.015f * u.viewSize.x; float4 m = max(mask.sample(linear, look / u.viewSize), 0.3f * pooled.sample(linear, near / u.viewSize)); float fog = saturate(m.r), hidden = saturate(m.g), requested = saturate(m.b); if (fog < 0.002f && requested < 0.002f) { return float4(0.0f); } float2 warp = float2(fbm(q + float2(3.1f, 1.7f)), fbm(q + float2(-2.3f, 5.2f))); float n = fbm(q + 0.8f * warp); - float noiseTerm = 0.78f + 0.22f * n; + // The prototype's texture is barely there; deeper troughs read as dirt on the lens. + float noiseTerm = 0.88f + 0.12f * n; float breathing = 1.0f + u.motion * 0.04f * sin(2.0f * M_PI_F * 0.08f * u.time); - // Part-lifted fog breaks up along the noise; full fog and clear stay as they are. - float shaped = saturate(fog + (n - 0.5f) * 1.6f * fog * (1.0f - fog)); + // Lifting fog breaks up along the noise, as mist burning off; fog that holds still (a cell + // seen once) stays an even, thinner veil, since held blotches read as a smudge. + float erosion = mix(0.4f, 1.6f, saturate(m.a)); + float shaped = saturate(fog + (n - 0.5f) * erosion * fog * (1.0f - fog)); float a = saturate(0.66f * noiseTerm * shaped * breathing * mix(1.0f, 0.72f, hidden)); + // Hidden is a tint of real violet, the tape map's colour, not a grey veil over the thing + // in front: half-mixed at 0.75 fog it read as a grey ghost over the bin. float3 chalk = float3(0xF7, 0xF5, 0xEF) / 255.0f; float3 cool = float3(0xD9, 0xE2, 0xEC) / 255.0f; - float3 violet = float3(0xB4, 0x9C, 0xFF) / 255.0f; - float3 color = mix(mix(chalk, cool, 0.35f), violet, 0.5f * hidden); + float3 violet = float3(0xA4, 0x8A, 0xFA) / 255.0f; + float3 color = mix(mix(chalk, cool, 0.2f), violet, 0.9f * hidden); float pulse = 1.0f - u.motion * 0.14f * (0.5f + 0.5f * sin(2.0f * M_PI_F * u.time / 1.8f)); - float r = saturate(0.36f * requested * noiseTerm * pulse); + float r = saturate(0.28f * requested * noiseTerm * pulse); float3 rgb = color * a; rgb = u.requested.rgb * r + rgb * (1.0f - r); @@ -236,15 +244,17 @@ enum LiveFogShaders { float veil = violetOne ? 1.0f : mix(1.0f, 0.2f, saturate(fog)); float alpha = evidence * birth * fade * veil; - // Flat 2.5 pt and edge 3.5 pt from 2.5 m out, growing to 4 and 6 pt at 1 m. - float far = clamp((length(s.a.xyz - u.cameraAndTime.xyz) - 1.0f) / 1.5f, 0.0f, 1.0f); - float size = mix(mix(4.0f, 2.5f, far), mix(6.0f, 3.5f, far), edge); + // Flat 3.5 pt and edge 5 pt from 4 m out, growing to 5 and 7.5 pt at 1 m. The prototype's + // 2.5 to 4 pt over 1 to 2.5 m left dots at walking distance (1.5 to 3 m) as pinpricks. + float far = clamp((length(s.a.xyz - u.cameraAndTime.xyz) - 1.0f) / 3.0f, 0.0f, 1.0f); + float size = mix(mix(5.0f, 3.5f, far), mix(7.5f, 5.0f, far), edge); float inner = 0.7f; float3 color = hologram; if (halo) { - // Edge 4x at 22%, flat 2x at 8%; a flat dot turning edge grows into it. - size *= mix(2.0f, 4.0f, edge); - alpha *= mix(0.08f, 0.22f, edge); + // Edge 4x at 32%, flat 2.6x at 16%; a flat dot turning edge grows into it. The glow + // is what makes the dots read as light; the prototype's 8% flat halo didn't show. + size *= mix(2.6f, 4.0f, edge); + alpha *= mix(0.16f, 0.32f, edge); inner = 0.0f; color = glow; } diff --git a/ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift b/ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift index 47a3542f6..28b75abab 100644 --- a/ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift +++ b/ios/HouseScan/UI/Camera/LiveFog/LiveFogView.swift @@ -35,6 +35,7 @@ final class LiveFogSupport { await MainActor.run { switch result { case let .success(gpu): + RuntimeLog.engine.info("live fog ready") LiveFogSupport.shared.status = .ready(gpu) case let .failure(error): // Logged as an error, not trapped: a trap in a Debug build would stop every UI