Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions ios/HouseScan/Contract/ScanContract.swift
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,31 @@ struct CoverageStrip: Equatable, Sendable {
}
}

/// LiDAR phones: dots on the surfaces depth has measured, drawn over the camera by the live fog.
/// Guidance, never evidence: `coverage` alone decides what the scan has seen, and the fog lifts
/// only where it says so. Built from the same kept keyframes as coverage (`LiveDotsFeed`).
struct LiveDots: Sendable {
struct Dot: Sendable, Equatable {
/// Stable while the dot exists, so the renderer can tell a birth from a survivor.
var id: UInt64
var position: SIMD3<Float>
/// On a crease or a silhouette; drawn larger, with a stronger glow.
var isEdge: Bool
/// On something standing in front of the wall (a bin, a bush): drawn violet, the hidden
/// state's colour.
var onOccluder: Bool
/// 0.45 seen from one direction, up to 0.9 from four.
var opacity: Float
}

/// At most `SurfaceDotConfig.maxDots` (5,000), those within 6 m of the latest keyframe.
var dots: [Dot] = []
/// Increments on every change, so the renderer rebuilds its buffer only then.
var revision = 0

static let empty = LiveDots()
}

// MARK: - Guidance

enum WallSide: String, Equatable, Sendable {
Expand Down Expand Up @@ -698,6 +723,8 @@ final class ScanViewState {

var wall: WallGeometry?
var coverage: CoverageStrip = .empty
/// LiDAR phones: the live dots. Empty without depth.
var liveDots: LiveDots = .empty
var closeUp: CloseUpState = .aiming(hold: 0, problem: nil)
/// Close-up attempts that ended without a usable photo. "Can't get a clear shot" appears
/// from the second one on, never earlier.
Expand Down
12 changes: 8 additions & 4 deletions ios/HouseScan/HouseScanApp.swift
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,15 @@ import SwiftUI
struct HouseScanApp: App {
var body: some Scene {
WindowGroup {
if ProcessInfo.processInfo.arguments.contains("-uiDemo") {
UIDemo.makeRoot()
} else {
AppRuntime.makeRoot()
Group {
if ProcessInfo.processInfo.arguments.contains("-uiDemo") {
UIDemo.makeRoot()
} else {
AppRuntime.makeRoot()
}
}
// Compiles the live fog's shaders in the background long before the walk needs them.
.task { LiveFogSupport.shared.prepare() }
}
}
}
81 changes: 81 additions & 0 deletions ios/HouseScan/Runtime/LiveDotsFeed.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
import Foundation
import HouseScanKit
import os

/// Runs the LiDAR dot field (`SurfaceDots`) off the main actor and publishes its dots for the
/// camera overlay (`ScanViewState.liveDots`).
///
/// The engine hands it each kept keyframe that has depth, the same frames coverage observes, so
/// the dots and the fog move together. Fusing a 256 x 192 map takes a few milliseconds (3.4 ms
/// median on an M4 Pro in a Release build; not measured on a phone), so it runs on its own
/// serial queue. A keyframe arriving while two are still waiting is dropped: the next one covers
/// the same wall. An anchor correction is a matrix product (`SurfaceDots.apply`) and republishes
/// only when no keyframe is waiting to publish anyway, so neither kind of work can pile up.
final class LiveDotsFeed: @unchecked Sendable {
// `@unchecked Sendable`: everything below `queue` is read and written only on it; `pending`
// is a lock.
private let queue = DispatchQueue(label: "dev.housescanning.housescan.live-dots", qos: .utility)
private var field = SurfaceDots()
private var revision = 0
private var last: (eye: SIMD3<Float>, wall: WallFrame)?
private let pending = OSAllocatedUnfairLock(initialState: 0)
private let publish: @MainActor @Sendable (LiveDots, _ generation: Int) -> Void

/// Keyframes waiting on the queue past which new ones are dropped.
static let maxPending = 2

/// `publish` runs on the main actor with the scan generation the work was queued under, so
/// the engine can ignore dots from before a Start over.
init(publish: @escaping @MainActor @Sendable (LiveDots, _ generation: Int) -> Void) {
self.publish = publish
}

func integrate(camera: CameraFrame, depth: DepthImage, wall: WallFrame, generation: Int) {
let accepted = pending.withLock { count -> Bool in
guard count < Self.maxPending else { return false }
count += 1
return true
}
guard accepted else {
RuntimeLog.capture.info("live dots: dropped a keyframe, two still waiting")
return
}
queue.async { [self] in
field.integrate(camera: camera, depth: depth)
last = (camera.position, wall)
pending.withLock { $0 -= 1 }
send(generation)
}
}

/// ARKit's correction to the meter's anchor, applied as coverage applies it.
func apply(_ correction: YawCorrection, wall: WallFrame, generation: Int) {
queue.async { [self] in
field.apply(correction)
if let eye = last?.eye { last = (correction.point(eye), wall) }
if pending.withLock({ $0 }) == 0 { send(generation) }
}
}

func reset() {
queue.async { [self] in
field = SurfaceDots()
last = nil
}
}

/// On `queue`.
private func send(_ generation: Int) {
guard let last else { return }
revision += 1
let dots = LiveDots(
dots: field.dots(near: last.eye, wall: last.wall).map {
LiveDots.Dot(id: $0.id, position: $0.position, isEdge: $0.isEdge, onOccluder: $0.onOccluder, opacity: $0.opacity)
},
revision: revision)
let publish = publish
DispatchQueue.main.async {
MainActor.assumeIsolated { publish(dots, generation) }
}
}
}
15 changes: 14 additions & 1 deletion ios/HouseScan/Runtime/ScanEngine.swift
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,12 @@ final class ScanEngine {
/// The packet's sensor streams for the current world frame.
private(set) var recorder: CaptureRecorder
private let motion = MotionSource()
/// LiDAR phones: the dots over the camera, fused off the main actor from the keyframes
/// coverage observes. Guidance for the overlay only; nothing reads them back.
private lazy var liveDots = LiveDotsFeed { [weak self] dots, scan in
guard let self, scan == self.generation else { return }
self.state.liveDots = dots
}
/// Every request the homeowner was shown, for the packet.
var guidanceLog = GuidanceLog()
/// The spot check (`ScanEngine+Confirm.swift`).
Expand Down Expand Up @@ -394,6 +400,7 @@ final class ScanEngine {
guard outcome.correction != nil || outcome.groundChanged else { return }
coverage = map
if let correction = outcome.correction {
liveDots.apply(correction, wall: map.wall, generation: generation)
var features = state.features
for index in features.indices { features[index].points = features[index].points.map(correction.point) }
if features != state.features { state.features = features }
Expand Down Expand Up @@ -669,7 +676,11 @@ final class ScanEngine {
// The frame's own time lets the walked path join only poses kept close together in time.
// With LiDAR depth, a cell counts only where depth confirms the camera saw it.
// Corrected for anchor moves made while the photo was saving.
let delta = coverage?.observe(correctedCamera(frame), trackingNormal: frame.tracking == .normal, time: frame.timestamp, depth: frame.depth, segment: segment)
let camera = correctedCamera(frame)
let delta = coverage?.observe(camera, trackingNormal: frame.tracking == .normal, time: frame.timestamp, depth: frame.depth, segment: segment)
if let depth = frame.depth, frame.tracking == .normal, let wall = coverage?.wall {
liveDots.integrate(camera: camera, depth: depth, wall: wall, generation: scan)
}
RuntimeLog.capture.info("stored \(frame.id, privacy: .public) as keyframe \(index)\(frame.depth == nil ? "" : " with depth", privacy: .public): \(delta?.newlySeen ?? 0) cells newly seen, \(delta?.newlyCovered ?? 0) newly covered, \(delta?.newlyHidden ?? 0) newly hidden")
if overhead { recordOverhead(frame) }
state.captureCount += 1
Expand Down Expand Up @@ -1584,6 +1595,8 @@ final class ScanEngine {
uploadTask?.cancel()
replay?.stop()
coverage = nil
liveDots.reset()

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Dots survive spatial reset When relocalization times out during a wall walk, resetSpatialState discards the old wall and coverage, but unlike this reset path, it does not reset the dot field or clear state.liveDots. When the homeowner starts again from the meter, new keyframes fuse depth into dots from the old world frame, so dots can appear in the wrong places. Reset the feed and published dots during spatial reset too.

Prompt To Fix With AI
This is a comment left during a code review.
Path: ios/HouseScan/Runtime/ScanEngine.swift
Line: 1598

Comment:
**Dots survive spatial reset** When relocalization times out during a wall walk, `resetSpatialState` discards the old wall and coverage, but unlike this reset path, it does not reset the dot field or clear `state.liveDots`. When the homeowner starts again from the meter, new keyframes fuse depth into dots from the old world frame, so dots can appear in the wrong places. Reset the feed and published dots during spatial reset too.

---

For each issue above, determine whether it is valid and should be fixed. If so, fix it directly.

Fix in Codex Fix in Claude Code

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Caretaker check at f57310c: confirmed, and new in this PR. A relocalization timeout calls resetSpatialState (ScanEngine.swift:1113-1169), which restarts ARKit with .resetTracking but keeps LiveDotsFeed's SurfaceDots field and state.liveDots. Depth from the new world frame then fuses into old-frame dots. resetAll already clears both (lines 1598-1599). The effect is on the overlay only; capture, coverage and export never read the dots. This is a reset bug, separate from the deferred dots/veil redesign.

The three-line fix (the same two calls in resetSpatialState) is prepared on a caretaker branch for the owner to pick up; it isn't pushed here. The app has no unit-test target, so the fix has only been checked against the source: the serial queue and the generation guard make reset() drop queued old-frame publishes, as they do for Start over.

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Follow-up: the prepared fix, 010f322 on the caretaker branch codex/greptile-158-dot-reset (local, not pushed), builds. A Debug iphonesimulator xcodebuild of the HouseScan scheme succeeded through the shared QA queue (job greptile-158-build-010f322-r2, 20:07–20:09 UTC). That shows it compiles, not that the reset behaves correctly on a device.

state.liveDots = .empty
meterAnchorID.map { live?.removeAnchor($0) }
meterAnchorID = nil
meterPlaneSource = .detectedPlane
Expand Down
33 changes: 30 additions & 3 deletions ios/HouseScan/UI/Camera/CameraOverlays.swift
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ struct CameraOverlays: View {
let state: ScanViewState
var highlight: GapRequest?

@Environment(\.accessibilityReduceMotion) private var reduceMotion
@Environment(\.self) private var environment

var body: some View {
ZStack {
overlays
Expand All @@ -14,11 +17,11 @@ struct CameraOverlays: View {

@ViewBuilder
private var overlays: some View {
// Hidden while tracking isn't normal: drawn from a pose the phone doesn't trust, the haze,
// path and pins would sit in the wrong place (checklist T3). They fade back on recovery.
// Hidden while tracking isn't normal: drawn from a pose the phone doesn't trust, the fog,
// path and marks would sit in the wrong place (checklist T3). They fade back on recovery.
if state.tracking == .normal, let projection = state.projection, let wall = state.wall {
ZStack {
FogOverlay(coverage: state.coverage, wall: wall, projection: projection, highlight: highlight)
fog(projection: projection, wall: wall)
.ignoresSafeArea()
WallMarksOverlay(
projection: projection,
Expand All @@ -34,4 +37,28 @@ struct CameraOverlays: View {
}
}
}

/// The live fog, with dots when the phone has depth. The frosted strip stands in while the
/// shaders compile at launch, and for good if Metal fails, so unseen wall never shows clear.
@ViewBuilder
private func fog(projection: CameraProjection, wall: WallGeometry) -> some View {
switch LiveFogSupport.shared.status {
case let .ready(gpu):
ZStack {
LiveFogView(gpu: gpu, input: LiveFogInput(
projection: projection, wall: wall, coverage: state.coverage, highlight: highlight,
dots: state.depthAvailable ? state.liveDots : .empty, reduceMotion: reduceMotion,
requestedColor: requestedColor))
FogMarks(coverage: state.coverage, wall: wall, projection: projection, highlight: highlight)
}
.transition(.opacity)
case .preparing, .failed:
FogOverlay(coverage: state.coverage, wall: wall, projection: projection, highlight: highlight)
}
}

private var requestedColor: SIMD3<Float> {
let resolved = Palette.caution.resolve(in: environment)
return SIMD3(resolved.red, resolved.green, resolved.blue)
}
}
113 changes: 113 additions & 0 deletions ios/HouseScan/UI/Camera/FogMarks.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
import SwiftUI
import simd

/// The crisp lines over the live fog, which a soft mask can't draw: a dashed violet outline
/// round each stretch hidden behind something, a slate hatch where the homeowner said "I can't
/// get there", and an amber outline round the stretch a gap request asks for. The same marks
/// and dash the tape map uses, so the camera and the map read alike.
struct FogMarks: View {
var coverage: CoverageStrip
var wall: WallGeometry
var projection: CameraProjection
var highlight: GapRequest?

var body: some View {
Canvas(rendersAsynchronously: false) { context, size in
let geometry = WallProjection(projection: projection, wall: wall, size: size)
let runs = runs()
// The map's dash, lengthened for the larger shapes on the camera.
let dash = StrokeStyle(lineWidth: 2, lineCap: .round, lineJoin: .round, dash: Palette.hiddenDash.map { $0 * 1.6 })
for run in runs.hidden {
guard let outline = outline(run.band, run.edges, geometry) else { continue }
// A dark halo under the dash keeps it legible on a sunlit wall.
context.stroke(outline, with: .color(.black.opacity(0.35)), style: StrokeStyle(lineWidth: 4, lineJoin: .round))
context.stroke(outline, with: .color(Palette.hidden), style: dash)
}
for run in runs.skipped {
guard let quad = outline(run.band, run.edges, geometry) else { continue }
context.fill(quad, with: .color(Palette.skipped.opacity(0.3)))
var hatched = context
hatched.clip(to: quad)
let bounds = quad.boundingRect
var lines = Path()
var x = bounds.minX - bounds.height
while x < bounds.maxX {
lines.move(to: CGPoint(x: x, y: bounds.maxY))
lines.addLine(to: CGPoint(x: x + bounds.height, y: bounds.minY))
x += 10
}
hatched.stroke(lines, with: .color(.white.opacity(0.55)), lineWidth: 1.5)
}
for run in runs.requested {
guard let outline = outline(run.band, run.edges, geometry) else { continue }
context.stroke(outline, with: .color(.black.opacity(0.3)), style: StrokeStyle(lineWidth: 4, lineJoin: .round))
context.stroke(outline, with: .color(Palette.caution), style: StrokeStyle(lineWidth: 2, lineJoin: .round))
}
}
.allowsHitTesting(false)
.accessibilityHidden(true)
}

private enum Kind { case hidden, skipped, requested }

private struct Run {
var band: CoverageBand
/// The s of every cell edge along the run, clipped to the visible range.
var edges: [Float]
}

/// Runs of hidden, of skipped, and of requested-but-not-covered cells in each band. Coverage
/// filling the middle of a request splits it, and each piece keeps its outline.
private func runs() -> (hidden: [Run], skipped: [Run], requested: [Run]) {
var hidden: [Run] = [], skipped: [Run] = [], requested: [Run] = []
let visible = coverage.visibleRange
guard visible.upperBound > visible.lowerBound else { return ([], [], []) }
for band in CoverageBand.allCases {
let states = coverage.cells(band)
var current: (kind: Kind, run: Run)?
func close() {
if let current {
switch current.kind {
case .hidden: hidden.append(current.run)
case .skipped: skipped.append(current.run)
case .requested: requested.append(current.run)
}
}
current = nil
}
for index in states.indices {
let range = coverage.cellRange(index)
guard range.upperBound > visible.lowerBound, range.lowerBound < visible.upperBound else { close(); continue }
let s0 = max(range.lowerBound, visible.lowerBound), s1 = min(range.upperBound, visible.upperBound)
let state = states[index]
let isRequested = highlight.map { $0.band == band && $0.span.lowerBound < range.upperBound && range.lowerBound < $0.span.upperBound } ?? false
let kind: Kind? = if isRequested, state != .covered { .requested }
else if state == .hidden { .hidden }
else if state == .skipped { .skipped }
else { nil }
guard let kind else { close(); continue }
if var open = current, open.kind == kind {
open.run.edges.append(s1)
current = open
} else {
close()
current = (kind, Run(band: band, edges: [s0, s1]))
}
}
close()
}
return (hidden, skipped, requested)
}

/// The outline of a run through every cell edge, so it follows the wall round a corner. Nil
/// when any point is behind the camera.
private func outline(_ band: CoverageBand, _ edges: [Float], _ geometry: WallProjection) -> Path? {
let corners: [SIMD3<Float>] = switch band {
case .wall:
edges.map { wall.world(s: $0, height: coverage.wallBandHeight) } + edges.reversed().map { wall.world(s: $0, height: 0) }
case .ground:
edges.map { wall.world(s: $0, height: 0, out: coverage.groundBandDepth) } + edges.reversed().map { wall.world(s: $0, height: 0) }
}
return geometry.polygon(corners)
}
}
4 changes: 3 additions & 1 deletion ios/HouseScan/UI/Camera/FogOverlay.swift
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import SwiftUI
import simd

/// The haze over the parts of the wall and ground the phone hasn't seen yet.
/// The haze over the parts of the wall and ground the phone hasn't seen yet: the frosted strip
/// that `LiveFogView` replaced. It still draws while the live fog's shaders compile at launch,
/// and for good if Metal fails, so unseen wall never shows clear.
///
/// Unseen cells carry a soft frosted haze, seen cells a thinner one, covered cells none. When a
/// cell's state changes (a new `coverage.revision`), its haze fades and drifts upward over
Expand Down
Loading
Loading