Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions ios/HouseScan/Runtime/LiveCapture.swift
Original file line number Diff line number Diff line change
Expand Up @@ -164,9 +164,11 @@ final class LiveCapture {
/// Whether RealityKit can be drawing the shown result now: its anchor entity is anchored to
/// the meter's ARKit anchor, it is enabled, and the result's focus point lands in front of
/// the camera and inside the view. A check of the chain, not of pixels: RealityKit says
/// nothing about what it drew. The findings are logged when they change.
func resultIsDrawn() -> Bool {
guard let result, let resultFocus else { return false }
/// nothing about what it drew. The findings are logged when they change. `held` is the chain
/// without the view: the AR scene still has the result, whether or not it is in view
/// (`ResultOverlayPolicy.update(drawn:held:time:)`).
func resultIsDrawn() -> (drawn: Bool, held: Bool) {
guard let result, let resultFocus else { return (false, false) }
let id = delegate.shared.withLock { $0.meterAnchorID }
let inSession = id.map { id in arView.session.currentFrame?.anchors.contains(where: { $0.identifier == id }) == true } ?? false
let anchored = result.isAnchored
Expand All @@ -181,7 +183,7 @@ final class LiveCapture {
let at = point.map { String(format: "(%.0f, %.0f) in %.0f x %.0f", Double($0.x), Double($0.y), Double(arView.bounds.width), Double(arView.bounds.height)) } ?? "none"
RuntimeLog.engine.info("AR result: \(report, privacy: .public), projected at \(at, privacy: .public)")
}
return anchored && enabled && onScreen
return (anchored && enabled && onScreen, anchored && enabled)
}

func hideResult() {
Expand Down
2 changes: 1 addition & 1 deletion ios/HouseScan/Runtime/ScanEngine+GuidanceLog.swift
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ extension ScanEngine {
}
return GuidanceLog.Request(
topic: .gap(id: gap.id), kind: pastEndSide == nil ? .gapBand : .gapPastEnd, origin: gap.origin == .server ? .server : .phone,
message: Self.text(ScanCopy.gap(gap, ends: (left: state.wall?.leftEnd, right: state.wall?.rightEnd))), band: pastEndSide == nil ? band : nil, span: plan.span
message: Self.text(ScanCopy.gap(gap)), band: pastEndSide == nil ? band : nil, span: plan.span
)
case .onboarding, .findMeter, .markFeatures, .uploading, .result, .resultAR, .unsupported:
return nil
Expand Down
78 changes: 70 additions & 8 deletions ios/HouseScan/Runtime/ScanEngine.swift
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ final class ScanEngine {
/// The meter anchor's pose the wall and everything captured agree with, and the corrections
/// still to apply to them as ARKit refines it (`refreshMeterFromAnchor`). Nil on a replay.
private var meterTracking: MeterAnchorTracking?
/// Which frames carried the meter's anchor since it was anchored, for the drift log
/// (`noteMeterAnchor`). Starts again with `meterTracking`.
private var meterAnchorPresence = MeterAnchorPresence()
/// Whether a frame source may start, and what a failure and Start over do to it.
private var sourceState = CaptureSourceState()
/// Detected horizontal planes, with their classes and outlines.
Expand Down Expand Up @@ -243,8 +246,10 @@ final class ScanEngine {
breakWalkedPath(because: "the walk paused (\(state.phase.rawValue) -> \(phase.rawValue))")
}
if state.phase == .resultAR { hideResultInCamera() }
let previous = state.phase
state.phase = phase
RuntimeLog.state.info("STATE=\(phase.rawValue, privacy: .public)")
logMeterAnchorSummary(from: previous, to: phase)
switch phase {
case .findMeter:
state.guidance = .findMeter
Expand Down Expand Up @@ -416,6 +421,7 @@ final class ScanEngine {
wallPlanes = frame.wallPlanes
refitWallToDetectedPlane()
}
noteMeterAnchor(frame)
// A frame made before the meter was anchored again carries the old anchor's pose.
if frame.meterAnchorID == meterAnchorID { refreshMeterFromAnchor(frame) }
guard !frame.isPoseOnly else { return }
Expand Down Expand Up @@ -513,24 +519,76 @@ final class ScanEngine {
if features != state.features { state.features = features }
publishWall()
publishCoverage()
logAnchorCorrection(correction, step: step)
logAnchorCorrection(correction, step: step, frame: frame)
}

/// One line per correction applied, with the total since the meter was anchored, for telling
/// a map ARKit corrected (the marks and the meter move together) from drift it never
/// corrected (no line, and the marks sit off their objects) on a walk away and back (#73).
// The drift log (#73). Its lines are `.notice`, which the unified log keeps on the device, so
// `log collect` after a walk and Console without "Include Info Messages" both show them. On a
// walk away and back, the lines that are there settle it, never a line being absent:
// corrections with the marks on their objects mean ARKit corrected its map and the marks
// followed; frames that had the anchor, no corrections, and marks off their objects mean
// drift ARKit never corrected, which no anchor can fix; frames that lost the anchor mean
// nothing could follow it. `t` is the capture clock the packet's manifest times use.

/// One line per correction applied, with the total since the meter was anchored.
/// `MeterAnchorTracking` already limits the rate: a correction is applied only once the anchor
/// has moved 2 cm or turned 0.4 degrees since the last one.
private func logAnchorCorrection(_ correction: YawCorrection, step: Float) {
private func logAnchorCorrection(_ correction: YawCorrection, step: Float, frame: SourceFrame) {
guard let tracking = meterTracking else { return }
let total = tracking.sinceAnchored
let turned = correction.yaw * 180 / .pi
let totalMoved = simd_length(total.moved)
let totalTurned = total.yaw * 180 / .pi
let count = tracking.corrections
let marks = state.features.count
RuntimeLog.capture.info(
"meter anchor corrected: moved \(step, format: .fixed(precision: 3)) m, turned \(turned, format: .fixed(precision: 2)) degrees; since anchored (\(count) corrections): moved \(totalMoved, format: .fixed(precision: 3)) m (x \(total.moved.x, format: .fixed(precision: 3)), y \(total.moved.y, format: .fixed(precision: 3)), z \(total.moved.z, format: .fixed(precision: 3))), turned \(totalTurned, format: .fixed(precision: 2)) degrees; \(marks) marks moved with it (phase \(self.state.phase.rawValue, privacy: .public))"
let t = frame.timestamp
let fromMeter = coverage.map { simd_distance(frame.camera.position, $0.wall.meter) } ?? 0
let phase = state.phase.rawValue
RuntimeLog.capture.notice(
"meter anchor corrected at t=\(t, format: .fixed(precision: 2)) s, phone \(fromMeter, format: .fixed(precision: 1)) m from the meter: moved \(step, format: .fixed(precision: 3)) m, turned \(turned, format: .fixed(precision: 2)) degrees; since anchored (\(count) corrections): moved \(totalMoved, format: .fixed(precision: 3)) m (x \(total.moved.x, format: .fixed(precision: 3)), y \(total.moved.y, format: .fixed(precision: 3)), z \(total.moved.z, format: .fixed(precision: 3))), turned \(totalTurned, format: .fixed(precision: 2)) degrees; \(marks) marks moved with it (phase \(phase, privacy: .public))"
)
}

/// One line when the frames stop carrying the meter's anchor, and one when they carry it
/// again: while it is missing, no correction can reach the wall or the marks. The first frame
/// after the meter is anchored writes one too. Live only: a replay has no anchor.
private func noteMeterAnchor(_ frame: SourceFrame) {
guard let id = meterAnchorID, let tracking = meterTracking else { return }
let sighting: MeterAnchorPresence.Sighting =
frame.meterAnchorID != id ? .otherAnchor : frame.meterAnchor == nil ? .missing : .present
guard let change = meterAnchorPresence.observe(sighting) else { return }
let t = frame.timestamp
let fromMeter = coverage.map { simd_distance(frame.camera.position, $0.wall.meter) } ?? 0
let count = tracking.corrections
let phase = state.phase.rawValue
RuntimeLog.capture.notice(
"meter anchor in frames: \(change.rawValue, privacy: .public) at t=\(t, format: .fixed(precision: 2)) s, phone \(fromMeter, format: .fixed(precision: 1)) m from the meter (\(count) corrections so far, phase \(phase, privacy: .public))"
)
}

/// The tracking's totals on every phase change once the meter is anchored, zero corrections
/// included, and whether the latest frame carried the anchor. Sending the scan and opening the
/// AR result always write one, with no anchor too (a replay), so a walk's log ends with it.
private func logMeterAnchorSummary(from previous: ScanPhase, to phase: ScanPhase) {
let from = previous.rawValue, to = phase.rawValue
let t = captureClock.map { String(format: "%.2f", $0) } ?? "none"
guard let tracking = meterTracking else {
guard phase == .uploading || phase == .resultAR else { return }
RuntimeLog.capture.notice(
"meter anchor at \(from, privacy: .public) -> \(to, privacy: .public), t=\(t, privacy: .public): not tracked (no meter anchor)"
)
return
}
let total = tracking.sinceAnchored
let totalMoved = simd_length(total.moved)
let totalTurned = total.yaw * 180 / .pi
let count = tracking.corrections
let frames = meterAnchorPresence
let last = frames.last?.rawValue ?? "no frame yet"
let wall = coverage == nil ? "no wall" : "wall set"
let marks = state.features.count
RuntimeLog.capture.notice(
"meter anchor at \(from, privacy: .public) -> \(to, privacy: .public), t=\(t, privacy: .public): \(count) corrections since anchored, moved \(totalMoved, format: .fixed(precision: 3)) m (x \(total.moved.x, format: .fixed(precision: 3)), y \(total.moved.y, format: .fixed(precision: 3)), z \(total.moved.z, format: .fixed(precision: 3))), turned \(totalTurned, format: .fixed(precision: 2)) degrees; latest frame: \(last, privacy: .public); frames with the anchor \(frames.present), without it \(frames.missing), naming another \(frames.otherAnchor); \(wall, privacy: .public), \(marks) marks"
)
}

Expand Down Expand Up @@ -1296,6 +1354,7 @@ final class ScanEngine {
meterAnchorID.map { live?.removeAnchor($0) }
meterAnchorID = nil
meterTracking = nil
meterAnchorPresence = MeterAnchorPresence()
meterPlaneSource = .detectedPlane
state.wall = nil
state.coverage = .empty
Expand Down Expand Up @@ -1410,7 +1469,8 @@ final class ScanEngine {
var policy = ResultOverlayPolicy()
while !Task.isCancelled {
guard let self, self.state.phase == .resultAR, let live = self.live else { return }
let usesRealityKit = policy.update(drawn: live.resultIsDrawn(), time: self.screenTime)
let look = live.resultIsDrawn()
let usesRealityKit = policy.update(drawn: look.drawn, held: look.held, time: self.screenTime)
if self.state.resultInCamera != usesRealityKit {
RuntimeLog.engine.info("AR result drawn by \(usesRealityKit ? "the AR scene" : "the screen overlay", privacy: .public)")
self.state.resultInCamera = usesRealityKit
Expand Down Expand Up @@ -1836,6 +1896,7 @@ final class ScanEngine {
meterAnchorID.map { live?.removeAnchor($0) }
meterAnchorID = nil
meterTracking = nil
meterAnchorPresence = MeterAnchorPresence()
meterPlaneSource = .detectedPlane
store = KeyframeStore()
recorder = Self.makeRecorder(store)
Expand Down Expand Up @@ -1905,6 +1966,7 @@ final class ScanEngine {
func setMeterAnchor(_ id: UUID?, pose: simd_float4x4?) {
meterAnchorID = id
meterTracking = pose.map(MeterAnchorTracking.init)
meterAnchorPresence = MeterAnchorPresence()
}

var detectedGroundPlanes: [GroundPlaneEvidence] { groundPlanes }
Expand Down
27 changes: 16 additions & 11 deletions ios/HouseScan/UI/Copy/Distance.swift
Original file line number Diff line number Diff line change
Expand Up @@ -68,18 +68,23 @@ enum Distance {

/// "from 4 ft to 19 ft right of your meter", or "from 3 ft left to 2 ft right of your meter":
/// a stretch by its two ends in whole feet, nearer end first, for a request that can run
/// along much of the wall (issue #75). It is clipped to the wall's marked ends, so a request
/// running past one names only the wall there is; one lying wholly past an end is named as
/// it is. Ends that round to the same foot read "about 5 ft right of your meter".
static func range(_ span: ClosedRange<Float>, clippedTo ends: (left: Float?, right: Float?) = (nil, nil)) -> String {
var low = max(span.lowerBound, ends.left ?? -.infinity)
var high = min(span.upperBound, ends.right ?? .infinity)
if low > high {
low = span.lowerBound
high = span.upperBound
}
/// along much of the wall (issue #75). Ends on the same foot read "about 5 ft right of your
/// meter".
///
/// The whole stretch is named, as asked: a server request is met only over its whole span
/// (`GapPlanner.progress`), and the one kind raised past a marked end, ground past a limit
/// end, asks for the ground there too (#47). Clipped to the ends, a request for 4 to 33 ft
/// with a limit end at 20 ft read "from 4 ft to 20 ft right", and showing that much left the
/// bar near half way. For the same reason the ends round outward, the lower one down and the
/// upper one up: "from 2 ft to 7 ft" for 2.4...7.4 ft left the last 0.4 ft unasked. An end
/// within 0.001 ft of a whole foot counts as on it, so float noise (0.6096 m is 2.0000001 ft)
/// doesn't add a foot. 2.4...7.4 ft reads "from 2 ft to 8 ft right", -7.4...-2.4 ft "from 2 ft
/// to 8 ft left", and -2.6...1.2 ft "from 3 ft left to 2 ft right".
static func range(_ span: ClosedRange<Float>) -> String {
let foot = metersPerInch * 12
let lowFeet = Int((low / foot).rounded()), highFeet = Int((high / foot).rounded())
let slack: Float = 0.001
let lowFeet = Int((span.lowerBound / foot + slack).rounded(.down))
let highFeet = max(lowFeet, Int((span.upperBound / foot - slack).rounded(.up)))
switch (lowFeet, highFeet) {
case (0, 0):
return "around your meter"
Expand Down
8 changes: 4 additions & 4 deletions ios/HouseScan/UI/Copy/ScanCopy.swift
Original file line number Diff line number Diff line change
Expand Up @@ -332,11 +332,11 @@ enum ScanCopy {
// MARK: Gap

/// The card for a gap request. A server request can run along much of the wall, so its
/// stretch is named by its two ends, clipped to the wall's marked `ends` (issue #75); the
/// phone's own requests are short and named by their middle.
static func gap(_ gap: GapRequest, ends: (left: Float?, right: Float?) = (nil, nil)) -> Instruction {
/// stretch is named by its two ends, all of it (issue #75, `Distance.range`); the phone's own
/// requests are short and named by their middle.
static func gap(_ gap: GapRequest) -> Instruction {
let place = Distance.aroundFromMeter(gap.span)
let stretch = Distance.range(gap.span, clippedTo: ends)
let stretch = Distance.range(gap.span)
switch gap.reason {
case .groundNearCandidate:
return Instruction(title: "Show the ground \(place)", detail: "This might be a spot for the battery, so the ground there needs a clear look from two places.")
Expand Down
2 changes: 1 addition & 1 deletion ios/HouseScan/UI/Screens/GapRequestScreen.swift
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ struct GapRequestScreen: View {
if gap.isSatisfied {
return Instruction(title: "Got it, thanks", detail: followUps > 0 ? "Updating your result." : "That's the view we needed.")
}
return ScanCopy.withCoaching(ScanCopy.gap(gap, ends: (left: state.wall?.leftEnd, right: state.wall?.rightEnd)), coaching)
return ScanCopy.withCoaching(ScanCopy.gap(gap), coaching)
}

/// The coaching on the card, marked with its symbol (`tone`). Once the view is in, the
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,9 @@ public struct MeterAnchorTracking: Sendable, Equatable {

/// All the corrections applied since the meter was anchored, as one: how far ARKit has moved
/// the meter (world x, y, z, in meters) and turned the wall about gravity (radians). What
/// hasn't reached `minimumMove` or `minimumTurn` yet isn't in it. A device log of it tells a
/// map ARKit corrected from drift it never corrected (#73): the marks and the result follow
/// the first, and nothing on the phone can follow the second.
/// hasn't reached `minimumMove` or `minimumTurn` yet isn't in it. A device log of it, with
/// `MeterAnchorPresence`, tells a map ARKit corrected from drift it never corrected (#73): the
/// marks and the result follow the first, and nothing on the phone can follow the second.
public var sinceAnchored: (moved: SIMD3<Float>, yaw: Float) {
(Self.origin(pose) - Self.origin(anchoredPose), YawCorrection(from: anchoredPose, to: pose).yaw)
}
Expand All @@ -115,6 +115,43 @@ public struct MeterAnchorTracking: Sendable, Equatable {
}
}

/// Whether the frames since the meter was anchored carried its anchor, for the drift log (#73).
/// A correction can only reach the marks from a frame that carries the anchor, so a walk with no
/// corrections logged is read against this: frames that had the anchor and never moved it
/// (drift ARKit never corrected), or frames that lost it (nothing could follow it).
public struct MeterAnchorPresence: Sendable, Equatable {
public enum Sighting: String, Sendable, CaseIterable {
/// The frame carries the anchor's pose.
case present
/// The frame names the anchor, but ARKit no longer lists it among the frame's anchors.
case missing
/// The frame names another anchor or none: made before the meter was anchored again, or
/// after the session let the anchor go.
case otherAnchor = "other anchor"
}

/// What the latest frame showed; nil before the first frame since the meter was anchored.
public private(set) var last: Sighting?
public private(set) var present = 0
public private(set) var missing = 0
public private(set) var otherAnchor = 0

public init() {}

/// Counts a frame. Returns `sighting` when it differs from the frame before's (the first
/// frame included), nil when it is the same.
public mutating func observe(_ sighting: Sighting) -> Sighting? {
switch sighting {
case .present: present += 1
case .missing: missing += 1
case .otherAnchor: otherAnchor += 1
}
let changed = sighting != last
last = sighting
return changed ? sighting : nil
}
}

extension WallFrame {
/// Moves the wall with the world: the meter and the ground with the translation, the meter's
/// piece and every corner's piece turned by the yaw. Every s (the ends, the corners, what was
Expand Down
Loading
Loading