From c4dec62010f20dd94881978a1dc7bce6a47d8920 Mon Sep 17 00:00:00 2001 From: AidenJohnston Date: Sun, 27 Sep 2026 00:13:52 -0500 Subject: [PATCH 1/5] ios: gate coaching keeps the task card, waits out dark flicker, and says "Turn more slowly" for turns The walk's capture-gate coaching was debounced by time only and restarted its clock whenever the gate's per-frame reason changed. At night the kept frames' luma sat on the gate's 40, and a tilt swapped "too dark" for "moving", so the card replaced the task and came back every few seconds (#80). Turning the phone in place also read as walking too fast (#26). - AutoCapture splits SkipReason.moving into movingFast and turningFast. - New HouseScanKit CoachingDebouncer: each problem keeps its own clock; darkness comes from the frames' luma with hysteresis (on above 50% dark frames in 2 s, off below 20% and above 48 luma for 1.5 s), and after about 20 s of mostly dark frames it says once that it's too dark to measure; "Slow down" is not said on aim, tilt, step-back, see-behind or marking steps; blur shows after 2 s. Thresholds are guesses for a night run. - ScanEngine.walkCoaching feeds it the gate reason and the frame's luma. The tracking coaching and the past-the-end debounce are unchanged. - WallWalkScreen keeps the task title for gate coaching and puts the coaching on the second line with its symbol; tracking problems and past-the-end still replace the card. - New copy for too dark, too dark to measure and turn more slowly. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/Contract/ScanContract.swift | 10 +- ios/HouseScan/Runtime/ScanEngine.swift | 68 ++++-- ios/HouseScan/UI/Copy/ScanCopy.swift | 18 +- ios/HouseScan/UI/Preview/DemoEngine.swift | 2 + ios/HouseScan/UI/Preview/UIDemo.swift | 2 +- ios/HouseScan/UI/Screens/WallWalkScreen.swift | 29 ++- .../HouseScanKit/Capture/AutoCapture.swift | 10 +- .../Guidance/CoachingDebouncer.swift | 200 ++++++++++++++++++ .../HouseScanKitTests/AutoCaptureTests.swift | 37 +++- .../CoachingDebouncerTests.swift | 161 ++++++++++++++ .../ScreenStatesUITests.swift | 21 ++ 11 files changed, 522 insertions(+), 36 deletions(-) create mode 100644 ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift create mode 100644 ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift diff --git a/ios/HouseScan/Contract/ScanContract.swift b/ios/HouseScan/Contract/ScanContract.swift index 55f2b8c72..04a23cdf7 100644 --- a/ios/HouseScan/Contract/ScanContract.swift +++ b/ios/HouseScan/Contract/ScanContract.swift @@ -314,13 +314,21 @@ enum GuidanceStep: Equatable, Sendable { case seeBehind(s: Float) } -/// A problem that overrides guidance until it clears. UI/ owns the words. +/// A problem to fix while it lasts. Tracking problems and `pastWallEnd` override guidance; the +/// capture gate's (`slowDown`, `turnSlowly`, `holdSteady`, `tooDark`, `tooDarkToMeasure`) and +/// `needsTexture` ride along with it on the walk. UI/ owns the words. enum Coaching: Equatable, Sendable { case initializing case slowDown case needsTexture case tooDark + /// Frames have been mostly too dark for a long while: waiting won't help, daylight will + /// (`CoachingDebouncer.GateProblem.persistentlyDark`). + case tooDarkToMeasure case holdSteady + /// Turning or tilting the phone faster than the capture gate keeps photos at, as opposed to + /// walking too fast (#26). + case turnSlowly case relocalizing case trackingLost /// The phone stands past an end it can't see back from, so the walk keeps no photos: nothing diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index feeb3e252..8781baa55 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -31,9 +31,11 @@ final class ScanEngine { // Stored evidence private(set) var store: KeyframeStore private var keptSourceIDs: Set = [] - /// Debounces coaching that comes from the capture gate (see `gateCoaching`). + /// Debounces "past the end of the wall" during the walk (see `walkCoaching`). private var gateProblem: (coaching: Coaching, since: Double)? private var gateClearSince: Double? + /// Debounces the capture gate's own coaching during the walk (see `walkCoaching`). + private var gateCoaching = CoachingDebouncer() private var closeUpPending = false /// Why the last close-up has to be retaken (from the meter-number reader, or "None of /// these"), and when that was said, in screen seconds. Shown until the next shot fires. @@ -561,33 +563,32 @@ final class ScanEngine { keptSourceIDs.insert(frame.id) keep(frame) } - state.coaching = walkCoaching(tracking: frame.tracking, skip: skip, pastEnd: pastEnd != nil, time: frame.timestamp) + state.coaching = walkCoaching(tracking: frame.tracking, skip: skip, meanLuma: frame.quality?.meanLuma, pastEnd: pastEnd != nil, time: frame.timestamp) afterCoverageChange(camera: frame.camera, time: frame.timestamp) askOverheadIfTiltedUp(frame) } - /// Tracking problems show at once. A problem the capture gate reports (moving, blurry, too - /// dark) shows only once it has lasted `showAfter` seconds and clears after `clearAfter` - /// seconds of frames without it: the gate judges every frame, and one fast frame at 30 fps - /// would otherwise flash a prompt for a single frame. During the walk moving and blurry read - /// as "Slow down", never "Hold steady", which would tell a walking homeowner to stop. Both - /// durations are guesses to try on a phone, not measured. Standing past an end the phone - /// can't see back from (`pastEnd`) is debounced the same way and comes before the gate's - /// reasons: no photo is kept there whatever the gate says. - private func walkCoaching(tracking: TrackingQuality, skip: CaptureDecision.SkipReason?, pastEnd: Bool, time: Double) -> Coaching? { + /// Tracking problems show at once. The capture gate's problems go through `gateCoaching` + /// (`CoachingDebouncer`): each has its own clock, darkness is judged from the frames' luma + /// with hysteresis, and "Slow down" is kept for walking, so while the homeowner stands and + /// aims (`isAiming`) only turning and blur are said. Blur reads as "Slow down" while walking + /// and "Hold steady" while aiming. Standing past an end the phone can't see back from + /// (`pastEnd`) shows once it has lasted `showAfter` seconds, clears after `clearAfter` + /// seconds without it, and comes before the gate's problems: no photo is kept there whatever + /// the gate says. Both durations are guesses to try on a phone, not measured. + private func walkCoaching(tracking: TrackingQuality, skip: CaptureDecision.SkipReason?, meanLuma: Double?, pastEnd: Bool, time: Double) -> Coaching? { let showAfter = 0.7 let clearAfter = 0.5 + let aiming = isAiming + let gate = gateCoaching.update(time: time, skip: skip, meanLuma: meanLuma, aiming: aiming) guard tracking == .normal else { gateProblem = nil gateClearSince = nil + // The light is what it was, but the motion from before the phone lost its place is not. + gateCoaching.forgetMotion() return coaching(for: tracking, skip: nil) } - var candidate: Coaching? = switch skip { - case .moving?, .blurry?: .slowDown - case .tooDark?: .tooDark - default: nil - } - if pastEnd { candidate = .pastWallEnd } + let candidate: Coaching? = pastEnd ? .pastWallEnd : nil if let problem = gateProblem, time < problem.since { gateProblem = nil } // replay restarted if let candidate { gateClearSince = nil @@ -602,8 +603,32 @@ final class ScanEngine { gateClearSince = nil } } - guard let problem = gateProblem, time - problem.since >= showAfter else { return nil } - return problem.coaching + if let problem = gateProblem, time - problem.since >= showAfter { return problem.coaching } + return gate.map { Self.coaching(for: $0, aiming: aiming) } + } + + /// Whether the homeowner is asked to stand and aim rather than walk: an aim, tilt, step-back, + /// see-behind or marking step, a question on screen, or a gap request's view. Walking too + /// fast is not coached then (#26). Read before this frame's guidance update, so it is the + /// step on screen when the frame arrived. + private var isAiming: Bool { + if state.marking != nil || state.endQuestion != nil || state.overheadQuestion { return true } + switch state.guidance { + case .aimAtGround, .aimAtWall, .tiltUp, .stepBack, .seeBehind, .markEnd, .markNextWall, .gap: return true + case .findMeter, .aimAtWallForMeter, .holdOnMeter, .walk, .walkComplete: return false + } + } + + /// The coaching for a gate problem. Blur reads as "Slow down" while walking and "Hold + /// steady" while aiming, where "Slow down" would tell someone standing still to walk slower. + static func coaching(for problem: CoachingDebouncer.GateProblem, aiming: Bool) -> Coaching { + switch problem { + case .tooDark: .tooDark + case .persistentlyDark: .tooDarkToMeasure + case .movingFast: .slowDown + case .turningFast: .turnSlowly + case .blurry: aiming ? .holdSteady : .slowDown + } } private func afterCoverageChange(camera: CameraFrame?, time: Double) { @@ -1130,6 +1155,7 @@ final class ScanEngine { state.lastCapture = nil gateProblem = nil gateClearSince = nil + gateCoaching = CoachingDebouncer() autoCapture.reset() planner.reset() go(.findMeter) @@ -1547,6 +1573,7 @@ final class ScanEngine { resetPacketLog() keptSourceIDs = [] autoCapture.reset() + gateCoaching = CoachingDebouncer() planner.reset() closeUpGate = CloseUpGate() gapPlan = nil @@ -1737,7 +1764,8 @@ extension ScanEngine { case .limited(.unknown): return .trackingLost case .normal: switch skip { - case .moving?, .blurry?: return .holdSteady + case .movingFast?, .blurry?: return .holdSteady + case .turningFast?: return .turnSlowly case .tooDark?: return .tooDark default: return nil } diff --git a/ios/HouseScan/UI/Copy/ScanCopy.swift b/ios/HouseScan/UI/Copy/ScanCopy.swift index cc4e82396..bb40f4fee 100644 --- a/ios/HouseScan/UI/Copy/ScanCopy.swift +++ b/ios/HouseScan/UI/Copy/ScanCopy.swift @@ -75,9 +75,13 @@ enum ScanCopy { case .needsTexture: Instruction(title: "Aim at a corner or somewhere with more texture", detail: "A plain wall or the sky gives your phone nothing to follow.") case .tooDark: - Instruction(title: "It's too dark to see the wall", detail: "Turn on a porch light, or try again in daylight.") + Instruction(title: "It's dark here", detail: "Some photos won't count. Try your phone's flashlight, or come back in daylight.") + case .tooDarkToMeasure: + Instruction(title: "It's too dark to measure here", detail: "Try in daylight.") case .holdSteady: Instruction(title: "Hold steady", detail: nil) + case .turnSlowly: + Instruction(title: "Turn more slowly", detail: "Photos taken while turning come out blurred.") case .relocalizing: Instruction(title: "Point at the meter like this.", detail: "Your phone lost its place for a moment.") case .trackingLost: @@ -93,13 +97,23 @@ enum ScanCopy { case .initializing: "iphone.gen3.radiowaves.left.and.right" case .slowDown: "tortoise.fill" case .needsTexture: "square.grid.3x3.middle.filled" - case .tooDark: "moon.fill" + case .tooDark, .tooDarkToMeasure: "moon.fill" case .holdSteady: "hand.raised.fill" + case .turnSlowly: "arrow.clockwise" case .relocalizing, .trackingLost: "location.slash.fill" case .pastWallEnd: "arrow.uturn.backward" } } + /// One line for coaching that rides along with the task instead of replacing it (the walk's + /// capture-gate coaching): the task's title stays, and this takes its second line. + static func coachingNote(_ coaching: Coaching) -> String { + let words = ScanCopy.coaching(coaching) + let title = words.title.hasSuffix(".") ? words.title : "\(words.title)." + guard let detail = words.detail else { return title } + return "\(title) \(detail)" + } + // MARK: Wall ends /// Under the wall map when ending the wall at the dashed line would cut off part of the walk. diff --git a/ios/HouseScan/UI/Preview/DemoEngine.swift b/ios/HouseScan/UI/Preview/DemoEngine.swift index 2ef5cdc1b..139a2927a 100644 --- a/ios/HouseScan/UI/Preview/DemoEngine.swift +++ b/ios/HouseScan/UI/Preview/DemoEngine.swift @@ -897,7 +897,9 @@ final class DemoEngine: ScanActions { case "slowDown": .slowDown case "needsTexture": .needsTexture case "tooDark": .tooDark + case "tooDarkToMeasure": .tooDarkToMeasure case "holdSteady": .holdSteady + case "turnSlowly": .turnSlowly case "relocalizing": .relocalizing case "trackingLost": .trackingLost case "pastWallEnd": .pastWallEnd diff --git a/ios/HouseScan/UI/Preview/UIDemo.swift b/ios/HouseScan/UI/Preview/UIDemo.swift index adba5d550..2f9a9b351 100644 --- a/ios/HouseScan/UI/Preview/UIDemo.swift +++ b/ios/HouseScan/UI/Preview/UIDemo.swift @@ -7,7 +7,7 @@ import SwiftUI /// - `-uiDemoFreeze`: don't run the timed scripts, so the screen holds still. /// - `-uiDemoMarking `: open the walk in marking mode. /// - `-uiDemoRefusal`: the marking shows a refusal. -/// - `-uiDemoCoaching `. +/// - `-uiDemoCoaching `. /// - `-uiDemoCloseUpFailed`: the close-up has failed twice, so the way out shows. /// - `-uiDemoMeterChoose`: the close-up asks which of three made-up readings is the meter number. /// - `-uiDemoGroundQuestion`: open the feature review with the ground question unanswered. diff --git a/ios/HouseScan/UI/Screens/WallWalkScreen.swift b/ios/HouseScan/UI/Screens/WallWalkScreen.swift index 4bb4073b8..2a6c452f7 100644 --- a/ios/HouseScan/UI/Screens/WallWalkScreen.swift +++ b/ios/HouseScan/UI/Screens/WallWalkScreen.swift @@ -4,8 +4,9 @@ import SwiftUI /// /// Over the camera: haze on what the phone hasn't seen, a blue dotted path on the ground, a /// ring on the next thing to aim at, pins on what's been marked. At the bottom: the tape map -/// and at most two actions. At the top: one instruction, replaced by coaching while there is a -/// problem, or by the marking prompt while marking. +/// and at most two actions. At the top: one instruction, with coaching on its second line while +/// the photos have a problem, replaced while tracking has one, or by the marking prompt while +/// marking. struct WallWalkScreen: View { let state: ScanViewState let actions: any ScanActions @@ -81,9 +82,27 @@ struct WallWalkScreen: View { } if let side = state.endQuestion { return ScanCopy.endQuestion(side) } if state.overheadQuestion { return ScanCopy.overheadQuestion } - if let coaching { return ScanCopy.coaching(coaching) } - if state.wallTooShort { return Instruction(title: ScanCopy.wallTooShort, detail: ScanCopy.guidance(state.guidance).title) } - return ScanCopy.guidance(state.guidance) + if let coaching, Self.replacesTask(coaching) { return ScanCopy.coaching(coaching) } + if state.wallTooShort { return withCoaching(Instruction(title: ScanCopy.wallTooShort, detail: ScanCopy.guidance(state.guidance).title)) } + return withCoaching(ScanCopy.guidance(state.guidance)) + } + + /// Coaching about how the photos come out (the capture gate's, and too little texture) rides + /// along with the task: the task's title stays, the coaching takes the second line, and its + /// symbol marks the card (`tone`). Replacing the whole card hid the task each time the + /// coaching came up (#80). + private func withCoaching(_ task: Instruction) -> Instruction { + guard let coaching else { return task } + return Instruction(title: task.title, detail: ScanCopy.coachingNote(coaching)) + } + + /// Tracking problems and standing past the end replace the task: nothing the task asks for + /// counts until they clear. + private static func replacesTask(_ coaching: Coaching) -> Bool { + switch coaching { + case .initializing, .relocalizing, .trackingLost, .pastWallEnd: true + case .slowDown, .needsTexture, .tooDark, .tooDarkToMeasure, .holdSteady, .turnSlowly: false + } } /// "Slow down" is for walking. With the tray open the homeowner has stopped to pick a mark diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Capture/AutoCapture.swift b/ios/HouseScanKit/Sources/HouseScanKit/Capture/AutoCapture.swift index d16e9c9f6..81d4cb2bb 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Capture/AutoCapture.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Capture/AutoCapture.swift @@ -66,7 +66,12 @@ public enum CaptureDecision: Sendable, Equatable { public enum SkipReason: Sendable, Equatable { case trackingNotReady case tooSoon - case moving + /// The phone moved faster than `AutoCaptureConfig.maxSpeed`: walking too fast. + case movingFast + /// The phone turned faster than `AutoCaptureConfig.maxAngularSpeed` without moving too + /// fast: a tilt or a pan, often while standing still. Kept apart from `movingFast` so + /// coaching doesn't tell someone standing still to walk slower (#26). + case turningFast case blurry case tooDark case tooBright @@ -121,7 +126,8 @@ public struct AutoCapture: Sendable { let dt = Float(frame.timestamp - previous.timestamp) let speed = simd_distance(frame.camera.position, previous.camera.position) / dt let turn = frame.camera.rotationAngle(to: previous.camera) / dt - if speed > config.maxSpeed || turn > config.maxAngularSpeed { return .skip(.moving) } + if speed > config.maxSpeed { return .skip(.movingFast) } + if turn > config.maxAngularSpeed { return .skip(.turningFast) } } if let quality = frame.quality ?? lastQuality { if quality.meanLuma < config.minMeanLuma { return .skip(.tooDark) } diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift b/ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift new file mode 100644 index 000000000..e88988ba0 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift @@ -0,0 +1,200 @@ +import Foundation + +/// Turns the capture gate's per-frame verdicts into coaching that stays up long enough to read. +/// +/// The gate judges about ten frames a second and gives one reason per frame: the first check +/// that fails (`AutoCapture.evaluate`). Shown as it comes, that reason flickers. In the 4.1 field +/// test (#80) a night scene's mean luma sat right on the gate's 40, so "too dark" came and went +/// every few seconds, and a tilt in the dark swapped it for "Slow down", because motion is +/// checked before exposure. Tilting and turning in place also read as walking too fast (#26). +/// +/// So each problem keeps its own clock, and a frame that reports another problem never restarts +/// a problem that is showing: +/// - Darkness is judged from the frames' own mean luma, not from the gate's reason, with +/// hysteresis: it shows once most recent frames are dark, and clears only once nearly none +/// are and the light has stayed up for a while. It comes first: while it is dark, the motion +/// problems don't take its place. +/// - Moving fast, turning fast and blur each show after their own spell of frames that have +/// them, and clear a moment after the last such frame. A spell that ends before it shows says +/// nothing. +/// - Walking too fast is not raised while the homeowner is asked to stand and aim (`aiming`): +/// the motion there is turning the phone, and "walk slower" is the wrong advice. +/// +/// Every threshold is a guess to try on a phone at night, not measured. +public struct CoachingDebouncer: Sendable { + /// What the capture gate's coaching can say. + public enum GateProblem: Sendable, Equatable, Hashable { + /// Most recent frames are too dark to count. + case tooDark + /// Frames have been mostly dark for a long time. Said in place of `tooDark` until the + /// light comes back, and once per debouncer (one scan). + case persistentlyDark + /// Walking faster than the gate keeps photos at. + case movingFast + /// Turning or tilting the phone faster than the gate keeps photos at. + case turningFast + /// Frames much less sharp than the recent ones, for a while. + case blurry + } + + public struct Config: Sendable, Equatable { + /// A frame is dark below this mean luma: the gate's own `AutoCaptureConfig.minMeanLuma`. + public var darkLuma: Double = AutoCaptureConfig().minMeanLuma + /// Darkness is judged over the measured frames of the last 2 s. + public var darkWindow: Double = 2 + /// Fewer measured frames than this in the window (about half a second of them) say + /// nothing about darkness, so the first frames of a walk can't raise it alone. + public var minDarkSamples = 5 + /// Shows once more than half the measured frames in the window are dark. + public var showDarkShare: Double = 0.5 + /// Clears once fewer than a fifth of them are dark... + public var clearDarkShare: Double = 0.2 + /// ...and each measured frame is above 48, well clear of `darkLuma`... + public var clearLuma: Double = 48 + /// ...for 1.5 s in a row. + public var clearDarkAfter: Double = 1.5 + /// Dark shown for 20 s, with more than half the measured frames of those 20 s dark, + /// becomes `persistentlyDark`: waiting for the light won't help. + public var persistentDarkAfter: Double = 20 + /// Moving or turning fast shows after 0.7 s of it: one fast frame is a jolt, not a habit. + public var motionShowAfter: Double = 0.7 + /// Blur shows only after 2 s of it. The gate's blur test compares a frame with the recent + /// median, and a tilt changes the view enough to trip it with no real blur. + public var blurShowAfter: Double = 2 + /// A motion or blur problem that shows clears 0.5 s after the last frame that had it. + public var clearAfter: Double = 0.5 + + public init() {} + } + + /// One continuous run of frames with a motion or blur problem. + private struct Spell: Sendable, Equatable { + var since: Double + var lastSeen: Double + } + + private enum PersistentDark: Sendable, Equatable { + case notYet + case showing + /// Said, and the light came back since: not said again. + case said + } + + public let config: Config + private var lastTime: Double? + private var spells: [GateProblem: Spell] = [:] + /// Measured frames of the last `persistentDarkAfter` seconds: time and whether it was dark. + private var luma: [(time: Double, dark: Bool)] = [] + /// When the dark coaching went up, nil while it is down. + private var darkSince: Double? + /// Since when the light has been back, while the dark coaching is up. + private var brightSince: Double? + private var persistentDark = PersistentDark.notYet + + public init(config: Config = Config()) { + self.config = config + } + + /// Whether the dark coaching (`tooDark` or `persistentlyDark`) is up. + public var isDark: Bool { darkSince != nil } + + /// Takes one judged frame and returns the problem to coach, if any. + /// + /// - `time`: the frame's timestamp in seconds. A time before the last one (a replay that + /// restarted) starts over. + /// - `skip`: the gate's reason for skipping the frame, nil when the gate passed it. + /// - `meanLuma`: the frame's own mean luma, nil when it wasn't measured. + /// - `aiming`: the homeowner is asked to stand and aim (an aim, tilt, step-back, see-behind + /// or marking step), so walking too fast is not raised. + public mutating func update(time: Double, skip: CaptureDecision.SkipReason?, meanLuma: Double?, aiming: Bool) -> GateProblem? { + if let lastTime, time < lastTime { self = CoachingDebouncer(config: config) } + lastTime = time + updateDarkness(time: time, meanLuma: meanLuma) + updateMotion(time: time, skip: skip) + if darkSince != nil { return persistentDark == .showing ? .persistentlyDark : .tooDark } + if isShown(.turningFast, at: time) { return .turningFast } + if !aiming, isShown(.movingFast, at: time) { return .movingFast } + if isShown(.blurry, at: time) { return .blurry } + return nil + } + + /// Forgets the motion and blur spells, for example while tracking is lost. Darkness is kept: + /// the light hasn't changed because the phone lost its place. + public mutating func forgetMotion() { + spells = [:] + } + + // MARK: Darkness + + private mutating func updateDarkness(time: Double, meanLuma: Double?) { + guard let meanLuma else { return } + luma.append((time: time, dark: meanLuma < config.darkLuma)) + let keepFrom = time - max(config.persistentDarkAfter, config.darkWindow) + if let firstKept = luma.firstIndex(where: { $0.time > keepFrom }), firstKept > 0 { + luma.removeFirst(firstKept) + } + let recent = luma.filter { $0.time > time - config.darkWindow } + let share = Double(recent.filter { $0.dark }.count) / Double(max(recent.count, 1)) + guard let since = darkSince else { + if recent.count >= config.minDarkSamples, share > config.showDarkShare { + darkSince = time + brightSince = nil + } + return + } + if share < config.clearDarkShare, meanLuma > config.clearLuma { + if brightSince == nil { brightSince = time } + } else { + brightSince = nil + } + if let bright = brightSince, time - bright >= config.clearDarkAfter { + darkSince = nil + brightSince = nil + if persistentDark == .showing { persistentDark = .said } + return + } + if persistentDark == .notYet, time - since >= config.persistentDarkAfter { + let longShare = Double(luma.filter { $0.dark }.count) / Double(max(luma.count, 1)) + if longShare > config.showDarkShare { persistentDark = .showing } + } + } + + // MARK: Motion and blur + + private mutating func updateMotion(time: Double, skip: CaptureDecision.SkipReason?) { + for (problem, seen) in Self.evidence(skip) { + if seen { + spells[problem] = Spell(since: spells[problem]?.since ?? time, lastSeen: time) + } else if let spell = spells[problem], !reachedShow(problem, spell) { + // A spell that ends before it shows says nothing. + spells[problem] = nil + } + } + spells = spells.filter { time - $0.value.lastSeen < config.clearAfter } + } + + private func isShown(_ problem: GateProblem, at time: Double) -> Bool { + guard let spell = spells[problem] else { return false } + return reachedShow(problem, spell) && time - spell.lastSeen < config.clearAfter + } + + private func reachedShow(_ problem: GateProblem, _ spell: Spell) -> Bool { + let after = problem == .blurry ? config.blurShowAfter : config.motionShowAfter + return spell.lastSeen - spell.since >= after + } + + /// What one verdict says about each motion and blur problem: true when the frame had it, + /// false when the gate checked and the frame didn't, and nothing when a check that comes + /// first failed (`AutoCapture.evaluate` checks tracking, then speed, then turning, then + /// exposure, then sharpness). + private static func evidence(_ skip: CaptureDecision.SkipReason?) -> [GateProblem: Bool] { + switch skip { + case .trackingNotReady?: [:] + case .movingFast?: [.movingFast: true] + case .turningFast?: [.movingFast: false, .turningFast: true] + case .tooDark?, .tooBright?: [.movingFast: false, .turningFast: false] + case .blurry?: [.movingFast: false, .turningFast: false, .blurry: true] + case .tooSoon?, .redundant?, nil: [.movingFast: false, .turningFast: false, .blurry: false] + } + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift index 96434e86a..7860603d6 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift @@ -1,10 +1,15 @@ +import Foundation import HouseScanKit import simd import Testing @Suite struct AutoCaptureTests { - static func frame(_ t: Double, x: Float = 0, tracking: TrackingStatus = .normal, quality: FrameQuality? = nil) -> FrameSample { - let camera = portraitCamera(at: SIMD3(x, 1.4, 2.6), forward: forwardFacingWall(pitchedDown: 20)) + /// A camera at `x` along the wall, pitched down 20° and turned `yaw` degrees about the vertical. + static func frame(_ t: Double, x: Float = 0, yaw: Float = 0, tracking: TrackingStatus = .normal, quality: FrameQuality? = nil) -> FrameSample { + let f = forwardFacingWall(pitchedDown: 20) + let a = yaw * .pi / 180 + let forward = SIMD3(f.x * cos(a) + f.z * sin(a), f.y, -f.x * sin(a) + f.z * cos(a)) + let camera = portraitCamera(at: SIMD3(x, 1.4, 2.6), forward: forward) return FrameSample(timestamp: t, camera: camera, tracking: tracking, quality: quality) } @@ -47,11 +52,33 @@ import Testing #expect(gate.evaluate(Self.frame(0.6), newlySeenCells: 10) == .skip(.tooSoon)) } - @Test func fastWalkingIsMoving() { + @Test func fastWalkingIsMovingFast() { var gate = Self.afterFirstKeep() _ = gate.evaluate(Self.frame(0.6), newlySeenCells: 0) - // 1 m since the frame at 0.6, in 0.4 s: 2.5 m/s > maxSpeed 1.5. - #expect(gate.evaluate(Self.frame(1.0, x: 1), newlySeenCells: 10) == .skip(.moving)) + // 0.8 m since the frame at 0.6, in 0.4 s: 2 m/s > maxSpeed 1.5, with no turn. + #expect(gate.evaluate(Self.frame(1.0, x: 0.8), newlySeenCells: 10) == .skip(.movingFast)) + } + + /// #26: turning the phone in place is not walking too fast. + @Test func turningInPlaceIsTurningFast() { + var gate = Self.afterFirstKeep() + _ = gate.evaluate(Self.frame(0.6), newlySeenCells: 0) + // 36° since the frame at 0.6, in 0.4 s: 90°/s > maxAngularSpeed 60°/s, with no move. + #expect(gate.evaluate(Self.frame(1.0, yaw: 36), newlySeenCells: 10) == .skip(.turningFast)) + } + + /// Speed is checked first: walking fast while turning is moving fast. + @Test func walkingAndTurningFastIsMovingFast() { + var gate = Self.afterFirstKeep() + _ = gate.evaluate(Self.frame(0.6), newlySeenCells: 0) + #expect(gate.evaluate(Self.frame(1.0, x: 0.8, yaw: 36), newlySeenCells: 10) == .skip(.movingFast)) + } + + /// A slow turn is neither. + @Test func slowTurnIsKept() { + var gate = Self.afterFirstKeep() + // 20° in 0.5 s is 40°/s, under 60°/s, and 20° >= spacingRadians 15°. + #expect(gate.evaluate(Self.frame(1.0, yaw: 20), newlySeenCells: 0) == .keep(.spacing)) } @Test func keepsOnSpacing() { diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift new file mode 100644 index 000000000..9374feb34 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift @@ -0,0 +1,161 @@ +import Foundation +import HouseScanKit +import Testing + +/// Synthetic frame sequences at the gate's pace, about ten judged frames a second. +@Suite struct CoachingDebouncerTests { + typealias Problem = CoachingDebouncer.GateProblem + + /// One judged frame. + struct Frame { + var time: Double + var skip: CaptureDecision.SkipReason? + var luma: Double? = 128 + var aiming = false + } + + /// What the debouncer says for each frame. + static func run(_ frames: [Frame], on debouncer: inout CoachingDebouncer) -> [Problem?] { + frames.map { debouncer.update(time: $0.time, skip: $0.skip, meanLuma: $0.luma, aiming: $0.aiming) } + } + + static func run(_ frames: [Frame]) -> [Problem?] { + var debouncer = CoachingDebouncer() + return run(frames, on: &debouncer) + } + + /// Frames every 0.1 s from `start` for `seconds`. + static func times(from start: Double = 0, for seconds: Double) -> [Double] { + (0..) -> (shows: Int, clears: Int) { + var shows = 0 + var clears = 0 + var up = false + for answer in said { + let now = answer.map { problems.contains($0) } ?? false + if now, !up { shows += 1 } + if !now, up { clears += 1 } + up = now + } + return (shows, clears) + } + + static let dark: Set = [.tooDark, .persistentlyDark] + + /// #80: a night scene's luma sitting on the gate's 40 made the dark card come and go every few + /// seconds. With hysteresis it shows once and clears once, when the light is really back. + @Test func lumaOscillatingAroundTheGateShowsOnceAndClearsOnce() { + // 40 ± 6 luma, a few seconds a swing, for 30 s; then 60 luma for 4 s. + let night = Self.times(for: 30).map { t -> Frame in + let luma = 40 + 6 * sin(t * 1.7) + return Frame(time: t, skip: luma < 40 ? .tooDark : .redundant, luma: luma) + } + let light = Self.times(from: 30, for: 4).map { Frame(time: $0, skip: .redundant, luma: 60) } + let said = Self.run(night + light) + let (shows, clears) = Self.transitions(said, of: Self.dark) + #expect(shows == 1) + #expect(clears == 1) + #expect(said.last! == nil) + } + + /// #80: a tilt in the dark turned the gate's reason from too dark to moving and back, which + /// swapped the card. Darkness is judged from luma, so the flip neither hides it nor restarts it. + @Test func tooDarkAndMovingFlipsDontRestartTheDarkCoaching() { + let frames = Self.times(for: 6).enumerated().map { index, t in + Frame(time: t, skip: index % 3 == 0 ? .movingFast : .tooDark, luma: 30) + } + let said = Self.run(frames) + let first = said.firstIndex { $0 == .tooDark } + #expect(first != nil) + // Up within the first second, and never down or replaced after that. + #expect((first ?? .max) <= 10) + #expect(said[(first ?? 0)...].allSatisfy { $0 == .tooDark }) + } + + /// Once shown, a different problem on some frames doesn't restart moving fast. + @Test func movingFastStaysThroughOtherReasons() { + var debouncer = CoachingDebouncer() + let fast = Self.times(for: 1).map { Frame(time: $0, skip: .movingFast) } + #expect(Self.run(fast, on: &debouncer).last! == .movingFast) + // Fast frames with a blurry one in between (a frame the gate found not moving, 0.1 s). + let mixed = Self.times(from: 1, for: 1).enumerated().map { index, t in + Frame(time: t, skip: index == 4 ? .blurry : .movingFast) + } + #expect(Self.run(mixed, on: &debouncer).allSatisfy { $0 == .movingFast }) + } + + /// A single fast frame is a jolt: nothing shows. + @Test func shortSpellsSayNothing() { + let frames = Self.times(for: 3).enumerated().map { index, t in + Frame(time: t, skip: index % 5 == 0 ? .movingFast : .redundant) + } + #expect(Self.run(frames).allSatisfy { $0 == nil }) + } + + /// #26: during an aim or a tilt the motion is the phone turning, so "Slow down" is not said. + @Test func slowDownIsSuppressedWhileAiming() { + let aiming = Self.times(for: 3).map { Frame(time: $0, skip: .movingFast, aiming: true) } + #expect(Self.run(aiming).allSatisfy { $0 == nil }) + let walking = Self.times(for: 3).map { Frame(time: $0, skip: .movingFast, aiming: false) } + #expect(Self.run(walking).last! == .movingFast) + } + + /// Turning fast is its own problem, said while aiming too. + @Test func turningFastShowsAfterItsSpell() { + let frames = Self.times(for: 2).map { Frame(time: $0, skip: .turningFast, aiming: true) } + let said = Self.run(frames) + #expect(said.prefix(7).allSatisfy { $0 == nil }) + #expect(said.last! == .turningFast) + } + + /// Blur from a tilt that changes the view: under 2 s it says nothing. + @Test func shortBlurSaysNothing() { + let blur = Self.times(for: 1.8).map { Frame(time: $0, skip: .blurry) } + let after = Self.times(from: 1.8, for: 1).map { Frame(time: $0, skip: .redundant) } + #expect(Self.run(blur + after).allSatisfy { $0 == nil }) + let long = Self.times(for: 2.5).map { Frame(time: $0, skip: .blurry) } + #expect(Self.run(long).last! == .blurry) + } + + /// A shown problem clears half a second after its last frame. + @Test func motionClearsAfterItsLastFrame() { + let fast = Self.times(for: 1).map { Frame(time: $0, skip: .movingFast) } + let calm = Self.times(from: 1, for: 1).map { Frame(time: $0, skip: .redundant) } + let said = Self.run(fast + calm) + #expect(said[10] == .movingFast) + #expect(said.last! == nil) + } + + /// Mostly dark for more than 20 s: said once, instead of "It's dark here", until it's light. + @Test func persistentDarkIsSaidOnce() { + var debouncer = CoachingDebouncer() + let night = Self.times(for: 25).map { Frame(time: $0, skip: .tooDark, luma: 25) } + let said = Self.run(night, on: &debouncer) + #expect(said[30] == .tooDark) + #expect(said.last! == .persistentlyDark) + let light = Self.times(from: 25, for: 4).map { Frame(time: $0, skip: .redundant, luma: 90) } + #expect(Self.run(light, on: &debouncer).last! == nil) + let nightAgain = Self.times(from: 29, for: 25).map { Frame(time: $0, skip: .tooDark, luma: 25) } + let again = Self.run(nightAgain, on: &debouncer) + #expect(again.last! == .tooDark) + #expect(!again.contains { $0 == .persistentlyDark }) + } + + /// Frames without a luma measurement don't count toward darkness either way. + @Test func unmeasuredFramesDontRaiseDarkness() { + let frames = Self.times(for: 3).map { Frame(time: $0, skip: .tooDark, luma: nil) } + #expect(Self.run(frames).allSatisfy { $0 == nil }) + } + + /// A replay that restarts (time going back) starts over. + @Test func timeGoingBackStartsOver() { + var debouncer = CoachingDebouncer() + let night = Self.times(for: 2).map { Frame(time: $0, skip: .tooDark, luma: 25) } + #expect(Self.run(night, on: &debouncer).last! == .tooDark) + #expect(debouncer.update(time: 0, skip: nil, meanLuma: 128, aiming: false) == nil) + #expect(!debouncer.isDark) + } +} diff --git a/ios/HouseScanUITests/ScreenStatesUITests.swift b/ios/HouseScanUITests/ScreenStatesUITests.swift index 22ea26c38..060dfa3de 100644 --- a/ios/HouseScanUITests/ScreenStatesUITests.swift +++ b/ios/HouseScanUITests/ScreenStatesUITests.swift @@ -18,6 +18,8 @@ final class ScreenStatesUITests: XCTestCase { ("meterCloseUp-chooseNumber", ["-uiDemoPhase", "meterCloseUp", "-uiDemoMeterChoose"], "meterCloseUp"), ("wallWalk", ["-uiDemoPhase", "wallWalk"], "wallWalk"), ("wallWalk-slowDown", ["-uiDemoPhase", "wallWalk", "-uiDemoCoaching", "slowDown"], "wallWalk"), + ("wallWalk-tooDark", ["-uiDemoPhase", "wallWalk", "-uiDemoCoaching", "tooDark"], "wallWalk"), + ("wallWalk-turnSlowly", ["-uiDemoPhase", "wallWalk", "-uiDemoCoaching", "turnSlowly"], "wallWalk"), ("wallWalk-needsTexture", ["-uiDemoPhase", "wallWalk", "-uiDemoCoaching", "needsTexture"], "wallWalk"), ("wallWalk-relocalizing", ["-uiDemoPhase", "wallWalk", "-uiDemoCoaching", "relocalizing"], "wallWalk"), ("wallWalk-markingRefused", ["-uiDemoPhase", "wallWalk", "-uiDemoMarking", "window", "-uiDemoRefusal"], "wallWalk"), @@ -70,6 +72,10 @@ final class ScreenStatesUITests: XCTestCase { private static let expectations: [String: (identifier: String?, text: String)] = [ "wallWalk-hidden": ("wallTape", "2 sections hidden behind something"), "wallWalk-seeBehind": ("instruction", "Something is in front of the wall here"), + // #80, #26: the gate's coaching rides on the task card. + "wallWalk-slowDown": ("instruction", "Walk slowly to your right"), + "wallWalk-tooDark": ("instruction", "It's dark here"), + "wallWalk-turnSlowly": ("instruction", "Turn more slowly"), "gapRequest-followUp": ("instruction", "One more view to finish"), "uploading-followUp": (nil, "One more view to finish"), "markFeatures-groundQuestion": (nil, "What's on the ground along this wall?"), @@ -167,6 +173,21 @@ final class ScreenStatesUITests: XCTestCase { XCTAssertTrue(element(app, "screen.onboarding").waitForExistence(timeout: 10)) } + /// #80: the capture gate's coaching keeps the walk's task on the card and adds its own line, + /// instead of replacing the card. + @MainActor + func testGateCoachingKeepsTheTaskOnTheCard() throws { + continueAfterFailure = false + let app = XCUIApplication() + app.launchArguments = ["-uiDemo", "-uiDemoFreeze", "-uiDemoPhase", "wallWalk", "-uiDemoCoaching", "tooDark"] + app.launch() + XCTAssertTrue(element(app, "screen.wallWalk").waitForExistence(timeout: 15)) + let card = element(app, "instruction") + XCTAssertTrue(card.waitForExistence(timeout: 5)) + XCTAssertTrue(card.label.contains("Walk slowly to your right"), "the task must stay on the card, got \(card.label)") + XCTAssertTrue(card.label.contains("It's dark here"), "the coaching must show on the card, got \(card.label)") + } + /// B-09: "Add something" on the review opens the camera with the marking prompt, and the /// review comes back with the new item once it is marked, or unchanged after Cancel. @MainActor From 9b0acc5851e7fa69a1da560724c919f1d63bcc5e Mon Sep 17 00:00:00 2001 From: AidenJohnston Date: Sun, 27 Sep 2026 00:23:18 -0500 Subject: [PATCH 2/5] ios: gate coaching keeps the task's second line, refusals keep their tone, tray hides turn/steady coaching Ride-along coaching no longer replaces the task's detail: on aim steps that line is the only thing that says where to aim, and the dark coaching can now stay up for a whole night walk. The coaching's title goes on its own line under it. A refusal on the task keeps the red triangle over ride-along coaching, and with the mark tray open "Turn more slowly" and "Hold steady" are hidden like "Slow down" was. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/UI/Copy/ScanCopy.swift | 11 +++--- ios/HouseScan/UI/Screens/WallWalkScreen.swift | 37 ++++++++++++------- .../ScreenStatesUITests.swift | 1 + 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/ios/HouseScan/UI/Copy/ScanCopy.swift b/ios/HouseScan/UI/Copy/ScanCopy.swift index bb40f4fee..2e81b6f2e 100644 --- a/ios/HouseScan/UI/Copy/ScanCopy.swift +++ b/ios/HouseScan/UI/Copy/ScanCopy.swift @@ -105,13 +105,12 @@ enum ScanCopy { } } - /// One line for coaching that rides along with the task instead of replacing it (the walk's - /// capture-gate coaching): the task's title stays, and this takes its second line. + /// One short line for coaching that rides along with the task instead of replacing it (the + /// walk's capture-gate coaching): the task's title and second line stay, and this goes under + /// them. Only the coaching's title, so the task's own words stay the bigger part of the card. static func coachingNote(_ coaching: Coaching) -> String { - let words = ScanCopy.coaching(coaching) - let title = words.title.hasSuffix(".") ? words.title : "\(words.title)." - guard let detail = words.detail else { return title } - return "\(title) \(detail)" + let title = ScanCopy.coaching(coaching).title + return title.hasSuffix(".") ? title : "\(title)." } // MARK: Wall ends diff --git a/ios/HouseScan/UI/Screens/WallWalkScreen.swift b/ios/HouseScan/UI/Screens/WallWalkScreen.swift index 2a6c452f7..a36052acb 100644 --- a/ios/HouseScan/UI/Screens/WallWalkScreen.swift +++ b/ios/HouseScan/UI/Screens/WallWalkScreen.swift @@ -4,9 +4,9 @@ import SwiftUI /// /// Over the camera: haze on what the phone hasn't seen, a blue dotted path on the ground, a /// ring on the next thing to aim at, pins on what's been marked. At the bottom: the tape map -/// and at most two actions. At the top: one instruction, with coaching on its second line while -/// the photos have a problem, replaced while tracking has one, or by the marking prompt while -/// marking. +/// and at most two actions. At the top: one instruction, with coaching on a line of its own +/// under it while the photos have a problem, replaced while tracking has one, or by the marking +/// prompt while marking. struct WallWalkScreen: View { let state: ScanViewState let actions: any ScanActions @@ -88,12 +88,14 @@ struct WallWalkScreen: View { } /// Coaching about how the photos come out (the capture gate's, and too little texture) rides - /// along with the task: the task's title stays, the coaching takes the second line, and its - /// symbol marks the card (`tone`). Replacing the whole card hid the task each time the - /// coaching came up (#80). + /// along with the task: the task's title and second line both stay (on an aim step the second + /// line is the only thing that says where to aim), the coaching adds its own short line under + /// them, and its symbol marks the card (`tone`). Replacing the whole card hid the task each + /// time the coaching came up (#80), and the dark coaching can stay up for a whole night walk. private func withCoaching(_ task: Instruction) -> Instruction { guard let coaching else { return task } - return Instruction(title: task.title, detail: ScanCopy.coachingNote(coaching)) + let detail = [task.detail, ScanCopy.coachingNote(coaching)].compactMap { $0 }.joined(separator: "\n") + return Instruction(title: task.title, detail: detail) } /// Tracking problems and standing past the end replace the task: nothing the task asks for @@ -105,21 +107,28 @@ struct WallWalkScreen: View { } } - /// "Slow down" is for walking. With the tray open the homeowner has stopped to pick a mark - /// and is only turning the phone, which the gate also reads as moving (field test run 1). - /// Tracking problems still show. + /// "Slow down", "Turn more slowly" and "Hold steady" are for walking and aiming. With the tray + /// open the homeowner has stopped to pick a mark and is only turning the phone, which the gate + /// also reads as moving or turning (field test run 1). Tracking problems still show. private var coaching: Coaching? { - if trayOpen, state.coaching == .slowDown { return nil } - return state.coaching + switch state.coaching { + case .slowDown?, .turnSlowly?, .holdSteady?: trayOpen ? nil : state.coaching + default: state.coaching + } } + /// Coaching that replaces the task marks the card with its symbol. A refusal on the task + /// (the next wall wasn't marked, the wall is too short) keeps its red triangle over coaching + /// that only rides along: the refusal's words are on the card, so its tone should match. private var tone: InstructionCard.Tone { if state.marking?.refusal != nil { return .refusal } - if state.marking == nil, state.endQuestion == nil, !state.overheadQuestion, let coaching { + let coachingShows = state.marking == nil && state.endQuestion == nil && !state.overheadQuestion + if coachingShows, let coaching, Self.replacesTask(coaching) { return .coaching(symbol: ScanCopy.coachingSymbol(coaching)) } if state.marking == nil, case .markNextWall(_, _?) = state.guidance { return .refusal } - if state.marking == nil, state.endQuestion == nil, !state.overheadQuestion, state.wallTooShort { return .refusal } + if coachingShows, state.wallTooShort { return .refusal } + if coachingShows, let coaching { return .coaching(symbol: ScanCopy.coachingSymbol(coaching)) } return .normal } diff --git a/ios/HouseScanUITests/ScreenStatesUITests.swift b/ios/HouseScanUITests/ScreenStatesUITests.swift index 060dfa3de..ccca0f5f2 100644 --- a/ios/HouseScanUITests/ScreenStatesUITests.swift +++ b/ios/HouseScanUITests/ScreenStatesUITests.swift @@ -185,6 +185,7 @@ final class ScreenStatesUITests: XCTestCase { let card = element(app, "instruction") XCTAssertTrue(card.waitForExistence(timeout: 5)) XCTAssertTrue(card.label.contains("Walk slowly to your right"), "the task must stay on the card, got \(card.label)") + XCTAssertTrue(card.label.contains("Keep the wall and the ground in view"), "the task's second line must stay on the card, got \(card.label)") XCTAssertTrue(card.label.contains("It's dark here"), "the coaching must show on the card, got \(card.label)") } From 633c26d751fc24ecf8968756a29139fa6b969126 Mon Sep 17 00:00:00 2001 From: AidenJohnston Date: Sun, 27 Sep 2026 01:52:59 -0500 Subject: [PATCH 3/5] ios: gate coaching rides along on the gap request, walk-out requests still say slow down - WallWalkScreen.coachingHidesReply (from the base's f14947e) lists turnSlowly and tooDarkToMeasure in its keep-the-reply arm, so the merged switch stays exhaustive. - GapRequestScreen: tracking problems and past-the-end still replace the request; the capture gate's coaching now rides along under it with its symbol as the tone, as on the walk. Once the view is in, gate coaching no longer shows. The replace/ride-along rule and the ride-along copy move to ScanCopy so both screens share them. - ScanEngine.isAiming: a gap request counts as aiming unless it asks to walk (GapPlan.Need.asksToWalk, walkOut only), and markNextWall ("walk round it") counts as walking, so hurrying there is told to slow down. - CoachingDebouncer.Config.blurShowAfter says plainly that it turns blur coaching off in practice (the gate's median-based blur test ends a spell within about 8 frames). - Tests: asksToWalk routing in CoachingDebouncerTests, a gapRequest-tooDark screen state and a test that the request stays on the card; the walk test reads the card once from a snapshot. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/Runtime/ScanEngine.swift | 14 +++++--- ios/HouseScan/UI/Copy/ScanCopy.swift | 20 ++++++++++++ .../UI/Screens/GapRequestScreen.swift | 21 +++++++++--- ios/HouseScan/UI/Screens/WallWalkScreen.swift | 32 +++++-------------- .../Guidance/CoachingDebouncer.swift | 21 ++++++++++++ .../CoachingDebouncerTests.swift | 14 ++++++++ .../ScreenStatesUITests.swift | 28 ++++++++++++++-- 7 files changed, 114 insertions(+), 36 deletions(-) diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index 8781baa55..3d2be57aa 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -608,14 +608,18 @@ final class ScanEngine { } /// Whether the homeowner is asked to stand and aim rather than walk: an aim, tilt, step-back, - /// see-behind or marking step, a question on screen, or a gap request's view. Walking too - /// fast is not coached then (#26). Read before this frame's guidance update, so it is the - /// step on screen when the frame arrived. + /// see-behind or mark-the-end step, marking, a question on screen, or a gap request's view. + /// Walking too fast is not coached then (#26). Two steps ask for walking and so still say + /// "Slow down": the corner ("walk round it, aim at the next wall and mark it"; the mark + /// itself is `state.marking`, which aims) and a request to walk the stretch far enough out + /// (`GapPlan.Need.asksToWalk`). Read before this frame's guidance update, so it is the step + /// on screen when the frame arrived. private var isAiming: Bool { if state.marking != nil || state.endQuestion != nil || state.overheadQuestion { return true } switch state.guidance { - case .aimAtGround, .aimAtWall, .tiltUp, .stepBack, .seeBehind, .markEnd, .markNextWall, .gap: return true - case .findMeter, .aimAtWallForMeter, .holdOnMeter, .walk, .walkComplete: return false + case .aimAtGround, .aimAtWall, .tiltUp, .stepBack, .seeBehind, .markEnd: return true + case .gap: return !(gapPlan?.need.asksToWalk ?? false) + case .findMeter, .aimAtWallForMeter, .holdOnMeter, .walk, .walkComplete, .markNextWall: return false } } diff --git a/ios/HouseScan/UI/Copy/ScanCopy.swift b/ios/HouseScan/UI/Copy/ScanCopy.swift index 2e81b6f2e..fffa4f480 100644 --- a/ios/HouseScan/UI/Copy/ScanCopy.swift +++ b/ios/HouseScan/UI/Copy/ScanCopy.swift @@ -113,6 +113,26 @@ enum ScanCopy { return title.hasSuffix(".") ? title : "\(title)." } + /// Tracking problems and standing past the end replace the task on the card: nothing the task + /// asks for counts until they clear. The capture gate's coaching and too little texture ride + /// along with it instead (`withCoaching`). No `default`, so a new case has to pick a side. + static func coachingReplacesTask(_ coaching: Coaching) -> Bool { + switch coaching { + case .initializing, .relocalizing, .trackingLost, .pastWallEnd: true + case .slowDown, .needsTexture, .tooDark, .tooDarkToMeasure, .holdSteady, .turnSlowly: false + } + } + + /// The task with ride-along coaching under it: the task's title and second line both stay (on + /// an aim step the second line is the only thing that says where to aim), and the coaching adds + /// its own short line (`coachingNote`). Replacing the whole card hid the task each time the + /// coaching came up (#80), and the dark coaching can stay up for a whole night walk. + static func withCoaching(_ task: Instruction, _ coaching: Coaching?) -> Instruction { + guard let coaching else { return task } + let detail = [task.detail, coachingNote(coaching)].compactMap { $0 }.joined(separator: "\n") + return Instruction(title: task.title, detail: detail) + } + // MARK: Wall ends /// Under the wall map when ending the wall at the dashed line would cut off part of the walk. diff --git a/ios/HouseScan/UI/Screens/GapRequestScreen.swift b/ios/HouseScan/UI/Screens/GapRequestScreen.swift index eaf00f2e4..1bcd01ed2 100644 --- a/ios/HouseScan/UI/Screens/GapRequestScreen.swift +++ b/ios/HouseScan/UI/Screens/GapRequestScreen.swift @@ -26,7 +26,7 @@ struct GapRequestScreen: View { } CameraChrome( instruction: instruction, - tone: asking ? .normal : state.coaching.map { .coaching(symbol: ScanCopy.coachingSymbol($0)) } ?? .normal, + tone: asking ? .normal : coaching.map { .coaching(symbol: ScanCopy.coachingSymbol($0)) } ?? .normal, reply: state.gap?.isSatisfied == true || asking ? nil : InstructionCard.Reply( title: "I can't get there", identifier: "action.skipGap", @@ -77,14 +77,27 @@ struct GapRequestScreen: View { return max(1, result.missing.filter(\.capturable).count) } + /// As on the walk: tracking problems and standing past an end replace the request, and the + /// capture gate's coaching rides along under it (`ScanCopy.withCoaching`). The dark coaching + /// can stay up for a whole night request, and replacing the card hid the request and "Got it, + /// thanks" for as long as it did (field test 4.1, run 3). private var instruction: Instruction { if asking { return ScanCopy.overheadQuestion } - if let coaching = state.coaching { return ScanCopy.coaching(coaching) } - guard let gap = state.gap else { return ScanCopy.guidance(.gap) } + if let coaching, ScanCopy.coachingReplacesTask(coaching) { return ScanCopy.coaching(coaching) } + guard let gap = state.gap else { return ScanCopy.withCoaching(ScanCopy.guidance(.gap), coaching) } if gap.isSatisfied { return Instruction(title: "Got it, thanks", detail: followUps > 0 ? "Updating your result." : "That's the view we needed.") } - return ScanCopy.gap(gap) + return ScanCopy.withCoaching(ScanCopy.gap(gap), coaching) + } + + /// The coaching on the card, marked with its symbol (`tone`). Once the view is in, the + /// capture gate's coaching has nothing left to ask for, so only coaching that replaces the + /// card still shows. + private var coaching: Coaching? { + guard let coaching = state.coaching else { return nil } + if state.gap?.isSatisfied == true, !ScanCopy.coachingReplacesTask(coaching) { return nil } + return coaching } } diff --git a/ios/HouseScan/UI/Screens/WallWalkScreen.swift b/ios/HouseScan/UI/Screens/WallWalkScreen.swift index cc828b3e6..f8f13ed5c 100644 --- a/ios/HouseScan/UI/Screens/WallWalkScreen.swift +++ b/ios/HouseScan/UI/Screens/WallWalkScreen.swift @@ -82,29 +82,13 @@ struct WallWalkScreen: View { } if let side = state.endQuestion { return ScanCopy.endQuestion(side) } if state.overheadQuestion { return ScanCopy.overheadQuestion } - if let coaching, Self.replacesTask(coaching) { return ScanCopy.coaching(coaching) } - if state.wallTooShort { return withCoaching(Instruction(title: ScanCopy.wallTooShort, detail: ScanCopy.guidance(state.guidance).title)) } - return withCoaching(ScanCopy.guidance(state.guidance)) - } - - /// Coaching about how the photos come out (the capture gate's, and too little texture) rides - /// along with the task: the task's title and second line both stay (on an aim step the second - /// line is the only thing that says where to aim), the coaching adds its own short line under - /// them, and its symbol marks the card (`tone`). Replacing the whole card hid the task each - /// time the coaching came up (#80), and the dark coaching can stay up for a whole night walk. - private func withCoaching(_ task: Instruction) -> Instruction { - guard let coaching else { return task } - let detail = [task.detail, ScanCopy.coachingNote(coaching)].compactMap { $0 }.joined(separator: "\n") - return Instruction(title: task.title, detail: detail) - } - - /// Tracking problems and standing past the end replace the task: nothing the task asks for - /// counts until they clear. - private static func replacesTask(_ coaching: Coaching) -> Bool { - switch coaching { - case .initializing, .relocalizing, .trackingLost, .pastWallEnd: true - case .slowDown, .needsTexture, .tooDark, .tooDarkToMeasure, .holdSteady, .turnSlowly: false + if let coaching, ScanCopy.coachingReplacesTask(coaching) { return ScanCopy.coaching(coaching) } + // Coaching about how the photos come out (the capture gate's, and too little texture) + // rides along with the task (`ScanCopy.withCoaching`), and its symbol marks the card (`tone`). + if state.wallTooShort { + return ScanCopy.withCoaching(Instruction(title: ScanCopy.wallTooShort, detail: ScanCopy.guidance(state.guidance).title), coaching) } + return ScanCopy.withCoaching(ScanCopy.guidance(state.guidance), coaching) } /// "Slow down", "Turn more slowly" and "Hold steady" are for walking and aiming. With the tray @@ -123,7 +107,7 @@ struct WallWalkScreen: View { private var tone: InstructionCard.Tone { if state.marking?.refusal != nil { return .refusal } let coachingShows = state.marking == nil && state.endQuestion == nil && !state.overheadQuestion - if coachingShows, let coaching, Self.replacesTask(coaching) { + if coachingShows, let coaching, ScanCopy.coachingReplacesTask(coaching) { return .coaching(symbol: ScanCopy.coachingSymbol(coaching)) } if state.marking == nil, case .markNextWall(_, _?) = state.guidance { return .refusal } @@ -297,7 +281,7 @@ struct WallWalkScreen: View { /// place, as does being past an end. private var coachingHidesReply: Bool { switch state.coaching { - case nil, .slowDown?, .needsTexture?, .tooDark?, .holdSteady?: false + case nil, .slowDown?, .needsTexture?, .tooDark?, .tooDarkToMeasure?, .holdSteady?, .turnSlowly?: false case .initializing?, .relocalizing?, .trackingLost?, .pastWallEnd?: true } } diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift b/ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift index e88988ba0..5bc135fd4 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift @@ -60,6 +60,13 @@ public struct CoachingDebouncer: Sendable { public var motionShowAfter: Double = 0.7 /// Blur shows only after 2 s of it. The gate's blur test compares a frame with the recent /// median, and a tilt changes the view enough to trip it with no real blur. + /// + /// In practice this turns blur coaching off. The median is over the last 15 measured + /// frames (`AutoCaptureConfig.sharpnessWindow`), so after about 8 blurry frames in a row + /// the median is itself blurry and the next frame passes, ending the spell: at about ten + /// measured frames a second an unbroken spell lasts under a second. Steady blur (shaky + /// hands at night, a long exposure) never says "Hold steady". Whether blur should be judged + /// by its share of a window instead, as darkness is, is for the device run. public var blurShowAfter: Double = 2 /// A motion or blur problem that shows clears 0.5 s after the last frame that had it. public var clearAfter: Double = 0.5 @@ -198,3 +205,17 @@ public struct CoachingDebouncer: Sendable { } } } + +extension GapPlan.Need { + /// Whether the request asks the homeowner to walk rather than stand and aim. Only a walk past + /// the stretch far enough out does ("Walk along this stretch... Follow the dotted line"), so + /// on it the capture gate's coaching says "Slow down" as on the walk; on the others walking + /// speed isn't coached (`CoachingDebouncer.update`'s `aiming`). No `default`, so a new need + /// has to pick a side. + public var asksToWalk: Bool { + switch self { + case .walkOut: true + case .cells, .groundOut, .overhead, .wallUp: false + } + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift index 9374feb34..24189fc0f 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift @@ -103,6 +103,20 @@ import Testing #expect(Self.run(walking).last! == .movingFast) } + /// A gap request to walk the stretch far enough out asks for walking, so hurrying along it is + /// still told to slow down. The other requests ask the homeowner to stand and aim. + @Test func aWalkOutRequestStillSaysSlowDown() { + #expect(GapPlan.Need.walkOut(1.5).asksToWalk) + let standing: [GapPlan.Need] = [.cells, .groundOut(1.5), .overhead(nil), .overhead(2), .wallUp(2)] + for need in standing { + #expect(!need.asksToWalk, "\(need)") + } + let walkOut = Self.times(for: 3).map { Frame(time: $0, skip: .movingFast, aiming: !GapPlan.Need.walkOut(1.5).asksToWalk) } + #expect(Self.run(walkOut).last! == .movingFast) + let groundOut = Self.times(for: 3).map { Frame(time: $0, skip: .movingFast, aiming: !GapPlan.Need.groundOut(1.5).asksToWalk) } + #expect(Self.run(groundOut).allSatisfy { $0 == nil }) + } + /// Turning fast is its own problem, said while aiming too. @Test func turningFastShowsAfterItsSpell() { let frames = Self.times(for: 2).map { Frame(time: $0, skip: .turningFast, aiming: true) } diff --git a/ios/HouseScanUITests/ScreenStatesUITests.swift b/ios/HouseScanUITests/ScreenStatesUITests.swift index 2220fd743..35360b9bc 100644 --- a/ios/HouseScanUITests/ScreenStatesUITests.swift +++ b/ios/HouseScanUITests/ScreenStatesUITests.swift @@ -40,6 +40,7 @@ final class ScreenStatesUITests: XCTestCase { ("gapRequest", ["-uiDemoPhase", "gapRequest"], "gapRequest"), ("gapRequest-groundOut", ["-uiDemoPhase", "gapRequest", "-uiDemoGap", "groundOut"], "gapRequest"), ("gapRequest-walkOut", ["-uiDemoPhase", "gapRequest", "-uiDemoGap", "walkOut"], "gapRequest"), + ("gapRequest-tooDark", ["-uiDemoPhase", "gapRequest", "-uiDemoCoaching", "tooDark"], "gapRequest"), ("gapRequest-overhead", ["-uiDemoPhase", "gapRequest", "-uiDemoGap", "overhead"], "gapRequest"), ("gapRequest-overheadQuestion", ["-uiDemoPhase", "gapRequest", "-uiDemoGap", "overhead", "-uiDemoOverheadQuestion"], "gapRequest"), ("gapRequest-followUp", ["-uiDemoPhase", "gapRequest", "-uiDemoFollowUp"], "gapRequest"), @@ -76,6 +77,7 @@ final class ScreenStatesUITests: XCTestCase { "wallWalk-slowDown": ("instruction", "Walk slowly to your right"), "wallWalk-tooDark": ("instruction", "It's dark here"), "wallWalk-turnSlowly": ("instruction", "Turn more slowly"), + "gapRequest-tooDark": ("instruction", "Show the ground"), "gapRequest-followUp": ("instruction", "One more view to finish"), "uploading-followUp": (nil, "One more view to finish"), "markFeatures-groundQuestion": (nil, "What's on the ground along this wall?"), @@ -184,9 +186,29 @@ final class ScreenStatesUITests: XCTestCase { XCTAssertTrue(element(app, "screen.wallWalk").waitForExistence(timeout: 15)) let card = element(app, "instruction") XCTAssertTrue(card.waitForExistence(timeout: 5)) - XCTAssertTrue(card.label.contains("Walk slowly to your right"), "the task must stay on the card, got \(card.label)") - XCTAssertTrue(card.label.contains("Keep the wall and the ground in view"), "the task's second line must stay on the card, got \(card.label)") - XCTAssertTrue(card.label.contains("It's dark here"), "the coaching must show on the card, got \(card.label)") + // One snapshot read, never live (the #24 flake fix): a live read of a gone element + // records a failure that can't be caught. + let label = ElementRead.snapshot(card)?.label ?? "" + XCTAssertTrue(label.contains("Walk slowly to your right"), "the task must stay on the card, got \(label)") + XCTAssertTrue(label.contains("Keep the wall and the ground in view"), "the task's second line must stay on the card, got \(label)") + XCTAssertTrue(label.contains("It's dark here"), "the coaching must show on the card, got \(label)") + } + + /// #80, as on the walk: the capture gate's coaching keeps a gap request on the card. The dark + /// coaching can stay up for a whole night request (field test 4.1, run 3). + @MainActor + func testGateCoachingKeepsTheGapRequestOnTheCard() throws { + continueAfterFailure = false + let app = XCUIApplication() + app.launchArguments = ["-uiDemo", "-uiDemoFreeze", "-uiDemoPhase", "gapRequest", "-uiDemoCoaching", "tooDark"] + app.launch() + XCTAssertTrue(element(app, "screen.gapRequest").waitForExistence(timeout: 15)) + let card = element(app, "instruction") + XCTAssertTrue(card.waitForExistence(timeout: 5)) + let label = ElementRead.snapshot(card)?.label ?? "" + XCTAssertTrue(label.contains("Show the ground"), "the request must stay on the card, got \(label)") + XCTAssertTrue(label.contains("a clear look from two places"), "the request's second line must stay on the card, got \(label)") + XCTAssertTrue(label.contains("It's dark here"), "the coaching must show on the card, got \(label)") } /// B-09: "Add something" on the review opens the camera with the marking prompt, and the From 040d9a61cd8c6fd43784c3d7575c00fd991cf182 Mon Sep 17 00:00:00 2001 From: AidenJohnston Date: Sun, 27 Sep 2026 05:49:52 -0500 Subject: [PATCH 4/5] ios: aiming builds no walking-speed spell, and the dark note says what helps Moving fast while aiming now drops the walking-speed spell instead of keeping it hidden, so the next walking step can't open with "Slow down" on a calm frame. The ride-along dark coaching keeps its advice (flashlight, daylight), since it can stay up for a whole night walk. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/UI/Copy/ScanCopy.swift | 13 ++++++++++--- .../HouseScanKit/Guidance/CoachingDebouncer.swift | 8 ++++++-- .../HouseScanKitTests/CoachingDebouncerTests.swift | 13 +++++++++++++ ios/HouseScanUITests/ScreenStatesUITests.swift | 2 ++ 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/ios/HouseScan/UI/Copy/ScanCopy.swift b/ios/HouseScan/UI/Copy/ScanCopy.swift index 3b743681b..34a8edc85 100644 --- a/ios/HouseScan/UI/Copy/ScanCopy.swift +++ b/ios/HouseScan/UI/Copy/ScanCopy.swift @@ -108,10 +108,17 @@ enum ScanCopy { /// One short line for coaching that rides along with the task instead of replacing it (the /// walk's capture-gate coaching): the task's title and second line stay, and this goes under - /// them. Only the coaching's title, so the task's own words stay the bigger part of the card. + /// them. Mostly only the coaching's title, so the task's own words stay the bigger part of the + /// card. The dark coaching keeps what to do about it: it can stay up for a whole night walk, and + /// "It's dark here" alone doesn't say what would help. static func coachingNote(_ coaching: Coaching) -> String { - let title = ScanCopy.coaching(coaching).title - return title.hasSuffix(".") ? title : "\(title)." + switch coaching { + case .tooDark: return "It's dark here. Try your phone's flashlight, or come back in daylight." + case .tooDarkToMeasure: return "It's too dark to measure here. Try in daylight." + default: + let title = ScanCopy.coaching(coaching).title + return title.hasSuffix(".") ? title : "\(title)." + } } /// Tracking problems and standing past the end replace the task on the card: nothing the task diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift b/ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift index 5bc135fd4..eac8462ac 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift @@ -17,8 +17,9 @@ import Foundation /// - Moving fast, turning fast and blur each show after their own spell of frames that have /// them, and clear a moment after the last such frame. A spell that ends before it shows says /// nothing. -/// - Walking too fast is not raised while the homeowner is asked to stand and aim (`aiming`): -/// the motion there is turning the phone, and "walk slower" is the wrong advice. +/// - Walking too fast is not raised, or counted toward a spell, while the homeowner is asked to +/// stand and aim (`aiming`): the motion there is turning the phone, and "walk slower" is the +/// wrong advice. /// /// Every threshold is a guess to try on a phone at night, not measured. public struct CoachingDebouncer: Sendable { @@ -118,6 +119,9 @@ public struct CoachingDebouncer: Sendable { lastTime = time updateDarkness(time: time, meanLuma: meanLuma) updateMotion(time: time, skip: skip) + // Walking speed isn't coached while aiming, so a spell there mustn't build up either: it + // would say "Slow down" on the first frames of the next walking step, however calm. + if aiming { spells[.movingFast] = nil } if darkSince != nil { return persistentDark == .showing ? .persistentlyDark : .tooDark } if isShown(.turningFast, at: time) { return .turningFast } if !aiming, isShown(.movingFast, at: time) { return .movingFast } diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift index 24189fc0f..d383f01dd 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift @@ -103,6 +103,19 @@ import Testing #expect(Self.run(walking).last! == .movingFast) } + /// Moving fast while aiming builds no spell, so the next walking step starts clean: a calm + /// frame says nothing, and walking fast again needs its own 0.7 s before "Slow down". + @Test func movingFastWhileAimingDoesntCarryIntoTheNextStep() { + var debouncer = CoachingDebouncer() + let aiming = Self.times(for: 2).map { Frame(time: $0, skip: .movingFast, aiming: true) } + #expect(Self.run(aiming, on: &debouncer).allSatisfy { $0 == nil }) + #expect(debouncer.update(time: 2, skip: .redundant, meanLuma: 128, aiming: false) == nil) + let walking = Self.times(from: 2.1, for: 1).map { Frame(time: $0, skip: .movingFast) } + let said = Self.run(walking, on: &debouncer) + #expect(said.prefix(7).allSatisfy { $0 == nil }) + #expect(said.last! == .movingFast) + } + /// A gap request to walk the stretch far enough out asks for walking, so hurrying along it is /// still told to slow down. The other requests ask the homeowner to stand and aim. @Test func aWalkOutRequestStillSaysSlowDown() { diff --git a/ios/HouseScanUITests/ScreenStatesUITests.swift b/ios/HouseScanUITests/ScreenStatesUITests.swift index c5f29498d..7f84a8add 100644 --- a/ios/HouseScanUITests/ScreenStatesUITests.swift +++ b/ios/HouseScanUITests/ScreenStatesUITests.swift @@ -228,6 +228,7 @@ final class ScreenStatesUITests: XCTestCase { XCTAssertTrue(label.contains("Walk slowly to your right"), "the task must stay on the card, got \(label)") XCTAssertTrue(label.contains("Keep the wall and the ground in view"), "the task's second line must stay on the card, got \(label)") XCTAssertTrue(label.contains("It's dark here"), "the coaching must show on the card, got \(label)") + XCTAssertTrue(label.contains("flashlight"), "the dark coaching must say what would help, got \(label)") } /// #80, as on the walk: the capture gate's coaching keeps a gap request on the card. The dark @@ -245,6 +246,7 @@ final class ScreenStatesUITests: XCTestCase { XCTAssertTrue(label.contains("Show the ground"), "the request must stay on the card, got \(label)") XCTAssertTrue(label.contains("a clear look from two places"), "the request's second line must stay on the card, got \(label)") XCTAssertTrue(label.contains("It's dark here"), "the coaching must show on the card, got \(label)") + XCTAssertTrue(label.contains("flashlight"), "the dark coaching must say what would help, got \(label)") } /// B-09: "Add something" on the review opens the camera with the marking prompt, and the From a097c6692411ac615e9774fdbe43935f12314745 Mon Sep 17 00:00:00 2001 From: AidenJohnston Date: Sun, 27 Sep 2026 08:51:03 -0500 Subject: [PATCH 5/5] ios: steady blur is coached, judged against the sharpness before it The gate compares each frame with the median of its last 15, so steady blur stops reading as blurry within a second, and with blur shown only after 2 s its coaching never came up. The debouncer now also counts a frame as blurry when it is under the gate's ratio of the median sharpness from the 1.5 s before the spell began, held fixed while the spell lasts. A short drop, such as a tilt, still says nothing, and steady blur shows from 2 s in until the photos are sharp again. A package test runs the real gate into the debouncer. The dark walk card is also checked at AX5. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/Runtime/ScanEngine.swift | 6 +- .../Guidance/CoachingDebouncer.swift | 56 ++++++++++++++++--- .../CoachingDebouncerTests.swift | 42 +++++++++++++- .../ScreenStatesUITests.swift | 2 + 4 files changed, 93 insertions(+), 13 deletions(-) diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index 83031dea4..1b295096a 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -582,7 +582,7 @@ final class ScanEngine { keptSourceIDs.insert(frame.id) keep(frame) } - state.coaching = walkCoaching(tracking: frame.tracking, skip: skip, meanLuma: frame.quality?.meanLuma, pastEnd: pastEnd != nil, time: frame.timestamp) + state.coaching = walkCoaching(tracking: frame.tracking, skip: skip, meanLuma: frame.quality?.meanLuma, sharpness: frame.quality?.sharpness, pastEnd: pastEnd != nil, time: frame.timestamp) afterCoverageChange(camera: frame.camera, time: frame.timestamp) askOverheadIfTiltedUp(frame) } @@ -595,11 +595,11 @@ final class ScanEngine { /// (`pastEnd`) shows once it has lasted `showAfter` seconds, clears after `clearAfter` /// seconds without it, and comes before the gate's problems: no photo is kept there whatever /// the gate says. Both durations are guesses to try on a phone, not measured. - private func walkCoaching(tracking: TrackingQuality, skip: CaptureDecision.SkipReason?, meanLuma: Double?, pastEnd: Bool, time: Double) -> Coaching? { + private func walkCoaching(tracking: TrackingQuality, skip: CaptureDecision.SkipReason?, meanLuma: Double?, sharpness: Double?, pastEnd: Bool, time: Double) -> Coaching? { let showAfter = 0.7 let clearAfter = 0.5 let aiming = isAiming - let gate = gateCoaching.update(time: time, skip: skip, meanLuma: meanLuma, aiming: aiming) + let gate = gateCoaching.update(time: time, skip: skip, meanLuma: meanLuma, sharpness: sharpness, aiming: aiming) guard tracking == .normal else { gateProblem = nil gateClearSince = nil diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift b/ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift index eac8462ac..be84e1478 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift @@ -17,6 +17,10 @@ import Foundation /// - Moving fast, turning fast and blur each show after their own spell of frames that have /// them, and clear a moment after the last such frame. A spell that ends before it shows says /// nothing. +/// - Blur is also judged from the frames' own sharpness against the sharpness from just before +/// the blur began. The gate compares each frame with the median of its last 15, so it stops +/// calling steady blur blurry within a second; held against the earlier frames, steady blur +/// stays a spell until the photos are sharp again. /// - Walking too fast is not raised, or counted toward a spell, while the homeowner is asked to /// stand and aim (`aiming`): the motion there is turning the phone, and "walk slower" is the /// wrong advice. @@ -62,13 +66,20 @@ public struct CoachingDebouncer: Sendable { /// Blur shows only after 2 s of it. The gate's blur test compares a frame with the recent /// median, and a tilt changes the view enough to trip it with no real blur. /// - /// In practice this turns blur coaching off. The median is over the last 15 measured + /// The gate alone can't make a spell that long: its median is over the last 15 measured /// frames (`AutoCaptureConfig.sharpnessWindow`), so after about 8 blurry frames in a row - /// the median is itself blurry and the next frame passes, ending the spell: at about ten - /// measured frames a second an unbroken spell lasts under a second. Steady blur (shaky - /// hands at night, a long exposure) never says "Hold steady". Whether blur should be judged - /// by its share of a window instead, as darkness is, is for the device run. + /// the median is itself blurry and the next frame passes. So a frame also counts as + /// blurry when it is under `blurRatio` of `blurReference`, the sharpness from before the + /// spell began, and steady blur (shaky hands at night, a long exposure) keeps its spell. public var blurShowAfter: Double = 2 + /// A frame under this share of the sharpness from before the blur counts as blurry: the + /// gate's own `AutoCaptureConfig.sharpnessRatio`. + public var blurRatio: Double = AutoCaptureConfig().sharpnessRatio + /// The sharpness from before the blur is the median of the measured frames of the 1.5 s + /// before its spell began, about the gate's 15 frames, and needs `minDarkSamples` of them. + /// It stays fixed while the spell lasts, so it doesn't catch up with the blur as the + /// gate's median does. + public var blurReferenceWindow: Double = 1.5 /// A motion or blur problem that shows clears 0.5 s after the last frame that had it. public var clearAfter: Double = 0.5 @@ -93,6 +104,11 @@ public struct CoachingDebouncer: Sendable { private var spells: [GateProblem: Spell] = [:] /// Measured frames of the last `persistentDarkAfter` seconds: time and whether it was dark. private var luma: [(time: Double, dark: Bool)] = [] + /// Measured frames of the last `blurReferenceWindow` seconds: time and sharpness. + private var sharpness: [(time: Double, value: Double)] = [] + /// The median sharpness from before the current blur spell (`Config.blurReferenceWindow`), + /// nil while there is no spell. + private var blurReference: Double? /// When the dark coaching went up, nil while it is down. private var darkSince: Double? /// Since when the light has been back, while the dark coaching is up. @@ -112,13 +128,15 @@ public struct CoachingDebouncer: Sendable { /// restarted) starts over. /// - `skip`: the gate's reason for skipping the frame, nil when the gate passed it. /// - `meanLuma`: the frame's own mean luma, nil when it wasn't measured. + /// - `sharpness`: the frame's own sharpness (`FrameQuality.sharpness`), nil when it wasn't + /// measured. /// - `aiming`: the homeowner is asked to stand and aim (an aim, tilt, step-back, see-behind /// or marking step), so walking too fast is not raised. - public mutating func update(time: Double, skip: CaptureDecision.SkipReason?, meanLuma: Double?, aiming: Bool) -> GateProblem? { + public mutating func update(time: Double, skip: CaptureDecision.SkipReason?, meanLuma: Double?, sharpness: Double? = nil, aiming: Bool) -> GateProblem? { if let lastTime, time < lastTime { self = CoachingDebouncer(config: config) } lastTime = time updateDarkness(time: time, meanLuma: meanLuma) - updateMotion(time: time, skip: skip) + updateMotion(time: time, skip: skip, sharpness: sharpness) // Walking speed isn't coached while aiming, so a spell there mustn't build up either: it // would say "Slow down" on the first frames of the next walking step, however calm. if aiming { spells[.movingFast] = nil } @@ -133,6 +151,8 @@ public struct CoachingDebouncer: Sendable { /// the light hasn't changed because the phone lost its place. public mutating func forgetMotion() { spells = [:] + sharpness = [] + blurReference = nil } // MARK: Darkness @@ -172,8 +192,14 @@ public struct CoachingDebouncer: Sendable { // MARK: Motion and blur - private mutating func updateMotion(time: Double, skip: CaptureDecision.SkipReason?) { - for (problem, seen) in Self.evidence(skip) { + private mutating func updateMotion(time: Double, skip: CaptureDecision.SkipReason?, sharpness value: Double?) { + var evidence = Self.evidence(skip) + // The sharpness from before any blur: fixed while a spell lasts, else the recent median. + let reference = blurReference ?? Self.median(sharpness.map(\.value), atLeast: config.minDarkSamples) + if skip != .trackingNotReady, let value, let reference, value < reference * config.blurRatio { + evidence[.blurry] = true + } + for (problem, seen) in evidence { if seen { spells[problem] = Spell(since: spells[problem]?.since ?? time, lastSeen: time) } else if let spell = spells[problem], !reachedShow(problem, spell) { @@ -182,6 +208,18 @@ public struct CoachingDebouncer: Sendable { } } spells = spells.filter { time - $0.value.lastSeen < config.clearAfter } + blurReference = spells[.blurry] == nil ? nil : reference + if let value { + sharpness.append((time: time, value: value)) + if let firstKept = sharpness.firstIndex(where: { $0.time > time - config.blurReferenceWindow }), firstKept > 0 { + sharpness.removeFirst(firstKept) + } + } + } + + private static func median(_ values: [Double], atLeast count: Int) -> Double? { + guard values.count >= count else { return nil } + return values.sorted()[values.count / 2] } private func isShown(_ problem: GateProblem, at time: Double) -> Bool { diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift index d383f01dd..209e08c87 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift @@ -12,11 +12,12 @@ import Testing var skip: CaptureDecision.SkipReason? var luma: Double? = 128 var aiming = false + var sharpness: Double? = nil } /// What the debouncer says for each frame. static func run(_ frames: [Frame], on debouncer: inout CoachingDebouncer) -> [Problem?] { - frames.map { debouncer.update(time: $0.time, skip: $0.skip, meanLuma: $0.luma, aiming: $0.aiming) } + frames.map { debouncer.update(time: $0.time, skip: $0.skip, meanLuma: $0.luma, sharpness: $0.sharpness, aiming: $0.aiming) } } static func run(_ frames: [Frame]) -> [Problem?] { @@ -147,6 +148,45 @@ import Testing #expect(Self.run(long).last! == .blurry) } + /// The real gate feeding the debouncer, frame by frame: 3 s of sharp frames, 4 s at under a + /// third of that sharpness (shaky hands, a long exposure), then sharp again. The gate calls + /// only the first second of the blur blurry, because its 15-frame median catches up; the + /// coaching still says blur from 2 s in until the photos are sharp again. + @Test func sustainedBlurThroughTheGateIsCoached() { + var gate = AutoCapture() + var debouncer = CoachingDebouncer() + var skips: [CaptureDecision.SkipReason?] = [] + var said: [Problem?] = [] + for t in Self.times(for: 8) { + let sharpness: Double = (3..<7).contains(t) ? 30 : 100 + let frame = AutoCaptureTests.frame(t, quality: AutoCaptureTests.good(sharpness: sharpness)) + let decision = gate.evaluate(frame, newlySeenCells: 0) + var skip: CaptureDecision.SkipReason? + switch decision { + case .keep: gate.didKeep(frame) + case .skip(let reason): skip = reason + } + skips.append(skip) + said.append(debouncer.update(time: t, skip: skip, meanLuma: 128, sharpness: sharpness, aiming: true)) + } + // The gate adapts: by 4.5 s it no longer calls the blurry frames blurry. + #expect(skips[30] == .blurry) + #expect(skips[45...69].allSatisfy { $0 != .blurry }) + // The coaching says nothing for the first 2 s of blur, then blur until it's sharp again. + #expect(said[..<50].allSatisfy { $0 == nil }) + #expect(said[50..<70].allSatisfy { $0 == .blurry }) + #expect(said.last! == nil) + } + + /// A tilt that makes the view much less sharp for under 2 s says nothing, also when judged by + /// sharpness against the frames before it. + @Test func shortSharpnessDropSaysNothing() { + let before = Self.times(for: 2).map { Frame(time: $0, skip: .redundant, sharpness: 100) } + let tilt = Self.times(from: 2, for: 1.8).map { Frame(time: $0, skip: .redundant, sharpness: 30) } + let after = Self.times(from: 3.8, for: 2).map { Frame(time: $0, skip: .redundant, sharpness: 100) } + #expect(Self.run(before + tilt + after).allSatisfy { $0 == nil }) + } + /// A shown problem clears half a second after its last frame. @Test func motionClearsAfterItsLastFrame() { let fast = Self.times(for: 1).map { Frame(time: $0, skip: .movingFast) } diff --git a/ios/HouseScanUITests/ScreenStatesUITests.swift b/ios/HouseScanUITests/ScreenStatesUITests.swift index 7f84a8add..3b7bfdef8 100644 --- a/ios/HouseScanUITests/ScreenStatesUITests.swift +++ b/ios/HouseScanUITests/ScreenStatesUITests.swift @@ -75,6 +75,8 @@ final class ScreenStatesUITests: XCTestCase { "wallWalk-hidden", "wallWalk-seeBehind", "gapRequest-followUp", "uploading-followUp", "markFeatures-groundQuestion", "markFeatures-groundAnswered", "markFeatures-lostPlace", "spotConfirm", "spotConfirm-answered", + // The longest walk card: the task, its second line and the dark coaching's advice (#80). + "wallWalk-tooDark", ] /// Words a state must show: in the named element's label or value, or with no identifier,