diff --git a/ios/HouseScan/Contract/ScanContract.swift b/ios/HouseScan/Contract/ScanContract.swift index 6df2b8294..226ec6eea 100644 --- a/ios/HouseScan/Contract/ScanContract.swift +++ b/ios/HouseScan/Contract/ScanContract.swift @@ -317,13 +317,21 @@ enum GuidanceStep: Equatable, Sendable { case seeBehind(s: Float) } -/// A problem that overrides guidance until it clears. UI/ owns the words. +/// A problem to fix while it lasts. Tracking problems and `pastWallEnd` override guidance; the +/// capture gate's (`slowDown`, `turnSlowly`, `holdSteady`, `tooDark`, `tooDarkToMeasure`) and +/// `needsTexture` ride along with it on the walk. UI/ owns the words. enum Coaching: Equatable, Sendable { case initializing case slowDown case needsTexture case tooDark + /// Frames have been mostly too dark for a long while: waiting won't help, daylight will + /// (`CoachingDebouncer.GateProblem.persistentlyDark`). + case tooDarkToMeasure case holdSteady + /// Turning or tilting the phone faster than the capture gate keeps photos at, as opposed to + /// walking too fast (#26). + case turnSlowly case relocalizing case trackingLost /// The phone stands past an end it can't see back from, so the walk keeps no photos: nothing diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index 60013075d..a46d7e840 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -31,9 +31,11 @@ final class ScanEngine { // Stored evidence private(set) var store: KeyframeStore private var keptSourceIDs: Set = [] - /// Debounces coaching that comes from the capture gate (see `gateCoaching`). + /// Debounces "past the end of the wall" during the walk (see `walkCoaching`). private var gateProblem: (coaching: Coaching, since: Double)? private var gateClearSince: Double? + /// Debounces the capture gate's own coaching during the walk (see `walkCoaching`). + private var gateCoaching = CoachingDebouncer() private var closeUpPending = false /// Why the last close-up has to be retaken (from the meter-number reader, or "None of /// these"), and when that was said, in screen seconds. Shown until the next shot fires. @@ -596,33 +598,32 @@ final class ScanEngine { keptSourceIDs.insert(frame.id) keep(frame) } - state.coaching = walkCoaching(tracking: frame.tracking, skip: skip, pastEnd: pastEnd != nil, time: frame.timestamp) + state.coaching = walkCoaching(tracking: frame.tracking, skip: skip, meanLuma: frame.quality?.meanLuma, sharpness: frame.quality?.sharpness, pastEnd: pastEnd != nil, time: frame.timestamp) afterCoverageChange(camera: frame.camera, time: frame.timestamp) askOverheadIfTiltedUp(frame) } - /// Tracking problems show at once. A problem the capture gate reports (moving, blurry, too - /// dark) shows only once it has lasted `showAfter` seconds and clears after `clearAfter` - /// seconds of frames without it: the gate judges every frame, and one fast frame at 30 fps - /// would otherwise flash a prompt for a single frame. During the walk moving and blurry read - /// as "Slow down", never "Hold steady", which would tell a walking homeowner to stop. Both - /// durations are guesses to try on a phone, not measured. Standing past an end the phone - /// can't see back from (`pastEnd`) is debounced the same way and comes before the gate's - /// reasons: no photo is kept there whatever the gate says. - private func walkCoaching(tracking: TrackingQuality, skip: CaptureDecision.SkipReason?, pastEnd: Bool, time: Double) -> Coaching? { + /// Tracking problems show at once. The capture gate's problems go through `gateCoaching` + /// (`CoachingDebouncer`): each has its own clock, darkness is judged from the frames' luma + /// with hysteresis, and "Slow down" is kept for walking, so while the homeowner stands and + /// aims (`isAiming`) only turning and blur are said. Blur reads as "Slow down" while walking + /// and "Hold steady" while aiming. Standing past an end the phone can't see back from + /// (`pastEnd`) shows once it has lasted `showAfter` seconds, clears after `clearAfter` + /// seconds without it, and comes before the gate's problems: no photo is kept there whatever + /// the gate says. Both durations are guesses to try on a phone, not measured. + private func walkCoaching(tracking: TrackingQuality, skip: CaptureDecision.SkipReason?, meanLuma: Double?, sharpness: Double?, pastEnd: Bool, time: Double) -> Coaching? { let showAfter = 0.7 let clearAfter = 0.5 + let aiming = isAiming + let gate = gateCoaching.update(time: time, skip: skip, meanLuma: meanLuma, sharpness: sharpness, aiming: aiming) guard tracking == .normal else { gateProblem = nil gateClearSince = nil + // The light is what it was, but the motion from before the phone lost its place is not. + gateCoaching.forgetMotion() return coaching(for: tracking, skip: nil) } - var candidate: Coaching? = switch skip { - case .moving?, .blurry?: .slowDown - case .tooDark?: .tooDark - default: nil - } - if pastEnd { candidate = .pastWallEnd } + let candidate: Coaching? = pastEnd ? .pastWallEnd : nil if let problem = gateProblem, time < problem.since { gateProblem = nil } // replay restarted if let candidate { gateClearSince = nil @@ -637,8 +638,36 @@ final class ScanEngine { gateClearSince = nil } } - guard let problem = gateProblem, time - problem.since >= showAfter else { return nil } - return problem.coaching + if let problem = gateProblem, time - problem.since >= showAfter { return problem.coaching } + return gate.map { Self.coaching(for: $0, aiming: aiming) } + } + + /// Whether the homeowner is asked to stand and aim rather than walk: an aim, tilt, step-back, + /// see-behind or mark-the-end step, marking, a question on screen, or a gap request's view. + /// Walking too fast is not coached then (#26). Two steps ask for walking and so still say + /// "Slow down": the corner ("walk round it, aim at the next wall and mark it"; the mark + /// itself is `state.marking`, which aims) and a request to walk the stretch far enough out + /// (`GapPlan.Need.asksToWalk`). Read before this frame's guidance update, so it is the step + /// on screen when the frame arrived. + private var isAiming: Bool { + if state.marking != nil || state.endQuestion != nil || state.overheadQuestion { return true } + switch state.guidance { + case .aimAtGround, .aimAtWall, .tiltUp, .stepBack, .seeBehind, .markEnd: return true + case .gap: return !(gapPlan?.need.asksToWalk ?? false) + case .findMeter, .aimAtWallForMeter, .holdOnMeter, .walk, .walkComplete, .markNextWall: return false + } + } + + /// The coaching for a gate problem. Blur reads as "Slow down" while walking and "Hold + /// steady" while aiming, where "Slow down" would tell someone standing still to walk slower. + static func coaching(for problem: CoachingDebouncer.GateProblem, aiming: Bool) -> Coaching { + switch problem { + case .tooDark: .tooDark + case .persistentlyDark: .tooDarkToMeasure + case .movingFast: .slowDown + case .turningFast: .turnSlowly + case .blurry: aiming ? .holdSteady : .slowDown + } } private func afterCoverageChange(camera: CameraFrame?, time: Double) { @@ -1182,6 +1211,7 @@ final class ScanEngine { state.lastCapture = nil gateProblem = nil gateClearSince = nil + gateCoaching = CoachingDebouncer() autoCapture.reset() planner.reset() go(.findMeter) @@ -1644,6 +1674,7 @@ final class ScanEngine { resetPacketLog() keptSourceIDs = [] autoCapture.reset() + gateCoaching = CoachingDebouncer() planner.reset() closeUpGate = CloseUpGate() gapPlan = nil @@ -1845,7 +1876,8 @@ extension ScanEngine { case .limited(.unknown): return .trackingLost case .normal: switch skip { - case .moving?, .blurry?: return .holdSteady + case .movingFast?, .blurry?: return .holdSteady + case .turningFast?: return .turnSlowly case .tooDark?: return .tooDark default: return nil } diff --git a/ios/HouseScan/UI/Copy/ScanCopy.swift b/ios/HouseScan/UI/Copy/ScanCopy.swift index c0684ca70..fa72a7b12 100644 --- a/ios/HouseScan/UI/Copy/ScanCopy.swift +++ b/ios/HouseScan/UI/Copy/ScanCopy.swift @@ -76,9 +76,13 @@ enum ScanCopy { case .needsTexture: Instruction(title: "Aim at a corner or somewhere with more texture", detail: "A plain wall or the sky gives your phone nothing to follow.") case .tooDark: - Instruction(title: "It's too dark to see the wall", detail: "Turn on a porch light, or try again in daylight.") + Instruction(title: "It's dark here", detail: "Some photos won't count. Try your phone's flashlight, or come back in daylight.") + case .tooDarkToMeasure: + Instruction(title: "It's too dark to measure here", detail: "Try in daylight.") case .holdSteady: Instruction(title: "Hold steady", detail: nil) + case .turnSlowly: + Instruction(title: "Turn more slowly", detail: "Photos taken while turning come out blurred.") case .relocalizing: Instruction(title: "Point at the meter like this.", detail: "Your phone lost its place for a moment.") case .trackingLost: @@ -94,13 +98,49 @@ enum ScanCopy { case .initializing: "iphone.gen3.radiowaves.left.and.right" case .slowDown: "tortoise.fill" case .needsTexture: "square.grid.3x3.middle.filled" - case .tooDark: "moon.fill" + case .tooDark, .tooDarkToMeasure: "moon.fill" case .holdSteady: "hand.raised.fill" + case .turnSlowly: "arrow.clockwise" case .relocalizing, .trackingLost: "location.slash.fill" case .pastWallEnd: "arrow.uturn.backward" } } + /// One short line for coaching that rides along with the task instead of replacing it (the + /// walk's capture-gate coaching): the task's title and second line stay, and this goes under + /// them. Mostly only the coaching's title, so the task's own words stay the bigger part of the + /// card. The dark coaching keeps what to do about it: it can stay up for a whole night walk, and + /// "It's dark here" alone doesn't say what would help. + static func coachingNote(_ coaching: Coaching) -> String { + switch coaching { + case .tooDark: return "It's dark here. Try your phone's flashlight, or come back in daylight." + case .tooDarkToMeasure: return "It's too dark to measure here. Try in daylight." + default: + let title = ScanCopy.coaching(coaching).title + return title.hasSuffix(".") ? title : "\(title)." + } + } + + /// Tracking problems and standing past the end replace the task on the card: nothing the task + /// asks for counts until they clear. The capture gate's coaching and too little texture ride + /// along with it instead (`withCoaching`). No `default`, so a new case has to pick a side. + static func coachingReplacesTask(_ coaching: Coaching) -> Bool { + switch coaching { + case .initializing, .relocalizing, .trackingLost, .pastWallEnd: true + case .slowDown, .needsTexture, .tooDark, .tooDarkToMeasure, .holdSteady, .turnSlowly: false + } + } + + /// The task with ride-along coaching under it: the task's title and second line both stay (on + /// an aim step the second line is the only thing that says where to aim), and the coaching adds + /// its own short line (`coachingNote`). Replacing the whole card hid the task each time the + /// coaching came up (#80), and the dark coaching can stay up for a whole night walk. + static func withCoaching(_ task: Instruction, _ coaching: Coaching?) -> Instruction { + guard let coaching else { return task } + let detail = [task.detail, coachingNote(coaching)].compactMap { $0 }.joined(separator: "\n") + return Instruction(title: task.title, detail: detail) + } + // MARK: Wall ends /// Under the wall map when ending the wall at the dashed line would cut off part of the walk. diff --git a/ios/HouseScan/UI/Preview/DemoEngine.swift b/ios/HouseScan/UI/Preview/DemoEngine.swift index 80fd8bfd5..9b5433a9f 100644 --- a/ios/HouseScan/UI/Preview/DemoEngine.swift +++ b/ios/HouseScan/UI/Preview/DemoEngine.swift @@ -976,7 +976,9 @@ final class DemoEngine: ScanActions { case "slowDown": .slowDown case "needsTexture": .needsTexture case "tooDark": .tooDark + case "tooDarkToMeasure": .tooDarkToMeasure case "holdSteady": .holdSteady + case "turnSlowly": .turnSlowly case "relocalizing": .relocalizing case "trackingLost": .trackingLost case "pastWallEnd": .pastWallEnd diff --git a/ios/HouseScan/UI/Preview/UIDemo.swift b/ios/HouseScan/UI/Preview/UIDemo.swift index b541f6422..34977cb3f 100644 --- a/ios/HouseScan/UI/Preview/UIDemo.swift +++ b/ios/HouseScan/UI/Preview/UIDemo.swift @@ -7,7 +7,7 @@ import SwiftUI /// - `-uiDemoFreeze`: don't run the timed scripts, so the screen holds still. /// - `-uiDemoMarking `: open the walk in marking mode. /// - `-uiDemoRefusal`: the marking shows a refusal. -/// - `-uiDemoCoaching `. +/// - `-uiDemoCoaching `. /// - `-uiDemoCloseUpFailed`: the close-up has failed twice, so the way out shows. /// - `-uiDemoMeterChoose`: the close-up asks which of three made-up readings is the meter number. /// - `-uiDemoOffline`: uploads fail offline. diff --git a/ios/HouseScan/UI/Screens/GapRequestScreen.swift b/ios/HouseScan/UI/Screens/GapRequestScreen.swift index eaf00f2e4..1bcd01ed2 100644 --- a/ios/HouseScan/UI/Screens/GapRequestScreen.swift +++ b/ios/HouseScan/UI/Screens/GapRequestScreen.swift @@ -26,7 +26,7 @@ struct GapRequestScreen: View { } CameraChrome( instruction: instruction, - tone: asking ? .normal : state.coaching.map { .coaching(symbol: ScanCopy.coachingSymbol($0)) } ?? .normal, + tone: asking ? .normal : coaching.map { .coaching(symbol: ScanCopy.coachingSymbol($0)) } ?? .normal, reply: state.gap?.isSatisfied == true || asking ? nil : InstructionCard.Reply( title: "I can't get there", identifier: "action.skipGap", @@ -77,14 +77,27 @@ struct GapRequestScreen: View { return max(1, result.missing.filter(\.capturable).count) } + /// As on the walk: tracking problems and standing past an end replace the request, and the + /// capture gate's coaching rides along under it (`ScanCopy.withCoaching`). The dark coaching + /// can stay up for a whole night request, and replacing the card hid the request and "Got it, + /// thanks" for as long as it did (field test 4.1, run 3). private var instruction: Instruction { if asking { return ScanCopy.overheadQuestion } - if let coaching = state.coaching { return ScanCopy.coaching(coaching) } - guard let gap = state.gap else { return ScanCopy.guidance(.gap) } + if let coaching, ScanCopy.coachingReplacesTask(coaching) { return ScanCopy.coaching(coaching) } + guard let gap = state.gap else { return ScanCopy.withCoaching(ScanCopy.guidance(.gap), coaching) } if gap.isSatisfied { return Instruction(title: "Got it, thanks", detail: followUps > 0 ? "Updating your result." : "That's the view we needed.") } - return ScanCopy.gap(gap) + return ScanCopy.withCoaching(ScanCopy.gap(gap), coaching) + } + + /// The coaching on the card, marked with its symbol (`tone`). Once the view is in, the + /// capture gate's coaching has nothing left to ask for, so only coaching that replaces the + /// card still shows. + private var coaching: Coaching? { + guard let coaching = state.coaching else { return nil } + if state.gap?.isSatisfied == true, !ScanCopy.coachingReplacesTask(coaching) { return nil } + return coaching } } diff --git a/ios/HouseScan/UI/Screens/WallWalkScreen.swift b/ios/HouseScan/UI/Screens/WallWalkScreen.swift index aec821cb0..f8f13ed5c 100644 --- a/ios/HouseScan/UI/Screens/WallWalkScreen.swift +++ b/ios/HouseScan/UI/Screens/WallWalkScreen.swift @@ -4,8 +4,9 @@ import SwiftUI /// /// Over the camera: haze on what the phone hasn't seen, a blue dotted path on the ground, a /// ring on the next thing to aim at, pins on what's been marked. At the bottom: the tape map -/// and at most two actions. At the top: one instruction, replaced by coaching while there is a -/// problem, or by the marking prompt while marking. +/// and at most two actions. At the top: one instruction, with coaching on a line of its own +/// under it while the photos have a problem, replaced while tracking has one, or by the marking +/// prompt while marking. struct WallWalkScreen: View { let state: ScanViewState let actions: any ScanActions @@ -81,26 +82,37 @@ struct WallWalkScreen: View { } if let side = state.endQuestion { return ScanCopy.endQuestion(side) } if state.overheadQuestion { return ScanCopy.overheadQuestion } - if let coaching { return ScanCopy.coaching(coaching) } - if state.wallTooShort { return Instruction(title: ScanCopy.wallTooShort, detail: ScanCopy.guidance(state.guidance).title) } - return ScanCopy.guidance(state.guidance) + if let coaching, ScanCopy.coachingReplacesTask(coaching) { return ScanCopy.coaching(coaching) } + // Coaching about how the photos come out (the capture gate's, and too little texture) + // rides along with the task (`ScanCopy.withCoaching`), and its symbol marks the card (`tone`). + if state.wallTooShort { + return ScanCopy.withCoaching(Instruction(title: ScanCopy.wallTooShort, detail: ScanCopy.guidance(state.guidance).title), coaching) + } + return ScanCopy.withCoaching(ScanCopy.guidance(state.guidance), coaching) } - /// "Slow down" is for walking. With the tray open the homeowner has stopped to pick a mark - /// and is only turning the phone, which the gate also reads as moving (field test run 1). - /// Tracking problems still show. + /// "Slow down", "Turn more slowly" and "Hold steady" are for walking and aiming. With the tray + /// open the homeowner has stopped to pick a mark and is only turning the phone, which the gate + /// also reads as moving or turning (field test run 1). Tracking problems still show. private var coaching: Coaching? { - if trayOpen, state.coaching == .slowDown { return nil } - return state.coaching + switch state.coaching { + case .slowDown?, .turnSlowly?, .holdSteady?: trayOpen ? nil : state.coaching + default: state.coaching + } } + /// Coaching that replaces the task marks the card with its symbol. A refusal on the task + /// (the next wall wasn't marked, the wall is too short) keeps its red triangle over coaching + /// that only rides along: the refusal's words are on the card, so its tone should match. private var tone: InstructionCard.Tone { if state.marking?.refusal != nil { return .refusal } - if state.marking == nil, state.endQuestion == nil, !state.overheadQuestion, let coaching { + let coachingShows = state.marking == nil && state.endQuestion == nil && !state.overheadQuestion + if coachingShows, let coaching, ScanCopy.coachingReplacesTask(coaching) { return .coaching(symbol: ScanCopy.coachingSymbol(coaching)) } if state.marking == nil, case .markNextWall(_, _?) = state.guidance { return .refusal } - if state.marking == nil, state.endQuestion == nil, !state.overheadQuestion, state.wallTooShort { return .refusal } + if coachingShows, state.wallTooShort { return .refusal } + if coachingShows, let coaching { return .coaching(symbol: ScanCopy.coachingSymbol(coaching)) } return .normal } @@ -269,7 +281,7 @@ struct WallWalkScreen: View { /// place, as does being past an end. private var coachingHidesReply: Bool { switch state.coaching { - case nil, .slowDown?, .needsTexture?, .tooDark?, .holdSteady?: false + case nil, .slowDown?, .needsTexture?, .tooDark?, .tooDarkToMeasure?, .holdSteady?, .turnSlowly?: false case .initializing?, .relocalizing?, .trackingLost?, .pastWallEnd?: true } } diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Capture/AutoCapture.swift b/ios/HouseScanKit/Sources/HouseScanKit/Capture/AutoCapture.swift index d16e9c9f6..81d4cb2bb 100644 --- a/ios/HouseScanKit/Sources/HouseScanKit/Capture/AutoCapture.swift +++ b/ios/HouseScanKit/Sources/HouseScanKit/Capture/AutoCapture.swift @@ -66,7 +66,12 @@ public enum CaptureDecision: Sendable, Equatable { public enum SkipReason: Sendable, Equatable { case trackingNotReady case tooSoon - case moving + /// The phone moved faster than `AutoCaptureConfig.maxSpeed`: walking too fast. + case movingFast + /// The phone turned faster than `AutoCaptureConfig.maxAngularSpeed` without moving too + /// fast: a tilt or a pan, often while standing still. Kept apart from `movingFast` so + /// coaching doesn't tell someone standing still to walk slower (#26). + case turningFast case blurry case tooDark case tooBright @@ -121,7 +126,8 @@ public struct AutoCapture: Sendable { let dt = Float(frame.timestamp - previous.timestamp) let speed = simd_distance(frame.camera.position, previous.camera.position) / dt let turn = frame.camera.rotationAngle(to: previous.camera) / dt - if speed > config.maxSpeed || turn > config.maxAngularSpeed { return .skip(.moving) } + if speed > config.maxSpeed { return .skip(.movingFast) } + if turn > config.maxAngularSpeed { return .skip(.turningFast) } } if let quality = frame.quality ?? lastQuality { if quality.meanLuma < config.minMeanLuma { return .skip(.tooDark) } diff --git a/ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift b/ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift new file mode 100644 index 000000000..be84e1478 --- /dev/null +++ b/ios/HouseScanKit/Sources/HouseScanKit/Guidance/CoachingDebouncer.swift @@ -0,0 +1,263 @@ +import Foundation + +/// Turns the capture gate's per-frame verdicts into coaching that stays up long enough to read. +/// +/// The gate judges about ten frames a second and gives one reason per frame: the first check +/// that fails (`AutoCapture.evaluate`). Shown as it comes, that reason flickers. In the 4.1 field +/// test (#80) a night scene's mean luma sat right on the gate's 40, so "too dark" came and went +/// every few seconds, and a tilt in the dark swapped it for "Slow down", because motion is +/// checked before exposure. Tilting and turning in place also read as walking too fast (#26). +/// +/// So each problem keeps its own clock, and a frame that reports another problem never restarts +/// a problem that is showing: +/// - Darkness is judged from the frames' own mean luma, not from the gate's reason, with +/// hysteresis: it shows once most recent frames are dark, and clears only once nearly none +/// are and the light has stayed up for a while. It comes first: while it is dark, the motion +/// problems don't take its place. +/// - Moving fast, turning fast and blur each show after their own spell of frames that have +/// them, and clear a moment after the last such frame. A spell that ends before it shows says +/// nothing. +/// - Blur is also judged from the frames' own sharpness against the sharpness from just before +/// the blur began. The gate compares each frame with the median of its last 15, so it stops +/// calling steady blur blurry within a second; held against the earlier frames, steady blur +/// stays a spell until the photos are sharp again. +/// - Walking too fast is not raised, or counted toward a spell, while the homeowner is asked to +/// stand and aim (`aiming`): the motion there is turning the phone, and "walk slower" is the +/// wrong advice. +/// +/// Every threshold is a guess to try on a phone at night, not measured. +public struct CoachingDebouncer: Sendable { + /// What the capture gate's coaching can say. + public enum GateProblem: Sendable, Equatable, Hashable { + /// Most recent frames are too dark to count. + case tooDark + /// Frames have been mostly dark for a long time. Said in place of `tooDark` until the + /// light comes back, and once per debouncer (one scan). + case persistentlyDark + /// Walking faster than the gate keeps photos at. + case movingFast + /// Turning or tilting the phone faster than the gate keeps photos at. + case turningFast + /// Frames much less sharp than the recent ones, for a while. + case blurry + } + + public struct Config: Sendable, Equatable { + /// A frame is dark below this mean luma: the gate's own `AutoCaptureConfig.minMeanLuma`. + public var darkLuma: Double = AutoCaptureConfig().minMeanLuma + /// Darkness is judged over the measured frames of the last 2 s. + public var darkWindow: Double = 2 + /// Fewer measured frames than this in the window (about half a second of them) say + /// nothing about darkness, so the first frames of a walk can't raise it alone. + public var minDarkSamples = 5 + /// Shows once more than half the measured frames in the window are dark. + public var showDarkShare: Double = 0.5 + /// Clears once fewer than a fifth of them are dark... + public var clearDarkShare: Double = 0.2 + /// ...and each measured frame is above 48, well clear of `darkLuma`... + public var clearLuma: Double = 48 + /// ...for 1.5 s in a row. + public var clearDarkAfter: Double = 1.5 + /// Dark shown for 20 s, with more than half the measured frames of those 20 s dark, + /// becomes `persistentlyDark`: waiting for the light won't help. + public var persistentDarkAfter: Double = 20 + /// Moving or turning fast shows after 0.7 s of it: one fast frame is a jolt, not a habit. + public var motionShowAfter: Double = 0.7 + /// Blur shows only after 2 s of it. The gate's blur test compares a frame with the recent + /// median, and a tilt changes the view enough to trip it with no real blur. + /// + /// The gate alone can't make a spell that long: its median is over the last 15 measured + /// frames (`AutoCaptureConfig.sharpnessWindow`), so after about 8 blurry frames in a row + /// the median is itself blurry and the next frame passes. So a frame also counts as + /// blurry when it is under `blurRatio` of `blurReference`, the sharpness from before the + /// spell began, and steady blur (shaky hands at night, a long exposure) keeps its spell. + public var blurShowAfter: Double = 2 + /// A frame under this share of the sharpness from before the blur counts as blurry: the + /// gate's own `AutoCaptureConfig.sharpnessRatio`. + public var blurRatio: Double = AutoCaptureConfig().sharpnessRatio + /// The sharpness from before the blur is the median of the measured frames of the 1.5 s + /// before its spell began, about the gate's 15 frames, and needs `minDarkSamples` of them. + /// It stays fixed while the spell lasts, so it doesn't catch up with the blur as the + /// gate's median does. + public var blurReferenceWindow: Double = 1.5 + /// A motion or blur problem that shows clears 0.5 s after the last frame that had it. + public var clearAfter: Double = 0.5 + + public init() {} + } + + /// One continuous run of frames with a motion or blur problem. + private struct Spell: Sendable, Equatable { + var since: Double + var lastSeen: Double + } + + private enum PersistentDark: Sendable, Equatable { + case notYet + case showing + /// Said, and the light came back since: not said again. + case said + } + + public let config: Config + private var lastTime: Double? + private var spells: [GateProblem: Spell] = [:] + /// Measured frames of the last `persistentDarkAfter` seconds: time and whether it was dark. + private var luma: [(time: Double, dark: Bool)] = [] + /// Measured frames of the last `blurReferenceWindow` seconds: time and sharpness. + private var sharpness: [(time: Double, value: Double)] = [] + /// The median sharpness from before the current blur spell (`Config.blurReferenceWindow`), + /// nil while there is no spell. + private var blurReference: Double? + /// When the dark coaching went up, nil while it is down. + private var darkSince: Double? + /// Since when the light has been back, while the dark coaching is up. + private var brightSince: Double? + private var persistentDark = PersistentDark.notYet + + public init(config: Config = Config()) { + self.config = config + } + + /// Whether the dark coaching (`tooDark` or `persistentlyDark`) is up. + public var isDark: Bool { darkSince != nil } + + /// Takes one judged frame and returns the problem to coach, if any. + /// + /// - `time`: the frame's timestamp in seconds. A time before the last one (a replay that + /// restarted) starts over. + /// - `skip`: the gate's reason for skipping the frame, nil when the gate passed it. + /// - `meanLuma`: the frame's own mean luma, nil when it wasn't measured. + /// - `sharpness`: the frame's own sharpness (`FrameQuality.sharpness`), nil when it wasn't + /// measured. + /// - `aiming`: the homeowner is asked to stand and aim (an aim, tilt, step-back, see-behind + /// or marking step), so walking too fast is not raised. + public mutating func update(time: Double, skip: CaptureDecision.SkipReason?, meanLuma: Double?, sharpness: Double? = nil, aiming: Bool) -> GateProblem? { + if let lastTime, time < lastTime { self = CoachingDebouncer(config: config) } + lastTime = time + updateDarkness(time: time, meanLuma: meanLuma) + updateMotion(time: time, skip: skip, sharpness: sharpness) + // Walking speed isn't coached while aiming, so a spell there mustn't build up either: it + // would say "Slow down" on the first frames of the next walking step, however calm. + if aiming { spells[.movingFast] = nil } + if darkSince != nil { return persistentDark == .showing ? .persistentlyDark : .tooDark } + if isShown(.turningFast, at: time) { return .turningFast } + if !aiming, isShown(.movingFast, at: time) { return .movingFast } + if isShown(.blurry, at: time) { return .blurry } + return nil + } + + /// Forgets the motion and blur spells, for example while tracking is lost. Darkness is kept: + /// the light hasn't changed because the phone lost its place. + public mutating func forgetMotion() { + spells = [:] + sharpness = [] + blurReference = nil + } + + // MARK: Darkness + + private mutating func updateDarkness(time: Double, meanLuma: Double?) { + guard let meanLuma else { return } + luma.append((time: time, dark: meanLuma < config.darkLuma)) + let keepFrom = time - max(config.persistentDarkAfter, config.darkWindow) + if let firstKept = luma.firstIndex(where: { $0.time > keepFrom }), firstKept > 0 { + luma.removeFirst(firstKept) + } + let recent = luma.filter { $0.time > time - config.darkWindow } + let share = Double(recent.filter { $0.dark }.count) / Double(max(recent.count, 1)) + guard let since = darkSince else { + if recent.count >= config.minDarkSamples, share > config.showDarkShare { + darkSince = time + brightSince = nil + } + return + } + if share < config.clearDarkShare, meanLuma > config.clearLuma { + if brightSince == nil { brightSince = time } + } else { + brightSince = nil + } + if let bright = brightSince, time - bright >= config.clearDarkAfter { + darkSince = nil + brightSince = nil + if persistentDark == .showing { persistentDark = .said } + return + } + if persistentDark == .notYet, time - since >= config.persistentDarkAfter { + let longShare = Double(luma.filter { $0.dark }.count) / Double(max(luma.count, 1)) + if longShare > config.showDarkShare { persistentDark = .showing } + } + } + + // MARK: Motion and blur + + private mutating func updateMotion(time: Double, skip: CaptureDecision.SkipReason?, sharpness value: Double?) { + var evidence = Self.evidence(skip) + // The sharpness from before any blur: fixed while a spell lasts, else the recent median. + let reference = blurReference ?? Self.median(sharpness.map(\.value), atLeast: config.minDarkSamples) + if skip != .trackingNotReady, let value, let reference, value < reference * config.blurRatio { + evidence[.blurry] = true + } + for (problem, seen) in evidence { + if seen { + spells[problem] = Spell(since: spells[problem]?.since ?? time, lastSeen: time) + } else if let spell = spells[problem], !reachedShow(problem, spell) { + // A spell that ends before it shows says nothing. + spells[problem] = nil + } + } + spells = spells.filter { time - $0.value.lastSeen < config.clearAfter } + blurReference = spells[.blurry] == nil ? nil : reference + if let value { + sharpness.append((time: time, value: value)) + if let firstKept = sharpness.firstIndex(where: { $0.time > time - config.blurReferenceWindow }), firstKept > 0 { + sharpness.removeFirst(firstKept) + } + } + } + + private static func median(_ values: [Double], atLeast count: Int) -> Double? { + guard values.count >= count else { return nil } + return values.sorted()[values.count / 2] + } + + private func isShown(_ problem: GateProblem, at time: Double) -> Bool { + guard let spell = spells[problem] else { return false } + return reachedShow(problem, spell) && time - spell.lastSeen < config.clearAfter + } + + private func reachedShow(_ problem: GateProblem, _ spell: Spell) -> Bool { + let after = problem == .blurry ? config.blurShowAfter : config.motionShowAfter + return spell.lastSeen - spell.since >= after + } + + /// What one verdict says about each motion and blur problem: true when the frame had it, + /// false when the gate checked and the frame didn't, and nothing when a check that comes + /// first failed (`AutoCapture.evaluate` checks tracking, then speed, then turning, then + /// exposure, then sharpness). + private static func evidence(_ skip: CaptureDecision.SkipReason?) -> [GateProblem: Bool] { + switch skip { + case .trackingNotReady?: [:] + case .movingFast?: [.movingFast: true] + case .turningFast?: [.movingFast: false, .turningFast: true] + case .tooDark?, .tooBright?: [.movingFast: false, .turningFast: false] + case .blurry?: [.movingFast: false, .turningFast: false, .blurry: true] + case .tooSoon?, .redundant?, nil: [.movingFast: false, .turningFast: false, .blurry: false] + } + } +} + +extension GapPlan.Need { + /// Whether the request asks the homeowner to walk rather than stand and aim. Only a walk past + /// the stretch far enough out does ("Walk along this stretch... Follow the dotted line"), so + /// on it the capture gate's coaching says "Slow down" as on the walk; on the others walking + /// speed isn't coached (`CoachingDebouncer.update`'s `aiming`). No `default`, so a new need + /// has to pick a side. + public var asksToWalk: Bool { + switch self { + case .walkOut: true + case .cells, .groundOut, .overhead, .wallUp: false + } + } +} diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift index 96434e86a..7860603d6 100644 --- a/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/AutoCaptureTests.swift @@ -1,10 +1,15 @@ +import Foundation import HouseScanKit import simd import Testing @Suite struct AutoCaptureTests { - static func frame(_ t: Double, x: Float = 0, tracking: TrackingStatus = .normal, quality: FrameQuality? = nil) -> FrameSample { - let camera = portraitCamera(at: SIMD3(x, 1.4, 2.6), forward: forwardFacingWall(pitchedDown: 20)) + /// A camera at `x` along the wall, pitched down 20° and turned `yaw` degrees about the vertical. + static func frame(_ t: Double, x: Float = 0, yaw: Float = 0, tracking: TrackingStatus = .normal, quality: FrameQuality? = nil) -> FrameSample { + let f = forwardFacingWall(pitchedDown: 20) + let a = yaw * .pi / 180 + let forward = SIMD3(f.x * cos(a) + f.z * sin(a), f.y, -f.x * sin(a) + f.z * cos(a)) + let camera = portraitCamera(at: SIMD3(x, 1.4, 2.6), forward: forward) return FrameSample(timestamp: t, camera: camera, tracking: tracking, quality: quality) } @@ -47,11 +52,33 @@ import Testing #expect(gate.evaluate(Self.frame(0.6), newlySeenCells: 10) == .skip(.tooSoon)) } - @Test func fastWalkingIsMoving() { + @Test func fastWalkingIsMovingFast() { var gate = Self.afterFirstKeep() _ = gate.evaluate(Self.frame(0.6), newlySeenCells: 0) - // 1 m since the frame at 0.6, in 0.4 s: 2.5 m/s > maxSpeed 1.5. - #expect(gate.evaluate(Self.frame(1.0, x: 1), newlySeenCells: 10) == .skip(.moving)) + // 0.8 m since the frame at 0.6, in 0.4 s: 2 m/s > maxSpeed 1.5, with no turn. + #expect(gate.evaluate(Self.frame(1.0, x: 0.8), newlySeenCells: 10) == .skip(.movingFast)) + } + + /// #26: turning the phone in place is not walking too fast. + @Test func turningInPlaceIsTurningFast() { + var gate = Self.afterFirstKeep() + _ = gate.evaluate(Self.frame(0.6), newlySeenCells: 0) + // 36° since the frame at 0.6, in 0.4 s: 90°/s > maxAngularSpeed 60°/s, with no move. + #expect(gate.evaluate(Self.frame(1.0, yaw: 36), newlySeenCells: 10) == .skip(.turningFast)) + } + + /// Speed is checked first: walking fast while turning is moving fast. + @Test func walkingAndTurningFastIsMovingFast() { + var gate = Self.afterFirstKeep() + _ = gate.evaluate(Self.frame(0.6), newlySeenCells: 0) + #expect(gate.evaluate(Self.frame(1.0, x: 0.8, yaw: 36), newlySeenCells: 10) == .skip(.movingFast)) + } + + /// A slow turn is neither. + @Test func slowTurnIsKept() { + var gate = Self.afterFirstKeep() + // 20° in 0.5 s is 40°/s, under 60°/s, and 20° >= spacingRadians 15°. + #expect(gate.evaluate(Self.frame(1.0, yaw: 20), newlySeenCells: 0) == .keep(.spacing)) } @Test func keepsOnSpacing() { diff --git a/ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift b/ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift new file mode 100644 index 000000000..209e08c87 --- /dev/null +++ b/ios/HouseScanKit/Tests/HouseScanKitTests/CoachingDebouncerTests.swift @@ -0,0 +1,228 @@ +import Foundation +import HouseScanKit +import Testing + +/// Synthetic frame sequences at the gate's pace, about ten judged frames a second. +@Suite struct CoachingDebouncerTests { + typealias Problem = CoachingDebouncer.GateProblem + + /// One judged frame. + struct Frame { + var time: Double + var skip: CaptureDecision.SkipReason? + var luma: Double? = 128 + var aiming = false + var sharpness: Double? = nil + } + + /// What the debouncer says for each frame. + static func run(_ frames: [Frame], on debouncer: inout CoachingDebouncer) -> [Problem?] { + frames.map { debouncer.update(time: $0.time, skip: $0.skip, meanLuma: $0.luma, sharpness: $0.sharpness, aiming: $0.aiming) } + } + + static func run(_ frames: [Frame]) -> [Problem?] { + var debouncer = CoachingDebouncer() + return run(frames, on: &debouncer) + } + + /// Frames every 0.1 s from `start` for `seconds`. + static func times(from start: Double = 0, for seconds: Double) -> [Double] { + (0..) -> (shows: Int, clears: Int) { + var shows = 0 + var clears = 0 + var up = false + for answer in said { + let now = answer.map { problems.contains($0) } ?? false + if now, !up { shows += 1 } + if !now, up { clears += 1 } + up = now + } + return (shows, clears) + } + + static let dark: Set = [.tooDark, .persistentlyDark] + + /// #80: a night scene's luma sitting on the gate's 40 made the dark card come and go every few + /// seconds. With hysteresis it shows once and clears once, when the light is really back. + @Test func lumaOscillatingAroundTheGateShowsOnceAndClearsOnce() { + // 40 ± 6 luma, a few seconds a swing, for 30 s; then 60 luma for 4 s. + let night = Self.times(for: 30).map { t -> Frame in + let luma = 40 + 6 * sin(t * 1.7) + return Frame(time: t, skip: luma < 40 ? .tooDark : .redundant, luma: luma) + } + let light = Self.times(from: 30, for: 4).map { Frame(time: $0, skip: .redundant, luma: 60) } + let said = Self.run(night + light) + let (shows, clears) = Self.transitions(said, of: Self.dark) + #expect(shows == 1) + #expect(clears == 1) + #expect(said.last! == nil) + } + + /// #80: a tilt in the dark turned the gate's reason from too dark to moving and back, which + /// swapped the card. Darkness is judged from luma, so the flip neither hides it nor restarts it. + @Test func tooDarkAndMovingFlipsDontRestartTheDarkCoaching() { + let frames = Self.times(for: 6).enumerated().map { index, t in + Frame(time: t, skip: index % 3 == 0 ? .movingFast : .tooDark, luma: 30) + } + let said = Self.run(frames) + let first = said.firstIndex { $0 == .tooDark } + #expect(first != nil) + // Up within the first second, and never down or replaced after that. + #expect((first ?? .max) <= 10) + #expect(said[(first ?? 0)...].allSatisfy { $0 == .tooDark }) + } + + /// Once shown, a different problem on some frames doesn't restart moving fast. + @Test func movingFastStaysThroughOtherReasons() { + var debouncer = CoachingDebouncer() + let fast = Self.times(for: 1).map { Frame(time: $0, skip: .movingFast) } + #expect(Self.run(fast, on: &debouncer).last! == .movingFast) + // Fast frames with a blurry one in between (a frame the gate found not moving, 0.1 s). + let mixed = Self.times(from: 1, for: 1).enumerated().map { index, t in + Frame(time: t, skip: index == 4 ? .blurry : .movingFast) + } + #expect(Self.run(mixed, on: &debouncer).allSatisfy { $0 == .movingFast }) + } + + /// A single fast frame is a jolt: nothing shows. + @Test func shortSpellsSayNothing() { + let frames = Self.times(for: 3).enumerated().map { index, t in + Frame(time: t, skip: index % 5 == 0 ? .movingFast : .redundant) + } + #expect(Self.run(frames).allSatisfy { $0 == nil }) + } + + /// #26: during an aim or a tilt the motion is the phone turning, so "Slow down" is not said. + @Test func slowDownIsSuppressedWhileAiming() { + let aiming = Self.times(for: 3).map { Frame(time: $0, skip: .movingFast, aiming: true) } + #expect(Self.run(aiming).allSatisfy { $0 == nil }) + let walking = Self.times(for: 3).map { Frame(time: $0, skip: .movingFast, aiming: false) } + #expect(Self.run(walking).last! == .movingFast) + } + + /// Moving fast while aiming builds no spell, so the next walking step starts clean: a calm + /// frame says nothing, and walking fast again needs its own 0.7 s before "Slow down". + @Test func movingFastWhileAimingDoesntCarryIntoTheNextStep() { + var debouncer = CoachingDebouncer() + let aiming = Self.times(for: 2).map { Frame(time: $0, skip: .movingFast, aiming: true) } + #expect(Self.run(aiming, on: &debouncer).allSatisfy { $0 == nil }) + #expect(debouncer.update(time: 2, skip: .redundant, meanLuma: 128, aiming: false) == nil) + let walking = Self.times(from: 2.1, for: 1).map { Frame(time: $0, skip: .movingFast) } + let said = Self.run(walking, on: &debouncer) + #expect(said.prefix(7).allSatisfy { $0 == nil }) + #expect(said.last! == .movingFast) + } + + /// A gap request to walk the stretch far enough out asks for walking, so hurrying along it is + /// still told to slow down. The other requests ask the homeowner to stand and aim. + @Test func aWalkOutRequestStillSaysSlowDown() { + #expect(GapPlan.Need.walkOut(1.5).asksToWalk) + let standing: [GapPlan.Need] = [.cells, .groundOut(1.5), .overhead(nil), .overhead(2), .wallUp(2)] + for need in standing { + #expect(!need.asksToWalk, "\(need)") + } + let walkOut = Self.times(for: 3).map { Frame(time: $0, skip: .movingFast, aiming: !GapPlan.Need.walkOut(1.5).asksToWalk) } + #expect(Self.run(walkOut).last! == .movingFast) + let groundOut = Self.times(for: 3).map { Frame(time: $0, skip: .movingFast, aiming: !GapPlan.Need.groundOut(1.5).asksToWalk) } + #expect(Self.run(groundOut).allSatisfy { $0 == nil }) + } + + /// Turning fast is its own problem, said while aiming too. + @Test func turningFastShowsAfterItsSpell() { + let frames = Self.times(for: 2).map { Frame(time: $0, skip: .turningFast, aiming: true) } + let said = Self.run(frames) + #expect(said.prefix(7).allSatisfy { $0 == nil }) + #expect(said.last! == .turningFast) + } + + /// Blur from a tilt that changes the view: under 2 s it says nothing. + @Test func shortBlurSaysNothing() { + let blur = Self.times(for: 1.8).map { Frame(time: $0, skip: .blurry) } + let after = Self.times(from: 1.8, for: 1).map { Frame(time: $0, skip: .redundant) } + #expect(Self.run(blur + after).allSatisfy { $0 == nil }) + let long = Self.times(for: 2.5).map { Frame(time: $0, skip: .blurry) } + #expect(Self.run(long).last! == .blurry) + } + + /// The real gate feeding the debouncer, frame by frame: 3 s of sharp frames, 4 s at under a + /// third of that sharpness (shaky hands, a long exposure), then sharp again. The gate calls + /// only the first second of the blur blurry, because its 15-frame median catches up; the + /// coaching still says blur from 2 s in until the photos are sharp again. + @Test func sustainedBlurThroughTheGateIsCoached() { + var gate = AutoCapture() + var debouncer = CoachingDebouncer() + var skips: [CaptureDecision.SkipReason?] = [] + var said: [Problem?] = [] + for t in Self.times(for: 8) { + let sharpness: Double = (3..<7).contains(t) ? 30 : 100 + let frame = AutoCaptureTests.frame(t, quality: AutoCaptureTests.good(sharpness: sharpness)) + let decision = gate.evaluate(frame, newlySeenCells: 0) + var skip: CaptureDecision.SkipReason? + switch decision { + case .keep: gate.didKeep(frame) + case .skip(let reason): skip = reason + } + skips.append(skip) + said.append(debouncer.update(time: t, skip: skip, meanLuma: 128, sharpness: sharpness, aiming: true)) + } + // The gate adapts: by 4.5 s it no longer calls the blurry frames blurry. + #expect(skips[30] == .blurry) + #expect(skips[45...69].allSatisfy { $0 != .blurry }) + // The coaching says nothing for the first 2 s of blur, then blur until it's sharp again. + #expect(said[..<50].allSatisfy { $0 == nil }) + #expect(said[50..<70].allSatisfy { $0 == .blurry }) + #expect(said.last! == nil) + } + + /// A tilt that makes the view much less sharp for under 2 s says nothing, also when judged by + /// sharpness against the frames before it. + @Test func shortSharpnessDropSaysNothing() { + let before = Self.times(for: 2).map { Frame(time: $0, skip: .redundant, sharpness: 100) } + let tilt = Self.times(from: 2, for: 1.8).map { Frame(time: $0, skip: .redundant, sharpness: 30) } + let after = Self.times(from: 3.8, for: 2).map { Frame(time: $0, skip: .redundant, sharpness: 100) } + #expect(Self.run(before + tilt + after).allSatisfy { $0 == nil }) + } + + /// A shown problem clears half a second after its last frame. + @Test func motionClearsAfterItsLastFrame() { + let fast = Self.times(for: 1).map { Frame(time: $0, skip: .movingFast) } + let calm = Self.times(from: 1, for: 1).map { Frame(time: $0, skip: .redundant) } + let said = Self.run(fast + calm) + #expect(said[10] == .movingFast) + #expect(said.last! == nil) + } + + /// Mostly dark for more than 20 s: said once, instead of "It's dark here", until it's light. + @Test func persistentDarkIsSaidOnce() { + var debouncer = CoachingDebouncer() + let night = Self.times(for: 25).map { Frame(time: $0, skip: .tooDark, luma: 25) } + let said = Self.run(night, on: &debouncer) + #expect(said[30] == .tooDark) + #expect(said.last! == .persistentlyDark) + let light = Self.times(from: 25, for: 4).map { Frame(time: $0, skip: .redundant, luma: 90) } + #expect(Self.run(light, on: &debouncer).last! == nil) + let nightAgain = Self.times(from: 29, for: 25).map { Frame(time: $0, skip: .tooDark, luma: 25) } + let again = Self.run(nightAgain, on: &debouncer) + #expect(again.last! == .tooDark) + #expect(!again.contains { $0 == .persistentlyDark }) + } + + /// Frames without a luma measurement don't count toward darkness either way. + @Test func unmeasuredFramesDontRaiseDarkness() { + let frames = Self.times(for: 3).map { Frame(time: $0, skip: .tooDark, luma: nil) } + #expect(Self.run(frames).allSatisfy { $0 == nil }) + } + + /// A replay that restarts (time going back) starts over. + @Test func timeGoingBackStartsOver() { + var debouncer = CoachingDebouncer() + let night = Self.times(for: 2).map { Frame(time: $0, skip: .tooDark, luma: 25) } + #expect(Self.run(night, on: &debouncer).last! == .tooDark) + #expect(debouncer.update(time: 0, skip: nil, meanLuma: 128, aiming: false) == nil) + #expect(!debouncer.isDark) + } +} diff --git a/ios/HouseScanUITests/ScreenStatesUITests.swift b/ios/HouseScanUITests/ScreenStatesUITests.swift index 65a6b8f40..24ae8e967 100644 --- a/ios/HouseScanUITests/ScreenStatesUITests.swift +++ b/ios/HouseScanUITests/ScreenStatesUITests.swift @@ -18,6 +18,8 @@ final class ScreenStatesUITests: XCTestCase { ("meterCloseUp-chooseNumber", ["-uiDemoPhase", "meterCloseUp", "-uiDemoMeterChoose"], "meterCloseUp"), ("wallWalk", ["-uiDemoPhase", "wallWalk"], "wallWalk"), ("wallWalk-slowDown", ["-uiDemoPhase", "wallWalk", "-uiDemoCoaching", "slowDown"], "wallWalk"), + ("wallWalk-tooDark", ["-uiDemoPhase", "wallWalk", "-uiDemoCoaching", "tooDark"], "wallWalk"), + ("wallWalk-turnSlowly", ["-uiDemoPhase", "wallWalk", "-uiDemoCoaching", "turnSlowly"], "wallWalk"), ("wallWalk-needsTexture", ["-uiDemoPhase", "wallWalk", "-uiDemoCoaching", "needsTexture"], "wallWalk"), ("wallWalk-relocalizing", ["-uiDemoPhase", "wallWalk", "-uiDemoCoaching", "relocalizing"], "wallWalk"), ("wallWalk-markingRefused", ["-uiDemoPhase", "wallWalk", "-uiDemoMarking", "window", "-uiDemoRefusal"], "wallWalk"), @@ -39,6 +41,7 @@ final class ScreenStatesUITests: XCTestCase { ("gapRequest", ["-uiDemoPhase", "gapRequest"], "gapRequest"), ("gapRequest-groundOut", ["-uiDemoPhase", "gapRequest", "-uiDemoGap", "groundOut"], "gapRequest"), ("gapRequest-walkOut", ["-uiDemoPhase", "gapRequest", "-uiDemoGap", "walkOut"], "gapRequest"), + ("gapRequest-tooDark", ["-uiDemoPhase", "gapRequest", "-uiDemoCoaching", "tooDark"], "gapRequest"), ("gapRequest-overhead", ["-uiDemoPhase", "gapRequest", "-uiDemoGap", "overhead"], "gapRequest"), ("gapRequest-overheadQuestion", ["-uiDemoPhase", "gapRequest", "-uiDemoGap", "overhead", "-uiDemoOverheadQuestion"], "gapRequest"), ("gapRequest-followUp", ["-uiDemoPhase", "gapRequest", "-uiDemoFollowUp"], "gapRequest"), @@ -77,6 +80,8 @@ final class ScreenStatesUITests: XCTestCase { "wallWalk-hidden", "wallWalk-seeBehind", "wallWalk-fullLegend", "gapRequest-followUp", "uploading-followUp", "markFeatures-lostPlace", "spotConfirm", "spotConfirm-which", "spotConfirm-ground", "spotConfirm-answered", "spotConfirm-cantMark", "spotConfirm-unconfirmable", + // The longest walk card: the task, its second line and the dark coaching's advice (#80). + "wallWalk-tooDark", ] /// Words a state must show: in the named element's label or value, or with no identifier, @@ -85,6 +90,11 @@ final class ScreenStatesUITests: XCTestCase { "wallWalk-hidden": ("wallTape", "2 sections hidden behind something"), "wallWalk-fullLegend": ("wallTape", "2 sections hidden behind something"), "wallWalk-seeBehind": ("instruction", "Something is in front of the wall here"), + // #80, #26: the gate's coaching rides on the task card. + "wallWalk-slowDown": ("instruction", "Walk slowly to your right"), + "wallWalk-tooDark": ("instruction", "It's dark here"), + "wallWalk-turnSlowly": ("instruction", "Turn more slowly"), + "gapRequest-tooDark": ("instruction", "Show the ground"), "gapRequest-followUp": ("instruction", "One more view to finish"), "uploading-followUp": (nil, "One more view to finish"), "markFeatures-lostPlace": ("review.lostPlace", "Your phone lost its place"), @@ -216,6 +226,44 @@ final class ScreenStatesUITests: XCTestCase { XCTAssertTrue(element(app, "screen.onboarding").waitForExistence(timeout: 10)) } + /// #80: the capture gate's coaching keeps the walk's task on the card and adds its own line, + /// instead of replacing the card. + @MainActor + func testGateCoachingKeepsTheTaskOnTheCard() throws { + continueAfterFailure = false + let app = XCUIApplication() + app.launchArguments = ["-uiDemo", "-uiDemoFreeze", "-uiDemoPhase", "wallWalk", "-uiDemoCoaching", "tooDark"] + app.launch() + XCTAssertTrue(element(app, "screen.wallWalk").waitForExistence(timeout: 15)) + let card = element(app, "instruction") + XCTAssertTrue(card.waitForExistence(timeout: 5)) + // One snapshot read, never live (the #24 flake fix): a live read of a gone element + // records a failure that can't be caught. + let label = ElementRead.snapshot(card)?.label ?? "" + XCTAssertTrue(label.contains("Walk slowly to your right"), "the task must stay on the card, got \(label)") + XCTAssertTrue(label.contains("Keep the wall and the ground in view"), "the task's second line must stay on the card, got \(label)") + XCTAssertTrue(label.contains("It's dark here"), "the coaching must show on the card, got \(label)") + XCTAssertTrue(label.contains("flashlight"), "the dark coaching must say what would help, got \(label)") + } + + /// #80, as on the walk: the capture gate's coaching keeps a gap request on the card. The dark + /// coaching can stay up for a whole night request (field test 4.1, run 3). + @MainActor + func testGateCoachingKeepsTheGapRequestOnTheCard() throws { + continueAfterFailure = false + let app = XCUIApplication() + app.launchArguments = ["-uiDemo", "-uiDemoFreeze", "-uiDemoPhase", "gapRequest", "-uiDemoCoaching", "tooDark"] + app.launch() + XCTAssertTrue(element(app, "screen.gapRequest").waitForExistence(timeout: 15)) + let card = element(app, "instruction") + XCTAssertTrue(card.waitForExistence(timeout: 5)) + let label = ElementRead.snapshot(card)?.label ?? "" + XCTAssertTrue(label.contains("Show the ground"), "the request must stay on the card, got \(label)") + XCTAssertTrue(label.contains("a clear look from two places"), "the request's second line must stay on the card, got \(label)") + XCTAssertTrue(label.contains("It's dark here"), "the coaching must show on the card, got \(label)") + XCTAssertTrue(label.contains("flashlight"), "the dark coaching must say what would help, got \(label)") + } + /// B-09: "Add something" on the review opens the camera with the marking prompt, and the /// review comes back with the new item once it is marked, or unchanged after Cancel. @MainActor