From ad942d8dfac8d31f04734a6a93e5ca8960bf8a76 Mon Sep 17 00:00:00 2001 From: AidenJohnston Date: Sun, 27 Sep 2026 06:03:54 -0500 Subject: [PATCH 1/4] ios tests: audit the walk once the autopilot holds it The full-flow tests audited the walk the moment it appeared, while the autopilot was still marking features, setting the ends and tilting up over a playing replay. Each audit pass caught a different crossfade; a copy fading out has no identifier, so transient issues from two passes matched on a label or on the "no element" key and failed as one lasting issue (runs 36304552823, 36305050676, 36307476187). The app now leaves .held in the gate folder while it waits for the test to finish with a screen, and the walk is audited once that file appears, on the settled "Done with this wall" state. Every state the walk passes through is still audited frozen by ScreenStatesUITests. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/Runtime/ScanEngine.swift | 7 +++++- ios/HouseScanUITests/FullFlowUITests.swift | 29 +++++++++++++++++----- ios/README.md | 2 +- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index e6a16637c..e4f61b24f 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -283,10 +283,15 @@ final class ScanEngine { private var autoAdvanceDelay: Double { options.autopilot ? max(1.2, options.autopilotHold) : 1.2 } /// With `-autopilotGate`, waits until the UI test has finished with `phase` (its file exists), - /// for at most two minutes so a lost gate file can't hang the app. + /// for at most two minutes so a lost gate file can't hang the app. While it waits, the file + /// `.held` says the autopilot has finished with the screen and holds it still, for a UI + /// test that must audit it settled rather than while the autopilot drives it. func waitForGate(_ phase: ScanPhase) async { guard options.autopilot, let gate = options.autopilotGate else { return } let file = gate.appending(path: phase.rawValue) + if !FileManager.default.fileExists(atPath: file.path) { + try? Data().write(to: gate.appending(path: "\(phase.rawValue).held")) + } let deadline = ContinuousClock.now + .seconds(120) while ContinuousClock.now < deadline, !FileManager.default.fileExists(atPath: file.path) { try? await Task.sleep(for: .milliseconds(100)) diff --git a/ios/HouseScanUITests/FullFlowUITests.swift b/ios/HouseScanUITests/FullFlowUITests.swift index afc9a8e38..331adc89e 100644 --- a/ios/HouseScanUITests/FullFlowUITests.swift +++ b/ios/HouseScanUITests/FullFlowUITests.swift @@ -14,6 +14,16 @@ final class FullFlowUITests: XCTestCase { /// Screens in the order the flow must show them. static let flow = ["onboarding", "findMeter", "meterCloseUp", "wallWalk", "markFeatures", "gapRequest", "uploading", "spotConfirm", "result", "resultAR"] + /// Screens audited only once the autopilot has finished with them and holds them still + /// (`.held` in the gate folder). On the walk the autopilot marks features, sets the + /// ends and tilts up while the replay plays, so the screen changes several times a second: + /// an audit started when the walk appeared read cards and buttons mid-crossfade, each pass a + /// different transition, and a copy fading out has no identifier, so two passes' transient + /// issues matched on the label or the "no element" key and failed as one lasting issue (CI + /// runs 36304552823, 36305050676, 36307476187). Held, the walk shows its last state, "Done + /// with this wall", and the states it passed through are audited frozen by ScreenStatesUITests. + static let auditedOnceHeld: Set = ["wallWalk"] + override func setUp() { continueAfterFailure = false } @@ -237,12 +247,13 @@ final class FullFlowUITests: XCTestCase { /// Audits the screen twice, six seconds apart, and fails on the issues found both times /// (`AccessibilityAudit.run`). /// - /// The walk keeps taking photos while it is audited, so the photo count's number is often - /// mid-roll, and the result's text is still fading in when the screen appears. A single audit - /// reports those passing frames as contrast failures; a real contrast or clipping problem is - /// still there later. The walk also rebuilds parts of the screen while it is audited, so an - /// issue's element can be gone before it is read: that one is attached, not failed on. Every - /// screen state is also audited frozen by ScreenStatesUITests. + /// Replay screens can still be taking photos while they are audited, so the photo count's + /// number is often mid-roll, and the result's text is still fading in when the screen appears. + /// A single audit reports those passing frames as contrast failures; a real contrast or + /// clipping problem is still there later. The walk is audited once the autopilot holds it + /// (`auditedOnceHeld`). A screen that rebuilds parts of itself while it is audited can lose + /// an issue's element before it is read: that one is attached, not failed on. Every screen + /// state is also audited frozen by ScreenStatesUITests. @MainActor private func audit(_ app: XCUIApplication, screen: String) throws { // Six seconds: long enough for a system notification banner to leave. On CI one slid in @@ -298,6 +309,12 @@ final class FullFlowUITests: XCTestCase { // The walk replays the whole recording; everything else takes seconds. let timeout: TimeInterval = phase == "markFeatures" || phase == "result" ? 150 : 60 XCTAssertTrue(screen.waitForExistence(timeout: timeout), "screen.\(phase) never appeared") + if Self.auditedOnceHeld.contains(phase) { + let held = gate.appending(path: "\(phase).held") + let deadline = Date().addingTimeInterval(150) + while !FileManager.default.fileExists(atPath: held.path), Date() < deadline { Thread.sleep(forTimeInterval: 0.2) } + XCTAssertTrue(FileManager.default.fileExists(atPath: held.path), "the autopilot never finished with screen.\(phase)") + } // Let the entrance animation finish so the screenshot and audit see the settled screen. Thread.sleep(forTimeInterval: 1.0) let shot = XCTAttachment(screenshot: app.screenshot()) diff --git a/ios/README.md b/ios/README.md index 5a1d44c50..81b93e19f 100644 --- a/ios/README.md +++ b/ios/README.md @@ -19,7 +19,7 @@ Native iPhone app for the AR capture walk. The homeowner marks the electric mete | `-autopilotHold ` | How long the autopilot leaves each screen up (default 1.2 s). | | `-autopilotCantGetThere` | With `-autopilot`, ends the walk the way device run 1 did. It answers "Can't get there" to the walk's first request, the ground in front of the meter, before the replay has shown it. Then, instead of marking the ends, it plays again the frame where the walk went farthest on each side and answers "Can't get there" when the walk asks to go on that way, so the end lands where the phone stands. `FullFlowUITests.testCantGetThereEndsWhereThePhoneIs` runs it. | | `-autopilotSomethingThere` | With `-autopilot`, answers the first spot check "Something's in the way" instead of "It's clear". The scan stops claiming that area, is uploaded again, and later checks get "It's clear". The stretch taken out (by this answer or by Continue) goes to the `-autopilotGate` folder as `spot-refusal.json` (`withdrawn_span_ft`, feet along the wall as in scene.json). `FullFlowUITests.testSomethingThereChecksTheWallAgain` runs it. | -| `-autopilotGate ` | Used by the UI tests. Before leaving a screen, the app waits for a file named after that phase in this folder. Once the result shows, the autopilot writes the scan's scene.json there. | +| `-autopilotGate ` | Used by the UI tests. Before leaving a screen, the app waits for a file named after that phase in this folder, and while it waits it leaves `.held` there, so a test can audit the screen once the autopilot has finished with it. Once the result shows, the autopilot writes the scan's scene.json there. | | `-uiDemo` | Runs the screens on a scripted fake engine instead of the capture engine, for design work and for auditing states a replay can't reach. The arguments it takes are listed in `HouseScan/UI/Preview/UIDemo.swift`; for example `-uiDemoPhase spotConfirm -uiDemoSpotStep ground` opens the spot check's ground question, `-uiDemoPhase meterCloseUp -uiDemoMeterChoose` shows the meter number picker, and `-uiDemoPhase result -uiDemoCorner` shows the result model of a wall that turns a corner. | **Spot check.** Before the result (`STATE=spotConfirm`), the app shows the kept photo that best sees the answer's spot and its front clearance, outlined, and asks at most two questions on it. First: is anything in the marked area? Photos without depth and the walked path claim wall, ground and clear space they never saw, and the server keeps clearances only from marked things. "It's clear" backs those claims for that spot; "Something's in the way" takes them back over the area, so scene.json reports it unseen; "A gas meter, AC, window or door" opens the camera to mark it. Then, once it is clear: what is the ground where the battery would stand? The answer goes to the server as a ground patch over the footprint and its error margin, the only ground patch the app sends. Every answer but "It's clear" with "Not sure" sends the scan again. When no kept photo shows the whole area, nothing is asked and the area is left out. The rules are in `Runtime/ScanEngine+Confirm.swift` and HouseScanKit's `Confirm/SpotConfirmation.swift`. From ee8cdcc1fd21bc39d16fd02ea299d57957fc9e1d Mon Sep 17 00:00:00 2001 From: AidenJohnston Date: Sun, 27 Sep 2026 07:40:04 -0500 Subject: [PATCH 2/4] ios tests: say what the held walk shows Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScanUITests/FullFlowUITests.swift | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ios/HouseScanUITests/FullFlowUITests.swift b/ios/HouseScanUITests/FullFlowUITests.swift index 331adc89e..041fb691c 100644 --- a/ios/HouseScanUITests/FullFlowUITests.swift +++ b/ios/HouseScanUITests/FullFlowUITests.swift @@ -20,8 +20,9 @@ final class FullFlowUITests: XCTestCase { /// an audit started when the walk appeared read cards and buttons mid-crossfade, each pass a /// different transition, and a copy fading out has no identifier, so two passes' transient /// issues matched on the label or the "no element" key and failed as one lasting issue (CI - /// runs 36304552823, 36305050676, 36307476187). Held, the walk shows its last state, "Done - /// with this wall", and the states it passed through are audited frozen by ScreenStatesUITests. + /// runs 36304552823, 36305050676, 36307476187). Held, the walk stays in its last state, both + /// ends set and "Done with this wall" on offer, and the states it passed through are audited + /// frozen by ScreenStatesUITests. static let auditedOnceHeld: Set = ["wallWalk"] override func setUp() { From 32f6e710b0c562f169c805a3ca0dcdebc8c9421d Mon Sep 17 00:00:00 2001 From: AidenJohnston Date: Sun, 27 Sep 2026 08:21:27 -0500 Subject: [PATCH 3/4] ios tests: audit the walk in motion too, to report The walk's audit in motion caught real bugs that only show mid-transition: the map legend's entries drawn over each other (1faa79b), the two reply titles half-drawn over each other (b1a2444) and the translucent reply fill (397b6d6). Auditing only the held walk would have hidden them. The walk is now audited on its first appearance as well, report-only: findings are attached as audit-wallWalk-moving and printed, as with HOUSESCAN_AUDIT_REPORT_ONLY, and the strict audit stays on the held walk. Also: LaunchOptions documents .held, and the LiDAR test's comment no longer times the walk. Co-Authored-By: Claude Opus 5.5 (1M context) --- ios/HouseScan/Runtime/LaunchOptions.swift | 6 ++- ios/HouseScanUITests/FullFlowUITests.swift | 54 +++++++++++++++------- 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/ios/HouseScan/Runtime/LaunchOptions.swift b/ios/HouseScan/Runtime/LaunchOptions.swift index e5deeb9c6..9d6daab4d 100644 --- a/ios/HouseScan/Runtime/LaunchOptions.swift +++ b/ios/HouseScan/Runtime/LaunchOptions.swift @@ -14,8 +14,10 @@ import OSLog /// UI tests raise it so each screen stays long enough to screenshot and audit. /// - `-autopilotGate `: before the flow leaves a screen, wait until a file named after /// that phase exists in the folder. UI tests write it once they have screenshotted and audited -/// the screen, so a slow audit can never miss a screen. After the result shows, the autopilot -/// also writes the scan's scene.json there, for the test to check. +/// the screen, so a slow audit can never miss a screen. While it waits, the app leaves +/// `.held` there, so a test can tell when the autopilot has finished with a screen and +/// holds it still. After the result shows, the autopilot also writes the scan's scene.json +/// there, for the test to check. /// - `-autopilotCantGetThere`: the autopilot ends the walk with "Can't get there" instead of /// marking the ends (`Autopilot.endWalkByCantGetThere`). /// - `-autopilotSomethingThere`: the autopilot answers the first spot check "Something's there" diff --git a/ios/HouseScanUITests/FullFlowUITests.swift b/ios/HouseScanUITests/FullFlowUITests.swift index 041fb691c..b71c85f41 100644 --- a/ios/HouseScanUITests/FullFlowUITests.swift +++ b/ios/HouseScanUITests/FullFlowUITests.swift @@ -14,15 +14,18 @@ final class FullFlowUITests: XCTestCase { /// Screens in the order the flow must show them. static let flow = ["onboarding", "findMeter", "meterCloseUp", "wallWalk", "markFeatures", "gapRequest", "uploading", "spotConfirm", "result", "resultAR"] - /// Screens audited only once the autopilot has finished with them and holds them still - /// (`.held` in the gate folder). On the walk the autopilot marks features, sets the - /// ends and tilts up while the replay plays, so the screen changes several times a second: - /// an audit started when the walk appeared read cards and buttons mid-crossfade, each pass a - /// different transition, and a copy fading out has no identifier, so two passes' transient - /// issues matched on the label or the "no element" key and failed as one lasting issue (CI - /// runs 36304552823, 36305050676, 36307476187). Held, the walk stays in its last state, both - /// ends set and "Done with this wall" on offer, and the states it passed through are audited - /// frozen by ScreenStatesUITests. + /// Screens audited strictly only once the autopilot has finished with them and holds them + /// still (`.held` in the gate folder). On the walk the autopilot marks features, sets + /// the ends and tilts up while the replay plays, so the screen changes several times a second + /// and two audit passes read two different moments. A finding that isn't the same element in + /// both passes can still match: a view fading out has no identifier, so it is keyed by its + /// label, or by "no element". The walk is therefore also audited on its first appearance, in + /// motion, in report-only mode (`audit-wallWalk-moving` attachments, and `AUDIT` lines in the + /// log), because that audit is what caught real bugs that only show mid-transition: the map's + /// legend entries drawn over each other (1faa79b, run 36307476187), "Can't get there" and + /// "Can't see past it" half-drawn over each other (b1a2444), and the reply's translucent fill + /// (397b6d6). Read those attachments when the walk's chrome or its animations change. The + /// strict audit then checks the held walk: both ends set and "Done with this wall" on offer. static let auditedOnceHeld: Set = ["wallWalk"] override func setUp() { @@ -102,8 +105,9 @@ final class FullFlowUITests: XCTestCase { try runFlow(replay: Self.lidarFixture) { app, phase in guard !showedHidden else { return } switch phase { - // At the autopilot's 3x the walk plays in about 6 s. No other frame sees the wall - // behind the bin, so its cells stay hidden once the bin has been in view. + // Read once the autopilot holds the walk, after the whole replay has played. No + // other frame sees the wall behind the bin, so its cells stay hidden once the bin has + // been in view. case "wallWalk": showedHidden = Self.wallTapeShowsHidden(app, timeout: 60) case "gapRequest": showedHidden = Self.wallTapeShowsHidden(app, timeout: 30) default: break @@ -251,12 +255,14 @@ final class FullFlowUITests: XCTestCase { /// Replay screens can still be taking photos while they are audited, so the photo count's /// number is often mid-roll, and the result's text is still fading in when the screen appears. /// A single audit reports those passing frames as contrast failures; a real contrast or - /// clipping problem is still there later. The walk is audited once the autopilot holds it - /// (`auditedOnceHeld`). A screen that rebuilds parts of itself while it is audited can lose - /// an issue's element before it is read: that one is attached, not failed on. Every screen - /// state is also audited frozen by ScreenStatesUITests. + /// clipping problem is still there later. The walk is audited strictly once the autopilot + /// holds it, and in motion only to report (`auditedOnceHeld`). With `reportOnly`, or + /// HOUSESCAN_AUDIT_REPORT_ONLY=1, the issues found twice are attached as `audit-` + /// and printed, not failed on. A screen that rebuilds parts of itself while it is audited can + /// lose an issue's element before it is read: that one is attached, not failed on. Every + /// screen state is also audited frozen by ScreenStatesUITests. @MainActor - private func audit(_ app: XCUIApplication, screen: String) throws { + private func audit(_ app: XCUIApplication, screen: String, reportOnly: Bool = false) throws { // Six seconds: long enough for a system notification banner to leave. On CI one slid in // over the find-meter screen and failed its contrast check twice a second apart (run // 36264789032), as ScreenStatesUITests' audit also allows for. @@ -269,7 +275,7 @@ final class FullFlowUITests: XCTestCase { } for (_, finding) in outcome.persistent { let text = "screen.\(screen): \(finding.message)" - if Self.environment["HOUSESCAN_AUDIT_REPORT_ONLY"] == "1" { + if reportOnly || Self.environment["HOUSESCAN_AUDIT_REPORT_ONLY"] == "1" { let note = XCTAttachment(string: text) note.name = "audit-\(screen)" note.lifetime = .keepAlways @@ -311,6 +317,20 @@ final class FullFlowUITests: XCTestCase { let timeout: TimeInterval = phase == "markFeatures" || phase == "result" ? 150 : 60 XCTAssertTrue(screen.waitForExistence(timeout: timeout), "screen.\(phase) never appeared") if Self.auditedOnceHeld.contains(phase) { + // In motion, report-only: see `auditedOnceHeld`. + Thread.sleep(forTimeInterval: 1.0) + let moving = XCTAttachment(screenshot: app.screenshot()) + moving.name = "\(phase)-moving" + moving.lifetime = .keepAlways + add(moving) + do { + try audit(app, screen: "\(phase)-moving", reportOnly: true) + } catch { + let note = XCTAttachment(string: "the audit couldn't read the screen in motion: \(error)") + note.name = "audit-\(phase)-moving-unread" + note.lifetime = .keepAlways + add(note) + } let held = gate.appending(path: "\(phase).held") let deadline = Date().addingTimeInterval(150) while !FileManager.default.fileExists(atPath: held.path), Date() < deadline { Thread.sleep(forTimeInterval: 0.2) } From 4925f37fb70cb875b34b16fa03eff75218c1905f Mon Sep 17 00:00:00 2001 From: AidenJohnston Date: Sun, 27 Sep 2026 08:23:52 -0500 Subject: [PATCH 4/4] ios: the autopilot's gate waits four minutes, past the test's work on a held screen A test audits a held screen only once .held appears, and the walk's map check after it can wait 90 s; the two-minute gate could expire first on a slow runner and let the app leave mid-audit (Greptile on #169). Co-Authored-By: Claude Opus 5.5 (1M context) (cherry picked from commit c3aeabea41a557d5a50cfedae4b3b07fd52d9687) --- ios/HouseScan/Runtime/ScanEngine.swift | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ios/HouseScan/Runtime/ScanEngine.swift b/ios/HouseScan/Runtime/ScanEngine.swift index 039c530fa..8c249e498 100644 --- a/ios/HouseScan/Runtime/ScanEngine.swift +++ b/ios/HouseScan/Runtime/ScanEngine.swift @@ -293,16 +293,19 @@ final class ScanEngine { private var autoAdvanceDelay: Double { options.autopilot ? max(1.2, options.autopilotHold) : 1.2 } /// With `-autopilotGate`, waits until the UI test has finished with `phase` (its file exists), - /// for at most two minutes so a lost gate file can't hang the app. While it waits, the file + /// for at most four minutes so a lost gate file can't hang the app. While it waits, the file /// `.held` says the autopilot has finished with the screen and holds it still, for a UI - /// test that must audit it settled rather than while the autopilot drives it. + /// test that must audit it settled rather than while the autopilot drives it. A test starts + /// that audit only once `.held` appears, then reads the screen (the walk's map check can wait + /// up to 90 s), so the wait must outlast all of that; at two minutes a slow runner could have + /// let the app leave mid-audit. func waitForGate(_ phase: ScanPhase) async { guard options.autopilot, let gate = options.autopilotGate else { return } let file = gate.appending(path: phase.rawValue) if !FileManager.default.fileExists(atPath: file.path) { try? Data().write(to: gate.appending(path: "\(phase.rawValue).held")) } - let deadline = ContinuousClock.now + .seconds(120) + let deadline = ContinuousClock.now + .seconds(240) while ContinuousClock.now < deadline, !FileManager.default.fileExists(atPath: file.path) { try? await Task.sleep(for: .milliseconds(100)) }